From 3759560f57ac00d56c48e80c027296f9d32fbe83 Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 18 Oct 2026 15:04:24 +0800 Subject: [PATCH 001/258] Implement RDMA transport and local loop testing --- container/oci-image/Dockerfile | 4 +- container/oci-image/collect-libs.sh | 19 + container/oci-image/transfer.py | 10 +- .../R238-rdma-registered-buffer-foundation.md | 58 + doc/backlog/R239-rdma-rpc-transport.md | 113 ++ doc/backlog/R240-rdma-datapath-stream.md | 58 + doc/backlog/R241-rdma-chunk-kv-access.md | 55 + doc/backlog/R242-rdma-fat-client-sdk.md | 61 + doc/backlog/R5-rdma-alloc.md | 321 +----- doc/backlog/backlog.md | 22 +- doc/design/rpc/design-crowdb-rpc-rdma.md | 217 +++- doc/working/plan-rdma-chunk-kv-access.md | 111 ++ doc/working/plan-rdma-datapath-stream.md | 113 ++ doc/working/plan-rdma-fat-client-sdk.md | 111 ++ doc/working/plan-rdma-registered-buffers.md | 120 ++ doc/working/plan-rdma-rpc-transport.md | 225 ++++ doc/working/plan-rdma.md | 231 ++++ lib/crowdb-rpc/ffi/src/buffer.rs | 41 + lib/crowdb-rpc/ffi/src/lib.rs | 2 +- lib/crowdb-rpc/ffi/src/sys.rs | 16 + lib/crowdb-rpc/include/crowdb-rpc/buffer.h | 92 +- lib/crowdb-rpc/include/crowdb-rpc/c_api.h | 11 + .../include/crowdb-rpc/connection.h | 40 +- lib/crowdb-rpc/include/crowdb-rpc/framing.h | 12 +- .../include/crowdb-rpc/server/server.h | 1 + .../crowdb-rpc/transport/epoll/epoll_engine.h | 3 + .../transport/rdma/rdma_capability.h | 92 ++ .../transport/rdma/rdma_transport.h | 114 +- .../crowdb-rpc/transport/socket_transport.h | 53 +- lib/crowdb-rpc/src/buffer.cpp | 152 +++ lib/crowdb-rpc/src/c_api.cpp | 55 + lib/crowdb-rpc/src/server/server.cpp | 29 +- .../src/transport/epoll/epoll_engine.cpp | 48 + .../src/transport/rdma/rdma_capability.cpp | 176 +++ .../src/transport/rdma/rdma_transport.cpp | 1025 +++++++++++++++-- .../src/transport/socket_transport.cpp | 130 +++ lib/crowdb-rpc/tests/buffer_test.cpp | 47 + lib/crowdb-rpc/tests/rdma_capability_test.cpp | 104 ++ lib/crowdb-rpc/tests/rdma_test.cpp | 314 +++++ lib/crowdb-rpc/tests/transport_test.cpp | 29 + tools/rdma/run-local-loop.sh | 168 +++ tools/rdma/run-oci-profile.sh | 54 + tools/rdma/setup-rdma.sh | 260 +++++ 43 files changed, 4474 insertions(+), 443 deletions(-) create mode 100644 doc/backlog/R238-rdma-registered-buffer-foundation.md create mode 100644 doc/backlog/R239-rdma-rpc-transport.md create mode 100644 doc/backlog/R240-rdma-datapath-stream.md create mode 100644 doc/backlog/R241-rdma-chunk-kv-access.md create mode 100644 doc/backlog/R242-rdma-fat-client-sdk.md create mode 100644 doc/working/plan-rdma-chunk-kv-access.md create mode 100644 doc/working/plan-rdma-datapath-stream.md create mode 100644 doc/working/plan-rdma-fat-client-sdk.md create mode 100644 doc/working/plan-rdma-registered-buffers.md create mode 100644 doc/working/plan-rdma-rpc-transport.md create mode 100644 doc/working/plan-rdma.md create mode 100644 lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_capability_test.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_test.cpp create mode 100755 tools/rdma/run-local-loop.sh create mode 100755 tools/rdma/run-oci-profile.sh create mode 100755 tools/rdma/setup-rdma.sh diff --git a/container/oci-image/Dockerfile b/container/oci-image/Dockerfile index cf1bd9e4d..9265e25b2 100644 --- a/container/oci-image/Dockerfile +++ b/container/oci-image/Dockerfile @@ -1,5 +1,5 @@ FROM ubuntu:24.04@sha256:496754492fb28b4d3049432f2ca787449331e23fb14f0dd3fffea86bf5a93eb4 -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates openssh-server util-linux curl tini && rm -rf /var/lib/apt/lists/* \ +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates openssh-server util-linux curl tini libibverbs1 librdmacm1 ibverbs-providers ibverbs-utils && rm -rf /var/lib/apt/lists/* \ && groupadd --system --gid 10001 crowdb \ && useradd --system --uid 10001 --gid 10001 --home-dir /opt/crowdb --shell /bin/sh crowdb \ && echo 'crowdb:crowdb' | chpasswd \ @@ -16,6 +16,7 @@ RUN --mount=type=bind,source=.,target=/staged,ro \ && test "$(cat /staged/VERSION)" = "$PREVIEW_VERSION" \ && mkdir -p /opt/crowdb/etc /opt/crowdb/data /opt/crowdb/run \ && cp -a /staged/bin /staged/lib /staged/ui /opt/crowdb/ \ + && if test -d /staged/etc; then cp -a /staged/etc /opt/crowdb/; fi \ && cp /staged/profile.toml /opt/crowdb/etc/profile.toml \ && cp /staged/entrypoint.sh /opt/crowdb/bin/entrypoint \ && chown -R crowdb:crowdb /opt/crowdb/data /opt/crowdb/run \ @@ -39,6 +40,7 @@ LABEL org.opencontainers.image.title="CROWDB Node" \ org.crowdb.startup.default="local" ENV PATH="/opt/crowdb/bin:${PATH}" \ LD_LIBRARY_PATH="/opt/crowdb/lib" \ + RDMAV_DRIVERS_DIR="/opt/crowdb/lib/libibverbs" \ CROWDB_RUNTIME_ROOT="/opt/crowdb/run" USER root VOLUME ["/opt/crowdb/data"] diff --git a/container/oci-image/collect-libs.sh b/container/oci-image/collect-libs.sh index c678702fd..6f8e8a1ad 100644 --- a/container/oci-image/collect-libs.sh +++ b/container/oci-image/collect-libs.sh @@ -10,6 +10,17 @@ case "${CROWDB_KEEP_DEBUG_INFO:-0}" in esac mkdir -p "$output/bin" "$output/lib" +# libibverbs loads HCA providers (mlx5, efa, ... ) dynamically, so they do +# not appear in ldd output. Keep the provider ABI matched with the pixi +# libibverbs runtime instead of relying on the base image's older providers. +provider_root="$build_root/.pixi/envs/default/lib/libibverbs" +provider_config="$build_root/.pixi/envs/default/etc/libibverbs.d" +if [[ -d "$provider_root" && -d "$provider_config" ]]; then + mkdir -p "$output/lib/libibverbs" "$output/etc/libibverbs.d" + cp -a "$provider_root"/*.so "$output/lib/libibverbs/" + cp -a "$provider_config"/*.driver "$output/etc/libibverbs.d/" +fi + for binary in \ crowdb-monitor crowdb-kv-server crowdb-diskdb crowdb-diskio \ crowdb-chunkdb crowdb-chunk-kv-server crowdb-access-server \ @@ -41,6 +52,12 @@ for binary in "$output"/bin/*; do continue fi resolved_path=$(readlink -f "$path") + # Use Ubuntu's libibverbs/librdmacm in the image so the apt-provided + # providers and config directory remain ABI-matched. The pixi copies + # are build-time dependencies and would otherwise shadow them. + if [[ "$name" == libibverbs.so* || "$name" == librdmacm.so* ]]; then + continue + fi case "$resolved_path" in "$build_root/.pixi/envs/default/lib/"*|"$build_root/target/release/"*) if ! cp -L "$resolved_path" "$output/lib/$name"; then @@ -61,12 +78,14 @@ if [[ ! -f "$output/lib/libcrypto.so.3" ]]; then exit 1 fi for library in "$output"/lib/*; do + [[ -f "$library" ]] || continue patchelf --set-rpath '/opt/crowdb/lib' "$library" done rm "$output/dependencies.txt" if [[ "$debug_info" == stripped ]]; then for artifact in "$output"/bin/* "$output"/lib/*; do + [[ -f "$artifact" ]] || continue strip --strip-debug "$artifact" done fi diff --git a/container/oci-image/transfer.py b/container/oci-image/transfer.py index cd957062d..5ce82c025 100644 --- a/container/oci-image/transfer.py +++ b/container/oci-image/transfer.py @@ -42,7 +42,7 @@ def main(): parser.add_argument("--authfile", type=Path) args = parser.parse_args() archive = args.archive.resolve() - receipt(archive) + verified = receipt(archive) if args.operation == "publish": if args.authfile is None or not args.authfile.is_file() or args.authfile.stat().st_mode & 0o077: parser.error("publication requires a private registry authfile") @@ -56,8 +56,12 @@ def main(): f"docker-archive:{converted}:{args.image}"], check=True) subprocess.run(["docker", "load", "--input", str(converted)], check=True) image = json.loads(subprocess.check_output(["docker", "image", "inspect", args.image], text=True))[0] - if image["Id"] != inspect(archive)["config_digest"]: - raise RuntimeError("imported Docker image config digest differs from the OCI artifact") + # Docker's local image store rebuilds the manifest and therefore gives + # the imported image a new image ID. The config itself is the stable + # part of the OCI artifact that must survive conversion. + expected_config = inspect(archive)["config"] + if image.get("Config") != expected_config: + raise RuntimeError("imported Docker image config differs from the OCI artifact") if __name__ == "__main__": diff --git a/doc/backlog/R238-rdma-registered-buffer-foundation.md b/doc/backlog/R238-rdma-registered-buffer-foundation.md new file mode 100644 index 000000000..77683dd50 --- /dev/null +++ b/doc/backlog/R238-rdma-registered-buffer-foundation.md @@ -0,0 +1,58 @@ + + + +### R238: RDMA — Registered host-buffer foundation + +## Problem + +R5 requires bounded registered host buffers for RDMA, but the current RPC and +access layers do not share a lifetime contract for buffers used by NIC, +DiskIO, SDK callers, and asynchronous completion. An unbounded or prematurely +reused buffer can corrupt a later stream. The foundation must be independent of +the selected RPC transport and must work when RDMA is unavailable. + +## Solution + +1. Define a process and device scoped registered-memory pool with a hard byte + budget and separate control and data capacity classes: 64 KiB, 256 KiB, and + 1 MiB. +2. Define the ownership states `available`, `posted_receive`, + `receive_in_flight`, `send_in_flight`, `disk_owned`, `caller_owned`, and + `quarantined`. A buffer has one owner and one release transition. +3. Expose an opaque buffer lease that records capacity, actual length, request + ID, stream sequence, logical offset, and release state. Release returns a + caller-owned buffer to its class pool only after all device access ends. +4. Make exhaustion bounded: producers wait or return backpressure, and no + replacement allocation bypasses the registered-memory budget. Unknown + completion state quarantines the buffer until connection quiescence. +5. Keep registration optional. A build or host without RDMA retains the TCP + pool and does not require an RDMA device at runtime. + +## Dependencies + +- R5 remains the umbrella registered-memory goal. +- R4 bounded memory pool supplies the budget and admission principles. +- The output is consumed by R239 RDMA RPC transport and R240/R241 data paths. + +## Acceptance + +- Given a configured byte budget, when all classes are allocated, the next + allocation returns bounded backpressure and never exceeds the budget. + Invariant: registered bytes are bounded. Unit test. +- Given a buffer in each ownership state, when the valid owner releases it, + the next state is deterministic and no second owner can release it. + Invariant: single ownership. Unit test. +- Given an uncertain completion, when the connection closes, the buffer is + quarantined and is not returned to a new stream. Invariant: no reuse before + quiescence. Integration test. +- Given RDMA is unavailable, when the pool is initialized, TCP allocation + remains usable without an RDMA runtime. Invariant: optional capability. + Integration test. + +Run: + +```bash +pixi run -- cargo fmt --all -- --check +pixi run -- cargo test -p crowdb-rpc --all-targets +pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings +``` diff --git a/doc/backlog/R239-rdma-rpc-transport.md b/doc/backlog/R239-rdma-rpc-transport.md new file mode 100644 index 000000000..9072175e8 --- /dev/null +++ b/doc/backlog/R239-rdma-rpc-transport.md @@ -0,0 +1,113 @@ + + + +### R239: RPC — RC SEND/RECV transport with TCP fallback + +## Problem + +CROWDB RPC has a common TCP transport but no usable RDMA transport. RDMA must +reuse RPC correlation and framing while combining epoll wakeups with bounded CQ +polling. It must be optional: unsupported endpoints must use TCP without +repeated RDMA attempts, and failures before payload submission must fall back +without changing request semantics. + +## Solution + +1. Add an RC SEND/RECV transport behind the common RPC `Transport` interface. + Data messages use one 1 KiB control SGE and one selected data SGE; no CPU + concatenation is introduced. +2. Use service discovery for coarse RDMA capability and TCP handshake for the + current CM endpoint, device/GID identity, capability generation, protocol + version, and supported data classes. Local configuration and probing can + disable RDMA before CM setup. +3. Register the RDMA completion and CM event descriptors with the existing + epoll engine. Epoll wakes the worker; the RPC library acknowledges events, + re-arms notifications, polls bounded CQ batches, and dispatches completions. +4. Select RDMA as the preferred path for one logical connection and TCP as its + one-way fallback. A TCP connection is never upgraded to RDMA. The shared + request ID is used for idempotent or duplicate-suppressed retry. +5. Define CM, QP, receive posting, CQ error, drain, and shutdown states. A + partial or uncertain RDMA stream fails as one operation and is never joined + with TCP bytes. +6. Provide OCI image test profiles. The image contains `libibverbs`, + `librdmacm`, and the provider libraries; the host supplies the kernel RDMA + driver and HCA devices. The first hardware profile uses two containers with + `--network=host`, `--privileged`, and `--ulimit memlock=-1`, pinning the + client to HCA 0 and DiskIO to HCA 1 through transport configuration. A + later restricted profile maps only the required `/dev/infiniband` devices + and uses `IPC_LOCK` plus a bounded memlock limit. +7. Keep three E2E topology profiles. The TCP-only profile exposes no RDMA + devices and verifies that capability discovery avoids all RDMA attempts. The + two-HCA profile runs client and DiskIO containers over the two connected + physical ports. The shared-HCA profile runs multiple logical node + containers on one host and verifies multiple QPs and fallback, while + explicitly not claiming physical NIC isolation. More than two physical + nodes require a shared RDMA fabric, switch, SR-IOV VFs, or a software RDMA + device; additional containers alone do not create physical RDMA nodes. + +## Dependencies + +- R238 registered host-buffer foundation. +- `design-crowdb-rpc.md` common frame and correlation contract. +- Existing TCP epoll worker and transport error semantics. +- R240 and R241 consume this transport. + +## Acceptance + +- Given both peers advertise RDMA, when the TCP handshake completes, the + client establishes the negotiated RC path and sends a valid two-SGE frame. + Invariant: shared RPC framing. Integration test. +- Given no RDMA capability or local RDMA configuration, when a connection is + created, TCP is selected without an RDMA attempt. Invariant: optional RDMA. + Integration test. +- Given RDMA setup fails before payload submission, when an idempotent request + is retried, TCP returns one correlated response. Invariant: one-way fallback. + Integration test. +- Given an RDMA payload partially completes, when the connection fails, no TCP + bytes are appended and uncertain buffers remain quarantined. Invariant: + partial transfer isolation. Integration test. +- Given CQ and CM events arrive, when epoll wakes the worker, all currently + available completions are drained without a private polling thread. + Invariant: event/poll composition. Integration test. +- Given an OCI image with no RDMA devices, when the TCP-only profile starts, + service discovery selects TCP without attempting RDMA. Invariant: optional + capability. E2E test. +- Given two connected host HCA ports, when the two-HCA profile starts client and + DiskIO containers, each endpoint uses its configured local HCA and completes + an RDMA stream. Invariant: explicit device selection. E2E test. +- Given multiple containers sharing one HCA, when the shared-HCA profile opens + multiple logical connections, each QP remains correlated and bounded, while + the test reports shared hardware rather than physical node isolation. + Invariant: topology declaration matches test claim. E2E test. +- Given a prepared OCI runtime with the required RDMA devices and memlock + limit, when the RPC flow runs as a non-root application user, the container + completes CM/QP/send/receive/repost without invoking `sudo`, loading modules, + moving links, or entering host namespaces. Invariant: privileged host setup + is separated from unprivileged application execution. E2E test. +- Given an application linked against `crowdb-rpc`, when deployment provides + verbs/rdmacm libraries, HCA device access, memlock, and peer reachability, + the application uses the ordinary RPC API and the library selects RDMA or + TCP without setup-script calls or namespace operations in application code. + Invariant: RDMA deployment concerns stay outside the library consumer. + Integration test and documented runtime contract. + +Run: + +```bash +pixi run -- cargo fmt --all -- --check +pixi run -- cargo test -p crowdb-rpc --all-targets +pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings +``` + +The initial local command shape is: + +```bash +docker run --rm --network=host --privileged --ulimit memlock=-1 \ + -e CROWDB_RDMA_DEVICE=mlx5_0 \ + -e CROWDB_RDMA_PORT=1 \ + crowdb:test +``` + +The second endpoint uses its own device and local address. The restricted +container profile is a follow-up hardening step after the two-HCA profile +passes. diff --git a/doc/backlog/R240-rdma-datapath-stream.md b/doc/backlog/R240-rdma-datapath-stream.md new file mode 100644 index 000000000..776339afd --- /dev/null +++ b/doc/backlog/R240-rdma-datapath-stream.md @@ -0,0 +1,58 @@ + + + +### R240: Access server — RDMA datapath stream + +## Problem + +The S3/HTTP and DiskIO path currently has no RDMA stream contract. The path +must preserve existing ChunkIO aggregation, use a 1 MiB data capacity, and +transfer ownership from HTTP input to DiskIO without an intermediate CPU +payload copy. DiskIO must not recycle a receive buffer while asynchronous disk +I/O still uses it. + +## Solution + +1. Feed already aggregated ChunkIO buffers to R239. RDMA does not aggregate or + flush; it preserves actual payload length and stream sequence. +2. Use a 1 KiB control block plus a 1 MiB data capacity for the datapath. A + logical object is an ordered sequence of messages when larger than one + message. +3. Let Access Server fill its registered send buffer from the HTTP path. Let + DiskIO own its registered receive buffer after CQ completion and submit that + buffer directly to asynchronous disk I/O. +4. Repost a DiskIO receive buffer only after disk completion. Apply bounded + receive-window backpressure when DiskIO or the SDK consumer is slow. +5. Preserve R239 fallback: before payload submission the operation may use TCP; + after partial RDMA transfer the stream fails without TCP byte splicing. + +## Dependencies + +- R238 registered buffers. +- R239 RDMA RPC transport. +- Existing ChunkIO aggregation, Access Server S3, DiskIO asynchronous I/O, and + integrity contracts. + +## Acceptance + +- Given an aggregated HTTP upload, when it is sent to DiskIO, the receiver + writes the registered buffer directly and reposts it only after disk + completion. Invariant: buffer lifetime covers disk I/O. E2E test. +- Given an object larger than 1 MiB, when it is transferred, ordered messages + reconstruct the exact object using sequence and logical offset. Invariant: + stream ordering. E2E test. +- Given a full DiskIO write queue, when receive buffers are consumed, sender + backpressure remains bounded and no replacement buffer is allocated. + Invariant: bounded receive window. Integration test. +- Given an RDMA failure after payload submission, when the operation ends, no + TCP bytes are appended to the partial object. Invariant: fallback isolation. + E2E test. + +Run: + +```bash +pixi run -- cargo fmt --all -- --check +pixi run -- cargo test -p crowdb-access-server --all-targets +pixi run -- cargo test -p crowdb-diskio-client --all-targets +pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings +``` diff --git a/doc/backlog/R241-rdma-chunk-kv-access.md b/doc/backlog/R241-rdma-chunk-kv-access.md new file mode 100644 index 000000000..46740045d --- /dev/null +++ b/doc/backlog/R241-rdma-chunk-kv-access.md @@ -0,0 +1,55 @@ + + + +### R241: Access server — RDMA Chunk-KV integration + +## Problem + +Access Server and Chunk-KV need RDMA for access operations, but their message +sizes differ from the S3/DiskIO datapath. Metadata is usually small, while an +aggregated result may be larger and must be returned as multiple messages +without changing access semantics. + +## Solution + +1. Route Access Server to Chunk-KV through R239 while retaining the existing + access planner, request correlation, generation checks, and error mapping. +2. Configure 64 KiB data capacity for metadata and 256 KiB for aggregated + Chunk-KV results. The transport is created with these defaults from config; + the access/aggregation layer selects the class for each operation. +3. Represent a result larger than its selected capacity as ordered messages + with request ID, stream ID, sequence, logical offset, and end/error flags. +4. Use the same TCP/RDMA request identity and one-way RDMA-to-TCP fallback. + Retry only when the existing operation is idempotent or duplicate-suppressed. +5. Keep aggregation above RDMA. The RDMA layer does not add flush timers, + coalescing, or a second buffer-copy path. + +## Dependencies + +- R238 and R239. +- Existing Chunk-KV client/server and Access Server access semantics. +- Existing ChunkIO aggregation and request-number system. + +## Acceptance + +- Given metadata smaller than 64 KiB, when Access Server selects RDMA, the + result matches TCP and uses the metadata class. Invariant: semantic parity. + Integration test. +- Given an aggregated Chunk-KV response larger than 256 KiB, when it is + returned, ordered messages reconstruct the exact response. Invariant: + bounded message capacity and ordering. Integration test. +- Given an RDMA setup failure before payload, when an idempotent access request + is retried, TCP returns one response with the same request identity. + Invariant: shared correlation and fallback. Integration test. +- Given a partial RDMA response, when the connection fails, the access result + is failed and no TCP suffix is accepted. Invariant: no mixed transport + result. Integration test. + +Run: + +```bash +pixi run -- cargo fmt --all -- --check +pixi run -- cargo test -p crowdb-chunk-kv-client --all-targets +pixi run -- cargo test -p crowdb-access-server --all-targets +pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings +``` diff --git a/doc/backlog/R242-rdma-fat-client-sdk.md b/doc/backlog/R242-rdma-fat-client-sdk.md new file mode 100644 index 000000000..64ded00cc --- /dev/null +++ b/doc/backlog/R242-rdma-fat-client-sdk.md @@ -0,0 +1,61 @@ + + + +### R242: Access SDK — RDMA fat client and buffer leases + +## Problem + +The current Access Server boundary adds an application-to-server socket and +async hop. A fat client should expose access semantics directly while reusing +the access core and allowing the same RDMA path used by the server. Returned +buffers need an explicit lifetime so the SDK can avoid copies without +recycling memory while the caller still reads it. + +## Solution + +1. Split the access core from the HTTP frontend and add a native SDK frontend. + Both frontends use the same planner, authorization, generation, aggregation, + Chunk-KV, DiskIO, and TCP/RDMA selection semantics. +2. Define an FFI-safe opaque buffer lease containing request identity, sequence, + logical offset, actual length, capacity class, and release operation. The + caller owns a completed host lease until release; release returns it to the + registered pool. +3. Deliver completion from the C++ RPC worker to Rust without exposing verbs + objects. Rust must not dereference a released lease, and C++ must not + recycle a caller-owned lease from a completion callback. +4. Support host memory first. A caller requesting GPU delivery receives host + leases and performs an explicit host-to-GPU copy; a future allocator can + supply GPU-capable memory behind the same lease contract. +5. Preserve bounded admission, cancellation, shutdown, and RDMA-to-TCP + fallback for SDK operations. + +## Dependencies + +- R238, R239, and R241. +- Existing Access Server access-core and FFI conventions. +- R220 remains a separate Dataset SDK plan; this requirement supplies the + transport and lease contract it may reuse. + +## Acceptance + +- Given a native SDK access request, when the operation completes through RDMA, + the caller receives the exact ordered host buffers and releases each lease. + Invariant: caller-owned lifetime. Integration test. +- Given the caller holds all available leases, when another buffer is needed, + admission applies bounded backpressure and resumes after release. Invariant: + bounded SDK memory. Integration test. +- Given a request with host-to-GPU delivery selected, when the operation + completes, host data remains valid until the explicit copy completes and + release follows the copy. Invariant: copy dependency lifetime. Integration test. +- Given RDMA is unavailable or fails before payload, when the same SDK request + executes, TCP produces the same access result and request identity. + Invariant: frontend transport parity. Integration test. + +Run: + +```bash +pixi run -- cargo fmt --all -- --check +pixi run -- cargo test -p crowdb-rpc-ffi --all-targets +pixi run -- cargo test -p crowdb-access-server --all-targets +pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings +``` diff --git a/doc/backlog/R5-rdma-alloc.md b/doc/backlog/R5-rdma-alloc.md index 4e13895d8..7d517487b 100644 --- a/doc/backlog/R5-rdma-alloc.md +++ b/doc/backlog/R5-rdma-alloc.md @@ -1,288 +1,85 @@ -### R5: RDMA — Registered host buffers and scheduled client writes +### R5: RDMA — Host-buffer RPC and data transport ## Status -Design proposal, not implemented. The original low-priority allocator seam -remains part of this requirement; the tree-specific RDMA `BlockPageStore` -medium is still not started. Its absence does not prevent a separate host-memory -transfer prototype. No acceptance result or production throughput is claimed. +This umbrella requirement is decomposed into R238 through R242. The child +requirements are independently implementable and retain this requirement's +registered-host-buffer, RDMA RPC, datapath, Chunk-KV, and fat-client goals. ## Problem -GPU compute needs large logical object ranges from storage. Without usable -GPUDirect RDMA, data must land in client host memory before a Host-to-Device -CUDA copy. Ordinary TCP RPC can add CPU protocol work and intermediate copies; -RDMA permits the RNIC to transfer payload directly between registered buffers. -It does not remove disk I/O, checksum work, network traffic, or the CUDA copy. - -The original `buffer::allocate` seam in -`lib/crowdb-tree/include/crowdb-tree/buffer.h` is intended to permit an allocator -other than `std::malloc`. Registered-memory allocation additionally needs an -owner, registration per RDMA device/protection domain, byte limits, and a -lifetime extending beyond every asynchronous operation. It cannot be modeled -as merely returning a different pointer from malloc. - -The deployment has two useful paths: - -- **AccessServer relay:** DiskIO transfers to AccessServer host memory; - AccessServer writes to the client's host buffer. RC connection counts are - controlled, and the gateway can remain the mandatory network boundary. -- **DiskIO direct write:** AccessServer forwards an authorized destination - descriptor and read plan; DiskIO reads, validates, and writes directly to the - client. This removes one payload transfer and the gateway memory bounce. - RC is sufficient when connections are bounded; DC is an optional scaling - mechanism, not a prerequisite for direct transfer. - -The core transport contract is [RPC RDMA](../design/rpc/design-crowdb-rpc-rdma.md). -This design is independent of cuObject: registered allocation, span scheduling, -relay/direct topology, and completion ownership do not require its protocol or -libraries. A native client library may use RC or DC directly. - -**cuObject is one optional adapter**, described by -[R170](R170-s3-cuobject-rdma.md) and -[Access Server accelerated transfer](../design/access-server/s3/design-crowdb-access-s3-rdma.md). -Those documents constrain only the cuObject option; their no-gateway-payload -invariant does not prohibit the generic relay path defined here. No transport -adapter may silently switch paths after a partially executed operation. +CROWDB needs an RDMA path for RPC and data transfer while retaining TCP as a +one-way fallback. The path must move bounded registered host buffers through +Access Server, Chunk-KV, DiskIO, and the native fat client without an +intermediate CPU payload copy. The caller must receive explicit buffer leases, +and unsupported RDMA must not affect TCP operation. The formal architecture is +defined in [`design-crowdb-rpc-rdma.md`](../design/rpc/design-crowdb-rpc-rdma.md). ## Solution -### 1. Registered allocation and ownership - -1. Preserve the tree allocator seam. Implement transport buffers through the - native `lib/crowdb-rpc` / `crowdb-rpc-ffi` ownership boundary; DiskIO and - AccessServer borrow owners rather than passing naked pointers across FFI. - Tree-medium integration remains conditional on that medium being designed. -2. Maintain bounded, reusable registered host-buffer pools in participating - processes. Distinguish CUDA pinned host memory from RDMA memory registration: - they serve different devices and neither automatically establishes the other. - Client buffers used for asynchronous CUDA copy need both. AccessServer and - DiskIO staging buffers need RDMA registration but not CUDA registration. -3. Register a relay buffer separately for each participating RDMA device/PD. - A buffer may use the same virtual allocation for backend receive and frontend - send, with distinct local registrations. Do not forward local handles/lkeys - to another process or assume keys are interchangeable across devices. -4. Track pool byte usage, in-flight transfers, queue depth, and CUDA-copy users. - Reuse pools to amortize registration; apply admission/backpressure before - accepting work that exceeds configured limits. No new hot-path lock is - selected by this design. - -### 2. Control request and RC relay - -The client prepares its destination and sends an ordinary authenticated request -with object/range, operation ID, capacity, and destination descriptor. The -request is distinct from an RDMA Read: a Read needs a remote address/rkey and -reads already prepared remote memory; it does not trigger application disk I/O. - -```text -Compute client AccessServer DiskIO - | request + destination | | - |---------------------->| authenticate/plan | - | |--- prepare range ------->| - | | | disk read/checksum - | |<-- ready source span ----| - | |=== backend RDMA Read ===>| registered source - | |<====== payload ==========| - |<== frontend Write ====| registered relay buffer | - | completion + result | | - | CUDA copy -> kernel | | -``` - -5. `crowdb-access-server` authenticates and resolves a stable logical generation - using existing S3/chunk-reader semantics. `crowdb-diskio` reads into a local - registered buffer and validates the relevant storage integrity metadata - before declaring the source ready. Merely computing a new checksum without - comparing trusted expected integrity information does not validate old data. -6. AccessServer waits for successful backend Read completion before submitting - frontend Write. DiskIO retains the source until backend transfer completion - is confirmed through the control protocol. AccessServer retains its relay - buffer until the frontend send completion. A separate outbound copy is not - needed if that allocation is registered for the sending RNIC. -7. Use a defined notification mechanism. With Write with Immediate, the client - posts Receive WQEs before issuing the request; the Write WR selects the - destination address, while the Receive WQE provides notification capacity. - With ordinary Write, use an explicit correctly ordered completion protocol. - Storage submits the Write WR, not the receiving client. - -At verbs level, an outbound Write names a local SGE (`addr`, `length`, `lkey`) -and remote destination (`remote_addr`, `rkey`). RC can submit it through -`ibv_post_send()` with `IBV_WR_RDMA_WRITE` or -`IBV_WR_RDMA_WRITE_WITH_IMM`. Submission success means queued, not completed. -[Public API documentation](https://man7.org/linux/man-pages/man3/ibv_post_send.3.html) - -### 3. Optional direct write and scheduler - -```text -Compute client AccessServer DiskIO A/B - | request + destination | | - |---------------------->| authenticate/plan | - | |-- authorized spans ---->| - |<====================== direct RC/DC Write ======| - | |<-- span completions ----| - |<-- aggregate result --| | -``` - -8. The coordinator groups source spans by DiskIO node and sends bounded batches - rather than one control RPC per fragment. Each task includes operation, - tenant, generation, logical source, destination offset, exact length, expiry, - and authority for that span. Storage physical layout remains internal. - Mirrored reads choose a healthy replica; EC reads reconstruct logical bytes - before writing object offsets. Metadata does not expose parity as object data. -9. Each DiskIO reads and validates its spans, then submits multiple Writes to - one destination allocation. For a 4 MiB destination, node A may write two - 1 MiB spans at offsets 0 and 1 MiB, and node B at offsets 2 and 3 MiB. - Every WR specifies its own source and `destination_base + offset`. Check - arithmetic overflow, MR bounds, operation bounds, and non-overlap before - submission. A shared rkey may cover all these offsets; application span - authority is still narrower than the MR's hardware permissions. -10. Reuse long-lived RC QPs when endpoint counts fit configured resource limits. - DC descriptors contain endpoint/path information as well as memory access - information; local DCI/channel handles stay local. On the sending DiskIO, - batching by destination client/DCT can reduce target changes. Grouping by - source node alone reduces control work, not necessarily DCI target changes. - Use bounded fairness so a busy destination cannot monopolize resources. -11. Direct writes require an allowed, reachable RDMA path from DiskIO to client. - DC does not bypass routing, authentication, or network isolation. If policy - requires all payload to cross AccessServer, select the relay path explicitly. - Keep the scheduler and buffer ownership independent of adapters. A native - RC/DC adapter uses its own defined destination descriptor; if cuObject is - selected, follow R170's opaque-descriptor and distributed validation - contracts. Do not assume these descriptors or protocols are interchangeable. - -DC is publicly exposed through rdma-core's mlx5 extensions and UCX, but depends -on supported hardware/firmware. It is not a portable replacement for RC on -arbitrary RNICs. DCI availability, target changes, handshake configuration, -and completion draining can all affect cost; no fixed microsecond overhead is -assumed. [DC API](https://github.com/linux-rdma/rdma-core/blob/master/providers/mlx5/man/mlx5dv_wr_post.3.md), -[UCX scheduling and handshake configuration](https://networking-docs.nvidia.com/hpcxum/2.50/unified-communication-x-framework-library). - -### 4. Completion, failure, and lifetime - -12. Track every span independently; writes from different nodes may complete in - any order. Return full-operation success only after all required spans and - transfer notifications succeed. The client starts CUDA copy only after the - selected interval is known complete, and starts its kernel after the CUDA - dependency. Release/reuse client host storage only after the copy completes. -13. Preserve source/MR/channel ownership until local DMA access has ended. - A timeout invalidates the request but does not prove that remote Writes have - stopped. Cancellation, connection teardown/drain, or access revocation must - establish quiescence before destination reuse. An operation ID in a late - notification alone cannot prevent a stale RNIC write into reused memory. -14. Bind delegation to tenant, generation, direction, interval, nonce, and - expiry. Never treat address/rkey as object authorization or log capabilities. - Rejection before submission may choose a separately negotiated fallback; - after partial transfer fail the operation and keep the destination invalid - until quiescence. Do not splice fallback bytes into partial RDMA results. - -### 5. Network cost and production verification - -The production scenario discussed here is **40 GB/s payload throughput**, not -40 Gb/s and not the development machine's rate. With 1 MiB transfers this is -about 38,147 transfers/s and 26.2 microseconds of ideal payload transmission -per transfer. These are arithmetic bounds, not DC measurements; line rate must -also accommodate headers and other overhead. Multiple in-flight operations may -hide some setup latency, so serialized per-request timing is not a throughput -prediction. - -15. Measure relay and direct paths under the same production workload. Relay - needs approximately 40 GB/s payload on each leg and adds roughly 40 GB/s - relay-memory writes plus 40 GB/s reads, excluding checksum and other work. - Separate physical backend/frontend fabrics isolate link consumption but - still converge at AccessServer NICs, PCIe, and memory. On one shared fabric, - the extra internal leg consumes shared resources; exact contention depends - on paths and topology, not an automatic halving of every link's bandwidth. -16. Pipeline bounded chunks so backend receive, frontend send, and client CUDA - processing overlap. Measure storage throughput, checksum CPU cost, NIC and - PCIe utilization, memory bandwidth, registered bytes, CQ/queue pressure, - throughput and p99 completion latency. Compare fixed-target RC, fixed-target - DC, and DC changing target per 1 MiB at representative concurrency. Set an - environment-specific SLO before claiming the 40 GB/s goal is met. - -As of 2026-10-04, the development host reports Huawei OEM ConnectX-5, -board ID `HUA0000000004`, firmware `16.21.3002`, MLNX_OFED -`24.10-5.1.6.1`, kernel `6.8.0-142-generic`, and UCX `1.18.0`. -Its two active Ethernet ports report 25 Gb/s. UCX exposes RC/UD but no DC -transport; a minimal temporary DCI creation probe returned `EOPNOTSUPP` on both -RDMA devices. This is a limited probe, not proof of OEM disablement or a firmware -root cause. Recheck creation parameters/capabilities and the OEM PSID support -matrix before choosing a matching firmware update. Production hardware remains -separate. The local UCX build reports `--without-cuda`; host staging experiments -must therefore manage the CUDA copy separately rather than assume UCX GPU support. +1. Implement the registered host-buffer foundation in + [R238](R238-rdma-registered-buffer-foundation.md), including 64 KiB, + 256 KiB, and 1 MiB classes, bounded ownership, and quarantine. +2. Implement the common RC SEND/RECV RPC transport in + [R239](R239-rdma-rpc-transport.md), using service discovery plus TCP + handshake endpoint exchange, epoll wakeups, bounded CQ polling, and + RDMA-to-TCP fallback. +3. Implement the S3/HTTP/DiskIO stream in + [R240](R240-rdma-datapath-stream.md), reusing existing ChunkIO aggregation + and 1 MiB datapath buffers. +4. Implement Access Server to Chunk-KV integration in + [R241](R241-rdma-chunk-kv-access.md), using 64 KiB metadata and 256 KiB + aggregated result messages. +5. Implement the native fat client and FFI lease API in + [R242](R242-rdma-fat-client-sdk.md), with host memory first and explicit + host-to-GPU copy for the initial GPU simulation. + +The base transfer mechanism is SEND/RECV. Large client buffers are split into +ordered messages with request ID, sequence, and logical offset. RDMA READ and +WRITE are outside this requirement. ## Dependencies -- Existing `lib/crowdb-rpc`, `crowdb-rpc-ffi`, `crowdb-diskio`, - `crowdb-access-server`, and chunk-reader ownership/integrity contracts. -- [R4 bounded mempool](R4-bounded-mempool.md): coordinate budgets without making - RDMA registrations an unbounded side pool. -- cuObject and R170 are optional, not prerequisites for this requirement. Only - selecting that adapter introduces its libraries, support matrix, and base S3 - milestone dependencies. Native RC/DC development can proceed independently. -- RDMA-capable devices and compatible drivers on participating hosts; CUDA - pinned-host support on compute clients. GPUDirect is a future optional path, - not a prerequisite. Unsupported builds retain the ordinary TCP service. -- Tree-specific RDMA `BlockPageStore` remains future work; preserve the allocator - seam without claiming that transport registration implements that medium. +- R4 bounded memory pool principles. +- Existing common RPC framing and request-number system. +- Existing ChunkIO aggregation, Access Server, Chunk-KV, DiskIO, and FFI + ownership contracts. +- R170 remains an optional cuObject adapter and must reuse the stream contract. ## Acceptance -- Given bounded registered pools, when allocations reach their limits or a - device registration fails, assert admission fails cleanly and all completed - owners return resources without altering the tree allocator seam. Invariant: - bounded allocation and exception-safe ownership. Integration test. -- Given a relay with two RDMA devices, when DiskIO prepares valid bytes and - AccessServer transfers them, assert separate registrations work and frontend - submission follows backend completion without an extra outbound memcpy. - Invariant: device-specific registration and ordered relay. Integration test. -- Given corrupted disk bytes and trusted checksum metadata, when DiskIO prepares - a read, assert failure before publishing ready or submitting client Write. - Invariant: validated logical bytes. Integration test. -- Given prepared notification capacity, when storage uses Write with Immediate - or the chosen ordinary-Write completion protocol, assert exact destination - bytes and completion before client consumption. Invariant: submission is not - completion. E2E test. -- Given four 1 MiB spans on two DiskIO nodes, when each executes a grouped task - with multiple Writes, assert exact disjoint offsets, one coherent generation, - mirror/EC logical semantics, and success only after all spans complete in any - order. Invariant: complete logical range assembly. E2E test. -- Given malformed, overlapping, overflowing, expired, replayed, wrong-tenant, - or wrong-generation spans, when delegation is checked, assert rejection before - submission and no capability leakage. Invariant: bounded authority. Integration test. -- Given network policy forbidding DiskIO-client access, when a request selects - its transport, assert explicit generic relay or rejection, with no hidden - cuObject relay. Invariant: policy and transport-contract preservation. E2E test. -- Given an in-flight timeout, late Write, partial failure, shutdown, or pending - CUDA copy, when cleanup runs, assert no buffer/MR reuse before quiescence, - no partial fallback mixing, and no kernel consumption of incomplete data. - Invariant: lifetime covers every device access. E2E test. -- Given supported RC/DC production equipment, when running fixed and rotating - targets with 1 MiB transfers, assert configured pool/queue bounds and record - throughput, p99, and per-leg resource usage against the agreed SLO. Invariant: - performance claims are measured, not inferred from API availability. E2E test. +- Given a host with no RDMA capability, when the services start, TCP remains + usable and no RDMA attempt is required. Invariant: optional transport. + Integration test. +- Given RDMA capability and configured registered memory, when an access stream + runs, all buffers remain within the budget and are released only by their + owner. Invariant: bounded ownership. E2E test. +- Given an RDMA setup failure before payload submission, when an idempotent + request is retried, TCP returns the same logical result. Invariant: one-way + fallback. E2E test. +- Given a result larger than its selected class, when it is split into ordered + messages, the SDK, Chunk-KV, or DiskIO consumer reconstructs exact bytes. + Invariant: stream equivalence. E2E test. +- Given an SDK caller holds returned leases, when the pool is exhausted, new + work applies bounded backpressure and resumes after release. Invariant: + caller-controlled lifetime. Integration test. ## Open Questions -- Which deployments require gateway payload isolation, and which permit - authenticated DiskIO-client direct writes? This selects relay versus direct. -- What is the first implementation boundary: a generic internal RC prototype - or an external client library? Implement the shared buffer/scheduler core - independently; native RC/DC and cuObject are alternative adapters, and - cuObject is selected only when its client compatibility is needed. -- Which integrity algorithm, client notification protocol, and quiescence - mechanism are selected? These must fit existing storage and transport semantics. -- Which production NICs, topology, concurrent clients, DCI budget if enabled, - and p99 target define the 40 GB/s acceptance environment? +- Which service-discovery record supplies the coarse RDMA capability and + capability generation? The TCP handshake remains the authoritative dynamic + endpoint exchange. +- Which deployments enable the RDMA transport by default after hardware + capability is detected? Unsupported deployments remain TCP-only. -Required implementation gates (not run for this documentation-only change): +Run: ```bash -pixi run test-rpc-ffi -pixi run test-diskio-ct -pixi run -- cargo test -p crowdb-access-server --all-targets pixi run -- cargo fmt --all -- --check -pixi run rs-lint -pixi run tree-lint +pixi run -- cargo test -p crowdb-rpc --all-targets +pixi run -- cargo test -p crowdb-access-server --all-targets +pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings ``` diff --git a/doc/backlog/backlog.md b/doc/backlog/backlog.md index fe4879e96..e74742514 100644 --- a/doc/backlog/backlog.md +++ b/doc/backlog/backlog.md @@ -32,6 +32,16 @@ live in the linked document; implementation follows - **[R170](R170-s3-cuobject-rdma.md)** — optional cuObject RDMA data plane. **Area:** access server / S3 / DiskIO / RDMA. **Complexity:** High. **Status:** Deferred until the basic TCP S3 path is stable and measured. +- **[R240](R240-rdma-datapath-stream.md)** — RDMA S3/HTTP/DiskIO stream using + existing ChunkIO aggregation. **Area:** access server / S3 / DiskIO / RDMA. + **Complexity:** High. **Dependencies:** R238 and R239. +- **[R241](R241-rdma-chunk-kv-access.md)** — RDMA Access Server to Chunk-KV + transport with bounded metadata and aggregate messages. **Area:** access + server / Chunk-KV / RDMA. **Complexity:** High. **Dependencies:** R238 and + R239. +- **[R242](R242-rdma-fat-client-sdk.md)** — native fat-client access SDK and + FFI buffer leases. **Area:** access server / SDK / RDMA. **Complexity:** High. + **Dependencies:** R238, R239, and R241. - **[R185](R185-access-iceberg-cache-invalidation.md)** — bounded Iceberg cache and invalidation. **Area:** access server / Iceberg / Group 0 / Chunk-KV. **Complexity:** Medium. **Status:** Deferred pending focused cache measurements. @@ -96,9 +106,15 @@ live in the linked document; implementation follows - **[R4](R4-bounded-mempool.md)** — bounded memory pool for tree allocations. **Area:** crowdb-tree engine. **Complexity:** Medium. -- **[R5](R5-rdma-alloc.md)** — RDMA-registered host-buffer allocation. - **Area:** crowdb-tree / RDMA. **Complexity:** High. **Status:** Design - proposal; blocked on an RDMA backend. +- **[R5](R5-rdma-alloc.md)** — RDMA host-buffer RPC and data transport umbrella. + **Area:** RPC / access server / Chunk-KV / DiskIO / SDK / RDMA. + **Complexity:** High. **Status:** Decomposed into R238–R242. +- **[R238](R238-rdma-registered-buffer-foundation.md)** — bounded registered + host-buffer classes, ownership, leases, and quarantine. **Area:** RPC / RDMA. + **Complexity:** High. **Dependencies:** R4 and R5. +- **[R239](R239-rdma-rpc-transport.md)** — RC SEND/RECV RPC transport with + epoll/CQ integration and TCP fallback. **Area:** RPC / RDMA. **Complexity:** + High. **Dependencies:** R238. - **[R60](R60-tree-scan-sibling-leaf-readahead.md)** — sibling-leaf readahead on cold scans. **Area:** crowdb-tree / scan / DiskIO. **Complexity:** Medium. **Status:** Deferred pending cold file/block-backed measurements. diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index 4c6bfca4b..ade470eba 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -1,66 +1,183 @@ -# CROWDB - Design: RPC RDMA Transport +# CROWDB - Design: RDMA RPC and stream transport -Depends on: [`design-crowdb-rpc.md`](design-crowdb-rpc.md) (root — `Transport` interface, `Connection`, `OutFrame`, `Buffer`, `FrameParser`) -Satisfies: `design-crowdb-rpc.md` §2 (transport interface + I/O loop decisions) +Depends on: [`design-crowdb-rpc.md`](design-crowdb-rpc.md) for common RPC +framing, request correlation, connection, buffer, and worker concepts. -RDMA is the target transport for the diskio data path — kernel bypass, -no syscall per send, pre-registered memory pools. RDMA is Linux-only -(no RNICs on macOS); the build gates it behind `#ifdef __linux__` + a -CMake `CROWDB_RPC_HAVE_RDMA` flag. +This design defines the RDMA transport used by CROWDB RPC and the stream +datapath. It uses RC SEND/RECV with registered host buffers. TCP is the +fallback transport for the same logical connection; RDMA never upgrades an +established TCP connection. ## Table of Contents -- [1. RdmaTransport](#1-rdmatransport) -- [2. CQ Poll Loop](#2-cq-poll-loop) -- [3. Connection Setup](#3-connection-setup) +- [1. Scope and goals](#1-scope-and-goals) +- [2. Endpoint discovery and path selection](#2-endpoint-discovery-and-path-selection) +- [3. Message and aggregation model](#3-message-and-aggregation-model) +- [4. Registered buffers and ownership](#4-registered-buffers-and-ownership) +- [5. Event and completion model](#5-event-and-completion-model) +- [6. Failure and fallback](#6-failure-and-fallback) +- [7. Service interactions](#7-service-interactions) +- [8. Tunables](#8-tunables) +- [9. Correctness invariants](#9-correctness-invariants) +- [10. Scope boundary](#10-scope-boundary) ---- +## 1. Scope and goals -## 1. RdmaTransport +RDMA removes kernel socket processing and CPU payload copies between registered +buffers. It does not change RPC correlation, access authorization, ChunkIO +aggregation, or DiskIO durability. The transport provides RC connection setup +through RDMA CM, two-sided SEND/RECV, registered host-buffer pools, event +wakeups through the existing epoll engine, bounded CQ polling, and completion +delivery to the C++ RPC layer and FFI callers. -`RdmaTransport` holds an `ibv_context`, protection domain (`ibv_pd`), -and two `RdmaBufferPool`s — one for send, one for recv — both -pre-registered at construction. `submit` builds a send work request -with the control and data buffers' `ibv_mr` handles and posts it via -`ibv_post_send`; the QP sends directly from registered memory — no -kernel involvement, no copy. `register_buffer` copies a System buffer -into `send_pool_` if not already `Registered`; callers on the RDMA path -allocate from `RdmaBufferPool` directly to avoid this. +The base transfer contract does not expose RDMA READ or RDMA WRITE. A large +logical result is an ordered stream of messages with logical offsets. -## 2. CQ Poll Loop +## 2. Endpoint discovery and path selection +Service discovery first publishes coarse capability: whether the endpoint is +RDMA-capable and whether an RDMA address is currently available. If either +side lacks capability, or local configuration or device probing disables RDMA, +the client uses TCP without attempting RDMA CM. + +When coarse capability permits RDMA, the TCP handshake exchanges the current +RDMA CM address and port, GID/device identity, capability generation, protocol +version, and supported data classes. The client verifies that the generation +and peer identity are current before creating the RDMA connection. + +One logical RPC connection chooses RDMA as its preferred path and TCP as its +one-way fallback. The path is selected before the first payload. TCP is chosen +when RDMA setup fails before payload submission; an established TCP connection +is never upgraded to RDMA. + +## 3. Message and aggregation model + +ChunkIO and Chunk-KV retain ownership of aggregation and flush policy. RDMA +receives an already aggregated buffer and transmits it without additional +waiting or coalescing. A data message has: + +```text +SGE 0: 1 KiB control block +SGE 1: selected data buffer ``` -Per worker thread: - cq = ibv_create_cq(context_, depth, ...) - post initial recv WRs (refill recv queue) - - loop: - ibv_poll_cq(cq, wcs) - for each wc: - if send completion: recycle send buffer; post next send WR if queued - if recv completion: feed recv buffer to parser → on_frame; post new recv WR - check for cross-thread submits → post send WRs + +The control block contains the common RPC header and control metadata. The +selected data capacity is 64 KiB for metadata, 256 KiB for aggregated +Chunk-KV results, or 1 MiB for the S3/HTTP/DiskIO datapath. The actual payload +length is carried in the header. A logical result larger than the selected +capacity is an ordered sequence identified by request ID, stream ID, sequence, +logical offset, and START/DATA/END/ERROR flags. + +The two-SGE layout is intentional. It keeps control and data ownership clear +and avoids CPU concatenation. A sender may inline the small control block when +the device permits it; receive buffers retain the two-SGE layout. + +## 4. Registered buffers and ownership + +Each process has a bounded registered host-memory budget. Data classes and the +control pool are shared by streams; a stream does not reserve a private set of +buffers for its lifetime. A buffer moves through these states: + +```text +available -> posted_receive -> receive_in_flight -> caller_owned +available -> send_in_flight -> available +receive_in_flight -> disk_owned -> available +any uncertain state -> quarantined ``` -Same submit/completion/dispatch logic as the socket transport. The I/O -primitive changes (CQ poll vs epoll), the buffer registration changes -(`ibv_reg_mr` vs noop), everything else is shared via the `Transport` / -`Connection` / `FrameParser` interfaces. An RNIC disconnect surfaces as -a CQ error completion → connection closes, reconnect triggers. A full -QP send queue returns `RpcError::SendQueueFull`; an empty recv queue -drops incoming sends (RDMA behavior), so the recv pool depth is sized -to avoid this under normal load. RDMA CM events (connection rejected, -addr unreachable) trigger reconnect with backoff, same as TCP. - -## 3. Connection Setup - -Connection establishment uses `librdmacm` (RDMA CM). The server -sequence is `rdma_create_id` → `rdma_bind_addr` → `rdma_listen` → -`rdma_get_request` → `rdma_create_qp` → `rdma_accept`; the client -sequence is `rdma_create_id` → `rdma_resolve_addr` → -`rdma_resolve_route` → `rdma_connect` → `rdma_create_qp`. After QP -creation, the worker posts initial receive WRs to refill the recv -queue so incoming sends have a destination buffer. +The owner that receives a completed SDK buffer must explicitly release its +lease. DiskIO reposts a receive buffer only after its asynchronous disk write +completes. An uncertain completion is quarantined until connection quiescence. +Registered memory is not reused merely because a deadline expired. + +Posted receives are the implicit data credits. Buffer exhaustion, a full SDK +lease budget, or a full DiskIO write queue applies backpressure to the existing +producer or aggregation layer. RDMA adds no application-level credit message. + +## 5. Event and completion model + +The RDMA completion-channel fd and RDMA-CM event fd are registered with the +existing epoll engine. Epoll provides wakeups; the RPC worker then: + +1. acknowledges and re-arms the channel event; +2. drains CM events and updates connection state; +3. polls bounded batches from the CQ; +4. resolves each work request through its `wr_id` record; +5. dispatches frame completion or buffer ownership transfer. + +The RPC library controls polling. FFI receives only a completion callback or a +buffer lease after C++ has validated the frame and resolved ownership. There is +no independent RDMA polling thread in the default path. + +## 6. Failure and fallback + +RDMA setup failure, missing capability, or a transport error before the first +payload may select TCP. The request ID is shared by both transports. Retry is +allowed only when existing request semantics provide idempotency or duplicate +suppression. + +After RDMA payload submission, an uncertain or partial stream fails as one +operation. TCP bytes are never appended to the partial RDMA result. Late CQ +completions cannot publish data into a reused buffer. QP, CM, and CQ errors +move the connection to draining or failed state and quarantine uncertain +buffers. + +## 7. Service interactions + +For S3/HTTP and DiskIO, HTTP or ChunkIO fills the Access Server registered send +buffer; DiskIO receives into its own registered buffer and submits that buffer +directly to asynchronous disk I/O. DiskIO returns it to the receive pool only +after disk completion. + +For Access Server and Chunk-KV, the existing access and aggregation layer +selects the 64 KiB or 256 KiB class. RDMA transports each aggregate with the +same request, sequence, and response semantics as TCP. + +The fat client uses the access core directly. Its caller owns each completed +host buffer lease until release. GPU support initially consists of an explicit +host-to-GPU copy; a future allocator may provide GPU-capable memory without +changing the lease contract. + +## 8. Tunables + +The transport configuration supplies defaults when the transport is created: + +```toml +stream_buffer_size = "1MiB" +chunk_kv_metadata_buffer_size = "64KiB" +chunk_kv_aggregate_buffer_size = "256KiB" +rpc_control_block_size = "1KiB" +rdma_registered_bytes = "64MiB" +rdma_max_inflight_bytes = "32MiB" +``` + +The effective pool counts are bounded by registered bytes, receive depth, send +depth, DiskIO queue bytes, and SDK-held bytes. A configured class is a capacity +limit, not a required wire length. + +## 9. Correctness invariants + +- **I1:** A message is published only after its control header and payload + length pass validation for the negotiated class. +- **I2:** A buffer has exactly one owner and is not reused before transport, + disk, and caller access have completed. +- **I3:** Posted receive capacity is the bounded receive window; no unbounded + allocation replaces a missing buffer. +- **I4:** A logical stream publishes messages in sequence order and never mixes + bytes from TCP and a partially completed RDMA stream. +- **I5:** The shared request ID and generation prevent an ambiguous retry from + being mistaken for a distinct operation. +- **I6:** FFI completion never transfers a buffer whose ownership or length is + unresolved. +- **I7:** Unsupported RDMA leaves the TCP service usable and does not require + RDMA libraries or devices at runtime. + +## 10. Scope boundary + +This design covers host-memory RC SEND/RECV RPC and data streams. RDMA READ, +RDMA WRITE, remote destination descriptors, and GPUDirect are outside the base +transport contract. A future memory adapter may add them only behind the same +request, ownership, completion, and fallback semantics. diff --git a/doc/working/plan-rdma-chunk-kv-access.md b/doc/working/plan-rdma-chunk-kv-access.md new file mode 100644 index 000000000..1e90d00fb --- /dev/null +++ b/doc/working/plan-rdma-chunk-kv-access.md @@ -0,0 +1,111 @@ + + + +# RDMA Chunk-KV access Plan + +Upstream: [R241](../backlog/R241-rdma-chunk-kv-access.md), +[root RDMA plan](plan-rdma.md), [R238](../backlog/R238-rdma-registered-buffer-foundation.md), +and [R239](../backlog/R239-rdma-rpc-transport.md). + +Goal: use the common RDMA stream for Access Server to Chunk-KV operations while +preserving existing aggregation, request numbering, generation checks, and TCP +semantic parity. + +## Phase 1: access contract mapping + +- [ ] **Map request/response types**: identify metadata, point, batch, range, + and aggregate operations; record idempotency, request ID, generation, error, + and retry behavior. Files: `lib/crowdb-chunk-kv-client/`, Chunk-KV server + modules, and Access Server access-core modules. +- [ ] **Map aggregation owner**: identify where Chunk-KV combines responses and + where it decides flush/end boundaries. RDMA must receive the aggregate after + that decision, never create another aggregation layer. +- [ ] **Define class selection**: metadata defaults to 64 KiB, aggregated + Chunk-KV results default to 256 KiB, and 1 MiB remains a configured option for + larger value/range results. Transport creation receives defaults from config. + +## Phase 2: common client/server wiring + +- [ ] **Add RDMA transport choice**: make the Chunk-KV client use the R239 + logical connection interface and preserve TCP as fallback. Do not duplicate + request correlation in the Chunk-KV layer. +- [ ] **Advertise selected class**: include class and capacity in stream setup + or request metadata, validate the peer's negotiated class, and reject a + payload larger than that class before publishing it. +- [ ] **Preserve request identity**: carry request ID, stream ID, sequence, + logical offset, generation, and operation type through RDMA completion and + response assembly. +- [ ] **Preserve access errors**: map transport, timeout, cancellation, + generation, integrity, and Chunk-KV errors identically for TCP and RDMA. + +## Phase 3: multi-message response assembly + +- [ ] **Split aggregate responses**: when an existing aggregate exceeds 64 KiB + or 256 KiB, emit ordered messages without changing its logical result. +- [ ] **Reassemble on client**: validate sequence, offset, total/end flags, and + duplicate request identity; reject gaps, overlaps, stale generations, and + wrong classes. +- [ ] **Bound assembly memory**: use R238 classes and caller/consumer release + to keep aggregate response memory within the configured budget. +- [ ] **Retry safely**: use shared request IDs and only retry operations whose + current semantics provide idempotency or duplicate suppression. + +## Phase 4: fallback and compatibility + +- [ ] **Pre-payload fallback**: when RDMA setup fails, retry the same eligible + request on TCP with the same request ID and generation. +- [ ] **Partial response failure**: fail the whole logical response after an + uncertain RDMA message; never append a TCP suffix or publish incomplete data. +- [ ] **Config compatibility**: preserve TCP defaults and allow RDMA classes, + endpoint selection, and enablement to be configured when creating the + transport. + +## Tests + +Unit tests: + +- class selection for metadata, aggregate, and large range operations; +- split and reassembly of responses at 64 KiB and 256 KiB boundaries; +- sequence gaps, overlap, duplicate, stale generation, and wrong-class errors; +- request identity and idempotency/retry decisions; +- TCP/RDMA error mapping parity. + +Integration tests: + +- metadata point operation over RDMA matches TCP; +- aggregate result larger than 256 KiB is reconstructed exactly; +- aggregate producer remains the only flush/aggregation owner; +- pre-payload RDMA failure retries eligible requests on TCP; +- partial RDMA response fails without accepting a TCP suffix; +- bounded consumer release applies backpressure to Chunk-KV response production. + +E2E tests: + +- single-node HCA local-loop OCI Access Server to Chunk-KV operations; +- TCP-only OCI profile with no RDMA attempts; +- single-node local-loop multi-container profile with multiple Chunk-KV connections; +- mixed metadata, point, batch, and range corpus compared byte-for-byte across + TCP and RDMA. + +## Consolidated files + +- `lib/crowdb-chunk-kv-client/src/` +- `lib/crowdb-chunk-kv-client/tests/` +- Chunk-KV server RPC/service modules +- Access Server access-core and configuration modules +- `lib/crowdb-rpc/` shared frame and transport modules +- OCI E2E test configuration + +## Verification gates + +```bash +pixi run -- cargo fmt --all -- --check +pixi run -- cargo test -p crowdb-chunk-kv-client --all-targets +pixi run -- cargo test -p crowdb-access-server --all-targets +pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings +pixi run rs-lint +pixi run tree-lint +``` + +Do not hand off to R242 until Chunk-KV response assembly and TCP/RDMA semantic +parity pass for metadata and multi-message aggregate responses. diff --git a/doc/working/plan-rdma-datapath-stream.md b/doc/working/plan-rdma-datapath-stream.md new file mode 100644 index 000000000..0cdfe602b --- /dev/null +++ b/doc/working/plan-rdma-datapath-stream.md @@ -0,0 +1,113 @@ + + + +# RDMA datapath stream Plan + +Upstream: [R240](../backlog/R240-rdma-datapath-stream.md), +[root RDMA plan](plan-rdma.md), [R238](../backlog/R238-rdma-registered-buffer-foundation.md), +and [R239](../backlog/R239-rdma-rpc-transport.md). + +Goal: connect existing HTTP/ChunkIO aggregation to the RDMA RPC stream and +DiskIO receive loop without introducing another aggregation or CPU payload +copy. + +## Phase 1: map existing data flow + +- [ ] **Trace upload flow**: identify where HTTP body bytes enter ChunkIO, + become aggregated buffers, are submitted to DiskIO, and are released after + durable completion. Files: `app/crowdb-access-server/`, + `lib/crowdb-chunk-client/`, `lib/crowdb-diskio-client/`, and ChunkIO designs. +- [ ] **Trace download flow**: identify how SDK/HTTP consumers receive data, + how read-ahead aggregation chooses message boundaries, and when a caller + releases each buffer. +- [ ] **Define transport adapter**: add the smallest Access/DiskIO adapter that + accepts an already formed buffer, actual length, request identity, sequence, + and logical offset. Do not add a second flush timer or aggregation queue. + +## Phase 2: upload to DiskIO + +- [ ] **Allocate 1 MiB datapath class**: select the configured 1 MiB data + capacity plus 1 KiB control buffer for S3/HTTP/DiskIO streams. +- [ ] **Fill registered send buffer**: make the HTTP/ChunkIO producer write + directly into the send-owned registered buffer and submit its actual length. + The producer must stop when the send/registered budget is exhausted. +- [ ] **Receive ownership handoff**: on RDMA receive completion, transfer the + DiskIO buffer to the asynchronous disk writer. Keep request, sequence, + logical offset, and expected length attached to the write. +- [ ] **Repost after disk completion**: only reset and repost the receive buffer + after the disk write and integrity result are complete. Propagate disk errors + to the stream and do not silently repost corrupted data. + +## Phase 3: download to SDK/HTTP + +- [ ] **Prepost receive buffers**: use the R238 pool and R239 receive window; + publish a `BufferLease` only after header/class/length validation. +- [ ] **Preserve caller ownership**: keep the lease caller-owned until explicit + release. Do not let CQ completion or a stream callback recycle it. +- [ ] **Implement multi-message assembly**: deliver messages in sequence and + logical offset order, preserving end/error flags and cancellation behavior. +- [ ] **Keep TCP parity**: use the same access result and error mapping when the + stream selects TCP before RDMA payload submission. + +## Phase 4: pressure and failure behavior + +- [ ] **Disk queue pressure**: stop reposting when the asynchronous disk queue + reaches its byte bound; let receive-window exhaustion backpressure the + sender. +- [ ] **SDK pressure**: stop publishing new leases when held bytes reach the + configured limit; resume after release. +- [ ] **Partial stream failure**: fail the logical object/request after partial + RDMA transfer and quarantine uncertain buffers; never append TCP bytes. +- [ ] **Cancellation**: stop new aggregation submissions, drain known WRs, + preserve in-flight buffer lifetimes, and return only buffers proven safe. + +## Tests + +Unit tests: + +- actual length below 1 MiB does not send padded bytes; +- sequence/offset/end/error parsing and multi-message assembly; +- DiskIO repost is rejected before disk completion; +- SDK lease release is the only return transition for caller-owned buffers; +- no second aggregation or flush timer is invoked by the RDMA adapter. + +Integration tests: + +- HTTP/ChunkIO aggregate enters a registered send buffer directly; +- DiskIO receive completion hands the same buffer to asynchronous disk I/O; +- full disk queue stops repost and resumes after completion; +- all SDK leases held causes bounded producer backpressure; +- TCP and RDMA return identical bytes, lengths, and errors; +- cancellation and partial RDMA failure leave no reusable uncertain buffer. + +E2E tests: + +- single-node HCA local-loop OCI upload into DiskIO and download back to a client; +- object larger than 1 MiB reconstructed from ordered messages; +- TCP-only OCI profile produces the same object and metadata; +- single-node local-loop multi-container profile reports bounded pressure without claiming physical + endpoint isolation. + +## Consolidated files + +- `app/crowdb-access-server/` +- `lib/crowdb-chunk-client/` +- `lib/crowdb-diskio-client/` +- `lib/crowdb-rpc/` +- `lib/crowdb-chunkio/` or the existing ChunkIO owner module +- relevant S3/DiskIO integration tests and OCI E2E runner + +## Verification gates + +```bash +pixi run -- cargo fmt --all -- --check +pixi run -- cargo test -p crowdb-access-server --all-targets +pixi run -- cargo test -p crowdb-diskio-client --all-targets +pixi run -- cargo test -p crowdb-chunk-client --all-targets +pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings +pixi run rs-lint +pixi run tree-lint +``` + +Do not hand off to R242 until both upload and download have explicit lease +ownership tests and the DiskIO receive loop proves repost-after-disk-completion. diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md new file mode 100644 index 000000000..e4b0301bf --- /dev/null +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -0,0 +1,111 @@ + + + +# RDMA fat client SDK and FFI lease Plan + +Upstream: [R242](../backlog/R242-rdma-fat-client-sdk.md), +[root RDMA plan](plan-rdma.md), [R238](../backlog/R238-rdma-registered-buffer-foundation.md), +[R239](../backlog/R239-rdma-rpc-transport.md), and +[R241](../backlog/R241-rdma-chunk-kv-access.md). + +Goal: expose Access Server semantics through a native fat-client SDK that can +consume the same RDMA stream directly, with caller-owned host buffer leases +and an explicit host-to-GPU copy simulation. + +## Phase 1: access-core split + +- [ ] **Map Access Server boundary**: identify HTTP-only parsing, authorization, + access planning, generation validation, aggregation, Chunk-KV calls, DiskIO + calls, and response publication. Files: `app/crowdb-access-server/` and + access design modules. +- [ ] **Extract access core**: move shared planning and operation semantics + behind a native interface while keeping HTTP frontend behavior unchanged. + The core must select the logical connection path through R239 and preserve + TCP fallback. +- [ ] **Define SDK operations**: specify open/read/write/range/metadata + operations, request ID, generation, cancellation, deadlines, and stream + sequence without exposing C++ transport or verbs types. + +## Phase 2: FFI lease interface + +- [ ] **Define opaque lease ABI**: expose handle, actual length, capacity, + request ID, sequence, logical offset, error/end status, and release. Use + fixed-width types and explicit null/error behavior. +- [ ] **Implement callback bridge**: C++ completion resolves ownership and + invokes the Rust FFI callback; Rust forwards to the future/stream consumer. + The callback must not block the RPC worker or retain a borrowed pointer past + release. +- [ ] **Implement release/drop**: make release idempotency and misuse visible, + return caller-owned buffers to R238 only after caller access ends, and keep + shutdown safe while leases remain held. +- [ ] **Implement cancellation**: stop new work, drain or fail submitted + requests, preserve in-flight buffer lifetime, and report cancellation once. + +## Phase 3: host and GPU-copy flow + +- [ ] **Host path**: deliver registered host leases directly to SDK callers; + do not copy payload into a second SDK buffer. +- [ ] **GPU-copy simulation**: provide an explicit copy hook or SDK operation + that holds the host lease until the caller confirms host-to-GPU copy + completion. Do not expose a GPU pointer in the base ABI. +- [ ] **Buffer pressure**: enforce SDK-held byte budget; pause producer/admission + when callers retain all leases and resume after release. + +## Phase 4: frontend parity + +- [ ] **HTTP parity**: run identical access requests through HTTP frontend and + native SDK and compare bytes, metadata, generation, errors, and cancellation. +- [ ] **Transport parity**: run SDK operations over RDMA and TCP with the same + request ID and verify fallback before payload submission. +- [ ] **Multi-message stream**: expose ordered 64 KiB metadata or 256 KiB + aggregate messages as one SDK stream without copying them into a monolithic + result. + +## Tests + +Unit tests: + +- opaque ABI layout, fixed-width fields, null/error behavior; +- lease accessors, single release, release-after-shutdown, and misuse; +- callback does not block and does not publish unresolved ownership; +- cancellation resolves exactly once; +- host-to-GPU copy dependency keeps lease alive until completion. + +Integration tests: + +- native SDK uses the same access-core semantics as HTTP frontend; +- caller holds all leases and observes bounded admission/backpressure; +- multi-message metadata and aggregate responses preserve order and offsets; +- RDMA-disabled host uses TCP without changing SDK result; +- pre-payload RDMA failure retries eligible request with shared request ID; +- partial RDMA failure fails stream without accepting a TCP suffix. + +E2E tests: + +- single-node HCA local-loop OCI client/Access Server/DiskIO flow; +- TCP-only OCI profile with no RDMA devices; +- single-node local-loop multi-container SDK pressure and multiple logical streams; +- explicit host-to-GPU copy simulation with release after copy completion. + +## Consolidated files + +- `app/crowdb-access-server/` +- Access core and native SDK modules +- `lib/crowdb-rpc/ffi/src/client.rs` +- `lib/crowdb-rpc/ffi/src/server.rs` +- `lib/crowdb-rpc/ffi/src/sys.rs` +- SDK integration tests and OCI E2E runner + +## Verification gates + +```bash +pixi run -- cargo fmt --all -- --check +pixi run -- cargo test -p crowdb-rpc-ffi --all-targets +pixi run -- cargo test -p crowdb-access-server --all-targets +pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings +pixi run rs-lint +pixi run tree-lint +``` + +Completion requires caller-controlled lease tests, HTTP/SDK parity, TCP/RDMA +parity, and safe shutdown with outstanding leases. diff --git a/doc/working/plan-rdma-registered-buffers.md b/doc/working/plan-rdma-registered-buffers.md new file mode 100644 index 000000000..dcd766673 --- /dev/null +++ b/doc/working/plan-rdma-registered-buffers.md @@ -0,0 +1,120 @@ + + + +# RDMA registered host-buffer foundation Plan + +Upstream: [R238](../backlog/R238-rdma-registered-buffer-foundation.md), +[R5](../backlog/R5-rdma-alloc.md), and +[RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md). + +Goal: provide bounded registered host buffers and an explicit lease/ownership +contract for every later RDMA RPC and data-path component. + +## Progress + +Capacity classes, ownership states, lease metadata, bounded atomic admission, +lock-free freelists, quarantine exclusion, and C/Rust metadata accessors are +implemented in `buffer.h`, `buffer.cpp`, `c_api.h`, and the FFI buffer wrapper. +The pool is device-independent so TCP-only builds retain the bounded contract. +Verbs MR attachment, transport handoff, and shutdown integration remain open. + +The verbs pool now pre-registers one region and carves aligned capacity-class +slices from it. Each lease records the region MR handle and lkey, and +quarantined leases are retained out of the reusable free lists until pool +shutdown. + +## Phase 1: inventory and contract + +- [ ] **Map existing buffer ownership**: document the current `Buffer`, + `BufferPool`, `OutFrame`, callback, and FFI ownership transitions before + adding registration. Files: `lib/crowdb-rpc/include/crowdb-rpc/buffer.h`, + `lib/crowdb-rpc/include/crowdb-rpc/transport.h`, + `lib/crowdb-rpc/include/crowdb-rpc/connection.h`, + `lib/crowdb-rpc/ffi/src/sys.rs`. +- [ ] **Define capacity classes**: add named constants/configuration for 64 KiB, + 256 KiB, and 1 MiB data capacities plus the 1 KiB control capacity. Reject + zero, overflowed, and unsupported capacities before pool admission. Files: + `lib/crowdb-rpc/include/crowdb-rpc/buffer.h`, configuration headers, and + corresponding Rust FFI types. +- [ ] **Define lease metadata**: specify request ID, stream ID, sequence, + logical offset, actual length, capacity, registration identity, and release + state. Keep the handle opaque across FFI. Files: buffer and FFI headers, + `lib/crowdb-rpc/ffi/src/sys.rs`. + +## Phase 2: registered pool + +- [ ] **Implement device registration owner**: create and destroy a pool for a + protection domain, registered region, lkey, and device identity. Registration + failure must leave the TCP pool usable. Files: + `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_buffer_pool.h`, + `lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp`. +- [ ] **Implement class freelists**: carve buffers from bounded registered + regions instead of registering every allocation. Keep control and data pools + separate, enforce total bytes and class counts, and avoid an unbounded + fallback allocation. Files: same pool files and pool tests. +- [ ] **Implement ownership transitions**: make acquire, post, complete, disk + handoff, caller release, and quarantine explicit operations. Reject a second + release or recycle before completion. Files: pool/lease implementation. +- [ ] **Implement shutdown and quarantine**: drain known operations, mark + uncertain buffers quarantined, and deregister only after no device access can + remain. Files: pool, connection shutdown, and test fixtures. +- [ ] **Review hot-path synchronization**: keep allocation/recycle compatible + with the repository's lock-free rule; isolate lifecycle-only coordination and + record any unavoidable synchronization boundary. Files: pool implementation. + +## Phase 3: FFI lease surface + +- [ ] **Expose lease accessors**: return opaque handle, data pointer, actual + length, capacity, sequence, and logical offset without exposing verbs structs. + Files: `lib/crowdb-rpc/ffi/src/sys.rs`, `client.rs`, `server.rs`, and C ABI + declarations. +- [ ] **Enforce release ordering**: caller release returns a buffer only after + the C++ owner confirms completion; Rust must not retain a dereferenceable + view after release. Files: FFI wrapper and ownership tests. + +## Tests + +Unit tests: + +- class rounding/selection and overflow rejection; +- byte-budget admission and per-class counts; +- every valid ownership transition and invalid double release; +- quarantine exclusion and shutdown accounting; +- lease metadata and opaque-handle accessors. + +Integration tests: + +- registration failure falls back to the ordinary TCP pool; +- a completed receive lease is not reusable until explicit release; +- a disk-owned lease is not reusable until disk completion is reported; +- concurrent acquire/release does not exceed the configured byte budget. + +E2E tests: + +- run the TCP-only OCI profile without RDMA devices and confirm the service + remains usable; +- run the HCA local-loop profile and confirm registered buffers survive a complete + send, receive, and release cycle. + +## Consolidated files + +- `lib/crowdb-rpc/include/crowdb-rpc/buffer.h` +- `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_buffer_pool.h` +- `lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp` +- `lib/crowdb-rpc/ffi/src/sys.rs` +- `lib/crowdb-rpc/ffi/src/client.rs` +- `lib/crowdb-rpc/ffi/src/server.rs` +- `lib/crowdb-rpc/tests/` buffer and lease tests + +## Verification gates + +```bash +pixi run -- cargo fmt --all -- --check +pixi run -- cargo test -p crowdb-rpc --all-targets +pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings +pixi run tree-lint +``` + +Completion requires all ownership, budget, shutdown, and TCP-only tests to +pass. Do not begin R239 transport work while the pool can recycle an uncertain +buffer. diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md new file mode 100644 index 000000000..64626925a --- /dev/null +++ b/doc/working/plan-rdma-rpc-transport.md @@ -0,0 +1,225 @@ + + + +# RDMA RPC transport Plan + +Upstream: [R239](../backlog/R239-rdma-rpc-transport.md), +[root RDMA plan](plan-rdma.md), and +[RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md). + +Goal: add RC SEND/RECV to the common RPC transport with epoll/CQ event +composition and one-way RDMA-to-TCP fallback. + +## Progress + +The transport foundation now includes validated endpoint capability data +(protocol, generation, device, GID, port, and supported capacity classes), +explicit RDMA path states, and a pre-payload-only TCP fallback predicate. +Implemented since this checkpoint: active/passive RDMA-CM setup, RC QP +creation, two-SGE SEND construction, bounded receive posting, completion-channel +polling with CQ acknowledgement, error quarantine, validated receive-frame +dispatch into `Connection::on_frame`, and epoll external-fd registration. +The CQ path is now drained by an epoll worker through `completion_fd()` and +`drain_cq()` rather than a private polling thread. Capability handshake bytes +have a fixed wire representation with round-trip tests. Negotiated capability +exchange through the existing TCP handshake, common transport path +selection/fallback, receive repost after owner release, and two-peer OCI E2E +execution are now wired at the RPC seam: TCP connections exchange the fixed +capability frame, connections retain the peer capability, RDMA-enabled +connections are selected before TCP fallback, and completed RDMA frames retain +their buffers until the owning `Frame` is destroyed, then repost the receive +pair. The live host HCA local-loop CM/QP/send/receive flow now passes, including +receive repost and connection-level CQ handling. Remaining validation includes +executing the OCI profiles against a built `crowdb:test` image and exercising +the actual RPC worker path. + +## R239 status checkpoint + +Completed: + +- active and passive RDMA-CM address, route, connect, and accept flow; +- connection-level PD, CQ, completion channel, and registered send/receive + pools; +- initial receive posting, CQ completion dispatch, owner-release repost, and + RNR retry configuration; +- RDMA-to-TCP capability negotiation and pre-payload fallback wiring; +- epoll registration for CM, transport CQ, and connection CQ events; +- live HCA local-loop CM/QP/send/receive E2E with all frames received and clean CQ + status, and clean client/server exit; +- non-root runner selection and the library consumer/deployment contract. + +Remaining: + +- run the same flow through the actual `RpcServer`/`RpcClient` worker path, + including frame dispatch and request correlation rather than the transport + harness only; +- add mock/integration coverage for CM disconnect, QP error, CQ flush/error, + epoll wakeup, and pre-payload TCP fallback; +- exercise multiple sequential frames on one QP to verify receive repost and + buffer reuse under the worker; +- run single-node HCA local-loop multi-container and physical multi-node OCI profiles against the built image; +- remove temporary hardware diagnostics, normalize failure logging, and run + the full C++ format/tree-lint gates. + +## Phase 1: common transport seam + +- [ ] **Trace TCP submission**: map `Transport::submit`, `Connection`, + `OutFrame`, parser dispatch, completion callbacks, and connection shutdown. + Files: `lib/crowdb-rpc/include/crowdb-rpc/transport.h`, `connection.h`, + `out_frame.h`, TCP transport and worker sources. +- [ ] **Add RDMA capability state**: define unavailable, probing, connecting, + ready, draining, failed, and TCP-fallback states. Keep path state per logical + connection and prevent TCP-to-RDMA upgrade. Files: transport/connection + headers and state tests. +- [ ] **Define endpoint handshake data**: represent coarse discovery result, + CM address/port, GID/device identity, capability generation, protocol + version, and supported data classes. Validate stale generation and peer + identity before CM setup. Files: RPC control schema and endpoint config. + +## Phase 2: CM and QP lifecycle + +- [ ] **Implement device selection**: select configured HCA, port, local + address, and GID; fail closed to TCP when local probing finds no usable + device. Files: `rdma_transport.h/.cpp` and configuration adapter. +- [ ] **Implement active/passive CM flow**: create IDs, bind/listen or resolve + address/route, create QP, connect/accept, and publish ready only after initial + receive buffers are posted. Files: RDMA transport sources. +- [ ] **Implement connection teardown**: map CM disconnect, QP error, and CQ + error to one drain/fail transition. Stop new work, resolve known completions, + quarantine uncertain buffers, and wake blocked submitters. + +## Phase 3: event/poll loop + +- [ ] **Extend epoll registration**: add external event-fd registration and + removal to the existing `EpollEngine` without changing TCP readiness rules. + Files: `lib/crowdb-rpc/include/crowdb-rpc/transport/epoll/epoll_engine.h`, + `lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp`. +- [ ] **Drain CM events**: acknowledge all received CM events, update the + connection state, and re-arm the CM channel where required. +- [ ] **Drain CQ events**: acknowledge and re-arm completion notifications, then + poll bounded batches until the current CQ work is drained or the worker + budget is reached. No independent polling thread. +- [ ] **Make stop wakeable**: stop must wake epoll, stop new submissions, drain + known work, and join without waiting forever on an idle channel. + +## Phase 4: frame submission and completion + +- [ ] **Build two-SGE WRs**: map the 1 KiB control buffer and selected data + buffer to one SEND WR. Inline control only when the configured device limit + allows; never concatenate control and data in the CPU. +- [ ] **Post receives**: prepost control/data receive buffers, associate each + with a stable `wr_id`, and replenish only after the owner releases it. +- [ ] **Resolve WR identity**: `wr_id` must identify connection, request ID, + stream ID, sequence, logical offset, buffers, and ownership state. Handle + success, flush, local protection, retry, and remote access errors distinctly. +- [ ] **Dispatch common callbacks**: validate lengths and class before parser + publication, then invoke the existing C++ completion path and FFI callback. + +## Phase 5: fallback and observability + +- [ ] **Implement pre-payload fallback**: if CM/QP setup or first submission + fails before payload, select TCP with the same request identity. +- [ ] **Reject mixed completion**: after any payload submission, fail the stream + on uncertainty and never enqueue a TCP suffix. +- [ ] **Add metrics**: expose RDMA capability, selected device, QP state, CQ + depth, posted receives, RNR/retry errors, fallback count, and quarantined + bytes. + +## Tests + +Unit tests: + +- endpoint capability and generation validation; +- connection-state transitions and illegal TCP-to-RDMA upgrade; +- frame class/length validation and two-SGE construction; +- `wr_id` mapping and completion state transitions; +- fallback boundary and request-ID reuse rules. + +Integration tests: + +- RDMA-disabled build and host selects TCP without CM calls; +- mocked CM/CQ setup, reconnect, CQ error, flush completion, and shutdown; +- epoll external event wakes worker and drains all current CQ completions; +- send/receive sequence preserves exact control/data lengths; +- pre-payload failure retries idempotent request on TCP; +- partial payload failure never appends TCP data. + +E2E tests: + +- TCP-only OCI profile; +- single-node HCA local-loop profile with multiple QPs; +- physical multi-node profile with one RDMA NIC bound per container. +- non-root OCI profile: host prepares RDMA and network topology, the container + receives only the required `/dev/infiniband` devices and memlock allowance, + and the RPC flow runs as the application user without `sudo`, module loads, + link moves, or host namespace entry. + +## Consolidated files + +- `lib/crowdb-rpc/include/crowdb-rpc/transport.h` +- `lib/crowdb-rpc/include/crowdb-rpc/connection.h` +- `lib/crowdb-rpc/include/crowdb-rpc/transport/epoll/epoll_engine.h` +- `lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp` +- `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h` +- `lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp` +- `lib/crowdb-rpc/tests/` transport, fallback, and event-loop tests +- OCI image and E2E runner configuration + +## Verification gates + +```bash +pixi run -- cargo fmt --all -- --check +pixi run -- cargo test -p crowdb-rpc --all-targets +pixi run -- cargo test -p crowdb-rpc-ffi --all-targets +pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings +pixi run rs-lint +pixi run tree-lint +``` + +Do not hand off to R240/R241 until a framed RPC stream works over RDMA and TCP +fallback has passed the partial-transfer boundary tests. + +## Container launch contract + +The host performs privileged RDMA setup with `setup-rdma.sh local`. Each +container receives the required `/dev/infiniband` devices, device-access group, +the selected HCA mapping, and bounded memlock limit. The RPC process runs as +the application user and does not use `sudo`, load kernel modules, move links, +or enter host namespaces. + +The local hardware smoke test is: + +```bash +sudo tools/rdma/setup-rdma.sh local +tools/rdma/run-local-loop.sh local +``` + +Multiple containers on one node use the same HCA local-loop model and open +independent QPs. Multi-node OCI runs use the same application contract, with +each container bound to the RDMA NIC on its own physical node. No namespace +operation is part of the test or application path. + +## Library consumer contract + +An application linking `crowdb-rpc` does not run the RDMA setup script and does +not need to enter a network namespace. Deployment prepares the host or +container once; the application then uses the normal RPC server/client API. + +The deployment layer must provide: + +- a loaded RDMA kernel provider and an `ACTIVE` HCA port with a routable local + address/GID; +- `libibverbs`, `librdmacm`, and the matching provider library in the runtime + image; +- access for the application user to the required `/dev/infiniband` devices; +- a sufficient memlock limit for registered buffers; +- peer reachability on the selected RDMA address and port. + +The application layer only supplies the normal RPC endpoint configuration. The +library discovers usable devices and selects RDMA before TCP when the peer +advertises compatible capability. `CROWDB_RDMA_DEVICE`, `CROWDB_RDMA_PORT`, and +`CROWDB_RDMA_SOURCE_ADDR` are deployment overrides, not required API calls. +When any local capability, CM, QP, or pre-payload setup step is unavailable, +the library reports the reason and uses TCP without requiring application code +changes. After payload submission, the connection fails as one stream and does +not append a TCP suffix. diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md new file mode 100644 index 000000000..3cf3dc1ba --- /dev/null +++ b/doc/working/plan-rdma.md @@ -0,0 +1,231 @@ + + + +# CROWDB RDMA implementation Plan + +Upstream: [R5](../backlog/R5-rdma-alloc.md), +[RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md), and child +backlogs [R238](../backlog/R238-rdma-registered-buffer-foundation.md), +[R239](../backlog/R239-rdma-rpc-transport.md), +[R240](../backlog/R240-rdma-datapath-stream.md), +[R241](../backlog/R241-rdma-chunk-kv-access.md), and +[R242](../backlog/R242-rdma-fat-client-sdk.md). + +Goal: implement the host-memory RDMA SEND/RECV path end to end, with RDMA as +the preferred connection path, TCP as the one-way fallback, shared aggregation +semantics, bounded buffer leases, and automated OCI verification using HCA local +loop on one node and one RDMA NIC per container across physical nodes. + +## Current implementation status + +R238 foundation work has started. The common RPC buffer layer now has the four +capacity classes, bounded registered-buffer admission, explicit ownership +transitions, quarantine handling, and C/Rust lease metadata accessors. The +host setup and local-loop smoke test are now explicit: `setup-rdma.sh local` +prepares one HCA port, and `run-local-loop.sh local` drives both roles on that +port. Namespace setup is outside the supported test model. + +This is the root plan. The five child plans contain implementation details; +only this plan controls phase order, cross-backlog decisions, and final +acceptance. No child plan may broaden the transport contract without updating +the formal design first. + +## Fixed design decisions + +- Use one common RPC abstraction for TCP and RDMA. +- Use RC SEND/RECV, one 1 KiB control SGE and one selected data SGE. +- Use service discovery for coarse RDMA capability and TCP handshake for the + current RDMA endpoint, capability generation, and data classes. +- Select RDMA before the first payload; fallback is RDMA to TCP only. +- Reuse existing ChunkIO and Chunk-KV aggregation. RDMA does not aggregate or + flush independently. +- Use 64 KiB metadata, 256 KiB aggregated Chunk-KV, and 1 MiB datapath + capacities, configured as transport defaults. +- Use posted receives as implicit data credit; do not add an extra credit + protocol. +- Expose host buffer leases first. GPU delivery initially means an explicit + host-to-GPU copy. RDMA READ/WRITE is outside this implementation. +- Use `setup-rdma.sh local` as the standard host setup. A single node runs + client and server through the same HCA port and local IP; multiple containers + on that node use the same local-loop model and open independent QPs. +- For physical multi-node tests, each container is bound to the RDMA NIC on its + own node. The test harness supplies the selected device and address; it does + not move links into network namespaces. +- Treat `run-local-loop.sh local` as the smoke test for every local hardware + setup. Promote the same test flow to a multi-node container test by changing + only the endpoint placement and device binding. + +## Dependency graph + +```text +R238 registered buffers + | + v +R239 common RDMA RPC transport + / \ + v v +R240 R241 +datapath Chunk-KV/access + \ / + v +R242 fat client SDK and FFI lease +``` + +R240 and R241 may run in parallel after R239 passes its transport acceptance. +R242 starts only after R241 establishes the access semantics and after the FFI +lease contract from R238 is stable. + +## Phase 0: baseline and decision lock + +- [x] **Lock hardware test model**: use `setup-rdma.sh local` and + `run-local-loop.sh local` for single-node validation; do not use namespace + setup in product or acceptance tests. Multiple containers on one node share + the local HCA loop, while multi-node containers bind one NIC per node. +- [ ] **Record baseline**: run the existing TCP RPC, Access Server, DiskIO, and + Chunk-KV gates before modifying production code. Root files: affected crate + manifests and CI configuration. +- [ ] **Confirm host capabilities**: record `ibv_devices`, `rdma link`, the + selected HCA port, GID/IP mapping, and whether the local HCA loop is usable. + For multi-node runs, record one selected NIC and address per container. + Do not treat an unavailable host device as an implementation failure. +- [ ] **Lock configuration names**: agree on RDMA enablement, device/port/local + address, control/data class defaults, registered bytes, receive depth, and + fallback policy. Root files: configuration schema and formal design. +- [ ] **Prepare OCI image**: include `libibverbs`, `librdmacm`, and provider + libraries; keep a TCP-only image invocation that exposes no RDMA device. + Root files: Dockerfile/OCI test scripts and CI documentation. + +Exit condition: TCP baseline passes, the host topology is recorded, and the +configuration contract is stable. If the host has no RDMA, continue with the +TCP-only profile and build/mocked tests. + +## Phase 1: R238 registered buffers + +- [ ] **Execute R238 child plan**: implement classes, budgets, lease metadata, + ownership transitions, quarantine, and shutdown. +- [ ] **Run R238 gates**: unit and integration tests pass with RDMA disabled + and enabled build configurations. +- [ ] **Review handoff**: verify that R239 can obtain control/data buffers and + that every asynchronous owner has a release transition. + +Exit condition: no buffer can be recycled while transport, disk, or caller +ownership remains unresolved. + +## Phase 2: R239 common RDMA RPC + +- [ ] **Execute R239 child plan**: implement endpoint capability exchange, CM + and QP lifecycle, two-SGE SEND/RECV, epoll/CQ event composition, request + correlation, fallback, and shutdown. +- [ ] **Mock verification**: cover CQ success/error/flush, CM transitions, + receive exhaustion, fallback before payload, and no mixed TCP/RDMA stream. +- [ ] **Local-loop verification**: run `setup-rdma.sh local` followed by + `run-local-loop.sh local`; verify both CM roles reach `ESTABLISHED`, all + frames are received, and exact lengths and sequence values are preserved. +- [ ] **Container local-loop verification**: start multiple containers on one + node after the local setup; verify each container opens independent QPs over + the same HCA port and that request correlation and bounded buffers remain + correct. + +Exit condition: R239 can carry a framed stream, report completion to the common + RPC layer, and leave TCP fully usable when RDMA is absent or fails. + +## Phase 3: R240 and R241 service integration + +- [ ] **Execute R240 child plan**: connect ChunkIO-produced 1 MiB buffers to + Access Server/DiskIO, retain DiskIO buffers through disk completion, and test + large multi-message streams. +- [ ] **Execute R241 child plan**: connect Access Server/Chunk-KV with 64 KiB + metadata and 256 KiB aggregate classes, preserving existing request IDs, + ordering, and retry semantics. +- [ ] **Run semantic parity**: for the same test corpus, compare TCP and RDMA + results byte-for-byte and compare errors, sequence, generation, and + cancellation behavior. + +Exit condition: S3/DiskIO and Chunk-KV paths use RDMA without a second +aggregation layer, without an intermediate CPU payload copy, and with TCP +fallback before payload submission. + +## Phase 4: R242 fat client SDK + +- [ ] **Execute R242 child plan**: split access core from HTTP frontend, expose + FFI-safe host leases, and connect SDK operations to R241/R240 transports. +- [ ] **Host-to-GPU simulation**: hold each lease until the explicit host-to-GPU + copy completes, then release it; no GPU pointer is exposed by the base API. +- [ ] **SDK pressure tests**: hold all returned leases, verify bounded admission, + cancellation, shutdown, and recovery after release. + +Exit condition: native SDK and HTTP frontend return equivalent access results, +and the caller-controlled lease lifetime is visible and testable through FFI. + +## Phase 5: OCI E2E matrix + +- [ ] **TCP-only profile**: run OCI containers without RDMA devices and assert + service discovery skips RDMA entirely. +- [ ] **Single-node HCA local-loop profile**: run `setup-rdma.sh local`, start + one or more containers on the node, and run the client/server flow against + the same HCA port. Assert multiple QPs, request correlation, bounded buffers, + and TCP fallback. +- [ ] **Physical multi-node profile**: start one or more containers per node, + bind each container to that node's RDMA NIC, and run the same RPC/data-path + corpus across the fabric. No network namespace setup is part of this profile. +- [ ] **Automated topology selection**: make the test runner execute the + TCP-only and local-loop profiles automatically when their prerequisites are + present, and schedule the physical multi-node profile only when its node/NIC + inventory is available. + +Exit condition: every profile reports its topology, RDMA capability, selected +device, selected path, fallback count, buffer pressure, and final result. + +## Cross-phase invariants + +- [ ] **No mixed stream**: never append TCP bytes to a partially completed RDMA + stream. +- [ ] **No hidden aggregation**: only ChunkIO/Chunk-KV decides message boundary + and flush; RDMA transports the result. +- [ ] **No premature reuse**: completion, disk completion, and caller release + all precede buffer reuse for their respective ownership path. +- [ ] **Bounded memory**: registered bytes, posted receives, send depth, disk + queue bytes, and SDK-held bytes remain within configuration. +- [ ] **Capability optionality**: a host or OCI container without RDMA remains + a supported TCP deployment. +- [ ] **NIC ownership**: each physical-node container receives one explicit + RDMA device/address mapping; no test depends on moving a link or entering a + network namespace. +- [ ] **Shared identity**: request ID, generation, stream sequence, and logical + offset retain their meaning across TCP and RDMA. + +## Final verification + +Unit and integration gates: + +```bash +pixi run -- cargo fmt --all -- --check +pixi run -- cargo test -p crowdb-rpc --all-targets +pixi run -- cargo test -p crowdb-rpc-ffi --all-targets +pixi run -- cargo test -p crowdb-chunk-kv-client --all-targets +pixi run -- cargo test -p crowdb-access-server --all-targets +pixi run -- cargo test -p crowdb-diskio-client --all-targets +pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings +pixi run rs-lint +pixi run tree-lint +``` + +OCI E2E verification must run the TCP-only and single-node HCA local-loop +profiles when their prerequisites are available. The physical multi-node +profile reuses the same test corpus with one RDMA NIC bound per container and +is run when the node inventory is available. Namespace profiles are not part +of acceptance. + +## Test and setup files + +- `tools/rdma/setup-rdma.sh` +- `tools/rdma/run-local-loop.sh` +- OCI image and multi-container runner configuration + +## Handoff and blocking rules + +Each child plan is marked `[x]` only after its own acceptance cases and gates +pass. If a child discovers a protocol ambiguity, stop that child, record the +question here, and resolve it before changing the formal design. If the host +cannot provide RDMA, do not block R238/R239 mock work or TCP fallback tests; +mark only hardware E2E as pending. diff --git a/lib/crowdb-rpc/ffi/src/buffer.rs b/lib/crowdb-rpc/ffi/src/buffer.rs index 7c518e281..26ffb52fa 100644 --- a/lib/crowdb-rpc/ffi/src/buffer.rs +++ b/lib/crowdb-rpc/ffi/src/buffer.rs @@ -13,6 +13,16 @@ pub struct Buffer { handle: sys::crowdb_rpc_buffer_t, } +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct BufferLeaseMetadata { + pub request_id: u64, + pub stream_id: u64, + pub sequence: u64, + pub logical_offset: u64, + pub capacity: u32, + pub registration_id: u32, +} + /// An immutable, bounded scatter/gather data payload. /// /// Every view owns its underlying allocation until the RPC transport finishes @@ -166,6 +176,37 @@ impl Buffer { } } + #[must_use] + pub fn lease_metadata(&self) -> BufferLeaseMetadata { + unsafe { + BufferLeaseMetadata { + request_id: sys::crowdb_rpc_buffer_request_id(self.handle), + stream_id: sys::crowdb_rpc_buffer_stream_id(self.handle), + sequence: sys::crowdb_rpc_buffer_sequence(self.handle), + logical_offset: sys::crowdb_rpc_buffer_logical_offset(self.handle), + capacity: sys::crowdb_rpc_buffer_capacity(self.handle), + registration_id: sys::crowdb_rpc_buffer_registration_id(self.handle), + } + } + } + + pub fn set_lease_metadata(&mut self, metadata: BufferLeaseMetadata, registration_id: u32) { + unsafe { + sys::crowdb_rpc_buffer_set_metadata( + self.handle, + metadata.request_id, + metadata.stream_id, + metadata.sequence, + metadata.logical_offset, + registration_id, + ); + } + } + + pub fn quarantine(&mut self) -> bool { + !self.handle.is_null() && unsafe { sys::crowdb_rpc_buffer_quarantine(self.handle) != 0 } + } + /// Take ownership of the handle (prevents Drop from releasing it). pub fn into_raw(mut self) -> sys::crowdb_rpc_buffer_t { let h = self.handle; diff --git a/lib/crowdb-rpc/ffi/src/lib.rs b/lib/crowdb-rpc/ffi/src/lib.rs index 50ddb0d1e..f0ff137b9 100644 --- a/lib/crowdb-rpc/ffi/src/lib.rs +++ b/lib/crowdb-rpc/ffi/src/lib.rs @@ -21,7 +21,7 @@ mod owned_route; mod server; pub mod sys; -pub use buffer::{Buffer, BufferChain, BufferChainError, BufferPool}; +pub use buffer::{Buffer, BufferChain, BufferChainError, BufferLeaseMetadata, BufferPool}; pub use client::{noop_completion, CallFuture, ClientRequest, Response, RpcClient}; pub use connection_pool::{ConnectionPoolError, ConnectionPoolIndex, SelectedConnection}; pub use cpp_global_metrics::{cpp_global_metrics_max_name_len, flush_cpp_global_metrics}; diff --git a/lib/crowdb-rpc/ffi/src/sys.rs b/lib/crowdb-rpc/ffi/src/sys.rs index 1f5d6e8dc..735637037 100644 --- a/lib/crowdb-rpc/ffi/src/sys.rs +++ b/lib/crowdb-rpc/ffi/src/sys.rs @@ -124,6 +124,22 @@ extern "C" { pub fn crowdb_rpc_buffer_write(buf: crowdb_rpc_buffer_t, data: *const u8, len: u32); pub fn crowdb_rpc_buffer_data(buf: crowdb_rpc_buffer_t) -> *const u8; pub fn crowdb_rpc_buffer_len(buf: crowdb_rpc_buffer_t) -> u32; + pub fn crowdb_rpc_buffer_capacity(buf: crowdb_rpc_buffer_t) -> u32; + pub fn crowdb_rpc_buffer_request_id(buf: crowdb_rpc_buffer_t) -> u64; + pub fn crowdb_rpc_buffer_stream_id(buf: crowdb_rpc_buffer_t) -> u64; + pub fn crowdb_rpc_buffer_sequence(buf: crowdb_rpc_buffer_t) -> u64; + pub fn crowdb_rpc_buffer_logical_offset(buf: crowdb_rpc_buffer_t) -> u64; + pub fn crowdb_rpc_buffer_registration_id(buf: crowdb_rpc_buffer_t) -> u32; + pub fn crowdb_rpc_buffer_set_metadata( + buf: crowdb_rpc_buffer_t, + request_id: u64, + stream_id: u64, + sequence: u64, + logical_offset: u64, + registration_id: u32, + ); + pub fn crowdb_rpc_buffer_transition(buf: crowdb_rpc_buffer_t, from: i32, to: i32) -> i32; + pub fn crowdb_rpc_buffer_quarantine(buf: crowdb_rpc_buffer_t) -> i32; pub fn crowdb_rpc_buffer_detach(buf: crowdb_rpc_buffer_t) -> bool; pub fn crowdb_rpc_buffer_ref(buf: crowdb_rpc_buffer_t) -> crowdb_rpc_buffer_t; pub fn crowdb_rpc_buffer_release(buf: crowdb_rpc_buffer_t); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h index a83207c30..cf1e772bf 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h @@ -3,6 +3,7 @@ #pragma once +#include #include #include #include @@ -19,6 +20,25 @@ enum class BufferType : uint8_t { Registered, }; +enum class BufferOwnership : uint8_t { + Available, + PostedReceive, + ReceiveInFlight, + SendInFlight, + DiskOwned, + CallerOwned, + Quarantined, +}; + +enum class BufferCapacityClass : uint8_t { Control, Metadata, Aggregate, Datapath }; + +constexpr uint32_t RDMA_CONTROL_CAPACITY = 1U << 10; +constexpr uint32_t RDMA_METADATA_CAPACITY = 64U << 10; +constexpr uint32_t RDMA_AGGREGATE_CAPACITY = 256U << 10; +constexpr uint32_t RDMA_DATAPATH_CAPACITY = 1U << 20; + +[[nodiscard]] bool classify_rdma_capacity(uint32_t requested, BufferCapacityClass *out, uint32_t *capacity); + // A ref-counted, pool-owned byte buffer. The buffer is allocated from a // BufferPool, written once, then shared across consumers via ref(). Each // consumer calls release() when done; the buffer recycles to the pool when @@ -30,12 +50,22 @@ enum class BufferType : uint8_t { // recycle to pool when ref == 0. struct Buffer { - uint8_t *data = nullptr; - uint32_t len = 0; // bytes written (payload length) - uint32_t capacity = 0; // allocated capacity - BufferType type = BufferType::System; - class BufferPool *pool = nullptr; - std::atomic *ref = nullptr; // shared refcount (pool-allocated or standalone) + uint8_t *data = nullptr; + uint32_t len = 0; // bytes written (payload length) + uint32_t capacity = 0; // allocated capacity + BufferType type = BufferType::System; + class BufferPool *pool = nullptr; + std::atomic *ref = nullptr; // shared refcount (pool-allocated or standalone) + std::atomic ownership{BufferOwnership::Available}; + BufferCapacityClass capacity_class = BufferCapacityClass::Control; + uint64_t request_id = 0; + uint64_t stream_id = 0; + uint64_t sequence = 0; + uint64_t logical_offset = 0; + uint32_t registration_id = 0; + uint32_t registration_key = 0; + uintptr_t registration_handle = 0; + Buffer *next_pool = nullptr; // External free callback: when set, release() calls free_cb(free_ctx) // instead of freeing `data` directly. Used for zero-copy buffers that @@ -52,6 +82,10 @@ struct Buffer // Decrement refcount. On ref == 0, recycle to pool or call free_cb. void release(); + + [[nodiscard]] bool transition(BufferOwnership from, BufferOwnership to); + [[nodiscard]] bool quarantine(); + void set_metadata(uint64_t request, uint64_t stream, uint64_t seq, uint64_t offset, uint32_t registration); }; // BufferPool allocates and recycles Buffer objects. The pool reuses @@ -100,4 +134,50 @@ class SystemBufferPool : public BufferPool std::atomic outstanding_{0}; // in-use buffers (allocated, not yet freed) }; +// Bounded registered-memory abstraction. The pool remains usable without an +// RDMA device; a transport may attach registration metadata later. +class RegisteredBufferPool : public BufferPool +{ + public: + RegisteredBufferPool(uint64_t max_bytes, uint32_t max_buffers = 1024, uint32_t registration_id = 0); + ~RegisteredBufferPool() override; + + Buffer *alloc(uint32_t capacity) override; + void recycle(Buffer *buf) override; + + [[nodiscard]] uint64_t committed_bytes() const + { + return committed_bytes_.load(); + } + + [[nodiscard]] uint64_t quarantined_bytes() const + { + return quarantined_bytes_.load(); + } + + [[nodiscard]] uint32_t registration_id() const + { + return registration_id_; + } + + private: + static constexpr size_t CLASS_COUNT = 4; + uint64_t max_bytes_; + uint32_t max_buffers_; + uint32_t registration_id_; + std::atomic committed_bytes_{0}; + std::atomic outstanding_{0}; + std::array, CLASS_COUNT> free_heads_{}; + std::atomic quarantine_head_{nullptr}; + std::atomic quarantined_bytes_{0}; + + static size_t index(BufferCapacityClass cls) + { + return static_cast(cls); + } + + Buffer *pop(std::atomic &head); + void push(std::atomic &head, Buffer *buf); +}; + } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h index 6dc5269b3..85b138205 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h @@ -37,6 +37,17 @@ crowdb_rpc_buffer_t crowdb_rpc_buffer_alloc(crowdb_rpc_pool_t pool, uint32_t cap void crowdb_rpc_buffer_write(crowdb_rpc_buffer_t buf, const uint8_t *data, uint32_t len); const uint8_t *crowdb_rpc_buffer_data(crowdb_rpc_buffer_t buf); uint32_t crowdb_rpc_buffer_len(crowdb_rpc_buffer_t buf); +uint32_t crowdb_rpc_buffer_capacity(crowdb_rpc_buffer_t buf); +uint64_t crowdb_rpc_buffer_request_id(crowdb_rpc_buffer_t buf); +uint64_t crowdb_rpc_buffer_stream_id(crowdb_rpc_buffer_t buf); +uint64_t crowdb_rpc_buffer_sequence(crowdb_rpc_buffer_t buf); +uint64_t crowdb_rpc_buffer_logical_offset(crowdb_rpc_buffer_t buf); +uint32_t crowdb_rpc_buffer_registration_id(crowdb_rpc_buffer_t buf); +void crowdb_rpc_buffer_set_metadata(crowdb_rpc_buffer_t buf, uint64_t request_id, + uint64_t stream_id, uint64_t sequence, + uint64_t logical_offset, uint32_t registration_id); +int crowdb_rpc_buffer_transition(crowdb_rpc_buffer_t buf, int from, int to); +int crowdb_rpc_buffer_quarantine(crowdb_rpc_buffer_t buf); bool crowdb_rpc_buffer_detach(crowdb_rpc_buffer_t buf); crowdb_rpc_buffer_t crowdb_rpc_buffer_ref(crowdb_rpc_buffer_t buf); void crowdb_rpc_buffer_release(crowdb_rpc_buffer_t buf); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/connection.h b/lib/crowdb-rpc/include/crowdb-rpc/connection.h index 443f32bfa..4f1d2fe16 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/connection.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/connection.h @@ -8,6 +8,7 @@ #include "crowdb-rpc/connection_callback.h" #include "crowdb-rpc/framing.h" #include "crowdb-rpc/transport.h" +#include "crowdb-rpc/transport/rdma/rdma_capability.h" #include @@ -24,6 +25,8 @@ namespace crowdb::rpc // Alias of the shared crowdb-common primitive specialized for OutFrame*. using SendQueue = crowdb::common::MpscQueue; +enum class ConnectionPath : uint8_t { Tcp, Rdma }; + // Forward declaration — Connection::try_send records latency here. struct TransportStats; @@ -53,6 +56,32 @@ class Connection : public std::enable_shared_from_this return id_; } + [[nodiscard]] ConnectionPath path() const + { + return path_; + } + + void set_path(ConnectionPath path) + { + path_ = path; + } + + [[nodiscard]] bool peer_handshake_received() const + { + return peer_handshake_received_.load(std::memory_order_acquire); + } + + void set_peer_capability(const RdmaEndpointCapability &capability) + { + peer_capability_ = capability; + peer_handshake_received_.store(true, std::memory_order_release); + } + + [[nodiscard]] const RdmaEndpointCapability &peer_capability() const + { + return peer_capability_; + } + [[nodiscard]] const std::string &name() const { return name_; @@ -221,10 +250,13 @@ class Connection : public std::enable_shared_from_this } private: - int64_t id_; - std::string name_; - BufferPool *pool_; - FrameParser parser_; + int64_t id_; + std::string name_; + BufferPool *pool_; + ConnectionPath path_ = ConnectionPath::Tcp; + std::atomic peer_handshake_received_{false}; + RdmaEndpointCapability peer_capability_; + FrameParser parser_; static constexpr uint32_t SOCKET_CLOSED = uint32_t{1} << 31; std::atomic socket_users_{0}; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/framing.h b/lib/crowdb-rpc/include/crowdb-rpc/framing.h index 826792e1c..dd5a9bd8d 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/framing.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/framing.h @@ -48,10 +48,12 @@ struct Header struct Frame { Header header; - uint64_t request_id = 0; // extracted from control during parse - uint64_t rpc_create_nano = 0; // extracted from control during parse - uint64_t response_create_nano = 0; // extracted from control during parse (responses only) - Buffer *data_buf = nullptr; // pool-allocated; nullptr if control-only + uint64_t request_id = 0; // extracted from control during parse + uint64_t rpc_create_nano = 0; // extracted from control during parse + uint64_t response_create_nano = 0; // extracted from control during parse (responses only) + Buffer *data_buf = nullptr; // pool-allocated; nullptr if control-only + void (*release_callback)(void *, Buffer *) = nullptr; + void *release_context = nullptr; // Raw control message bytes (flatbuffer). Populated for all frames with // msg_size > 0. Common handlers (ping, unknown) use only request_id + @@ -61,6 +63,8 @@ struct Frame ~Frame() { + if (release_callback != nullptr) + release_callback(release_context, data_buf); if (data_buf != nullptr) { data_buf->release(); } diff --git a/lib/crowdb-rpc/include/crowdb-rpc/server/server.h b/lib/crowdb-rpc/include/crowdb-rpc/server/server.h index dd640c6e2..5cc8d70e5 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/server/server.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/server/server.h @@ -92,6 +92,7 @@ class RpcServer int listen_fd_ = -1; int listen_port_ = 0; + std::string listen_addr_; std::string listener_key_; int listener_lock_fd_ = -1; std::atomic running_{false}; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/epoll/epoll_engine.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/epoll/epoll_engine.h index 89f2073d1..3f729899f 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/epoll/epoll_engine.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/epoll/epoll_engine.h @@ -33,6 +33,8 @@ class EpollEngine : public SocketEngine void add_listen_fd(int fd) override; void remove_listen_fd(int fd) override; + void add_external_fd(int fd, uint64_t tag) override; + void remove_external_fd(int fd) override; void add_connection(int read_fd, int write_fd, Connection *conn) override; void remove_connection(int read_fd, int write_fd) override; void arm_read(int read_fd, Connection *conn) override; @@ -55,6 +57,7 @@ class EpollEngine : public SocketEngine // fd → Connection* map (only used for add/remove; wait() uses data.ptr). std::mutex conn_mu_; std::unordered_map connections_; + std::unordered_map external_fds_; void mod_fd(int fd, uint32_t events, Connection *conn) const; void mod_fd_write(int fd, uint32_t events, Connection *conn) const; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h new file mode 100644 index 000000000..68edc368f --- /dev/null +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h @@ -0,0 +1,92 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/buffer.h" + +#include +#include +#include +#include + +namespace crowdb::rpc +{ + +enum class RdmaPathState : uint8_t { + Unavailable, + Probing, + Connecting, + Ready, + Draining, + Failed, + TcpFallback, +}; + +struct RdmaEndpointCapability +{ + bool rdma_available = false; + uint16_t protocol_version = 0; + uint64_t generation = 0; + uint16_t port = 0; + std::string device; + std::string gid; + std::array data_classes{}; +}; + +constexpr uint16_t RDMA_HANDSHAKE_MAGIC = 0x5244; +constexpr uint16_t RDMA_HANDSHAKE_MSG_TYPE = 0xFFFE; +constexpr size_t RDMA_HANDSHAKE_SIZE = 128; + +[[nodiscard]] std::array +serialize_rdma_handshake(const RdmaEndpointCapability &capability); + +[[nodiscard]] bool parse_rdma_handshake(std::span bytes, RdmaEndpointCapability *out); + +struct RdmaSge +{ + uintptr_t address = 0; + uint32_t length = 0; + uint32_t lkey = 0; +}; + +struct RdmaFrameSges +{ + std::array entries{}; + uint8_t count = 0; +}; + +[[nodiscard]] bool build_rdma_frame_sges(const Buffer *control, const Buffer *data, RdmaFrameSges *out); + +[[nodiscard]] bool validate_rdma_capability(const RdmaEndpointCapability &capability, uint16_t expected_protocol, + uint64_t expected_generation, const std::string &expected_device); + +[[nodiscard]] bool can_fallback_to_tcp(RdmaPathState state, bool payload_submitted); + +class RdmaConnectionState +{ + public: + [[nodiscard]] RdmaPathState state() const + { + return state_; + } + + [[nodiscard]] bool payload_submitted() const + { + return payload_submitted_; + } + + bool begin_probe(); + bool begin_connect(); + bool mark_ready(); + bool mark_payload_submitted(); + bool fail(); + bool begin_drain(); + bool fallback_to_tcp(); + + private: + RdmaPathState state_ = RdmaPathState::Unavailable; + bool payload_submitted_ = false; +}; + +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index d30743a6a..45af15edb 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -6,6 +6,7 @@ #include "crowdb-rpc/buffer.h" #include "crowdb-rpc/connection.h" #include "crowdb-rpc/transport.h" +#include "crowdb-rpc/transport/rdma/rdma_capability.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -19,14 +20,18 @@ # include # include # include +# include namespace crowdb::rpc { +class SocketEngine; + // RdmaBufferPool: allocates buffers from ibv_mr-registered memory. // The pool pre-registers a large memory region and carves out buffers // from it, avoiding per-buffer ibv_reg_mr calls (which are expensive). -// Each buffer's `mr` field points to the sub-region's MR. +// Every buffer is a slice of the single MR, so registration is bounded by the +// pool region and deregistration happens exactly once at pool shutdown. class RdmaBufferPool : public BufferPool { public: @@ -39,13 +44,17 @@ class RdmaBufferPool : public BufferPool private: struct ibv_pd *pd_; - struct ibv_mr *mr_; + struct ibv_mr *mr_ = nullptr; + void *region_ = nullptr; uint32_t mr_size_; uint32_t max_buffers_; uint32_t outstanding_ = 0; + uint32_t next_offset_ = 0; std::mutex mu_; std::unordered_map> free_list_; + std::vector quarantine_; + std::vector all_buffers_; Buffer *alloc_fresh(uint32_t capacity); }; @@ -56,37 +65,114 @@ class RdmaBufferPool : public BufferPool class RdmaTransport : public Transport { public: + using ConnectionHandler = std::function)>; + struct Stats + { + std::atomic sends{0}; + std::atomic receives{0}; + std::atomic cq_errors{0}; + std::atomic fallback_count{0}; + std::atomic posted_receives{0}; + std::atomic quarantined_bytes{0}; + }; + RdmaTransport(); ~RdmaTransport() override; bool submit(Connection *conn, OutFrame *frame) override; + bool post_receive(Connection *conn, Buffer *control, Buffer *data); Buffer *register_buffer(Buffer *buf) override; void shutdown() override; // RDMA-specific: connect to a peer endpoint. // Returns a new Connection on success, nullptr on failure. - static std::shared_ptr connect(const std::string &addr, int port); + std::shared_ptr connect(const std::string &addr, int port); // RDMA-specific: listen on an endpoint (server side). - static bool listen(const std::string &addr, int port); - - // Start the CQ poll loop on worker threads. - void start(); - void stop(); + bool listen(const std::string &addr, int port); + + // Accept one pending CM request and create an RC QP-backed connection. + // The caller owns the returned logical connection and should register it + // with the common RPC worker before submitting frames. + std::shared_ptr accept(); + + // Enable completion handling. CQ draining is driven by the epoll worker. + void start(); + void stop(); + [[nodiscard]] int completion_fd() const; + [[nodiscard]] int cm_event_fd() const; + int drain_cq(uint32_t max_completions = 32); + void register_events(SocketEngine *engine); + int handle_external_event(uint64_t tag, uint32_t max_completions = 32); + void set_connection_handler(ConnectionHandler handler) + { + connection_handler_ = std::move(handler); + } + + [[nodiscard]] const Stats &stats() const + { + return stats_; + } + + Stats &stats() + { + return stats_; + } + + [[nodiscard]] bool ready() const + { + return context_ != nullptr && pd_ != nullptr && cq_ != nullptr; + } private: - struct ibv_context *context_ = nullptr; - struct ibv_pd *pd_ = nullptr; - struct rdma_event_channel *cm_channel_ = nullptr; - struct rdma_cm_id *listen_id_ = nullptr; + struct ibv_context *context_ = nullptr; + struct ibv_pd *pd_ = nullptr; + struct ibv_comp_channel *comp_channel_ = nullptr; + struct ibv_cq *cq_ = nullptr; + struct rdma_event_channel *cm_channel_ = nullptr; + struct rdma_cm_id *listen_id_ = nullptr; + uint8_t port_ = 1; + std::vector cm_ids_; + std::unordered_map cm_cqs_; + std::unordered_map cm_channels_; + SocketEngine *event_engine_ = nullptr; + std::mutex cm_mu_; std::unique_ptr send_pool_; std::unique_ptr recv_pool_; + ConnectionHandler connection_handler_; std::atomic running_{false}; - std::thread cq_thread_; - void cq_loop(); + bool prime_receive(const std::shared_ptr &connection); + Stats stats_; + + struct ReceiveWork + { + Connection *conn; + Buffer *control; + Buffer *data; + }; + + struct RepostContext + { + RdmaTransport *transport; + Connection *connection; + Buffer *control; + Buffer *data; + }; + + struct ConnectionPools + { + std::unique_ptr send; + std::unique_ptr recv; + }; + + std::unordered_map> cm_pools_; + + void release_completed_frame(struct ibv_wc &wc); + void release_completed_receive(struct ibv_wc &wc); + static void repost_frame_release(void *context, Buffer *data); }; } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h index d27463f16..ef8d29a69 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h @@ -23,6 +23,7 @@ namespace crowdb::rpc // references Worker for the owner-worker back-pointer. class SocketTransport; class Worker; +class Frame; // ── TransportStats: aggregation + latency counters ──────────────── // @@ -97,9 +98,10 @@ enum class SocketEvent { Readable, Writable, Error, - Notify, // cross-thread submit wakeup - Timer, // scheduled task deadline - Accept, // listen socket ready (acceptor only) + Notify, // cross-thread submit wakeup + Timer, // scheduled task deadline + Accept, // listen socket ready (acceptor only) + External, // transport-owned event descriptor (for example RDMA CM/CQ) }; struct EngineEvent @@ -109,6 +111,7 @@ struct EngineEvent Connection *conn; // resolved and retained by the worker before dispatch int64_t connection_id = 0; // kernel events carry identity, never an owning pointer bool write_side = false; + uint64_t external_tag = 0; }; class SocketEngine @@ -156,6 +159,19 @@ class SocketEngine (void)fd; } + // Register transport-owned descriptors such as an RDMA completion or CM + // channel. The engine only reports readiness; the owner drains it. + virtual void add_external_fd(int fd, uint64_t tag) + { + (void)fd; + (void)tag; + } + + virtual void remove_external_fd(int fd) + { + (void)fd; + } + // Connection management. read_fd and write_fd are separate fds // (write_fd = dup(read_fd)) so EPOLLONESHOT on read and write are // independent — arming write does not re-arm read. When write_fd @@ -356,6 +372,14 @@ class SocketTransport : public Transport // (nullptr on failure). The connection can both send and receive. std::shared_ptr connect(const std::string &addr, int port); + // Attach an RDMA connection to the common callback/registry lifecycle. + void attach_rdma_connection(const std::shared_ptr &conn, + Connection::OnFrameCallback on_frame = {}, + Connection::OnCloseCallback on_close = {}); + + using RdmaAcceptHandler = std::function &)>; + void set_rdma_accept_handler(RdmaAcceptHandler handler); + // Register a connection in the live-connection registry. Called // when a connection is created. The registry holds a weak_ptr so // that submit() on a stale handle returns false instead of crashing. @@ -465,8 +489,25 @@ class SocketTransport : public Transport // registered accept handler with the listen fd. void on_accept(int listen_fd); + void set_external_handler(std::function handler) + { + external_handler_ = std::move(handler); + } + + void on_external(uint64_t tag) + { + if (external_handler_) + external_handler_(tag); + } + + bool send_rdma_handshake(const std::shared_ptr &conn); + bool listen_rdma(const std::string &addr, int port); + private: - BufferPool *pool_; + BufferPool *pool_; +#ifdef CROWDB_RPC_HAVE_RDMA + std::unique_ptr rdma_transport_; +#endif std::vector> workers_; std::vector> engines_; // transport owns all engines std::atomic next_worker_{0}; @@ -491,7 +532,9 @@ class SocketTransport : public Transport // Accept handler — called by a worker thread when the listen fd // is readable. Set by the server via set_accept_handler. - std::function accept_handler_; + std::function accept_handler_; + std::function external_handler_; + RdmaAcceptHandler rdma_accept_handler_; // Connection registry: maps raw Connection* to shared_ptr, so // submit() can safely access connections from arbitrary threads diff --git a/lib/crowdb-rpc/src/buffer.cpp b/lib/crowdb-rpc/src/buffer.cpp index 545317569..3fa6d35c8 100644 --- a/lib/crowdb-rpc/src/buffer.cpp +++ b/lib/crowdb-rpc/src/buffer.cpp @@ -9,6 +9,31 @@ namespace crowdb::rpc { +bool classify_rdma_capacity(uint32_t requested, BufferCapacityClass *out, uint32_t *capacity) +{ + if (requested == 0 || out == nullptr || capacity == nullptr) + return false; + if (requested <= RDMA_CONTROL_CAPACITY) { + *out = BufferCapacityClass::Control; + *capacity = RDMA_CONTROL_CAPACITY; + } + else if (requested <= RDMA_METADATA_CAPACITY) { + *out = BufferCapacityClass::Metadata; + *capacity = RDMA_METADATA_CAPACITY; + } + else if (requested <= RDMA_AGGREGATE_CAPACITY) { + *out = BufferCapacityClass::Aggregate; + *capacity = RDMA_AGGREGATE_CAPACITY; + } + else if (requested <= RDMA_DATAPATH_CAPACITY) { + *out = BufferCapacityClass::Datapath; + *capacity = RDMA_DATAPATH_CAPACITY; + } + else + return false; + return true; +} + // ── Buffer ─────────────────────────────────────────────────────── void Buffer::write(const void *src, uint32_t n) @@ -56,6 +81,31 @@ void Buffer::release() } } +bool Buffer::transition(BufferOwnership from, BufferOwnership to) +{ + return ownership.compare_exchange_strong(from, to, std::memory_order_acq_rel, std::memory_order_acquire); +} + +bool Buffer::quarantine() +{ + BufferOwnership state = ownership.load(std::memory_order_acquire); + while (state != BufferOwnership::Quarantined) { + if (ownership.compare_exchange_weak(state, BufferOwnership::Quarantined, std::memory_order_acq_rel, + std::memory_order_acquire)) + return true; + } + return false; +} + +void Buffer::set_metadata(uint64_t request, uint64_t stream, uint64_t seq, uint64_t offset, uint32_t registration) +{ + request_id = request; + stream_id = stream; + sequence = seq; + logical_offset = offset; + registration_id = registration; +} + // ── SystemBufferPool ────────────────────────────────────────────── // // Direct heap allocation — no free-list, no mutex. Each alloc does @@ -111,4 +161,106 @@ bool SystemBufferPool::detach(Buffer *buf) return true; } +RegisteredBufferPool::RegisteredBufferPool(uint64_t max_bytes, uint32_t max_buffers, uint32_t registration_id) + : max_bytes_(max_bytes), + max_buffers_(max_buffers), + registration_id_(registration_id) +{ + for (auto &head : free_heads_) + head.store(nullptr, std::memory_order_relaxed); +} + +RegisteredBufferPool::~RegisteredBufferPool() +{ + auto destroy = [](Buffer *head) { + while (head != nullptr) { + Buffer *next = head->next_pool; + delete head->ref; + std::free(head->data); + delete head; + head = next; + } + }; + for (auto &head : free_heads_) + destroy(head.exchange(nullptr, std::memory_order_acq_rel)); + destroy(quarantine_head_.exchange(nullptr, std::memory_order_acq_rel)); +} + +Buffer *RegisteredBufferPool::pop(std::atomic &head) +{ + Buffer *current = head.load(std::memory_order_acquire); + while (current != nullptr && !head.compare_exchange_weak(current, current->next_pool, std::memory_order_acq_rel, + std::memory_order_acquire)) { + } + if (current != nullptr) + current->next_pool = nullptr; + return current; +} + +void RegisteredBufferPool::push(std::atomic &head, Buffer *buf) +{ + Buffer *current = head.load(std::memory_order_acquire); + do { + buf->next_pool = current; + } while (!head.compare_exchange_weak(current, buf, std::memory_order_acq_rel, std::memory_order_acquire)); +} + +Buffer *RegisteredBufferPool::alloc(uint32_t requested) +{ + BufferCapacityClass cls; + uint32_t capacity; + if (!classify_rdma_capacity(requested, &cls, &capacity)) + return nullptr; + Buffer *buf = pop(free_heads_[index(cls)]); + if (buf == nullptr) { + uint64_t bytes = committed_bytes_.load(std::memory_order_relaxed); + while (bytes + capacity <= max_bytes_ && + !committed_bytes_.compare_exchange_weak(bytes, bytes + capacity, std::memory_order_acq_rel, + std::memory_order_relaxed)) { + } + if (bytes + capacity > max_bytes_) + return nullptr; + uint32_t count = outstanding_.fetch_add(1, std::memory_order_acq_rel); + if (count >= max_buffers_) { + outstanding_.fetch_sub(1, std::memory_order_acq_rel); + committed_bytes_.fetch_sub(capacity, std::memory_order_acq_rel); + return nullptr; + } + void *data = nullptr; + if (posix_memalign(&data, 64, capacity) != 0) { + outstanding_.fetch_sub(1, std::memory_order_acq_rel); + committed_bytes_.fetch_sub(capacity, std::memory_order_acq_rel); + return nullptr; + } + buf = new Buffer; + buf->data = static_cast(data); + buf->capacity = capacity; + buf->type = BufferType::Registered; + buf->pool = this; + buf->ref = new std::atomic(1); + } + else { + outstanding_.fetch_add(1, std::memory_order_acq_rel); + buf->ref->store(1, std::memory_order_release); + } + buf->len = 0; + buf->capacity_class = cls; + buf->registration_id = registration_id_; + buf->ownership.store(BufferOwnership::Available, std::memory_order_release); + return buf; +} + +void RegisteredBufferPool::recycle(Buffer *buf) +{ + outstanding_.fetch_sub(1, std::memory_order_acq_rel); + if (buf->ownership.load(std::memory_order_acquire) == BufferOwnership::Quarantined) { + quarantined_bytes_.fetch_add(buf->capacity, std::memory_order_acq_rel); + push(quarantine_head_, buf); + return; + } + buf->len = 0; + buf->ownership.store(BufferOwnership::Available, std::memory_order_release); + push(free_heads_[index(buf->capacity_class)], buf); +} + } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/c_api.cpp b/lib/crowdb-rpc/src/c_api.cpp index 50909b1d3..64f0ca746 100644 --- a/lib/crowdb-rpc/src/c_api.cpp +++ b/lib/crowdb-rpc/src/c_api.cpp @@ -110,6 +110,61 @@ uint32_t crowdb_rpc_buffer_len(crowdb_rpc_buffer_t buf) } } +uint32_t crowdb_rpc_buffer_capacity(crowdb_rpc_buffer_t buf) +{ + return (buf != nullptr && buf->buf != nullptr) ? buf->buf->capacity : 0; +} + +uint64_t crowdb_rpc_buffer_request_id(crowdb_rpc_buffer_t buf) +{ + return (buf != nullptr && buf->buf != nullptr) ? buf->buf->request_id : 0; +} + +uint64_t crowdb_rpc_buffer_stream_id(crowdb_rpc_buffer_t buf) +{ + return (buf != nullptr && buf->buf != nullptr) ? buf->buf->stream_id : 0; +} + +uint64_t crowdb_rpc_buffer_sequence(crowdb_rpc_buffer_t buf) +{ + return (buf != nullptr && buf->buf != nullptr) ? buf->buf->sequence : 0; +} + +uint64_t crowdb_rpc_buffer_logical_offset(crowdb_rpc_buffer_t buf) +{ + return (buf != nullptr && buf->buf != nullptr) ? buf->buf->logical_offset : 0; +} + +uint32_t crowdb_rpc_buffer_registration_id(crowdb_rpc_buffer_t buf) +{ + return (buf != nullptr && buf->buf != nullptr) ? buf->buf->registration_id : 0; +} + +void crowdb_rpc_buffer_set_metadata(crowdb_rpc_buffer_t buf, uint64_t request_id, + uint64_t stream_id, uint64_t sequence, + uint64_t logical_offset, uint32_t registration_id) +{ + if (buf != nullptr && buf->buf != nullptr) { + buf->buf->set_metadata(request_id, stream_id, sequence, logical_offset, registration_id); + } +} + +int crowdb_rpc_buffer_transition(crowdb_rpc_buffer_t buf, int from, int to) +{ + if (buf == nullptr || buf->buf == nullptr || from < 0 || from > 6 || to < 0 || to > 6) { + return 0; + } + return buf->buf->transition(static_cast(from), + static_cast(to)) + ? 1 + : 0; +} + +int crowdb_rpc_buffer_quarantine(crowdb_rpc_buffer_t buf) +{ + return (buf != nullptr && buf->buf != nullptr && buf->buf->quarantine()) ? 1 : 0; +} + bool crowdb_rpc_buffer_detach(crowdb_rpc_buffer_t buf) { try { diff --git a/lib/crowdb-rpc/src/server/server.cpp b/lib/crowdb-rpc/src/server/server.cpp index a81259fba..c0a5a1ee4 100644 --- a/lib/crowdb-rpc/src/server/server.cpp +++ b/lib/crowdb-rpc/src/server/server.cpp @@ -7,6 +7,7 @@ #include "crowdb-rpc/client/client.h" // RpcClient, RpcError (for request_client_ + fail_all) #include "crowdb-rpc/rpc_metrics.h" #include "crowdb-rpc/server/handler.h" +#include "crowdb-rpc/transport/rdma/rdma_capability.h" #include "msg_type_generated.h" #include @@ -18,6 +19,7 @@ #include #include +#include #include #include #include @@ -61,7 +63,8 @@ RpcServer::~RpcServer() bool RpcServer::listen(const std::string &addr, int port) { - listen_fd_ = ::socket(AF_INET, SOCK_STREAM, 0); + listen_addr_ = addr.empty() ? "0.0.0.0" : addr; + listen_fd_ = ::socket(AF_INET, SOCK_STREAM, 0); if (listen_fd_ < 0) { const int error = errno; CRB_LOG_ERROR("rpc server socket failed: addr={} port={} errno={} ({})", addr, port, error, @@ -176,6 +179,21 @@ void RpcServer::start() return; } CRB_LOG_INFO("rpc server: starting"); + if (const char *enabled = std::getenv("CROWDB_RDMA_ENABLED"); + enabled != nullptr && std::strcmp(enabled, "1") == 0) { + transport_->set_rdma_accept_handler([this](const std::shared_ptr &conn) { + transport_->attach_rdma_connection( + conn, + [this](Frame *frame, Connection *c) { dispatch(frame, c); }, + [this](Connection *c) { + if (request_client_ != nullptr) + request_client_->fail_all(c, RpcError::ConnectionClosed); + }); + transport_->send_rdma_handshake(conn); + }); + if (!transport_->listen_rdma(listen_addr_, listen_port_)) + CRB_LOG_WARN("rpc server: RDMA listener unavailable; TCP remains active"); + } transport_->start(); // Register the listen fd with the I/O worker's epoll loop instead of // a dedicated acceptor thread. The worker's epoll loop is already @@ -267,6 +285,7 @@ void RpcServer::handle_accept(int listen_fd) request_client_->fail_all(c, RpcError::ConnectionClosed); } }); + transport_->send_rdma_handshake(conn); CRB_LOG_INFO("rpc server: connection accepted {}:{} -> conn_id={}", peer_ip, peer_port, static_cast(conn->id())); } @@ -277,6 +296,14 @@ void RpcServer::dispatch(Frame *frame, Connection *conn) uint16_t msg_type = frame->header.msg_type; bool is_one_way = (frame->header.flags & FLAG_ONE_WAY) != 0; + if (msg_type == RDMA_HANDSHAKE_MSG_TYPE) { + RdmaEndpointCapability capability; + if (parse_rdma_handshake(frame->control, &capability)) + conn->set_peer_capability(capability); + delete frame; + return; + } + // Try request dispatch first — if a handler is registered for // this msg_type, dispatch as a request. This ensures request // frames are not intercepted by on_response (which matches by diff --git a/lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp b/lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp index 69d70172f..3c0d253fd 100644 --- a/lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp +++ b/lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp @@ -69,6 +69,7 @@ void EpollEngine::set_oneshot(bool on) // Bit 0 of data.ptr marks write fds (buzz-cpp pattern). Connection* // is at least 2-byte aligned, so bit 0 is always 0 for read fds. static constexpr uint64_t CONNECTION_EVENT = uint64_t{1} << 63; +static constexpr uint64_t EXTERNAL_EVENT = uint64_t{1} << 62; static uint64_t event_token(Connection *conn, bool write_side) { @@ -115,6 +116,35 @@ void EpollEngine::remove_listen_fd(int fd) ::epoll_ctl(epoll_fd_, EPOLL_CTL_DEL, fd, nullptr); } +void EpollEngine::add_external_fd(int fd, uint64_t tag) +{ + if (fd < 0 || epoll_fd_ < 0 || (static_cast(fd) & EXTERNAL_EVENT) != 0) { + return; + } + { + std::scoped_lock lock(conn_mu_); + external_fds_[fd] = tag; + } + struct epoll_event ev; + std::memset(&ev, 0, sizeof(ev)); + ev.events = EPOLLIN; + ev.data.u64 = EXTERNAL_EVENT | static_cast(fd); + if (::epoll_ctl(epoll_fd_, EPOLL_CTL_ADD, fd, &ev) < 0) { + std::scoped_lock lock(conn_mu_); + external_fds_.erase(fd); + } +} + +void EpollEngine::remove_external_fd(int fd) +{ + if (fd < 0) { + return; + } + ::epoll_ctl(epoll_fd_, EPOLL_CTL_DEL, fd, nullptr); + std::scoped_lock lock(conn_mu_); + external_fds_.erase(fd); +} + void EpollEngine::add_connection(int read_fd, int write_fd, Connection *conn) { { @@ -233,6 +263,20 @@ int EpollEngine::wait(EngineEvent *out_events, int max_events, int timeout_ms) for (int i = 0; i < n && out < max_events; i++) { const struct epoll_event &ev = events[i]; + if ((ev.data.u64 & EXTERNAL_EVENT) != 0) { + const int fd = static_cast(ev.data.u64 & (EXTERNAL_EVENT - 1)); + uint64_t tag = 0; + { + std::scoped_lock lock(conn_mu_); + auto it = external_fds_.find(fd); + if (it != external_fds_.end()) { + tag = it->second; + } + } + out_events[out++] = {.type = SocketEvent::External, .fd = fd, .conn = nullptr, .external_tag = tag}; + continue; + } + if ((ev.data.u64 & CONNECTION_EVENT) != 0) { const auto id = static_cast((ev.data.u64 & ~CONNECTION_EVENT) >> 1); const bool write_side = (ev.data.u64 & 1) != 0; @@ -274,6 +318,10 @@ int EpollEngine::wait(EngineEvent *out_events, int max_events, int timeout_ms) void EpollEngine::shutdown() { + { + std::scoped_lock lock(conn_mu_); + external_fds_.clear(); + } if (notify_fd_ >= 0) { ::close(notify_fd_); notify_fd_ = -1; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp new file mode 100644 index 000000000..0a24b8aa4 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp @@ -0,0 +1,176 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/transport/rdma/rdma_capability.h" + +#include +#include + +namespace crowdb::rpc +{ + +std::array serialize_rdma_handshake(const RdmaEndpointCapability &capability) +{ + std::array bytes{}; + auto put16 = [&](size_t offset, uint16_t value) { + bytes[offset] = static_cast(value); + bytes[offset + 1] = static_cast(value >> 8); + }; + auto put64 = [&](size_t offset, uint64_t value) { + for (size_t i = 0; i < 8; ++i) + bytes[offset + i] = static_cast(value >> (i * 8)); + }; + put16(0, RDMA_HANDSHAKE_MAGIC); + put16(2, capability.protocol_version); + put64(4, capability.generation); + put16(12, capability.port); + bytes[14] = capability.rdma_available ? 1 : 0; + std::memcpy(bytes.data() + 16, capability.device.data(), std::min(32, capability.device.size())); + std::memcpy(bytes.data() + 48, capability.gid.data(), std::min(64, capability.gid.size())); + for (size_t i = 0; i < capability.data_classes.size(); ++i) { + const uint32_t value = capability.data_classes[i]; + const size_t offset = 112 + i * 4; + bytes[offset] = static_cast(value); + bytes[offset + 1] = static_cast(value >> 8); + bytes[offset + 2] = static_cast(value >> 16); + bytes[offset + 3] = static_cast(value >> 24); + } + return bytes; +} + +bool parse_rdma_handshake(std::span bytes, RdmaEndpointCapability *out) +{ + if (out == nullptr || bytes.size() < RDMA_HANDSHAKE_SIZE) + return false; + auto get16 = [&](size_t offset) { + return static_cast(bytes[offset]) | static_cast(bytes[offset + 1] << 8); + }; + auto get64 = [&](size_t offset) { + uint64_t value = 0; + for (size_t i = 0; i < 8; ++i) + value |= static_cast(bytes[offset + i]) << (i * 8); + return value; + }; + if (get16(0) != RDMA_HANDSHAKE_MAGIC) + return false; + out->protocol_version = get16(2); + out->generation = get64(4); + out->port = get16(12); + out->rdma_available = bytes[14] != 0; + out->device.assign(reinterpret_cast(bytes.data() + 16), 32); + out->gid.assign(reinterpret_cast(bytes.data() + 48), 64); + out->device.resize(out->device.find('\0') == std::string::npos ? out->device.size() : out->device.find('\0')); + out->gid.resize(out->gid.find('\0') == std::string::npos ? out->gid.size() : out->gid.find('\0')); + for (size_t i = 0; i < out->data_classes.size(); ++i) { + const size_t offset = 112 + i * 4; + out->data_classes[i] = static_cast(bytes[offset]) | (static_cast(bytes[offset + 1]) << 8) | + (static_cast(bytes[offset + 2]) << 16) | + (static_cast(bytes[offset + 3]) << 24); + } + return true; +} + +bool build_rdma_frame_sges(const Buffer *control, const Buffer *data, RdmaFrameSges *out) +{ + if (control == nullptr || out == nullptr || control->data == nullptr || control->len > RDMA_CONTROL_CAPACITY || + control->len > control->capacity) { + return false; + } + out->entries[0] = {.address = reinterpret_cast(control->data), + .length = control->len, + .lkey = control->registration_key}; + out->count = 1; + if (data == nullptr || data->len == 0) + return true; + if (data->data == nullptr || data->len > data->capacity) + return false; + BufferCapacityClass cls; + uint32_t capacity; + if (!classify_rdma_capacity(data->capacity, &cls, &capacity)) + return false; + (void)cls; + (void)capacity; + out->entries[1] = { + .address = reinterpret_cast(data->data), .length = data->len, .lkey = data->registration_key}; + out->count = 2; + return true; +} + +bool validate_rdma_capability(const RdmaEndpointCapability &capability, uint16_t expected_protocol, + uint64_t expected_generation, const std::string &expected_device) +{ + if (!capability.rdma_available || capability.protocol_version != expected_protocol || + capability.generation != expected_generation || capability.port == 0 || capability.device.empty() || + capability.gid.empty() || (!expected_device.empty() && capability.device != expected_device)) { + return false; + } + return capability.data_classes[0] == RDMA_CONTROL_CAPACITY && + capability.data_classes[1] == RDMA_METADATA_CAPACITY && + capability.data_classes[2] == RDMA_AGGREGATE_CAPACITY && + capability.data_classes[3] == RDMA_DATAPATH_CAPACITY; +} + +bool can_fallback_to_tcp(RdmaPathState state, bool payload_submitted) +{ + if (payload_submitted) + return false; + return state == RdmaPathState::Unavailable || state == RdmaPathState::Failed || state == RdmaPathState::Connecting; +} + +bool RdmaConnectionState::begin_probe() +{ + if (state_ != RdmaPathState::Unavailable) + return false; + state_ = RdmaPathState::Probing; + return true; +} + +bool RdmaConnectionState::begin_connect() +{ + if (state_ != RdmaPathState::Probing) + return false; + state_ = RdmaPathState::Connecting; + return true; +} + +bool RdmaConnectionState::mark_ready() +{ + if (state_ != RdmaPathState::Connecting) + return false; + state_ = RdmaPathState::Ready; + return true; +} + +bool RdmaConnectionState::mark_payload_submitted() +{ + if (state_ != RdmaPathState::Ready || payload_submitted_) + return false; + payload_submitted_ = true; + return true; +} + +bool RdmaConnectionState::fail() +{ + if (state_ != RdmaPathState::Connecting && state_ != RdmaPathState::Ready) + return false; + state_ = RdmaPathState::Failed; + return true; +} + +bool RdmaConnectionState::begin_drain() +{ + if (state_ != RdmaPathState::Ready && state_ != RdmaPathState::Failed) + return false; + state_ = RdmaPathState::Draining; + return true; +} + +bool RdmaConnectionState::fallback_to_tcp() +{ + if (!can_fallback_to_tcp(state_, payload_submitted_)) + return false; + state_ = RdmaPathState::TcpFallback; + return true; +} + +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index 101eeabf6..7abee30f8 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -3,16 +3,47 @@ #include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "crowdb-rpc/transport/socket_transport.h" +#include "crowdb-common/log.h" + #ifdef CROWDB_RPC_HAVE_RDMA +# include +# include +# include +# include # include +# include # include +# include # include namespace crowdb::rpc { +namespace +{ +void make_nonblocking(int fd) +{ + if (fd < 0) + return; + const int flags = ::fcntl(fd, F_GETFL, 0); + if (flags >= 0) + (void)::fcntl(fd, F_SETFL, flags | O_NONBLOCK); +} + +bool wait_cm_event(rdma_event_channel *channel, rdma_cm_event **event, int timeout_ms) +{ + if (channel == nullptr || event == nullptr) + return false; + pollfd descriptor{channel->fd, POLLIN, 0}; + if (::poll(&descriptor, 1, timeout_ms) <= 0 || (descriptor.revents & POLLIN) == 0) + return false; + return rdma_get_cm_event(channel, event) == 0; +} +} + // ── RdmaBufferPool ──────────────────────────────────────────────── RdmaBufferPool::RdmaBufferPool(struct ibv_pd *pd, uint32_t mr_size, uint32_t max_buffers) @@ -20,16 +51,19 @@ RdmaBufferPool::RdmaBufferPool(struct ibv_pd *pd, uint32_t mr_size, uint32_t max mr_size_(mr_size), max_buffers_(max_buffers) { - // Allocate and register a large memory region. - void *buf = std::malloc(mr_size); - if (buf == nullptr) { + if (pd_ == nullptr || mr_size_ == 0 || max_buffers_ == 0) { return; } - mr_ = ibv_reg_mr(pd_, buf, mr_size, + if (posix_memalign(®ion_, 64, mr_size_) != 0) { + region_ = nullptr; + return; + } + mr_ = ibv_reg_mr(pd_, region_, mr_size_, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_ATOMIC); if (mr_ == nullptr) { - std::free(buf); + std::free(region_); + region_ = nullptr; return; } } @@ -37,40 +71,37 @@ RdmaBufferPool::RdmaBufferPool(struct ibv_pd *pd, uint32_t mr_size, uint32_t max RdmaBufferPool::~RdmaBufferPool() { std::scoped_lock lock(mu_); - for (auto &[_, vec] : free_list_) { - for (Buffer *buf : vec) { - delete buf->ref; - delete buf; - } + if (outstanding_ != 0) { + // Keep the MR registered while a caller, disk operation, or device + // completion still owns a slice. Process teardown reclaims it. + return; + } + for (Buffer *buf : all_buffers_) { + delete buf->ref; + delete buf; } + all_buffers_.clear(); free_list_.clear(); + quarantine_.clear(); if (mr_ != nullptr) { - void *buf = mr_->addr; ibv_dereg_mr(mr_); - std::free(buf); + mr_ = nullptr; } -} - -static uint32_t bucket_capacity(uint32_t capacity) -{ - if (capacity == 0) { - return 1; + if (region_ != nullptr) { + std::free(region_); + region_ = nullptr; } - --capacity; - capacity |= capacity >> 1; - capacity |= capacity >> 2; - capacity |= capacity >> 4; - capacity |= capacity >> 8; - capacity |= capacity >> 16; - return capacity + 1; } Buffer *RdmaBufferPool::alloc_fresh(uint32_t capacity) { - if (outstanding_ >= max_buffers_ || mr_ == nullptr) { + if (outstanding_ >= max_buffers_ || mr_ == nullptr || region_ == nullptr) { + return nullptr; + } + const uint32_t aligned_offset = (next_offset_ + 63U) & ~63U; + if (capacity > mr_size_ - aligned_offset) { return nullptr; } - ++outstanding_; auto *buf = new Buffer; buf->capacity = capacity; @@ -78,56 +109,55 @@ Buffer *RdmaBufferPool::alloc_fresh(uint32_t capacity) buf->pool = this; buf->ref = new std::atomic(1); - // Carve out from the registered memory region. - // TODO: proper slab allocation from the MR. For now, malloc + reg - // per buffer (simpler, slower — will be optimized). - void *ptr = std::malloc(capacity); - if (ptr == nullptr) { - delete buf->ref; - delete buf; - --outstanding_; - return nullptr; - } - // Register this sub-region. - struct ibv_mr *sub_mr = ibv_reg_mr(pd_, ptr, capacity, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); - if (sub_mr == nullptr) { - std::free(ptr); - delete buf->ref; - delete buf; - --outstanding_; - return nullptr; - } - buf->data = static_cast(ptr); - buf->len = 0; - // Store the MR pointer in the buffer's pool field (cast — the Buffer - // struct doesn't have an mr field yet; this is a TODO for the full - // RDMA implementation). + buf->data = static_cast(region_) + aligned_offset; + buf->len = 0; + buf->registration_handle = reinterpret_cast(mr_); + buf->registration_key = mr_->lkey; + next_offset_ = aligned_offset + capacity; + ++outstanding_; + all_buffers_.push_back(buf); return buf; } Buffer *RdmaBufferPool::alloc(uint32_t capacity) { - uint32_t bucket = bucket_capacity(capacity); - { - std::scoped_lock lock(mu_); - auto it = free_list_.find(bucket); - if (it != free_list_.end() && !it->second.empty()) { - Buffer *buf = it->second.back(); - it->second.pop_back(); - buf->len = 0; - buf->ref->store(1, std::memory_order_relaxed); - return buf; - } + BufferCapacityClass cls; + uint32_t bucket; + if (!classify_rdma_capacity(capacity, &cls, &bucket)) { + return nullptr; } - return alloc_fresh(bucket); + std::scoped_lock lock(mu_); + auto it = free_list_.find(bucket); + if (it != free_list_.end() && !it->second.empty()) { + Buffer *buf = it->second.back(); + it->second.pop_back(); + buf->len = 0; + buf->ref->store(1, std::memory_order_relaxed); + buf->capacity_class = cls; + buf->ownership.store(BufferOwnership::Available, std::memory_order_release); + ++outstanding_; + return buf; + } + Buffer *buf = alloc_fresh(bucket); + if (buf != nullptr) { + buf->capacity_class = cls; + buf->registration_id = mr_ == nullptr ? 0 : mr_->lkey; + } + return buf; } void RdmaBufferPool::recycle(Buffer *buf) { { std::scoped_lock lock(mu_); - uint32_t bucket = bucket_capacity(buf->capacity); - free_list_[bucket].push_back(buf); + if (buf->ownership.load(std::memory_order_acquire) == BufferOwnership::Quarantined) { + quarantine_.push_back(buf); + } + else { + buf->len = 0; + buf->ownership.store(BufferOwnership::Available, std::memory_order_release); + free_list_[buf->capacity].push_back(buf); + } } --outstanding_; } @@ -136,69 +166,886 @@ void RdmaBufferPool::recycle(Buffer *buf) RdmaTransport::RdmaTransport() { - // TODO: open the IB device, allocate a PD, create the buffer pools. - // This is the stub — the full implementation requires Linux hardware. + int device_count = 0; + ibv_device **devices = ibv_get_device_list(&device_count); + if (devices == nullptr || device_count == 0) { + std::fprintf(stderr, "rdma init: no ibv devices count=%d errno=%d\\n", device_count, errno); + if (devices != nullptr) { + ibv_free_device_list(devices); + } + return; + } + const char *requested_device = std::getenv("CROWDB_RDMA_DEVICE"); + if (const char *requested_port = std::getenv("CROWDB_RDMA_PORT"); requested_port != nullptr) { + const int value = std::atoi(requested_port); + if (value > 0 && value <= 255) + port_ = static_cast(value); + } + ibv_device *selected = devices[0]; + if (requested_device != nullptr) { + selected = nullptr; + for (int i = 0; i < device_count; ++i) { + if (std::strcmp(ibv_get_device_name(devices[i]), requested_device) == 0) { + selected = devices[i]; + break; + } + } + if (selected == nullptr) { + std::fprintf(stderr, "rdma init: requested device not found name=%s count=%d\\n", requested_device, + device_count); + ibv_free_device_list(devices); + return; + } + } + context_ = ibv_open_device(selected); + ibv_free_device_list(devices); + if (context_ == nullptr) { + std::fprintf(stderr, "rdma init: ibv_open_device failed name=%s errno=%d\\n", + requested_device == nullptr ? "default" : requested_device, errno); + return; + } + ibv_port_attr port_attributes{}; + if (ibv_query_port(context_, port_, &port_attributes) != 0 || port_attributes.state != IBV_PORT_ACTIVE) { + std::fprintf(stderr, "rdma init: port unavailable port=%u state=%u errno=%d\\n", port_, + port_attributes.state, errno); + ibv_close_device(context_); + context_ = nullptr; + return; + } + pd_ = ibv_alloc_pd(context_); + if (pd_ == nullptr) { + std::fprintf(stderr, "rdma init: ibv_alloc_pd failed errno=%d\\n", errno); + ibv_close_device(context_); + context_ = nullptr; + return; + } + comp_channel_ = ibv_create_comp_channel(context_); + if (comp_channel_ == nullptr) { + std::fprintf(stderr, "rdma init: ibv_create_comp_channel failed errno=%d\\n", errno); + ibv_dealloc_pd(pd_); + pd_ = nullptr; + ibv_close_device(context_); + context_ = nullptr; + return; + } + const int channel_flags = ::fcntl(comp_channel_->fd, F_GETFL, 0); + if (channel_flags >= 0) + ::fcntl(comp_channel_->fd, F_SETFL, channel_flags | O_NONBLOCK); + cq_ = ibv_create_cq(context_, 2048, this, comp_channel_, 0); + if (cq_ == nullptr || ibv_req_notify_cq(cq_, 0) != 0) { + std::fprintf(stderr, "rdma init: CQ setup failed cq=%p errno=%d\\n", static_cast(cq_), errno); + if (cq_ != nullptr) + ibv_destroy_cq(cq_); + cq_ = nullptr; + ibv_destroy_comp_channel(comp_channel_); + comp_channel_ = nullptr; + ibv_dealloc_pd(pd_); + pd_ = nullptr; + ibv_close_device(context_); + context_ = nullptr; + return; + } + constexpr uint32_t pool_bytes = 64U << 20; + send_pool_ = std::make_unique(pd_, pool_bytes, 1024); + recv_pool_ = std::make_unique(pd_, pool_bytes, 1024); } RdmaTransport::~RdmaTransport() { - shutdown(); + RdmaTransport::shutdown(); + send_pool_.reset(); + recv_pool_.reset(); + if (cq_ != nullptr) { + ibv_destroy_cq(cq_); + cq_ = nullptr; + } + if (comp_channel_ != nullptr) { + ibv_destroy_comp_channel(comp_channel_); + comp_channel_ = nullptr; + } + if (pd_ != nullptr) { + ibv_dealloc_pd(pd_); + pd_ = nullptr; + } + if (context_ != nullptr) { + ibv_close_device(context_); + context_ = nullptr; + } +} + +bool RdmaTransport::submit(Connection *conn, OutFrame *frame) +{ + if (conn == nullptr || frame == nullptr || cq_ == nullptr || frame->control == nullptr || + conn->transport_handle == 0) { + return false; + } + auto *pools = static_cast(conn->user_data); + if (pools != nullptr && pools->send != nullptr && frame->control->pool != pools->send.get()) { + Buffer *copy = pools->send->alloc(frame->control->len); + if (copy == nullptr) + return false; + std::memcpy(copy->data, frame->control->data, frame->control->len); + copy->len = frame->control->len; + copy->request_id = frame->control->request_id; + copy->stream_id = frame->control->stream_id; + copy->sequence = frame->control->sequence; + copy->logical_offset = frame->control->logical_offset; + frame->control->release(); + frame->control = copy; + } + RdmaFrameSges frame_sges; + if (!build_rdma_frame_sges(frame->control, frame->data, &frame_sges)) { + return false; + } + ibv_sge sges[2]{}; + for (uint8_t i = 0; i < frame_sges.count; ++i) { + sges[i].addr = frame_sges.entries[i].address; + sges[i].length = frame_sges.entries[i].length; + sges[i].lkey = frame_sges.entries[i].lkey; + } + ibv_send_wr wr{}; + wr.wr_id = reinterpret_cast(frame); + wr.sg_list = sges; + wr.num_sge = frame_sges.count; + wr.opcode = IBV_WR_SEND; + wr.send_flags = IBV_SEND_SIGNALED; + ibv_send_wr *bad = nullptr; + if (ibv_post_send(reinterpret_cast(conn->transport_handle), &wr, &bad) != 0) { + std::fprintf(stderr, "rdma send: ibv_post_send failed qp=%p errno=%d\n", + reinterpret_cast(conn->transport_handle), errno); + return false; + } + std::fprintf(stderr, "rdma send: posted qp=%p frame=%p\n", + reinterpret_cast(conn->transport_handle), static_cast(frame)); + stats_.sends.fetch_add(1, std::memory_order_relaxed); + return true; } -bool RdmaTransport::submit(Connection * /*conn*/, OutFrame * /*frame*/) +bool RdmaTransport::post_receive(Connection *conn, Buffer *control, Buffer *data) { - // TODO: build a send WR with the control + data MRs, ibv_post_send. - return false; // stub + if (conn == nullptr || control == nullptr || control->data == nullptr || cq_ == nullptr || + conn->transport_handle == 0 || control->capacity < RDMA_CONTROL_CAPACITY) { + return false; + } + ibv_sge sges[2]{}; + sges[0].addr = reinterpret_cast(control->data); + sges[0].length = control->capacity; + sges[0].lkey = control->registration_key; + int num_sge = 1; + if (data != nullptr) { + if (data->data == nullptr || data->capacity == 0) + return false; + sges[1].addr = reinterpret_cast(data->data); + sges[1].length = data->capacity; + sges[1].lkey = data->registration_key; + num_sge = 2; + } + auto work = std::make_unique(ReceiveWork{conn, control, data}); + control->ref_clone(); + if (data != nullptr) + data->ref_clone(); + ibv_recv_wr wr{}; + wr.wr_id = reinterpret_cast(work.get()) | 1U; + wr.sg_list = sges; + wr.num_sge = num_sge; + ibv_recv_wr *bad = nullptr; + const bool control_posted = control->transition(BufferOwnership::Available, BufferOwnership::PostedReceive); + const bool data_posted = data == nullptr || + data->transition(BufferOwnership::Available, BufferOwnership::PostedReceive) || + data->transition(BufferOwnership::CallerOwned, BufferOwnership::PostedReceive); + int post_rc = -1; + if (control_posted && data_posted) + post_rc = ibv_post_recv(reinterpret_cast(conn->transport_handle), &wr, &bad); + if (!control_posted || !data_posted || post_rc != 0) { + std::fprintf(stderr, "rdma receive: ibv_post_recv failed qp=%p rc=%d errno=%d bad=%p control_state=%u data_state=%u\n", + reinterpret_cast(conn->transport_handle), post_rc, errno, static_cast(bad), + static_cast(control->ownership.load(std::memory_order_relaxed)), + data == nullptr ? 0U : static_cast(data->ownership.load(std::memory_order_relaxed))); + control->ownership.store(BufferOwnership::Available, std::memory_order_release); + if (data != nullptr) + data->ownership.store(BufferOwnership::Available, std::memory_order_release); + control->release(); + if (data != nullptr) + data->release(); + return false; + } + std::fprintf(stderr, "rdma receive: posted qp=%p wr_id=%#lx\n", + reinterpret_cast(conn->transport_handle), static_cast(wr.wr_id)); + stats_.posted_receives.fetch_add(1, std::memory_order_relaxed); + work.release(); + return true; } Buffer *RdmaTransport::register_buffer(Buffer *buf) { - if (buf->type == BufferType::Registered) { + if (buf == nullptr || buf->type == BufferType::Registered) { return buf; // already registered } - // Copy into the send pool (System → Registered). - // TODO: alloc from send_pool_, copy, release the original. - return buf; // stub + if (send_pool_ == nullptr || buf->len == 0) { + return nullptr; + } + Buffer *registered = send_pool_->alloc(buf->len); + if (registered == nullptr) { + return nullptr; + } + std::memcpy(registered->data, buf->data, buf->len); + registered->len = buf->len; + registered->request_id = buf->request_id; + registered->stream_id = buf->stream_id; + registered->sequence = buf->sequence; + registered->logical_offset = buf->logical_offset; + return registered; } void RdmaTransport::shutdown() { stop(); + { + std::scoped_lock lock(cm_mu_); + for (rdma_cm_id *id : cm_ids_) { + if (id->qp != nullptr) + rdma_destroy_qp(id); + if (auto it = cm_cqs_.find(id); it != cm_cqs_.end()) { + ibv_destroy_cq(it->second); + cm_cqs_.erase(it); + } + if (auto it = cm_channels_.find(id); it != cm_channels_.end()) { + if (event_engine_ != nullptr) + event_engine_->remove_external_fd(it->second->fd); + ibv_destroy_comp_channel(it->second); + cm_channels_.erase(it); + } + rdma_destroy_id(id); + } + cm_ids_.clear(); + cm_pools_.clear(); + } + if (listen_id_ != nullptr) { + if (listen_id_->qp != nullptr) + rdma_destroy_qp(listen_id_); + rdma_destroy_id(listen_id_); + listen_id_ = nullptr; + } + if (cm_channel_ != nullptr) { + rdma_destroy_event_channel(cm_channel_); + cm_channel_ = nullptr; + } } -std::shared_ptr RdmaTransport::connect(const std::string & /*addr*/, int /*port*/) +std::shared_ptr RdmaTransport::connect(const std::string &addr, int port) { - // TODO: rdma_create_id, rdma_resolve_addr, rdma_resolve_route, - // rdma_connect, rdma_create_qp. - return nullptr; // stub + if (pd_ == nullptr || cq_ == nullptr || addr.empty() || port <= 0 || port > 65535) { + std::fprintf(stderr, "rdma connect: transport unavailable pd=%p cq=%p addr=%s port=%d\\n", + static_cast(pd_), static_cast(cq_), addr.c_str(), port); + CRB_LOG_WARN("rdma connect: transport unavailable pd={} cq={} addr={} port={}", + static_cast(pd_), static_cast(cq_), addr, port); + return nullptr; + } + if (cm_channel_ == nullptr) + cm_channel_ = rdma_create_event_channel(); + if (cm_channel_ == nullptr) { + std::fprintf(stderr, "rdma connect: create event channel failed errno=%d\\n", errno); + return nullptr; + } + make_nonblocking(cm_channel_->fd); + rdma_cm_id *id = nullptr; + if (rdma_create_id(cm_channel_, &id, this, RDMA_PS_TCP) != 0) { + std::fprintf(stderr, "rdma connect: create id failed errno=%d\\n", errno); + return nullptr; + } + sockaddr_in destination{}; + destination.sin_family = AF_INET; + destination.sin_port = htons(static_cast(port)); + if (::inet_pton(AF_INET, addr.c_str(), &destination.sin_addr) != 1) { + CRB_LOG_WARN("rdma connect: invalid destination {}", addr); + rdma_destroy_id(id); + return nullptr; + } + sockaddr_in source{}; + sockaddr *source_address = nullptr; + if (const char *source_env = std::getenv("CROWDB_RDMA_SOURCE_ADDR"); source_env != nullptr && *source_env != '\0') { + source.sin_family = AF_INET; + if (::inet_pton(AF_INET, source_env, &source.sin_addr) != 1) { + CRB_LOG_WARN("rdma connect: invalid source address {}", source_env); + rdma_destroy_id(id); + return nullptr; + } + source_address = reinterpret_cast(&source); + if (rdma_bind_addr(id, source_address) != 0) { + std::fprintf(stderr, "rdma connect: bind source failed source=%s errno=%d\\n", source_env, errno); + CRB_LOG_WARN("rdma connect: bind source failed source={} errno={}", source_env, errno); + rdma_destroy_id(id); + return nullptr; + } + } + // When a source address was explicitly bound above, librdmacm expects a + // null source here; passing it twice can fail immediately with EINVAL. + if (rdma_resolve_addr(id, nullptr, reinterpret_cast(&destination), 2000) != 0) { + std::fprintf(stderr, "rdma connect: resolve_addr failed addr=%s errno=%d\\n", addr.c_str(), errno); + CRB_LOG_WARN("rdma connect: resolve_addr failed addr={} port={} errno={}", addr, port, errno); + rdma_destroy_id(id); + return nullptr; + } + rdma_cm_event *event = nullptr; + if (!wait_cm_event(cm_channel_, &event, 3000) || event == nullptr || event->event != RDMA_CM_EVENT_ADDR_RESOLVED) { + std::fprintf(stderr, "rdma connect: ADDR_RESOLVED missing event=%d errno=%d\n", event == nullptr ? -1 : event->event, errno); + CRB_LOG_WARN("rdma connect: ADDR_RESOLVED missing addr={} event={} errno={}", addr, + event == nullptr ? -1 : event->event, errno); + if (event != nullptr) + rdma_ack_cm_event(event); + rdma_destroy_id(id); + return nullptr; + } + rdma_ack_cm_event(event); + if (rdma_resolve_route(id, 2000) != 0) { + CRB_LOG_WARN("rdma connect: resolve_route failed addr={} errno={}", addr, errno); + rdma_destroy_id(id); + return nullptr; + } + if (!wait_cm_event(cm_channel_, &event, 3000) || event == nullptr || event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { + std::fprintf(stderr, "rdma connect: ROUTE_RESOLVED missing event=%d errno=%d\n", event == nullptr ? -1 : event->event, errno); + CRB_LOG_WARN("rdma connect: ROUTE_RESOLVED missing addr={} event={} errno={}", addr, + event == nullptr ? -1 : event->event, errno); + if (event != nullptr) + rdma_ack_cm_event(event); + rdma_destroy_id(id); + return nullptr; + } + rdma_ack_cm_event(event); + std::fprintf(stderr, "rdma connect: creating connection CQ verbs=%p\n", static_cast(id->verbs)); + ibv_comp_channel *connection_channel = ibv_create_comp_channel(id->verbs); + if (connection_channel != nullptr) + make_nonblocking(connection_channel->fd); + ibv_cq *connection_cq = connection_channel == nullptr + ? nullptr + : ibv_create_cq(id->verbs, 2048, nullptr, connection_channel, 0); + std::fprintf(stderr, "rdma connect: connection CQ=%p errno=%d\n", static_cast(connection_cq), errno); + if (connection_cq == nullptr) { + std::fprintf(stderr, "rdma connect: create CM CQ failed errno=%d\n", errno); + if (connection_channel != nullptr) + ibv_destroy_comp_channel(connection_channel); + rdma_destroy_id(id); + return nullptr; + } + ibv_device_attr device_attributes{}; + if (ibv_query_device(id->verbs, &device_attributes) != 0) { + std::fprintf(stderr, "rdma connect: query device failed errno=%d\n", errno); + ibv_destroy_cq(connection_cq); + ibv_destroy_comp_channel(connection_channel); + rdma_destroy_id(id); + return nullptr; + } + ibv_qp_init_attr qp_attr{}; + qp_attr.send_cq = connection_cq; + qp_attr.recv_cq = connection_cq; + qp_attr.qp_type = IBV_QPT_RC; + qp_attr.cap.max_send_wr = std::min(1024, device_attributes.max_qp_wr); + qp_attr.cap.max_recv_wr = std::min(1024, device_attributes.max_qp_wr); + qp_attr.cap.max_send_sge = std::min(2, device_attributes.max_sge); + qp_attr.cap.max_recv_sge = std::min(2, device_attributes.max_sge); + qp_attr.cap.max_inline_data = 0; + qp_attr.sq_sig_all = 1; + qp_attr.qp_context = id; + std::fprintf(stderr, "rdma connect: qp caps send_wr=%u recv_wr=%u send_sge=%u recv_sge=%u max_wr=%u max_sge=%u verbs_match=%d\n", + qp_attr.cap.max_send_wr, qp_attr.cap.max_recv_wr, qp_attr.cap.max_send_sge, + qp_attr.cap.max_recv_sge, device_attributes.max_qp_wr, device_attributes.max_sge, + id->verbs == context_ ? 1 : 0); + std::fprintf(stderr, "rdma connect: selected=%s transport=%s\n", + id->verbs == nullptr || id->verbs->device == nullptr ? "unknown" : ibv_get_device_name(id->verbs->device), + context_ == nullptr || context_->device == nullptr ? "unknown" : ibv_get_device_name(context_->device)); + std::fprintf(stderr, "rdma connect: creating QP id=%p pd=%p cq=%p\n", static_cast(id), + static_cast(id->pd), static_cast(connection_cq)); + if (rdma_create_qp(id, id->pd, &qp_attr) != 0) { + std::fprintf(stderr, "rdma connect: create qp failed errno=%d\n", errno); + ibv_destroy_cq(connection_cq); + ibv_destroy_comp_channel(connection_channel); + rdma_destroy_id(id); + return nullptr; + } + rdma_conn_param params{}; + params.initiator_depth = 1; + params.responder_resources = 1; + params.retry_count = 7; + params.rnr_retry_count = 7; + if (rdma_connect(id, ¶ms) != 0) { + std::fprintf(stderr, "rdma connect: rdma_connect failed addr=%s errno=%d\n", addr.c_str(), errno); + CRB_LOG_WARN("rdma connect: rdma_connect failed addr={} errno={}", addr, errno); + rdma_destroy_qp(id); + ibv_destroy_cq(connection_cq); + ibv_destroy_comp_channel(connection_channel); + rdma_destroy_id(id); + return nullptr; + } + if (!wait_cm_event(cm_channel_, &event, 5000) || event == nullptr || event->event != RDMA_CM_EVENT_ESTABLISHED) { + std::fprintf(stderr, "rdma connect: ESTABLISHED missing event=%d status=%d errno=%d\n", + event == nullptr ? -1 : event->event, event == nullptr ? 0 : event->status, errno); + CRB_LOG_WARN("rdma connect: ESTABLISHED missing addr={} event={} status={} errno={}", addr, + event == nullptr ? -1 : event->event, event == nullptr ? 0 : event->status, errno); + if (event != nullptr) + rdma_ack_cm_event(event); + rdma_destroy_qp(id); + ibv_destroy_cq(connection_cq); + rdma_destroy_id(id); + return nullptr; + } + rdma_ack_cm_event(event); + auto pools = std::make_unique(); + pools->send = std::make_unique(id->pd, 64U << 20, 1024); + pools->recv = std::make_unique(id->pd, 64U << 20, 1024); + if (pools->send == nullptr || pools->recv == nullptr) { + rdma_destroy_qp(id); + ibv_destroy_cq(connection_cq); + ibv_destroy_comp_channel(connection_channel); + rdma_destroy_id(id); + return nullptr; + } + auto connection = std::make_shared(reinterpret_cast(id), addr, pools->recv.get()); + connection->user_data = pools.get(); + connection->set_path(ConnectionPath::Rdma); + connection->transport_handle = reinterpret_cast(id->qp); + if (!prime_receive(connection)) { + std::fprintf(stderr, "rdma connect: prime_receive failed after ESTABLISHED errno=%d\n", errno); + rdma_destroy_qp(id); + rdma_destroy_id(id); + return nullptr; + } + { + std::scoped_lock lock(cm_mu_); + cm_ids_.push_back(id); + cm_cqs_.emplace(id, connection_cq); + cm_channels_.emplace(id, connection_channel); + cm_pools_.emplace(id, std::move(pools)); + if (event_engine_ != nullptr) + event_engine_->add_external_fd(connection_channel->fd, 3); + } + return connection; +} + +bool RdmaTransport::listen(const std::string &addr, int port) +{ + if (pd_ == nullptr || addr.empty() || port <= 0 || port > 65535 || listen_id_ != nullptr) + return false; + if (cm_channel_ == nullptr) + cm_channel_ = rdma_create_event_channel(); + if (cm_channel_ == nullptr || rdma_create_id(cm_channel_, &listen_id_, this, RDMA_PS_TCP) != 0) { + CRB_LOG_WARN("rdma listen: create CM id failed addr={} port={} errno={}", addr, port, errno); + return false; + } + make_nonblocking(cm_channel_->fd); + sockaddr_in local{}; + local.sin_family = AF_INET; + local.sin_port = htons(static_cast(port)); + if (::inet_pton(AF_INET, addr.c_str(), &local.sin_addr) != 1) { + CRB_LOG_WARN("rdma listen: invalid address {}", addr); + rdma_destroy_id(listen_id_); + listen_id_ = nullptr; + return false; + } + if (rdma_bind_addr(listen_id_, reinterpret_cast(&local)) != 0 || rdma_listen(listen_id_, 16) != 0) { + CRB_LOG_WARN("rdma listen: bind/listen failed addr={} port={} errno={}", addr, port, errno); + rdma_destroy_id(listen_id_); + listen_id_ = nullptr; + return false; + } + return true; } -bool RdmaTransport::listen(const std::string & /*addr*/, int /*port*/) +std::shared_ptr RdmaTransport::accept() { - // TODO: rdma_create_id, rdma_bind_addr, rdma_listen. - return false; // stub + if (listen_id_ == nullptr || cm_channel_ == nullptr || cq_ == nullptr) + return nullptr; + rdma_cm_event *event = nullptr; + if (rdma_get_cm_event(cm_channel_, &event) != 0 || event->event != RDMA_CM_EVENT_CONNECT_REQUEST) { + if (event != nullptr) { + rdma_cm_id *event_id = event->id; + rdma_ack_cm_event(event); + if (event_id != nullptr && event_id != listen_id_) { + if (event_id->qp != nullptr) + rdma_destroy_qp(event_id); + rdma_destroy_id(event_id); + } + } + return nullptr; + } + rdma_cm_id *id = event->id; + rdma_ack_cm_event(event); + std::fprintf(stderr, "rdma accept: creating connection CQ verbs=%p\n", static_cast(id->verbs)); + ibv_comp_channel *connection_channel = ibv_create_comp_channel(id->verbs); + if (connection_channel != nullptr) + make_nonblocking(connection_channel->fd); + ibv_cq *connection_cq = connection_channel == nullptr + ? nullptr + : ibv_create_cq(id->verbs, 2048, nullptr, connection_channel, 0); + std::fprintf(stderr, "rdma accept: connection CQ=%p errno=%d\n", static_cast(connection_cq), errno); + if (connection_cq == nullptr) { + std::fprintf(stderr, "rdma accept: create CM CQ failed errno=%d\n", errno); + if (connection_channel != nullptr) + ibv_destroy_comp_channel(connection_channel); + rdma_destroy_id(id); + return nullptr; + } + ibv_qp_init_attr qp_attr{}; + qp_attr.send_cq = connection_cq; + qp_attr.recv_cq = connection_cq; + qp_attr.qp_type = IBV_QPT_RC; + qp_attr.cap.max_send_wr = 1024; + qp_attr.cap.max_recv_wr = 1024; + qp_attr.cap.max_send_sge = 2; + qp_attr.cap.max_recv_sge = 2; + qp_attr.cap.max_inline_data = 0; + qp_attr.sq_sig_all = 1; + qp_attr.qp_context = id; + std::fprintf(stderr, "rdma accept: creating QP id=%p pd=%p cq=%p\n", static_cast(id), + static_cast(id->pd), static_cast(connection_cq)); + if (rdma_create_qp(id, id->pd, &qp_attr) != 0) { + ibv_destroy_cq(connection_cq); + ibv_destroy_comp_channel(connection_channel); + rdma_destroy_id(id); + return nullptr; + } + rdma_conn_param params{}; + params.initiator_depth = 1; + params.responder_resources = 1; + params.retry_count = 7; + params.rnr_retry_count = 7; + const int accept_rc = rdma_accept(id, ¶ms); + std::fprintf(stderr, "rdma accept: rdma_accept rc=%d errno=%d\n", accept_rc, errno); + if (accept_rc != 0) { + rdma_destroy_qp(id); + ibv_destroy_cq(connection_cq); + ibv_destroy_comp_channel(connection_channel); + rdma_destroy_id(id); + return nullptr; + } + event = nullptr; + const int event_rc = wait_cm_event(cm_channel_, &event, 5000) ? 0 : -1; + std::fprintf(stderr, "rdma accept: completion event rc=%d event=%d status=%d errno=%d\n", event_rc, + event == nullptr ? -1 : event->event, event == nullptr ? 0 : event->status, errno); + if (event_rc != 0 || event == nullptr || + event->event != RDMA_CM_EVENT_ESTABLISHED) { + if (event != nullptr) + rdma_ack_cm_event(event); + rdma_destroy_qp(id); + ibv_destroy_cq(connection_cq); + ibv_destroy_comp_channel(connection_channel); + rdma_destroy_id(id); + return nullptr; + } + rdma_ack_cm_event(event); + std::fprintf(stderr, "rdma accept: established, priming receive\n"); + auto pools = std::make_unique(); + pools->send = std::make_unique(id->pd, 64U << 20, 1024); + pools->recv = std::make_unique(id->pd, 64U << 20, 1024); + if (pools->send == nullptr || pools->recv == nullptr) { + rdma_destroy_qp(id); + ibv_destroy_cq(connection_cq); + ibv_destroy_comp_channel(connection_channel); + rdma_destroy_id(id); + return nullptr; + } + auto connection = std::make_shared(reinterpret_cast(id), "rdma-peer", pools->recv.get()); + connection->user_data = pools.get(); + connection->set_path(ConnectionPath::Rdma); + connection->transport_handle = reinterpret_cast(id->qp); + if (!prime_receive(connection)) { + rdma_destroy_qp(id); + ibv_destroy_cq(connection_cq); + ibv_destroy_comp_channel(connection_channel); + rdma_destroy_id(id); + return nullptr; + } + std::fprintf(stderr, "rdma accept: receive primed\n"); + { + std::scoped_lock lock(cm_mu_); + cm_ids_.push_back(id); + cm_cqs_.emplace(id, connection_cq); + cm_channels_.emplace(id, connection_channel); + cm_pools_.emplace(id, std::move(pools)); + if (event_engine_ != nullptr) + event_engine_->add_external_fd(connection_channel->fd, 3); + } + return connection; } void RdmaTransport::start() { running_.store(true, std::memory_order_relaxed); - // TODO: spawn CQ poll thread. } void RdmaTransport::stop() { - if (!running_.exchange(false, std::memory_order_acq_rel)) { + running_.store(false, std::memory_order_release); +} + +int RdmaTransport::completion_fd() const +{ + return comp_channel_ == nullptr ? -1 : comp_channel_->fd; +} + +int RdmaTransport::cm_event_fd() const +{ + return cm_channel_ == nullptr ? -1 : cm_channel_->fd; +} + +int RdmaTransport::drain_cq(uint32_t max_completions) +{ + if (!running_.load(std::memory_order_acquire)) + return 0; + uint32_t drained = 0; + std::array completions{}; + auto poll_one = [&](ibv_cq *cq) { + while (drained < max_completions && cq != nullptr) { + const int batch = ibv_poll_cq(cq, static_cast(std::min(32, max_completions - drained)), + completions.data()); + if (batch <= 0) + break; + for (int i = 0; i < batch; ++i) { + const auto opcode = completions[static_cast(i)].opcode; + const auto wr_id = completions[static_cast(i)].wr_id; + if ((wr_id & uintptr_t{1}) != 0 || opcode == IBV_WC_RECV || opcode == IBV_WC_RECV_RDMA_WITH_IMM) + release_completed_receive(completions[static_cast(i)]); + else if (opcode == IBV_WC_SEND) + release_completed_frame(completions[static_cast(i)]); + else { + std::fprintf(stderr, "rdma completion: unknown opcode=%u status=%u wr_id=%#lx\n", + static_cast(opcode), + static_cast(completions[static_cast(i)].status), + static_cast(completions[static_cast(i)].wr_id)); + } + } + drained += static_cast(batch); + } + }; + if (comp_channel_ != nullptr && cq_ != nullptr) { + ibv_cq *event_cq = nullptr; + void *context = nullptr; + if (ibv_get_cq_event(comp_channel_, &event_cq, &context) == 0) { + ibv_ack_cq_events(event_cq, 1); + ibv_req_notify_cq(event_cq, 0); + poll_one(event_cq); + } + } + { + std::scoped_lock lock(cm_mu_); + for (const auto &[id, cq] : cm_cqs_) { + (void)id; + poll_one(cq); + if (drained >= max_completions) + break; + } + } + return static_cast(drained); +} + +void RdmaTransport::register_events(SocketEngine *engine) +{ + if (engine == nullptr) return; + event_engine_ = engine; + if (completion_fd() >= 0) + engine->add_external_fd(completion_fd(), 1); + if (cm_event_fd() >= 0) + engine->add_external_fd(cm_event_fd(), 2); +} + +int RdmaTransport::handle_external_event(uint64_t tag, uint32_t max_completions) +{ + if (tag == 1) + return drain_cq(max_completions); + if (tag == 2) { + int accepted = 0; + while (true) { + auto connection = accept(); + if (connection == nullptr) + break; + ++accepted; + if (connection_handler_) + connection_handler_(connection); + } + return accepted; + } + if (tag == 3) { + { + std::scoped_lock lock(cm_mu_); + for (const auto &[id, channel] : cm_channels_) { + (void)id; + ibv_cq *event_cq = nullptr; + void *context = nullptr; + if (ibv_get_cq_event(channel, &event_cq, &context) == 0) { + ibv_ack_cq_events(event_cq, 1); + (void)ibv_req_notify_cq(event_cq, 0); + } + } + } + return drain_cq(max_completions); + } + return 0; +} + +bool RdmaTransport::prime_receive(const std::shared_ptr &connection) +{ + if (connection == nullptr || connection->pool() == nullptr) + { + std::fprintf(stderr, "rdma receive: missing connection/pool\\n"); + return false; } - if (cq_thread_.joinable()) { - cq_thread_.join(); + constexpr uint32_t kInitialReceiveDepth = 8; + BufferPool *pool = connection->pool(); + for (uint32_t i = 0; i < kInitialReceiveDepth; ++i) { + Buffer *control = pool->alloc(RDMA_CONTROL_CAPACITY); + Buffer *data = pool->alloc(1U << 20); + if (control == nullptr || data == nullptr) { + std::fprintf(stderr, "rdma receive: buffer allocation failed depth=%u control=%p data=%p\\n", i, + static_cast(control), static_cast(data)); + if (control != nullptr) + control->release(); + if (data != nullptr) + data->release(); + return false; + } + if (!post_receive(connection.get(), control, data)) { + std::fprintf(stderr, "rdma receive: initial ibv_post_recv failed depth=%u errno=%d\\n", i, errno); + control->release(); + data->release(); + return false; + } + // post_receive retains the work-request references; release the caller + // references so the completion owns the pair until it is consumed. + control->release(); + data->release(); } + return true; } -void RdmaTransport::cq_loop() +void RdmaTransport::release_completed_receive(ibv_wc &wc) { - // TODO: ibv_poll_cq, dispatch send/recv completions. - // This is the RDMA equivalent of the epoll/kqueue event loop. + std::fprintf(stderr, "rdma receive completion: wr_id=%#lx status=%u opcode=%u byte_len=%u\n", + static_cast(wc.wr_id), static_cast(wc.status), + static_cast(wc.opcode), wc.byte_len); + auto *work = reinterpret_cast(wc.wr_id & ~uintptr_t{1}); + if (work == nullptr) + return; + auto quarantine_and_release = [&] { + stats_.cq_errors.fetch_add(1, std::memory_order_relaxed); + work->conn->close(); + (void)work->control->quarantine(); + if (work->data != nullptr) + (void)work->data->quarantine(); + work->control->release(); + if (work->data != nullptr) + work->data->release(); + delete work; + }; + if (wc.status != IBV_WC_SUCCESS) { + quarantine_and_release(); + return; + } + if (!work->control->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight) || + (work->data != nullptr && + !work->data->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight))) { + quarantine_and_release(); + return; + } + const uint32_t control_bytes = work->control->capacity; + if (control_bytes < HEADER_SIZE) { + quarantine_and_release(); + return; + } + const Header header = parse_header(work->control->data); + const uint32_t control_end = static_cast(HEADER_SIZE) + header.msg_size; + const uint64_t expected_bytes = static_cast(control_end) + header.data_size; + if (header.magic != MAGIC || header.msg_offset < HEADER_SIZE || + static_cast(header.msg_offset) + header.msg_size > control_bytes || expected_bytes > wc.byte_len || + (header.data_size > 0 && (work->data == nullptr || header.data_size > work->data->capacity))) { + quarantine_and_release(); + return; + } + + work->control->len = control_end; + if (work->data != nullptr) + work->data->len = header.data_size; + auto *frame = new Frame; + frame->header = header; + if (header.msg_size != 0) { + frame->control.assign(work->control->data + header.msg_offset, + work->control->data + header.msg_offset + header.msg_size); + extract_control_fields(work->control->data + header.msg_offset, header.msg_size, frame->request_id, + frame->rpc_create_nano, frame->response_create_nano); + } + if (work->data != nullptr && header.data_size != 0) { + if (!work->data->transition(BufferOwnership::ReceiveInFlight, BufferOwnership::CallerOwned)) { + delete frame; + quarantine_and_release(); + return; + } + frame->data_buf = work->data; + work->data = nullptr; + } + Buffer *repost_control = work->control->ref_clone(); + Buffer *repost_data = (header.data_size == 0 && work->data != nullptr) ? work->data->ref_clone() : nullptr; + auto *repost = new RepostContext{this, work->conn, repost_control, repost_data}; + frame->release_callback = &RdmaTransport::repost_frame_release; + frame->release_context = repost; + work->control->ownership.store(BufferOwnership::Available, std::memory_order_release); + if (work->data != nullptr && header.data_size == 0) + work->data->ownership.store(BufferOwnership::Available, std::memory_order_release); + work->control->release(); + if (work->data != nullptr && header.data_size != 0) + work->data = nullptr; + if (work->data != nullptr) + work->data->release(); + work->conn->on_frame(frame); + stats_.receives.fetch_add(1, std::memory_order_relaxed); + delete work; +} + +void RdmaTransport::repost_frame_release(void *context, Buffer *data) +{ + auto *repost = static_cast(context); + if (repost == nullptr) + return; + Buffer *receive_data = data != nullptr ? data : repost->data; + if (repost->transport == nullptr || + !repost->transport->post_receive(repost->connection, repost->control, receive_data)) { + repost->control->release(); + if (receive_data != nullptr && receive_data != data) + receive_data->release(); + } + else { + // The callback owns the caller references while post_receive owns + // the WR references. + repost->control->release(); + if (receive_data != nullptr && receive_data != data) + receive_data->release(); + } + delete repost; +} + +void RdmaTransport::release_completed_frame(ibv_wc &wc) +{ + auto *frame = reinterpret_cast(wc.wr_id); + if (frame == nullptr) + return; + std::fprintf(stderr, "rdma send completion: wr_id=%#lx status=%u frame=%p control=%p\n", + static_cast(wc.wr_id), static_cast(wc.status), + static_cast(frame), static_cast(frame->control)); + if (wc.status != IBV_WC_SUCCESS) { + stats_.cq_errors.fetch_add(1, std::memory_order_relaxed); + // A failed SEND makes the stream uncertain; do not allow a TCP + // suffix to be appended to this RDMA connection. + if (frame->control != nullptr) + (void)frame->control->quarantine(); + // The completion is no longer trustworthy; the pool keeps these + // slices out of reuse until transport shutdown. + if (frame->control != nullptr) + (void)frame->control->quarantine(); + for (uint8_t i = 0; i < frame->data_views(); ++i) + (void)frame->data_view(i)->quarantine(); + } + if (frame->control != nullptr) + frame->control->release(); + frame->release_data(); + delete frame; } } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/socket_transport.cpp b/lib/crowdb-rpc/src/transport/socket_transport.cpp index 2abf11acf..15099d927 100644 --- a/lib/crowdb-rpc/src/transport/socket_transport.cpp +++ b/lib/crowdb-rpc/src/transport/socket_transport.cpp @@ -2,6 +2,9 @@ // Licensed under the Apache License, Version 2.0. #include "crowdb-rpc/transport/socket_transport.h" +#ifdef CROWDB_RPC_HAVE_RDMA +# include "crowdb-rpc/transport/rdma/rdma_transport.h" +#endif #ifdef __linux__ # include "crowdb-rpc/transport/epoll/epoll_engine.h" @@ -11,6 +14,8 @@ #include "crowdb-common/log.h" #include "crowdb-rpc/rpc_metrics.h" +#include "crowdb-rpc/server/message.h" +#include "crowdb-rpc/transport/rdma/rdma_capability.h" #include #include @@ -24,6 +29,7 @@ #include #include #include +#include #include namespace crowdb::rpc @@ -250,6 +256,9 @@ void Worker::run_loop() case SocketEvent::Accept: transport_->on_accept(ev.fd); break; + case SocketEvent::External: + transport_->on_external(ev.external_tag); + break; } } @@ -537,6 +546,12 @@ SocketTransport::SocketTransport(uint32_t io_engines, uint32_t io_workers, Buffe if (pool_ == nullptr) { pool_ = new SystemBufferPool(); // own a default pool } +#ifdef CROWDB_RPC_HAVE_RDMA + if (const char *enabled = std::getenv("CROWDB_RDMA_ENABLED"); + enabled != nullptr && std::strcmp(enabled, "1") == 0) { + rdma_transport_ = std::make_unique(); + } +#endif // Create N independent engines, each with M workers. When M>1, the // engine uses EV_ONESHOT/EPOLLONESHOT so only one worker wakes per // event; workers re-arm after processing. When M=1, no ONESHOT — @@ -555,7 +570,21 @@ SocketTransport::SocketTransport(uint32_t io_engines, uint32_t io_workers, Buffe auto worker = std::make_unique(worker_id, engine_ptr, &stats_, this); workers_.push_back(std::move(worker)); } +#ifdef CROWDB_RPC_HAVE_RDMA + if (rdma_transport_ != nullptr) + rdma_transport_->register_events(engine_ptr); +#endif + } +#ifdef CROWDB_RPC_HAVE_RDMA + if (rdma_transport_ != nullptr) { + rdma_transport_->set_connection_handler([this](const std::shared_ptr &conn) { + attach_rdma_connection(conn); + if (rdma_accept_handler_) + rdma_accept_handler_(conn); + }); + set_external_handler([this](uint64_t tag) { rdma_transport_->handle_external_event(tag); }); } +#endif } SocketTransport::SocketTransport(uint32_t num_workers, BufferPool *pool) : SocketTransport(1, num_workers, pool) @@ -569,6 +598,10 @@ SocketTransport::~SocketTransport() void SocketTransport::start() { +#ifdef CROWDB_RPC_HAVE_RDMA + if (rdma_transport_ != nullptr) + rdma_transport_->start(); +#endif for (auto &w : workers_) { w->start(); } @@ -588,6 +621,10 @@ void SocketTransport::stop() for (auto &w : workers_) { w->close_connections(); } +#ifdef CROWDB_RPC_HAVE_RDMA + if (rdma_transport_ != nullptr) + rdma_transport_->stop(); +#endif } void SocketTransport::add_listen_fd(int fd) @@ -624,6 +661,13 @@ void SocketTransport::shutdown() bool SocketTransport::submit(Connection *conn, OutFrame *frame) { +#ifdef CROWDB_RPC_HAVE_RDMA + if (conn != nullptr && conn->path() == ConnectionPath::Rdma && rdma_transport_ != nullptr) { + if (!rdma_transport_->submit(conn, frame)) + return false; + return true; + } +#endif if (workers_.empty()) { return false; } @@ -712,6 +756,10 @@ bool SocketTransport::submit(Connection *conn, OutFrame *frame) bool SocketTransport::submit_inline(Connection *conn, OutFrame *frame) { +#ifdef CROWDB_RPC_HAVE_RDMA + if (conn != nullptr && conn->path() == ConnectionPath::Rdma && rdma_transport_ != nullptr) + return rdma_transport_->submit(conn, frame); +#endif frame->create_nano = now_nano(); // Both paths: enqueue only. The post-event flush drains the queue // (retry_direct for Path A, flush_send for Path B), batching all @@ -820,6 +868,18 @@ size_t SocketTransport::connection_count() const std::shared_ptr SocketTransport::connect(const std::string &addr, int port) { +#ifdef CROWDB_RPC_HAVE_RDMA + if (rdma_transport_ != nullptr) { + if (auto rdma_conn = rdma_transport_->connect(addr, port); rdma_conn != nullptr) { + for (auto &engine : engines_) + rdma_transport_->register_events(engine.get()); + register_conn(rdma_conn); + attach_rdma_connection(rdma_conn); + return rdma_conn; + } + rdma_transport_->stats().fallback_count.fetch_add(1, std::memory_order_relaxed); + } +#endif int fd = ::socket(AF_INET, SOCK_STREAM, 0); if (fd < 0) { CRB_LOG_WARN("connect: socket() failed addr={} port={} errno={} ({})", addr, port, errno, std::strerror(errno)); @@ -864,11 +924,81 @@ std::shared_ptr SocketTransport::connect(const std::string &addr, in #endif auto conn = create_connection(fd, addr + ":" + std::to_string(port)); + send_rdma_handshake(conn); CRB_LOG_INFO("rpc transport: connection initiated -> {}:{} conn_id={}", addr, port, static_cast(conn->id())); return conn; } +void SocketTransport::attach_rdma_connection(const std::shared_ptr &conn, + Connection::OnFrameCallback on_frame, + Connection::OnCloseCallback on_close) +{ + if (conn == nullptr) + return; + conn->set_on_frame(std::move(on_frame)); + conn->set_on_close(std::move(on_close)); + register_conn(conn); + if (!workers_.empty()) { + Worker *worker = get_worker(); + conn->io_worker = worker; + conn->io_engine = worker == nullptr ? nullptr : worker->engine(); + } +} + +void SocketTransport::set_rdma_accept_handler(RdmaAcceptHandler handler) +{ + rdma_accept_handler_ = std::move(handler); +} + +bool SocketTransport::send_rdma_handshake(const std::shared_ptr &conn) +{ + if (!conn || pool_ == nullptr) + return false; + RdmaEndpointCapability capability; + const char *enabled = std::getenv("CROWDB_RDMA_ENABLED"); + capability.rdma_available = enabled != nullptr && std::strcmp(enabled, "1") == 0; + capability.protocol_version = 1; + capability.generation = 1; + capability.port = 1; + if (const char *device = std::getenv("CROWDB_RDMA_DEVICE"); device != nullptr) + capability.device = device; + if (const char *gid = std::getenv("CROWDB_RDMA_GID"); gid != nullptr) + capability.gid = gid; + capability.data_classes = {RDMA_CONTROL_CAPACITY, RDMA_METADATA_CAPACITY, RDMA_AGGREGATE_CAPACITY, + RDMA_DATAPATH_CAPACITY}; + const auto wire = serialize_rdma_handshake(capability); + Buffer *control = build_raw_control(pool_, wire.data(), static_cast(wire.size())); + if (control == nullptr) + return false; + OutFrame *frame = build_out_frame(0, RDMA_HANDSHAKE_MSG_TYPE, control, nullptr, FLAG_ONE_WAY); + if (frame == nullptr) { + control->release(); + return false; + } + if (!submit(conn.get(), frame)) { + control->release(); + delete frame; + return false; + } + return true; +} + +bool SocketTransport::listen_rdma(const std::string &addr, int port) +{ +#ifdef CROWDB_RPC_HAVE_RDMA + if (rdma_transport_ == nullptr || !rdma_transport_->listen(addr, port)) + return false; + for (auto &engine : engines_) + rdma_transport_->register_events(engine.get()); + return true; +#else + (void)addr; + (void)port; + return false; +#endif +} + std::unique_ptr SocketTransport::create_engine() { #ifdef __linux__ diff --git a/lib/crowdb-rpc/tests/buffer_test.cpp b/lib/crowdb-rpc/tests/buffer_test.cpp index dab729c92..f33c2a9fb 100644 --- a/lib/crowdb-rpc/tests/buffer_test.cpp +++ b/lib/crowdb-rpc/tests/buffer_test.cpp @@ -9,7 +9,10 @@ #include using crowdb::rpc::Buffer; +using crowdb::rpc::BufferCapacityClass; +using crowdb::rpc::BufferOwnership; using crowdb::rpc::BufferType; +using crowdb::rpc::RegisteredBufferPool; using crowdb::rpc::SystemBufferPool; TEST(BufferTest, AllocReturnsValidBuffer) @@ -103,3 +106,47 @@ TEST(BufferTest, DetachedReceiveBufferOutlivesPoolWithoutCopy) EXPECT_EQ(std::memcmp(buf->data, data, 4), 0); buf->release(); } + +TEST(BufferTest, RegisteredPoolRoundsClassesAndEnforcesByteBudget) +{ + RegisteredBufferPool pool(crowdb::rpc::RDMA_METADATA_CAPACITY, 4, 7); + Buffer *metadata = pool.alloc(1024); + ASSERT_NE(metadata, nullptr); + EXPECT_EQ(metadata->capacity, crowdb::rpc::RDMA_CONTROL_CAPACITY); + EXPECT_EQ(metadata->capacity_class, BufferCapacityClass::Control); + EXPECT_EQ(metadata->registration_id, 7U); + EXPECT_EQ(pool.alloc(crowdb::rpc::RDMA_METADATA_CAPACITY), nullptr); + metadata->release(); + Buffer *reused = pool.alloc(512); + ASSERT_NE(reused, nullptr); + reused->release(); +} + +TEST(BufferTest, RegisteredLeaseOwnershipAndQuarantine) +{ + RegisteredBufferPool pool(2 * crowdb::rpc::RDMA_CONTROL_CAPACITY, 2); + Buffer *buf = pool.alloc(1); + ASSERT_NE(buf, nullptr); + EXPECT_TRUE(buf->transition(BufferOwnership::Available, BufferOwnership::PostedReceive)); + EXPECT_TRUE(buf->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight)); + EXPECT_TRUE(buf->transition(BufferOwnership::ReceiveInFlight, BufferOwnership::CallerOwned)); + EXPECT_FALSE(buf->transition(BufferOwnership::ReceiveInFlight, BufferOwnership::Available)); + EXPECT_TRUE(buf->quarantine()); + buf->release(); + EXPECT_EQ(pool.quarantined_bytes(), crowdb::rpc::RDMA_CONTROL_CAPACITY); + Buffer *other = pool.alloc(1); + ASSERT_NE(other, nullptr); + EXPECT_NE(other, buf); + other->release(); + Buffer *reused = pool.alloc(1); + ASSERT_NE(reused, nullptr); + EXPECT_EQ(reused, other); + reused->release(); +} + +TEST(BufferTest, RegisteredCapacityRejectsUnsupportedValues) +{ + RegisteredBufferPool pool(8 * crowdb::rpc::RDMA_DATAPATH_CAPACITY, 8); + EXPECT_EQ(pool.alloc(crowdb::rpc::RDMA_DATAPATH_CAPACITY + 1), nullptr); + EXPECT_EQ(pool.alloc(0), nullptr); +} diff --git a/lib/crowdb-rpc/tests/rdma_capability_test.cpp b/lib/crowdb-rpc/tests/rdma_capability_test.cpp new file mode 100644 index 000000000..78137af9b --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_capability_test.cpp @@ -0,0 +1,104 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/transport/rdma/rdma_capability.h" + +#include + +using crowdb::rpc::RdmaConnectionState; +using crowdb::rpc::RdmaEndpointCapability; +using crowdb::rpc::RdmaPathState; +using crowdb::rpc::SystemBufferPool; + +TEST(RdmaCapabilityTest, ValidatesGenerationDeviceAndClasses) +{ + RdmaEndpointCapability capability{ + .rdma_available = true, + .protocol_version = 1, + .generation = 9, + .port = 1, + .device = "rocep35s0f0", + .gid = "fe80::1", + .data_classes = {1U << 10, 64U << 10, 256U << 10, 1U << 20} + }; + EXPECT_TRUE(crowdb::rpc::validate_rdma_capability(capability, 1, 9, "rocep35s0f0")); + EXPECT_FALSE(crowdb::rpc::validate_rdma_capability(capability, 1, 8, "rocep35s0f0")); + EXPECT_FALSE(crowdb::rpc::validate_rdma_capability(capability, 1, 9, "rocep35s0f1")); +} + +TEST(RdmaCapabilityTest, HandshakeRoundTripsEndpointIdentity) +{ + RdmaEndpointCapability source{ + .rdma_available = true, + .protocol_version = 2, + .generation = 17, + .port = 1, + .device = "rocep35s0f0", + .gid = "fe80::abcd", + .data_classes = {1024, 64U << 10, 256U << 10, 1U << 20} + }; + const auto wire = crowdb::rpc::serialize_rdma_handshake(source); + RdmaEndpointCapability decoded; + ASSERT_TRUE(crowdb::rpc::parse_rdma_handshake(wire, &decoded)); + EXPECT_EQ(decoded.protocol_version, source.protocol_version); + EXPECT_EQ(decoded.rdma_available, source.rdma_available); + EXPECT_EQ(decoded.generation, source.generation); + EXPECT_EQ(decoded.device, source.device); + EXPECT_EQ(decoded.gid, source.gid); + EXPECT_EQ(decoded.data_classes, source.data_classes); +} + +TEST(RdmaCapabilityTest, FallbackStopsAfterPayloadSubmission) +{ + EXPECT_TRUE(crowdb::rpc::can_fallback_to_tcp(RdmaPathState::Failed, false)); + EXPECT_TRUE(crowdb::rpc::can_fallback_to_tcp(RdmaPathState::Connecting, false)); + EXPECT_FALSE(crowdb::rpc::can_fallback_to_tcp(RdmaPathState::Ready, false)); + EXPECT_FALSE(crowdb::rpc::can_fallback_to_tcp(RdmaPathState::Failed, true)); +} + +TEST(RdmaCapabilityTest, ConnectionStateRejectsIllegalUpgradeAndFallback) +{ + RdmaConnectionState state; + EXPECT_EQ(state.state(), RdmaPathState::Unavailable); + EXPECT_TRUE(state.begin_probe()); + EXPECT_TRUE(state.begin_connect()); + EXPECT_TRUE(state.mark_ready()); + EXPECT_TRUE(state.mark_payload_submitted()); + EXPECT_FALSE(state.fallback_to_tcp()); + EXPECT_TRUE(state.fail()); + EXPECT_TRUE(state.begin_drain()); + EXPECT_FALSE(state.mark_ready()); +} + +TEST(RdmaCapabilityTest, SetupFailureCanFallbackBeforePayload) +{ + RdmaConnectionState state; + ASSERT_TRUE(state.begin_probe()); + ASSERT_TRUE(state.begin_connect()); + ASSERT_TRUE(state.fail()); + EXPECT_TRUE(state.fallback_to_tcp()); + EXPECT_EQ(state.state(), RdmaPathState::TcpFallback); +} + +TEST(RdmaCapabilityTest, BuildsControlAndDataSgesWithoutConcatenation) +{ + SystemBufferPool pool(2); + auto *control = pool.alloc(1024); + auto *data = pool.alloc(64U << 10); + ASSERT_NE(control, nullptr); + ASSERT_NE(data, nullptr); + const uint8_t control_bytes[] = {1, 2}; + const uint8_t data_bytes[] = {3, 4, 5}; + control->write(control_bytes, sizeof(control_bytes)); + data->write(data_bytes, sizeof(data_bytes)); + + crowdb::rpc::RdmaFrameSges sges; + ASSERT_TRUE(crowdb::rpc::build_rdma_frame_sges(control, data, &sges)); + EXPECT_EQ(sges.count, 2); + EXPECT_EQ(sges.entries[0].address, reinterpret_cast(control->data)); + EXPECT_EQ(sges.entries[0].length, sizeof(control_bytes)); + EXPECT_EQ(sges.entries[1].address, reinterpret_cast(data->data)); + EXPECT_EQ(sges.entries[1].length, sizeof(data_bytes)); + data->release(); + control->release(); +} diff --git a/lib/crowdb-rpc/tests/rdma_test.cpp b/lib/crowdb-rpc/tests/rdma_test.cpp new file mode 100644 index 000000000..04d255d8b --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_test.cpp @@ -0,0 +1,314 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "crowdb-rpc/client/client.h" +#include "crowdb-rpc/server/message.h" +#include "crowdb-rpc/server/server.h" +#include "crowdb-rpc/transport/socket_transport.h" + +#ifdef CROWDB_RPC_HAVE_RDMA + +# include + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +namespace +{ +void rdma_test_crash_handler(int signal_number) +{ + void *frames[64]; + const int count = ::backtrace(frames, 64); + const char prefix[] = "R239 RDMA crash backtrace:\n"; + (void)::write(STDERR_FILENO, prefix, sizeof(prefix) - 1); + ::backtrace_symbols_fd(frames, count, STDERR_FILENO); + std::_Exit(128 + signal_number); +} + +void install_rdma_test_crash_handler() +{ + static std::once_flag installed; + std::call_once(installed, [] { + std::signal(SIGSEGV, rdma_test_crash_handler); + std::signal(SIGABRT, rdma_test_crash_handler); + }); +} +} + +struct WorkerCompletion +{ + std::atomic completed{0}; + std::atomic failed{0}; +}; + +extern "C" void rdma_worker_completion(uint64_t, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, + crowdb_rpc_status status, void *opaque) +{ + auto *state = static_cast(opaque); + (status == CROWDB_RPC_OK ? state->completed : state->failed).fetch_add(1, std::memory_order_relaxed); + if (control != nullptr) + crowdb_rpc_buffer_release(control); + if (data != nullptr) + crowdb_rpc_buffer_release(data); +} + +using crowdb::rpc::Buffer; +using crowdb::rpc::BufferOwnership; +using crowdb::rpc::Connection; +using crowdb::rpc::Frame; +using crowdb::rpc::Header; +using crowdb::rpc::OutFrame; +using crowdb::rpc::RdmaBufferPool; +using crowdb::rpc::RdmaTransport; +using crowdb::rpc::RpcClient; +using crowdb::rpc::RpcServer; +using crowdb::rpc::SocketTransport; +using crowdb::rpc::SystemBufferPool; +using crowdb::rpc::HEADER_SIZE; +using crowdb::rpc::serialize_header; + +TEST(RdmaBufferPoolTest, SlicesOneRegisteredRegion) +{ + int device_count = 0; + ibv_device **devices = ibv_get_device_list(&device_count); + ASSERT_NE(devices, nullptr); + if (device_count == 0) { + ibv_free_device_list(devices); + GTEST_SKIP() << "no RDMA device is available"; + } + + ibv_context *context = ibv_open_device(devices[0]); + ASSERT_NE(context, nullptr); + ibv_pd *pd = ibv_alloc_pd(context); + ASSERT_NE(pd, nullptr); + + { + RdmaBufferPool pool(pd, 128U << 10, 4); + Buffer *control = pool.alloc(1); + ASSERT_NE(control, nullptr); + EXPECT_EQ(control->capacity, 1U << 10); + EXPECT_NE(control->registration_handle, 0U); + EXPECT_NE(control->registration_key, 0U); + EXPECT_TRUE(control->transition(BufferOwnership::Available, BufferOwnership::SendInFlight)); + EXPECT_TRUE(control->quarantine()); + control->release(); + Buffer *next = pool.alloc(1); + ASSERT_NE(next, nullptr); + EXPECT_NE(next, control); + next->release(); + } + + ibv_dealloc_pd(pd); + ibv_close_device(context); + ibv_free_device_list(devices); +} + +TEST(RdmaBufferPoolTest, TransportCopiesSystemBufferIntoRegisteredPool) +{ + RdmaTransport transport; + SystemBufferPool system_pool(1); + Buffer *source = system_pool.alloc(32); + ASSERT_NE(source, nullptr); + const uint8_t payload[] = {1, 2, 3, 4}; + source->write(payload, sizeof(payload)); + Buffer *registered = transport.register_buffer(source); + if (registered == nullptr) { + source->release(); + GTEST_SKIP() << "RDMA device registration is unavailable"; + } + EXPECT_EQ(registered->type, crowdb::rpc::BufferType::Registered); + EXPECT_EQ(registered->len, sizeof(payload)); + EXPECT_EQ(std::memcmp(registered->data, payload, sizeof(payload)), 0); + registered->release(); + source->release(); +} + +TEST(RdmaHardwareTest, CompleteCmQpSendReceiveFlow) +{ + constexpr uint64_t kFrameCount = 8; + constexpr int kTimeoutIterations = 500; + if (std::getenv("CROWDB_RDMA_E2E") == nullptr) + GTEST_SKIP() << "set CROWDB_RDMA_E2E=1 to run against a live HCA"; + install_rdma_test_crash_handler(); + const char *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + const char *connect_address = std::getenv("CROWDB_RDMA_E2E_CONNECT_ADDR"); + const int port = std::getenv("CROWDB_RDMA_E2E_PORT") == nullptr + ? 19100 + : std::atoi(std::getenv("CROWDB_RDMA_E2E_PORT")); + ASSERT_NE(address, nullptr); + if (connect_address == nullptr) + connect_address = address; + + const char *role = std::getenv("CROWDB_RDMA_E2E_ROLE"); + if (role != nullptr && std::strcmp(role, "server") == 0) { + RdmaTransport server; + ASSERT_TRUE(server.listen(address, port)); + server.start(); + std::shared_ptr accepted; + for (int i = 0; i < kTimeoutIterations && accepted == nullptr; ++i) { + accepted = server.accept(); + (void)server.drain_cq(32); + if (accepted == nullptr) + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + ASSERT_NE(accepted, nullptr); + accepted->set_on_frame([](Frame *frame, Connection *) { delete frame; }); + for (int i = 0; i < kTimeoutIterations && server.stats().receives.load() < kFrameCount; ++i) { + (void)server.drain_cq(32); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + std::fprintf(stderr, "R239 server stats: receives=%llu posted=%llu cq_errors=%llu\n", + static_cast(server.stats().receives.load()), + static_cast(server.stats().posted_receives.load()), + static_cast(server.stats().cq_errors.load())); + EXPECT_GE(server.stats().receives.load(), kFrameCount); + EXPECT_GE(server.stats().posted_receives.load(), kFrameCount); + server.stop(); + return; + } + if (role != nullptr && std::strcmp(role, "client") == 0) { + std::fprintf(stderr, "R239 hardware client: device=%s source=%s target=%s port=%d\\n", + std::getenv("CROWDB_RDMA_DEVICE") == nullptr ? "" : std::getenv("CROWDB_RDMA_DEVICE"), + std::getenv("CROWDB_RDMA_SOURCE_ADDR") == nullptr ? "" : std::getenv("CROWDB_RDMA_SOURCE_ADDR"), + connect_address, port); + RdmaTransport client; + std::fprintf(stderr, "R239 client transport ready=%d\\n", client.ready() ? 1 : 0); + client.start(); + auto outgoing = client.connect(connect_address, port); + ASSERT_NE(outgoing, nullptr); + EXPECT_GE(client.stats().posted_receives.load(), 1U); + SystemBufferPool system_pool(2); + for (uint64_t sequence = 0; sequence < kFrameCount; ++sequence) { + Buffer *raw = system_pool.alloc(HEADER_SIZE); + ASSERT_NE(raw, nullptr); + Header header; + header.msg_type = static_cast(1 + sequence); + serialize_header(raw->data, header); + raw->len = HEADER_SIZE; + Buffer *control = client.register_buffer(raw); + ASSERT_NE(control, nullptr); + auto *frame = new OutFrame; + frame->header = header; + frame->control = control; + ASSERT_TRUE(client.submit(outgoing.get(), frame)); + raw->release(); + } + for (int i = 0; i < 300; ++i) { + (void)client.drain_cq(32); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + client.stop(); + return; + } + + RdmaTransport server; + RdmaTransport client; + ASSERT_TRUE(server.listen(address, port)); + server.start(); + client.start(); + + std::shared_ptr accepted; + std::thread accept_thread([&] { + for (int i = 0; i < 200 && accepted == nullptr; ++i) { + accepted = server.accept(); + if (accepted == nullptr) + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + }); + auto outgoing = client.connect(connect_address, port); + accept_thread.join(); + ASSERT_NE(outgoing, nullptr); + ASSERT_NE(accepted, nullptr); + accepted->set_on_frame([](Frame *frame, Connection *) { delete frame; }); + EXPECT_GE(server.stats().posted_receives.load(), 1U); + EXPECT_GE(client.stats().posted_receives.load(), 1U); + + SystemBufferPool system_pool(2); + Buffer *raw = system_pool.alloc(HEADER_SIZE); + ASSERT_NE(raw, nullptr); + Header header; + header.msg_type = 1; + serialize_header(raw->data, header); + raw->len = HEADER_SIZE; + Buffer *control = client.register_buffer(raw); + ASSERT_NE(control, nullptr); + auto *frame = new OutFrame; + frame->header = header; + frame->control = control; + ASSERT_TRUE(client.submit(outgoing.get(), frame)); + raw->release(); + + for (int i = 0; i < 200 && server.stats().receives.load() == 0; ++i) { + (void)client.drain_cq(32); + (void)server.drain_cq(32); + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + EXPECT_GE(server.stats().receives.load(), 1U); + EXPECT_GE(server.stats().posted_receives.load(), 1U); + client.stop(); + server.stop(); +} + +TEST(RdmaHardwareTest, RpcServerClientWorkerRoundTrip) +{ + if (std::getenv("CROWDB_RDMA_RPC_E2E") == nullptr) + GTEST_SKIP() << "set CROWDB_RDMA_RPC_E2E=1 to run the worker flow"; + install_rdma_test_crash_handler(); + const char *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + const char *role = std::getenv("CROWDB_RDMA_E2E_ROLE"); + ASSERT_NE(address, nullptr); + const int port = std::getenv("CROWDB_RDMA_E2E_PORT") == nullptr ? 19101 : + std::atoi(std::getenv("CROWDB_RDMA_E2E_PORT")); + constexpr uint32_t kRequests = 8; + constexpr uint16_t kMessageType = 239; + + if (role != nullptr && std::strcmp(role, "server") == 0) { + RpcServer server; + ASSERT_TRUE(server.listen(address, port)); + std::atomic received{0}; + server.register_handler(kMessageType, [&received](Frame *frame, Connection *) { + received.fetch_add(1, std::memory_order_relaxed); + delete frame; + return static_cast(nullptr); + }); + server.start(); + for (int i = 0; i < 100 && received.load(std::memory_order_relaxed) < kRequests; ++i) + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + EXPECT_EQ(received.load(std::memory_order_relaxed), kRequests); + server.stop(); + return; + } + + ASSERT_NE(role, nullptr); + ASSERT_STREQ(role, "client"); + SocketTransport transport(1, 1); + transport.start(); + auto connection = transport.connect(address, port); + ASSERT_NE(connection, nullptr); + RpcClient client; + client.set_completion_pool_size(kRequests); + client.attach(connection.get()); + WorkerCompletion completion; + for (uint32_t i = 0; i < kRequests; ++i) { + Buffer *control = crowdb::rpc::build_ping_request(transport.pool(), i + 1, 0); + ASSERT_NE(control, nullptr); + ASSERT_TRUE(client.send(&transport, connection.get(), i + 1, control, nullptr, kMessageType, + rdma_worker_completion, &completion)); + } + for (int i = 0; i < 200 && completion.completed.load(std::memory_order_relaxed) < kRequests; ++i) + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + EXPECT_EQ(completion.completed.load(std::memory_order_relaxed), kRequests); + EXPECT_EQ(completion.failed.load(std::memory_order_relaxed), 0U); + transport.stop(); +} + +#endif diff --git a/lib/crowdb-rpc/tests/transport_test.cpp b/lib/crowdb-rpc/tests/transport_test.cpp index 2aed7818a..e34b1e053 100644 --- a/lib/crowdb-rpc/tests/transport_test.cpp +++ b/lib/crowdb-rpc/tests/transport_test.cpp @@ -5,6 +5,12 @@ #include "crowdb-rpc/framing.h" #include "crowdb-rpc/transport/socket_transport.h" +#ifdef __linux__ +# include "crowdb-rpc/transport/epoll/epoll_engine.h" + +# include +#endif + #include #include #include @@ -96,6 +102,29 @@ TEST_F(TransportLoopbackTest, StopWakesIdleWorker) EXPECT_LT(elapsed, std::chrono::milliseconds(500)); } +#ifdef __linux__ +TEST(EpollEngineTest, ExternalDescriptorWakesAndReportsTag) +{ + crowdb::rpc::EpollEngine engine; + ASSERT_EQ(engine.init(), 0); + const int fd = ::eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + ASSERT_GE(fd, 0); + engine.add_external_fd(fd, 0x55AA); + uint64_t one = 1; + ASSERT_EQ(::write(fd, &one, sizeof(one)), static_cast(sizeof(one))); + + crowdb::rpc::EngineEvent event{}; + ASSERT_EQ(engine.wait(&event, 1, 1000), 1); + EXPECT_EQ(event.type, crowdb::rpc::SocketEvent::External); + EXPECT_EQ(event.fd, fd); + EXPECT_EQ(event.external_tag, 0x55AAU); + + engine.remove_external_fd(fd); + ::close(fd); + engine.shutdown(); +} +#endif + TEST_F(TransportLoopbackTest, StopClosesRegisteredConnectionDescriptors) { SocketTransport transport(1, 1); diff --git a/tools/rdma/run-local-loop.sh b/tools/rdma/run-local-loop.sh new file mode 100755 index 000000000..d86582875 --- /dev/null +++ b/tools/rdma/run-local-loop.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# Run the RDMA hardware smoke test in local or namespace loop mode. + +set -Eeuo pipefail + +root_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +pixi_bin=${PIXI_BIN:-/home/cj/.pixi/bin/pixi} +test_bin=${CROWDB_RDMA_TEST_BIN:-$root_dir/lib/crowdb-rpc/build/crowdb_rpc_tests} +mode=${1:-local} +port=${CROWDB_RDMA_E2E_PORT:-19101} +filter=${CROWDB_RDMA_E2E_FILTER:-RdmaHardwareTest.CompleteCmQpSendReceiveFlow} +server_pid="" + +CLIENT_NS=ns1 +SERVER_NS=ns2 + +usage() { + cat <<'USAGE' +usage: run-local-loop.sh [local|namespace] + + local Run client and server in root on one RDMA port (default). + namespace Run client in ns1 and server in ns2 on the two configured ports. + +The setup must be prepared with: + sudo tools/rdma/setup-rdma.sh local + sudo tools/rdma/setup-rdma.sh namespace +USAGE +} + +die() { + echo "error: $*" >&2 + exit 1 +} + +if [[ "$mode" == "-h" || "$mode" == "--help" ]]; then + usage + exit 0 +fi +[[ "$mode" == local || "$mode" == namespace ]] || + die "unsupported mode '$mode'; use local or namespace" +[[ $# -eq 1 ]] || die "unexpected argument: ${2:-}" + +command -v ip >/dev/null || die "missing command: ip" +command -v rdma >/dev/null || die "missing command: rdma" +[[ -x "$pixi_bin" ]] || die "pixi executable not found: $pixi_bin" +[[ -x "$test_bin" ]] || die "test binary not found: $test_bin" + +if [[ "$mode" == namespace && $EUID -ne 0 ]]; then + exec sudo -E "$0" "$@" +fi + +run_in_ns() { + local ns="$1" + shift + if [[ "$ns" == root ]]; then + env "$@" + else + ip netns exec "$ns" env "$@" + fi +} + +ns_exists() { + ip netns list | awk '{print $1}' | grep -Fxq "$1" +} + +first_device() { + rdma link show | awk 'NR == 1 {sub("/1$", "", $2); print $2}' +} + +device_netdev() { + local device="$1" + rdma link show | awk -v device="$device" \ + '$2 ~ ("^" device "(/|$)") && $0 ~ / netdev / { + for (i = 1; i < NF; ++i) if ($i == "netdev") { print $(i + 1); exit } + }' +} + +root_netdev_for_device() { + local device="$1" path + for path in "/sys/class/infiniband/$device/device/net/"*; do + [[ -e "$path" ]] || continue + basename "$path" + return 0 + done +} + +ns_netdev_for_device() { + local ns="$1" device="$2" netdev + netdev=$(ip netns exec "$ns" rdma link show 2>/dev/null | + awk -v device="$device" '$2 ~ ("^" device "(/|$)") && $0 ~ / netdev / { + for (i = 1; i < NF; ++i) if ($i == "netdev") { print $(i + 1); exit } + }') + if [[ -z "$netdev" ]]; then + netdev=$(ip -n "$ns" -o link show | + awk -F': ' '$2 !~ /^lo([:@]|$)/ {sub(/@.*/, "", $2); print $2; exit}') + fi + printf '%s\n' "$netdev" +} + +addr_for() { + local ns="$1" dev="$2" + run_in_ns "$ns" ip -4 -o addr show dev "$dev" scope global | + awk 'NR == 1 {split($4, parts, "/"); print parts[1]}' +} + +if [[ "$mode" == local ]]; then + device=${CROWDB_RDMA_DEVICE:-$(first_device)} + netdev=${CROWDB_RDMA_LOCAL_IF:-$(device_netdev "$device")} + netdev=${netdev:-$(root_netdev_for_device "$device")} + address=${CROWDB_RDMA_LOCAL_ADDR:-$(addr_for root "$netdev")} + [[ -n "$device" ]] || die "could not discover an RDMA device" + [[ -n "$netdev" ]] || die "could not find the local Ethernet netdev" + [[ -n "$address" ]] || die "could not find an IPv4 address on $netdev" + client_ns=root + server_ns=root + client_device=$device + server_device=$device + client_addr=$address + server_addr=$address + client_if=$netdev + server_if=$netdev +else + ns_exists "$CLIENT_NS" || die "namespace does not exist: $CLIENT_NS" + ns_exists "$SERVER_NS" || die "namespace does not exist: $SERVER_NS" + devices=($(ls -1 /sys/class/infiniband | sort)) + (( ${#devices[@]} >= 2 )) || die "namespace mode requires two RDMA ports" + client_device=${CROWDB_RDMA_CLIENT_DEVICE:-${devices[0]}} + server_device=${CROWDB_RDMA_SERVER_DEVICE:-${devices[1]}} + client_if=${CROWDB_RDMA_CLIENT_IF:-$(ns_netdev_for_device "$CLIENT_NS" "$client_device")} + server_if=${CROWDB_RDMA_SERVER_IF:-$(ns_netdev_for_device "$SERVER_NS" "$server_device")} + client_addr=${CROWDB_RDMA_CLIENT_ADDR:-$(addr_for "$CLIENT_NS" "$client_if")} + server_addr=${CROWDB_RDMA_SERVER_ADDR:-$(addr_for "$SERVER_NS" "$server_if")} + [[ -n "$client_if" && -n "$server_if" ]] || die "could not find namespace netdevs" + [[ -n "$client_addr" && -n "$server_addr" ]] || die "could not find namespace IPv4 addresses" + client_ns=$CLIENT_NS + server_ns=$SERVER_NS +fi + +echo "RDMA $mode loop: client=$client_device/$client_if@$client_addr server=$server_device/$server_if@$server_addr port=$port" + +cleanup() { + if [[ -n "$server_pid" ]]; then + kill "$server_pid" 2>/dev/null || true + wait "$server_pid" 2>/dev/null || true + fi +} +trap cleanup EXIT INT TERM + +common_env=(CROWDB_RDMA_E2E=1 CROWDB_RDMA_E2E_PORT="$port") +run_in_ns "$server_ns" env "${common_env[@]}" \ + CROWDB_RDMA_E2E_ROLE=server \ + CROWDB_RDMA_E2E_LISTEN_ADDR="$server_addr" \ + CROWDB_RDMA_DEVICE="$server_device" \ + "$pixi_bin" run "$test_bin" --gtest_filter="$filter" & +server_pid=$! + +sleep 1 +run_in_ns "$client_ns" env "${common_env[@]}" \ + CROWDB_RDMA_E2E_ROLE=client \ + CROWDB_RDMA_E2E_LISTEN_ADDR="$server_addr" \ + CROWDB_RDMA_E2E_CONNECT_ADDR="$server_addr" \ + CROWDB_RDMA_SOURCE_ADDR="$client_addr" \ + CROWDB_RDMA_DEVICE="$client_device" \ + "$pixi_bin" run "$test_bin" --gtest_filter="$filter" + +wait "$server_pid" +server_pid="" +echo "RDMA $mode loop test passed." diff --git a/tools/rdma/run-oci-profile.sh b/tools/rdma/run-oci-profile.sh new file mode 100755 index 000000000..83f2f0810 --- /dev/null +++ b/tools/rdma/run-oci-profile.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Run the R239 OCI profiles. The host owns the kernel driver and HCA; the +# image supplies userspace verbs, CM, and provider libraries. +set -Eeuo pipefail + +profile="${1:-}" +image="${CROWDB_OCI_IMAGE:-crowdb:test}" +startup_mode="${CROWDB_STARTUP_MODE:-single}" +management_interface="${CROWDB_MANAGEMENT_INTERFACE:-eth0}" +if [[ -z "$profile" || "$profile" == "-h" || "$profile" == "--help" ]]; then + cat >&2 <<'USAGE' +usage: run-oci-profile.sh tcp-only|two-hca|shared-hca [image] + +tcp-only starts one container without RDMA devices or RDMA capability. +two-hca starts one privileged host-network container pinned to CROWDB_RDMA_DEVICE. +shared-hca starts one privileged host-network container using the shared HCA. +USAGE + exit 2 +fi +if [[ $# -ge 2 ]]; then image="$2"; fi + +case "$profile" in + tcp-only) + exec docker run --rm --network=host \ + -e CROWDB_RDMA_ENABLED=0 \ + -e CROWDB_STARTUP_MODE="$startup_mode" \ + -e CROWDB_MANAGEMENT_INTERFACE="$management_interface" \ + "$image" + ;; + two-hca) + device="${CROWDB_RDMA_DEVICE:-mlx5_0}" + exec docker run --rm --network=host --privileged --ulimit memlock=-1 \ + -e CROWDB_RDMA_ENABLED=1 \ + -e CROWDB_STARTUP_MODE="$startup_mode" \ + -e CROWDB_MANAGEMENT_INTERFACE="$management_interface" \ + -e CROWDB_RDMA_DEVICE="$device" \ + -e CROWDB_RDMA_PORT="${CROWDB_RDMA_PORT:-1}" \ + "$image" + ;; + shared-hca) + exec docker run --rm --network=host --privileged --ulimit memlock=-1 \ + -e CROWDB_RDMA_ENABLED=1 \ + -e CROWDB_RDMA_SHARED_HCA=1 \ + -e CROWDB_STARTUP_MODE="$startup_mode" \ + -e CROWDB_MANAGEMENT_INTERFACE="$management_interface" \ + -e CROWDB_RDMA_DEVICE="${CROWDB_RDMA_DEVICE:-mlx5_0}" \ + -e CROWDB_RDMA_PORT="${CROWDB_RDMA_PORT:-1}" \ + "$image" + ;; + *) + echo "unknown RDMA profile: $profile" >&2 + exit 2 + ;; +esac diff --git a/tools/rdma/setup-rdma.sh b/tools/rdma/setup-rdma.sh new file mode 100755 index 000000000..a299ffca7 --- /dev/null +++ b/tools/rdma/setup-rdma.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +# Prepare the two RDMA smoke-test modes for this ConnectX-5 host. +# +# local uses one port and one root-network address. namespace uses both ports +# and places the client and server netdevs in ns1 and ns2. + +set -Eeuo pipefail + +# Print a command first, then execute the same argv. This keeps the setup +# output readable without Bash xtrace echoing every internal shell statement. +run_cmd() { + printf '****> ' + printf '%q ' "$@" + printf '\n' + "$@" +} + +run_shell() { + local command="$1" + printf '****> %s\n' "$command" + bash -o pipefail -c "$command" +} + +CLIENT_NS="ns1" +SERVER_NS="ns2" +CLIENT_IF="${CROWDB_RDMA_CLIENT_IF:-enp35s0f0np0}" +SERVER_IF="${CROWDB_RDMA_SERVER_IF:-enp35s0f1np1}" +CLIENT_RDMA="" +SERVER_RDMA="" +CLIENT_IP="10.200.1.1/30" +SERVER_IP="10.200.1.2/30" +SERVER_ADDR="10.200.1.2" +SERVER_PID="" +MODE="${1:-local}" +NS_USER="${CROWDB_RDMA_NS_USER:-${SUDO_USER:-${USER:-cj}}}" + +sudo_run() { + run_cmd sudo "$@" +} + +ns_exists() { + ip netns list | awk '{print $1}' | grep -Fxq "$1" +} + +netdev_in_ns() { + local ns="$1" dev="$2" + sudo_run ip netns exec "$ns" ip link show dev "$dev" >/dev/null 2>&1 +} + +wait_for_rdma() { + local i + for i in $(seq 1 10); do + CLIENT_RDMA=$(rdma link show | awk -v netdev="$CLIENT_IF" \ + '$0 ~ " netdev " netdev " " {sub("/1$", "", $2); print $2; exit}') + if [[ "$MODE" == local ]]; then + SERVER_RDMA="$CLIENT_RDMA" + else + SERVER_RDMA=$(rdma link show | awk -v netdev="$SERVER_IF" \ + '$0 ~ " netdev " netdev " " {sub("/1$", "", $2); print $2; exit}') + fi + # Some rdma-core versions omit the netdev field. The device order is + # stable for this two-port adapter, so retain a safe fallback after + # confirming both expected netdevs exist in the root or namespaces. + if [[ -z "$CLIENT_RDMA" && -z "$SERVER_RDMA" ]] && + (ip link show dev "$CLIENT_IF" >/dev/null 2>&1 || netdev_in_ns ns1 "$CLIENT_IF") && + (ip link show dev "$SERVER_IF" >/dev/null 2>&1 || netdev_in_ns ns2 "$SERVER_IF"); then + CLIENT_RDMA="rocep35s0f0" + SERVER_RDMA="rocep35s0f1" + fi + if [[ -n "$CLIENT_RDMA" && -n "$SERVER_RDMA" && \ + -e "/sys/class/infiniband/${CLIENT_RDMA}" && \ + -e "/sys/class/infiniband/${SERVER_RDMA}" ]]; then + return 0 + fi + sleep 1 + done + echo "RDMA links for ${CLIENT_IF} and ${SERVER_IF} did not appear:" >&2 + rdma link show >&2 || true + return 1 +} + +check_commands() { + local cmd + for cmd in ip rdma ethtool modprobe ping setcap getcap realpath sysctl; do + command -v "$cmd" >/dev/null || { + echo "Missing command: $cmd" >&2 + return 1 + } + done +} + +enable_local_rdma_loopback() { + if [[ "$MODE" == local ]]; then + [[ -n "$CLIENT_IF" && -n "$SERVER_IF" ]] || return 0 + sudo_run sysctl -w net.ipv4.conf.all.accept_local=1 + sudo_run sysctl -w "net.ipv4.conf.${CLIENT_IF}.accept_local=1" + return 0 + fi + [[ -n "$CLIENT_IF" && -n "$SERVER_IF" ]] || return 0 + for ns in "$CLIENT_NS" "$SERVER_NS"; do + sudo_run ip netns exec "$ns" sysctl -w net.ipv4.conf.all.accept_local=1 + done + if [[ -n "$CLIENT_IF" ]]; then + sudo_run ip netns exec "$CLIENT_NS" sysctl -w "net.ipv4.conf.${CLIENT_IF}.accept_local=1" + fi + if [[ -n "$SERVER_IF" ]]; then + sudo_run ip netns exec "$SERVER_NS" sysctl -w "net.ipv4.conf.${SERVER_IF}.accept_local=1" + fi +} + +grant_namespace_access() { + local ip_bin + ip_bin=$(realpath "$(command -v ip)") + [[ -n "$NS_USER" && "$NS_USER" != root ]] || return 0 + echo "Granting $NS_USER access to enter RDMA test namespaces through $ip_bin" + sudo_run setcap cap_sys_admin+ep "$ip_bin" + sudo_run getcap "$ip_bin" +} + +check_netdev_driver() { + local ns="$1" dev="$2" + if netdev_in_ns "$ns" "$dev"; then + run_shell "sudo ip netns exec $ns ethtool -i $dev" + run_shell "sudo ip netns exec $ns ethtool $dev | grep -E 'Speed:|Link detected:'" || true + elif ip link show dev "$dev" >/dev/null 2>&1; then + run_shell "ethtool -i $dev" + run_shell "ethtool $dev | grep -E 'Speed:|Link detected:'" || true + else + echo "netdev not found in root or its namespace: $dev" >&2 + return 1 + fi +} + +check_rdma() { + sudo_run modprobe mlx5_core + sudo_run modprobe mlx5_ib + wait_for_rdma + echo "RDMA client=${CLIENT_RDMA} server=${SERVER_RDMA}" +} + +apply_namespaces() { + if [[ "$MODE" == local ]]; then + sudo_run ip link set dev "$CLIENT_IF" up + sudo_run ip link set dev "$CLIENT_IF" mtu 9000 + sudo_run ip addr replace "$CLIENT_IP" dev "$CLIENT_IF" + return 0 + fi + if [[ -z "$CLIENT_IF" || -z "$SERVER_IF" ]]; then + echo "RDMA devices are present, but no Ethernet netdev is associated with both ports." >&2 + return 1 + fi + for ns in "$CLIENT_NS" "$SERVER_NS"; do + if ! ns_exists "$ns"; then + sudo_run ip netns add "$ns" + fi + done + + # In shared mode the RDMA devices remain visible in both namespaces. + # Moving only the physical Ethernet netdevs isolates the IP/RDMA-CM paths. + if ! netdev_in_ns "$CLIENT_NS" "$CLIENT_IF"; then + if ip link show dev "$CLIENT_IF" >/dev/null 2>&1; then + sudo_run ip link set dev "$CLIENT_IF" netns "$CLIENT_NS" + fi + fi + if ! netdev_in_ns "$SERVER_NS" "$SERVER_IF"; then + if ip link show dev "$SERVER_IF" >/dev/null 2>&1; then + sudo_run ip link set dev "$SERVER_IF" netns "$SERVER_NS" + fi + fi + + sudo_run ip -n "$CLIENT_NS" link set lo up + sudo_run ip -n "$SERVER_NS" link set lo up + sudo_run ip -n "$CLIENT_NS" link set "$CLIENT_IF" up + sudo_run ip -n "$SERVER_NS" link set "$SERVER_IF" up + + # Jumbo Ethernet frames allow the RoCE provider to use a 4096-byte MTU. + sudo_run ip -n "$CLIENT_NS" link set "$CLIENT_IF" mtu 9000 + sudo_run ip -n "$SERVER_NS" link set "$SERVER_IF" mtu 9000 + + sudo_run ip -n "$CLIENT_NS" addr replace "$CLIENT_IP" dev "$CLIENT_IF" + sudo_run ip -n "$SERVER_NS" addr replace "$SERVER_IP" dev "$SERVER_IF" +} + +show_status() { + if [[ "$MODE" == local ]]; then + echo "=== local ===" + sudo_run ip -br addr show dev "$CLIENT_IF" + sudo_run rdma link show + sudo_run ethtool -i "$CLIENT_IF" + return 0 + fi + if [[ -z "$CLIENT_IF" || -z "$SERVER_IF" ]]; then + echo "No per-port netdevs to show; RDMA devices remain in shared host mode." + return 0 + fi + echo "=== ns1 / client ===" + sudo_run ip netns exec "$CLIENT_NS" ip -br addr + sudo_run ip netns exec "$CLIENT_NS" rdma link show + sudo_run ip netns exec "$CLIENT_NS" ethtool -i "$CLIENT_IF" + run_shell "sudo ip netns exec $CLIENT_NS ethtool $CLIENT_IF | grep -E 'Speed:|Link detected:'" || true + sudo_run ip netns exec "$CLIENT_NS" ip -d link show dev "$CLIENT_IF" + + echo "=== ns2 / server ===" + sudo_run ip netns exec "$SERVER_NS" ip -br addr + sudo_run ip netns exec "$SERVER_NS" rdma link show + sudo_run ip netns exec "$SERVER_NS" ethtool -i "$SERVER_IF" + run_shell "sudo ip netns exec $SERVER_NS ethtool $SERVER_IF | grep -E 'Speed:|Link detected:'" || true + sudo_run ip netns exec "$SERVER_NS" ip -d link show dev "$SERVER_IF" + + echo "=== connectivity ===" + sudo_run ip netns exec "$CLIENT_NS" ping -I "$CLIENT_IF" -c 3 -W 1 "$SERVER_ADDR" +} + +cleanup_namespaces() { + set +e + if [[ -n "$SERVER_PID" ]]; then + sudo_run kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + + if netdev_in_ns "$CLIENT_NS" "$CLIENT_IF"; then + sudo_run ip -n "$CLIENT_NS" link set "$CLIENT_IF" netns 1 + fi + if netdev_in_ns "$SERVER_NS" "$SERVER_IF"; then + sudo_run ip -n "$SERVER_NS" link set "$SERVER_IF" netns 1 + fi + + sudo_run ip netns del "$CLIENT_NS" 2>/dev/null || true + sudo_run ip netns del "$SERVER_NS" 2>/dev/null || true +} + + +setup() { + sudo_run -v + check_commands + check_rdma + apply_namespaces + enable_local_rdma_loopback + grant_namespace_access + show_status + echo + echo "One-shot $MODE-mode RDMA setup complete." +} + +clean() { + sudo_run -v + cleanup_namespaces +} + +case "$MODE" in + local|namespace) setup ;; + clean) clean ;; + -h|--help) + echo "Usage: $0 [local|namespace|clean]" + ;; + *) + echo "Usage: $0 [local|namespace|clean]" >&2 + exit 2 + ;; +esac From 19760ad96ad18202d8cf928ffaf5bc452644ce4d Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 18 Oct 2026 15:41:05 +0800 Subject: [PATCH 002/258] Complete R239 worker path and teardown --- doc/backlog/R239-rdma-rpc-transport.md | 42 +-- doc/working/plan-rdma-multi-node.md | 54 ++++ doc/working/plan-rdma-rpc-transport.md | 49 +-- doc/working/plan-rdma.md | 52 ++-- .../transport/rdma/rdma_transport.h | 35 ++- .../src/transport/rdma/rdma_transport.cpp | 279 ++++++++++-------- .../src/transport/socket_transport.cpp | 9 +- lib/crowdb-rpc/tests/rdma_test.cpp | 69 +++-- tools/rdma/run-local-loop.sh | 3 +- tools/rdma/run-oci-profile.sh | 21 +- 10 files changed, 354 insertions(+), 259 deletions(-) create mode 100644 doc/working/plan-rdma-multi-node.md diff --git a/doc/backlog/R239-rdma-rpc-transport.md b/doc/backlog/R239-rdma-rpc-transport.md index 9072175e8..8475ff75b 100644 --- a/doc/backlog/R239-rdma-rpc-transport.md +++ b/doc/backlog/R239-rdma-rpc-transport.md @@ -31,19 +31,21 @@ without changing request semantics. with TCP bytes. 6. Provide OCI image test profiles. The image contains `libibverbs`, `librdmacm`, and the provider libraries; the host supplies the kernel RDMA - driver and HCA devices. The first hardware profile uses two containers with - `--network=host`, `--privileged`, and `--ulimit memlock=-1`, pinning the - client to HCA 0 and DiskIO to HCA 1 through transport configuration. A - later restricted profile maps only the required `/dev/infiniband` devices - and uses `IPC_LOCK` plus a bounded memlock limit. -7. Keep three E2E topology profiles. The TCP-only profile exposes no RDMA - devices and verifies that capability discovery avoids all RDMA attempts. The - two-HCA profile runs client and DiskIO containers over the two connected - physical ports. The shared-HCA profile runs multiple logical node - containers on one host and verifies multiple QPs and fallback, while - explicitly not claiming physical NIC isolation. More than two physical - nodes require a shared RDMA fabric, switch, SR-IOV VFs, or a software RDMA - device; additional containers alone do not create physical RDMA nodes. + driver and HCA devices. The current hardware profile runs multiple + containers on one physical host with `--network=host`, `--privileged`, and + `--ulimit memlock=-1`, sharing one configured HCA and opening independent + QPs. A later restricted profile maps only the required `/dev/infiniband` + devices and uses `IPC_LOCK` plus a bounded memlock limit. +7. Keep two current E2E topology profiles. The TCP-only profile exposes no + RDMA devices and verifies that capability discovery avoids all RDMA + attempts. The shared-HCA profile runs multiple logical containers on one + physical host and verifies multiple QPs and fallback. Physical multi-node + topology is tracked by a separate follow-up plan and is outside R239 scope. +8. Use host networking for the current container profile. Every container + receives one `CROWDB_PORT_OFFSET`; each service applies that same offset to + its documented default port. The RDMA HCA device and physical HCA port are + shared and are not offset. The default RDMA CM service port is `19101`, so + the effective port is `19101 + CROWDB_PORT_OFFSET`. ## Dependencies @@ -72,12 +74,9 @@ without changing request semantics. - Given an OCI image with no RDMA devices, when the TCP-only profile starts, service discovery selects TCP without attempting RDMA. Invariant: optional capability. E2E test. -- Given two connected host HCA ports, when the two-HCA profile starts client and - DiskIO containers, each endpoint uses its configured local HCA and completes - an RDMA stream. Invariant: explicit device selection. E2E test. - Given multiple containers sharing one HCA, when the shared-HCA profile opens multiple logical connections, each QP remains correlated and bounded, while - the test reports shared hardware rather than physical node isolation. + the test reports a single physical host rather than physical node isolation. Invariant: topology declaration matches test claim. E2E test. - Given a prepared OCI runtime with the required RDMA devices and memlock limit, when the RPC flow runs as a non-root application user, the container @@ -108,6 +107,9 @@ docker run --rm --network=host --privileged --ulimit memlock=-1 \ crowdb:test ``` -The second endpoint uses its own device and local address. The restricted -container profile is a follow-up hardening step after the two-HCA profile -passes. +The second container uses the same physical host HCA and its own logical +connection. The restricted container profile is a follow-up hardening step +after the shared-HCA profile passes. Physical multi-node execution is specified +in the separate follow-up plan. With host networking, each container must use +a distinct port offset; the offset applies to TCP/RPC and RDMA CM service +ports, while `CROWDB_RDMA_PORT` remains the HCA physical port number. diff --git a/doc/working/plan-rdma-multi-node.md b/doc/working/plan-rdma-multi-node.md new file mode 100644 index 000000000..81230e323 --- /dev/null +++ b/doc/working/plan-rdma-multi-node.md @@ -0,0 +1,54 @@ + + + +# RDMA physical multi-node validation Plan + +Upstream: [R5](../backlog/R5-rdma-alloc.md), with transport behavior supplied +by [R239](../backlog/R239-rdma-rpc-transport.md) and the current +[RDMA implementation plan](plan-rdma.md). + +Goal: validate the already implemented RDMA RPC and data paths across physical +nodes after the single-node, multi-container acceptance is complete. This plan +does not add transport behavior or change the R239 backlog scope. + +## Scope + +- one or more containers per physical node; +- one explicitly selected RDMA NIC/device and address per container; +- shared RPC framing, request correlation, fallback, and bounded buffers; +- host-provided fabric, HCA, device access, and memlock setup; +- no network-namespace manipulation in the application or test path. + +The host-network port rule remains in force when this plan is executed: each +container receives a distinct port offset applied to all service ports. The +HCA physical port is not part of the offset. + +## Tasks + +- [ ] **Record node inventory**: capture each node's HCA, port, GID/IP + mapping, provider libraries, and reachable peer address. +- [ ] **Prepare multi-node OCI runtime**: provide the required devices, + userspace libraries, memlock allowance, and application-user access on each + node. +- [ ] **Bind endpoint devices**: configure one local device/address mapping per + container and verify that the negotiated peer identity matches the intended + node. +- [ ] **Run transport corpus**: exercise CM/QP setup, two-SGE frames, + receive repost, request correlation, disconnect/error handling, and + pre-payload TCP fallback across the fabric. +- [ ] **Run service corpus**: compare TCP and RDMA results for Access Server, + DiskIO, Chunk-KV, and SDK flows byte-for-byte and by error/sequence behavior. +- [ ] **Publish topology evidence**: report node count, device mapping, QP + count, fallback count, buffer pressure, and final result. + +## Exit condition + +The same RPC and service corpus passes across the available physical nodes, +with explicit device ownership and no namespace operations. Failures caused by +missing fabric or unavailable HCA remain environment blockers and do not alter +the R239 transport contract. + +## Verification + +Run the current R239 and service gates through `pixi run`, then execute the OCI +profile with the node inventory and device mappings recorded above. diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 64626925a..7e34b8db8 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -29,9 +29,13 @@ capability frame, connections retain the peer capability, RDMA-enabled connections are selected before TCP fallback, and completed RDMA frames retain their buffers until the owning `Frame` is destroyed, then repost the receive pair. The live host HCA local-loop CM/QP/send/receive flow now passes, including -receive repost and connection-level CQ handling. Remaining validation includes -executing the OCI profiles against a built `crowdb:test` image and exercising -the actual RPC worker path. +receive repost and connection-level CQ handling. The actual `RpcServer`/ +`RpcClient` worker path now passes a live request/response round trip with +multiple sequential frames on one QP, including CQ wakeup, frame dispatch, +request correlation, response completion, and clean teardown. Remaining +validation includes executing the single-node multi-container OCI profiles +against a built `crowdb:test` image. Physical +multi-node validation is deferred to [the separate follow-up plan](plan-rdma-multi-node.md). ## R239 status checkpoint @@ -46,18 +50,15 @@ Completed: - epoll registration for CM, transport CQ, and connection CQ events; - live HCA local-loop CM/QP/send/receive E2E with all frames received and clean CQ status, and clean client/server exit; +- actual `RpcServer`/`RpcClient` worker-path request/response round trip with + sequential frames and clean teardown; - non-root runner selection and the library consumer/deployment contract. Remaining: -- run the same flow through the actual `RpcServer`/`RpcClient` worker path, - including frame dispatch and request correlation rather than the transport - harness only; - add mock/integration coverage for CM disconnect, QP error, CQ flush/error, epoll wakeup, and pre-payload TCP fallback; -- exercise multiple sequential frames on one QP to verify receive repost and - buffer reuse under the worker; -- run single-node HCA local-loop multi-container and physical multi-node OCI profiles against the built image; +- run the single-node HCA local-loop multi-container OCI profile against the built image; - remove temporary hardware diagnostics, normalize failure logging, and run the full C++ format/tree-lint gates. @@ -78,10 +79,10 @@ Remaining: ## Phase 2: CM and QP lifecycle -- [ ] **Implement device selection**: select configured HCA, port, local +- [x] **Implement device selection**: select configured HCA, port, local address, and GID; fail closed to TCP when local probing finds no usable device. Files: `rdma_transport.h/.cpp` and configuration adapter. -- [ ] **Implement active/passive CM flow**: create IDs, bind/listen or resolve +- [x] **Implement active/passive CM flow**: create IDs, bind/listen or resolve address/route, create QP, connect/accept, and publish ready only after initial receive buffers are posted. Files: RDMA transport sources. - [ ] **Implement connection teardown**: map CM disconnect, QP error, and CQ @@ -90,13 +91,13 @@ Remaining: ## Phase 3: event/poll loop -- [ ] **Extend epoll registration**: add external event-fd registration and +- [x] **Extend epoll registration**: add external event-fd registration and removal to the existing `EpollEngine` without changing TCP readiness rules. Files: `lib/crowdb-rpc/include/crowdb-rpc/transport/epoll/epoll_engine.h`, `lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp`. - [ ] **Drain CM events**: acknowledge all received CM events, update the connection state, and re-arm the CM channel where required. -- [ ] **Drain CQ events**: acknowledge and re-arm completion notifications, then +- [x] **Drain CQ events**: acknowledge and re-arm completion notifications, then poll bounded batches until the current CQ work is drained or the worker budget is reached. No independent polling thread. - [ ] **Make stop wakeable**: stop must wake epoll, stop new submissions, drain @@ -104,15 +105,15 @@ Remaining: ## Phase 4: frame submission and completion -- [ ] **Build two-SGE WRs**: map the 1 KiB control buffer and selected data +- [x] **Build two-SGE WRs**: map the 1 KiB control buffer and selected data buffer to one SEND WR. Inline control only when the configured device limit allows; never concatenate control and data in the CPU. -- [ ] **Post receives**: prepost control/data receive buffers, associate each +- [x] **Post receives**: prepost control/data receive buffers, associate each with a stable `wr_id`, and replenish only after the owner releases it. - [ ] **Resolve WR identity**: `wr_id` must identify connection, request ID, stream ID, sequence, logical offset, buffers, and ownership state. Handle success, flush, local protection, retry, and remote access errors distinctly. -- [ ] **Dispatch common callbacks**: validate lengths and class before parser +- [x] **Dispatch common callbacks**: validate lengths and class before parser publication, then invoke the existing C++ completion path and FFI callback. ## Phase 5: fallback and observability @@ -147,8 +148,7 @@ Integration tests: E2E tests: - TCP-only OCI profile; -- single-node HCA local-loop profile with multiple QPs; -- physical multi-node profile with one RDMA NIC bound per container. +- single-node HCA local-loop profile with multiple containers sharing one HCA. - non-root OCI profile: host prepares RDMA and network topology, the container receives only the required `/dev/infiniband` devices and memlock allowance, and the RPC flow runs as the application user without `sudo`, module loads, @@ -163,7 +163,7 @@ E2E tests: - `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h` - `lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp` - `lib/crowdb-rpc/tests/` transport, fallback, and event-loop tests -- OCI image and E2E runner configuration +- OCI image and single-node multi-container E2E runner configuration ## Verification gates @@ -195,9 +195,9 @@ tools/rdma/run-local-loop.sh local ``` Multiple containers on one node use the same HCA local-loop model and open -independent QPs. Multi-node OCI runs use the same application contract, with -each container bound to the RDMA NIC on its own physical node. No namespace -operation is part of the test or application path. +independent QPs. Physical multi-node OCI runs are outside this plan and are +specified in the separate follow-up plan. No namespace operation is part of +the test or application path. ## Library consumer contract @@ -219,6 +219,11 @@ The application layer only supplies the normal RPC endpoint configuration. The library discovers usable devices and selects RDMA before TCP when the peer advertises compatible capability. `CROWDB_RDMA_DEVICE`, `CROWDB_RDMA_PORT`, and `CROWDB_RDMA_SOURCE_ADDR` are deployment overrides, not required API calls. +For host-network multi-container runs, deployment also supplies one +`CROWDB_PORT_OFFSET` per container. The library and service launchers apply the +same offset to every default TCP/RPC and RDMA CM service port; the HCA physical +port selected by `CROWDB_RDMA_PORT` is unchanged. The default RDMA CM service +port is `19101`. When any local capability, CM, QP, or pre-payload setup step is unavailable, the library reports the reason and uses TCP without requiring application code changes. After payload submission, the connection fails as one stream and does diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index 3cf3dc1ba..67b6fb0a9 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -13,8 +13,8 @@ backlogs [R238](../backlog/R238-rdma-registered-buffer-foundation.md), Goal: implement the host-memory RDMA SEND/RECV path end to end, with RDMA as the preferred connection path, TCP as the one-way fallback, shared aggregation -semantics, bounded buffer leases, and automated OCI verification using HCA local -loop on one node and one RDMA NIC per container across physical nodes. +semantics, bounded buffer leases, and automated OCI verification using a +single-node HCA local-loop with multiple containers. ## Current implementation status @@ -48,12 +48,15 @@ the formal design first. - Use `setup-rdma.sh local` as the standard host setup. A single node runs client and server through the same HCA port and local IP; multiple containers on that node use the same local-loop model and open independent QPs. -- For physical multi-node tests, each container is bound to the RDMA NIC on its - own node. The test harness supplies the selected device and address; it does - not move links into network namespaces. - Treat `run-local-loop.sh local` as the smoke test for every local hardware - setup. Promote the same test flow to a multi-node container test by changing - only the endpoint placement and device binding. + setup. The current acceptance scope is one physical node with multiple + containers sharing the configured HCA and opening independent QPs. +- Use host networking only for the multi-container profile. Assign each + container a `CROWDB_PORT_OFFSET`; apply it uniformly to all default + TCP/RPC and RDMA CM service ports, while keeping the HCA physical port + unchanged. +- Physical multi-node testing is deferred to [the separate follow-up plan](plan-rdma-multi-node.md) + and is not part of the current backlog scope. ## Dependency graph @@ -80,13 +83,13 @@ lease contract from R238 is stable. - [x] **Lock hardware test model**: use `setup-rdma.sh local` and `run-local-loop.sh local` for single-node validation; do not use namespace setup in product or acceptance tests. Multiple containers on one node share - the local HCA loop, while multi-node containers bind one NIC per node. + the local HCA loop and open independent QPs. - [ ] **Record baseline**: run the existing TCP RPC, Access Server, DiskIO, and Chunk-KV gates before modifying production code. Root files: affected crate manifests and CI configuration. - [ ] **Confirm host capabilities**: record `ibv_devices`, `rdma link`, the selected HCA port, GID/IP mapping, and whether the local HCA loop is usable. - For multi-node runs, record one selected NIC and address per container. + Record the container-to-QP mapping for the single-node shared-HCA run. Do not treat an unavailable host device as an implementation failure. - [ ] **Lock configuration names**: agree on RDMA enablement, device/port/local address, control/data class defaults, registered bytes, receive depth, and @@ -161,17 +164,13 @@ and the caller-controlled lease lifetime is visible and testable through FFI. - [ ] **TCP-only profile**: run OCI containers without RDMA devices and assert service discovery skips RDMA entirely. -- [ ] **Single-node HCA local-loop profile**: run `setup-rdma.sh local`, start - one or more containers on the node, and run the client/server flow against - the same HCA port. Assert multiple QPs, request correlation, bounded buffers, - and TCP fallback. -- [ ] **Physical multi-node profile**: start one or more containers per node, - bind each container to that node's RDMA NIC, and run the same RPC/data-path - corpus across the fabric. No network namespace setup is part of this profile. -- [ ] **Automated topology selection**: make the test runner execute the - TCP-only and local-loop profiles automatically when their prerequisites are - present, and schedule the physical multi-node profile only when its node/NIC - inventory is available. +- [ ] **Single-node multi-container profile**: run `setup-rdma.sh local`, start + multiple containers on one physical host, share the configured HCA, and + assert independent QPs, request correlation, bounded buffers, and TCP + fallback. +- [ ] **Automated local topology selection**: make the test runner execute the + TCP-only and single-node local-loop profiles when their prerequisites are + present. Exit condition: every profile reports its topology, RDMA capability, selected device, selected path, fallback count, buffer pressure, and final result. @@ -188,9 +187,9 @@ device, selected path, fallback count, buffer pressure, and final result. queue bytes, and SDK-held bytes remain within configuration. - [ ] **Capability optionality**: a host or OCI container without RDMA remains a supported TCP deployment. -- [ ] **NIC ownership**: each physical-node container receives one explicit - RDMA device/address mapping; no test depends on moving a link or entering a - network namespace. +- [ ] **NIC ownership**: each single-node container uses the explicitly + configured shared HCA; no test depends on moving a link or entering a network + namespace. - [ ] **Shared identity**: request ID, generation, stream sequence, and logical offset retain their meaning across TCP and RDMA. @@ -211,10 +210,9 @@ pixi run tree-lint ``` OCI E2E verification must run the TCP-only and single-node HCA local-loop -profiles when their prerequisites are available. The physical multi-node -profile reuses the same test corpus with one RDMA NIC bound per container and -is run when the node inventory is available. Namespace profiles are not part -of acceptance. +multi-container profiles when their prerequisites are available. Physical +multi-node verification is outside this plan and is covered by the separate +follow-up plan. Namespace profiles are not part of acceptance. ## Test and setup files diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index 45af15edb..256cf63f4 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -14,13 +14,13 @@ # include # include +# include # include # include # include # include # include # include -# include namespace crowdb::rpc { @@ -66,6 +66,7 @@ class RdmaTransport : public Transport { public: using ConnectionHandler = std::function)>; + struct Stats { std::atomic sends{0}; @@ -104,7 +105,8 @@ class RdmaTransport : public Transport int drain_cq(uint32_t max_completions = 32); void register_events(SocketEngine *engine); int handle_external_event(uint64_t tag, uint32_t max_completions = 32); - void set_connection_handler(ConnectionHandler handler) + + void set_connection_handler(ConnectionHandler handler) { connection_handler_ = std::move(handler); } @@ -125,18 +127,19 @@ class RdmaTransport : public Transport } private: - struct ibv_context *context_ = nullptr; - struct ibv_pd *pd_ = nullptr; - struct ibv_comp_channel *comp_channel_ = nullptr; - struct ibv_cq *cq_ = nullptr; - struct rdma_event_channel *cm_channel_ = nullptr; - struct rdma_cm_id *listen_id_ = nullptr; - uint8_t port_ = 1; - std::vector cm_ids_; - std::unordered_map cm_cqs_; - std::unordered_map cm_channels_; - SocketEngine *event_engine_ = nullptr; - std::mutex cm_mu_; + struct ibv_context *context_ = nullptr; + struct ibv_pd *pd_ = nullptr; + struct ibv_comp_channel *comp_channel_ = nullptr; + struct ibv_cq *cq_ = nullptr; + struct rdma_event_channel *cm_channel_ = nullptr; + struct rdma_cm_id *listen_id_ = nullptr; + uint8_t port_ = 1; + std::vector cm_ids_; + std::unordered_map> cm_connections_; + std::unordered_map cm_cqs_; + std::unordered_map cm_channels_; + SocketEngine *event_engine_ = nullptr; + std::mutex cm_mu_; std::unique_ptr send_pool_; std::unique_ptr recv_pool_; @@ -144,8 +147,8 @@ class RdmaTransport : public Transport std::atomic running_{false}; - bool prime_receive(const std::shared_ptr &connection); - Stats stats_; + bool prime_receive(const std::shared_ptr &connection); + Stats stats_; struct ReceiveWork { diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index 7abee30f8..1febd7ea5 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -3,8 +3,8 @@ #include "crowdb-rpc/transport/rdma/rdma_transport.h" -#include "crowdb-rpc/transport/socket_transport.h" #include "crowdb-common/log.h" +#include "crowdb-rpc/transport/socket_transport.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -16,8 +16,10 @@ # include # include +# include # include # include +# include namespace crowdb::rpc { @@ -42,7 +44,7 @@ bool wait_cm_event(rdma_event_channel *channel, rdma_cm_event **event, int timeo return false; return rdma_get_cm_event(channel, event) == 0; } -} +} // namespace // ── RdmaBufferPool ──────────────────────────────────────────────── @@ -169,7 +171,7 @@ RdmaTransport::RdmaTransport() int device_count = 0; ibv_device **devices = ibv_get_device_list(&device_count); if (devices == nullptr || device_count == 0) { - std::fprintf(stderr, "rdma init: no ibv devices count=%d errno=%d\\n", device_count, errno); + if (devices != nullptr) { ibv_free_device_list(devices); } @@ -191,8 +193,7 @@ RdmaTransport::RdmaTransport() } } if (selected == nullptr) { - std::fprintf(stderr, "rdma init: requested device not found name=%s count=%d\\n", requested_device, - device_count); + ibv_free_device_list(devices); return; } @@ -200,28 +201,26 @@ RdmaTransport::RdmaTransport() context_ = ibv_open_device(selected); ibv_free_device_list(devices); if (context_ == nullptr) { - std::fprintf(stderr, "rdma init: ibv_open_device failed name=%s errno=%d\\n", - requested_device == nullptr ? "default" : requested_device, errno); + return; } ibv_port_attr port_attributes{}; if (ibv_query_port(context_, port_, &port_attributes) != 0 || port_attributes.state != IBV_PORT_ACTIVE) { - std::fprintf(stderr, "rdma init: port unavailable port=%u state=%u errno=%d\\n", port_, - port_attributes.state, errno); + ibv_close_device(context_); context_ = nullptr; return; } pd_ = ibv_alloc_pd(context_); if (pd_ == nullptr) { - std::fprintf(stderr, "rdma init: ibv_alloc_pd failed errno=%d\\n", errno); + ibv_close_device(context_); context_ = nullptr; return; } comp_channel_ = ibv_create_comp_channel(context_); if (comp_channel_ == nullptr) { - std::fprintf(stderr, "rdma init: ibv_create_comp_channel failed errno=%d\\n", errno); + ibv_dealloc_pd(pd_); pd_ = nullptr; ibv_close_device(context_); @@ -233,7 +232,7 @@ RdmaTransport::RdmaTransport() ::fcntl(comp_channel_->fd, F_SETFL, channel_flags | O_NONBLOCK); cq_ = ibv_create_cq(context_, 2048, this, comp_channel_, 0); if (cq_ == nullptr || ibv_req_notify_cq(cq_, 0) != 0) { - std::fprintf(stderr, "rdma init: CQ setup failed cq=%p errno=%d\\n", static_cast(cq_), errno); + if (cq_ != nullptr) ibv_destroy_cq(cq_); cq_ = nullptr; @@ -279,20 +278,51 @@ bool RdmaTransport::submit(Connection *conn, OutFrame *frame) conn->transport_handle == 0) { return false; } + if (frame->data_views() > 1) { + return false; + } auto *pools = static_cast(conn->user_data); - if (pools != nullptr && pools->send != nullptr && frame->control->pool != pools->send.get()) { - Buffer *copy = pools->send->alloc(frame->control->len); - if (copy == nullptr) - return false; - std::memcpy(copy->data, frame->control->data, frame->control->len); - copy->len = frame->control->len; - copy->request_id = frame->control->request_id; - copy->stream_id = frame->control->stream_id; - copy->sequence = frame->control->sequence; - copy->logical_offset = frame->control->logical_offset; + if (pools == nullptr || pools->send == nullptr) { + return false; + } + + // SEND carries the RPC wire header and control message in one registered + // control SGE. TCP serializes the header in writev; RDMA must materialize + // the same bytes before posting the WR while keeping data in its own SGE. + Buffer *source_control = frame->control; + const uint32_t control_len = source_control->len; + if (source_control->data == nullptr || control_len > source_control->capacity || + control_len > RDMA_CONTROL_CAPACITY - HEADER_SIZE) { + CRB_LOG_WARN("rdma send: control buffer exceeds wire capacity len={} capacity={}", control_len, + source_control->capacity); + return false; + } + Buffer *wire_control = pools->send->alloc(RDMA_CONTROL_CAPACITY); + if (wire_control == nullptr) + return false; + frame->header.msg_size = control_len; + frame->header.data_size = frame->data == nullptr ? 0 : frame->data->len; + serialize_header(wire_control->data, frame->header); + if (control_len != 0) + std::memcpy(wire_control->data + HEADER_SIZE, source_control->data, control_len); + wire_control->len = HEADER_SIZE + control_len; + wire_control->request_id = source_control->request_id; + wire_control->stream_id = source_control->stream_id; + wire_control->sequence = source_control->sequence; + wire_control->logical_offset = source_control->logical_offset; + frame->control = wire_control; + source_control->release(); + + if (frame->data != nullptr && frame->data->len > frame->data->capacity) { frame->control->release(); - frame->control = copy; + frame->control = nullptr; + return false; + } + if (frame->data != nullptr && frame->data->len == 0) { + frame->data->release(); + frame->data = nullptr; } + RdmaFrameSges frame_sges; if (!build_rdma_frame_sges(frame->control, frame->data, &frame_sges)) { return false; @@ -311,12 +341,10 @@ bool RdmaTransport::submit(Connection *conn, OutFrame *frame) wr.send_flags = IBV_SEND_SIGNALED; ibv_send_wr *bad = nullptr; if (ibv_post_send(reinterpret_cast(conn->transport_handle), &wr, &bad) != 0) { - std::fprintf(stderr, "rdma send: ibv_post_send failed qp=%p errno=%d\n", - reinterpret_cast(conn->transport_handle), errno); + CRB_LOG_WARN("rdma send: post failed qp={} errno={}", reinterpret_cast(conn->transport_handle), errno); return false; } - std::fprintf(stderr, "rdma send: posted qp=%p frame=%p\n", - reinterpret_cast(conn->transport_handle), static_cast(frame)); + stats_.sends.fetch_add(1, std::memory_order_relaxed); return true; } @@ -353,12 +381,11 @@ bool RdmaTransport::post_receive(Connection *conn, Buffer *control, Buffer *data const bool data_posted = data == nullptr || data->transition(BufferOwnership::Available, BufferOwnership::PostedReceive) || data->transition(BufferOwnership::CallerOwned, BufferOwnership::PostedReceive); - int post_rc = -1; + int post_rc = -1; if (control_posted && data_posted) post_rc = ibv_post_recv(reinterpret_cast(conn->transport_handle), &wr, &bad); if (!control_posted || !data_posted || post_rc != 0) { - std::fprintf(stderr, "rdma receive: ibv_post_recv failed qp=%p rc=%d errno=%d bad=%p control_state=%u data_state=%u\n", - reinterpret_cast(conn->transport_handle), post_rc, errno, static_cast(bad), + CRB_LOG_WARN("rdma receive: post failed rc={} control_state={} data_state={}", post_rc, static_cast(control->ownership.load(std::memory_order_relaxed)), data == nullptr ? 0U : static_cast(data->ownership.load(std::memory_order_relaxed))); control->ownership.store(BufferOwnership::Available, std::memory_order_release); @@ -369,8 +396,7 @@ bool RdmaTransport::post_receive(Connection *conn, Buffer *control, Buffer *data data->release(); return false; } - std::fprintf(stderr, "rdma receive: posted qp=%p wr_id=%#lx\n", - reinterpret_cast(conn->transport_handle), static_cast(wr.wr_id)); + stats_.posted_receives.fetch_add(1, std::memory_order_relaxed); work.release(); return true; @@ -399,6 +425,14 @@ Buffer *RdmaTransport::register_buffer(Buffer *buf) void RdmaTransport::shutdown() { + // A QP may still own SEND or flushed receive WRs after the worker stops. + // Drain bounded CQ batches before destroying the IDs so provider teardown + // never observes live completion objects. + for (int attempt = 0; attempt < 100; ++attempt) { + const int drained = drain_cq(256); + if (drained == 0) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } stop(); { std::scoped_lock lock(cm_mu_); @@ -418,6 +452,7 @@ void RdmaTransport::shutdown() rdma_destroy_id(id); } cm_ids_.clear(); + cm_connections_.clear(); cm_pools_.clear(); } if (listen_id_ != nullptr) { @@ -435,22 +470,21 @@ void RdmaTransport::shutdown() std::shared_ptr RdmaTransport::connect(const std::string &addr, int port) { if (pd_ == nullptr || cq_ == nullptr || addr.empty() || port <= 0 || port > 65535) { - std::fprintf(stderr, "rdma connect: transport unavailable pd=%p cq=%p addr=%s port=%d\\n", - static_cast(pd_), static_cast(cq_), addr.c_str(), port); - CRB_LOG_WARN("rdma connect: transport unavailable pd={} cq={} addr={} port={}", - static_cast(pd_), static_cast(cq_), addr, port); + + CRB_LOG_WARN("rdma connect: transport unavailable pd={} cq={} addr={} port={}", static_cast(pd_), + static_cast(cq_), addr, port); return nullptr; } if (cm_channel_ == nullptr) cm_channel_ = rdma_create_event_channel(); if (cm_channel_ == nullptr) { - std::fprintf(stderr, "rdma connect: create event channel failed errno=%d\\n", errno); + return nullptr; } make_nonblocking(cm_channel_->fd); rdma_cm_id *id = nullptr; if (rdma_create_id(cm_channel_, &id, this, RDMA_PS_TCP) != 0) { - std::fprintf(stderr, "rdma connect: create id failed errno=%d\\n", errno); + return nullptr; } sockaddr_in destination{}; @@ -462,7 +496,7 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int return nullptr; } sockaddr_in source{}; - sockaddr *source_address = nullptr; + sockaddr *source_address = nullptr; if (const char *source_env = std::getenv("CROWDB_RDMA_SOURCE_ADDR"); source_env != nullptr && *source_env != '\0') { source.sin_family = AF_INET; if (::inet_pton(AF_INET, source_env, &source.sin_addr) != 1) { @@ -472,7 +506,7 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int } source_address = reinterpret_cast(&source); if (rdma_bind_addr(id, source_address) != 0) { - std::fprintf(stderr, "rdma connect: bind source failed source=%s errno=%d\\n", source_env, errno); + CRB_LOG_WARN("rdma connect: bind source failed source={} errno={}", source_env, errno); rdma_destroy_id(id); return nullptr; @@ -481,14 +515,14 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int // When a source address was explicitly bound above, librdmacm expects a // null source here; passing it twice can fail immediately with EINVAL. if (rdma_resolve_addr(id, nullptr, reinterpret_cast(&destination), 2000) != 0) { - std::fprintf(stderr, "rdma connect: resolve_addr failed addr=%s errno=%d\\n", addr.c_str(), errno); + CRB_LOG_WARN("rdma connect: resolve_addr failed addr={} port={} errno={}", addr, port, errno); rdma_destroy_id(id); return nullptr; } rdma_cm_event *event = nullptr; if (!wait_cm_event(cm_channel_, &event, 3000) || event == nullptr || event->event != RDMA_CM_EVENT_ADDR_RESOLVED) { - std::fprintf(stderr, "rdma connect: ADDR_RESOLVED missing event=%d errno=%d\n", event == nullptr ? -1 : event->event, errno); + CRB_LOG_WARN("rdma connect: ADDR_RESOLVED missing addr={} event={} errno={}", addr, event == nullptr ? -1 : event->event, errno); if (event != nullptr) @@ -503,7 +537,7 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int return nullptr; } if (!wait_cm_event(cm_channel_, &event, 3000) || event == nullptr || event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { - std::fprintf(stderr, "rdma connect: ROUTE_RESOLVED missing event=%d errno=%d\n", event == nullptr ? -1 : event->event, errno); + CRB_LOG_WARN("rdma connect: ROUTE_RESOLVED missing addr={} event={} errno={}", addr, event == nullptr ? -1 : event->event, errno); if (event != nullptr) @@ -512,51 +546,49 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int return nullptr; } rdma_ack_cm_event(event); - std::fprintf(stderr, "rdma connect: creating connection CQ verbs=%p\n", static_cast(id->verbs)); + ibv_comp_channel *connection_channel = ibv_create_comp_channel(id->verbs); if (connection_channel != nullptr) make_nonblocking(connection_channel->fd); - ibv_cq *connection_cq = connection_channel == nullptr - ? nullptr - : ibv_create_cq(id->verbs, 2048, nullptr, connection_channel, 0); - std::fprintf(stderr, "rdma connect: connection CQ=%p errno=%d\n", static_cast(connection_cq), errno); + ibv_cq *connection_cq = + connection_channel == nullptr ? nullptr : ibv_create_cq(id->verbs, 2048, nullptr, connection_channel, 0); + if (connection_cq == nullptr) { - std::fprintf(stderr, "rdma connect: create CM CQ failed errno=%d\n", errno); + if (connection_channel != nullptr) ibv_destroy_comp_channel(connection_channel); rdma_destroy_id(id); return nullptr; } + if (ibv_req_notify_cq(connection_cq, 0) != 0) { + CRB_LOG_WARN("rdma connect: arm connection CQ notifications failed errno={}", errno); + ibv_destroy_cq(connection_cq); + ibv_destroy_comp_channel(connection_channel); + rdma_destroy_id(id); + return nullptr; + } ibv_device_attr device_attributes{}; if (ibv_query_device(id->verbs, &device_attributes) != 0) { - std::fprintf(stderr, "rdma connect: query device failed errno=%d\n", errno); + ibv_destroy_cq(connection_cq); ibv_destroy_comp_channel(connection_channel); rdma_destroy_id(id); return nullptr; } ibv_qp_init_attr qp_attr{}; - qp_attr.send_cq = connection_cq; - qp_attr.recv_cq = connection_cq; - qp_attr.qp_type = IBV_QPT_RC; - qp_attr.cap.max_send_wr = std::min(1024, device_attributes.max_qp_wr); - qp_attr.cap.max_recv_wr = std::min(1024, device_attributes.max_qp_wr); - qp_attr.cap.max_send_sge = std::min(2, device_attributes.max_sge); - qp_attr.cap.max_recv_sge = std::min(2, device_attributes.max_sge); + qp_attr.send_cq = connection_cq; + qp_attr.recv_cq = connection_cq; + qp_attr.qp_type = IBV_QPT_RC; + qp_attr.cap.max_send_wr = std::min(1024, device_attributes.max_qp_wr); + qp_attr.cap.max_recv_wr = std::min(1024, device_attributes.max_qp_wr); + qp_attr.cap.max_send_sge = std::min(2, device_attributes.max_sge); + qp_attr.cap.max_recv_sge = std::min(2, device_attributes.max_sge); qp_attr.cap.max_inline_data = 0; - qp_attr.sq_sig_all = 1; - qp_attr.qp_context = id; - std::fprintf(stderr, "rdma connect: qp caps send_wr=%u recv_wr=%u send_sge=%u recv_sge=%u max_wr=%u max_sge=%u verbs_match=%d\n", - qp_attr.cap.max_send_wr, qp_attr.cap.max_recv_wr, qp_attr.cap.max_send_sge, - qp_attr.cap.max_recv_sge, device_attributes.max_qp_wr, device_attributes.max_sge, - id->verbs == context_ ? 1 : 0); - std::fprintf(stderr, "rdma connect: selected=%s transport=%s\n", - id->verbs == nullptr || id->verbs->device == nullptr ? "unknown" : ibv_get_device_name(id->verbs->device), - context_ == nullptr || context_->device == nullptr ? "unknown" : ibv_get_device_name(context_->device)); - std::fprintf(stderr, "rdma connect: creating QP id=%p pd=%p cq=%p\n", static_cast(id), - static_cast(id->pd), static_cast(connection_cq)); + qp_attr.sq_sig_all = 1; + qp_attr.qp_context = id; + if (rdma_create_qp(id, id->pd, &qp_attr) != 0) { - std::fprintf(stderr, "rdma connect: create qp failed errno=%d\n", errno); + ibv_destroy_cq(connection_cq); ibv_destroy_comp_channel(connection_channel); rdma_destroy_id(id); @@ -568,7 +600,7 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int params.retry_count = 7; params.rnr_retry_count = 7; if (rdma_connect(id, ¶ms) != 0) { - std::fprintf(stderr, "rdma connect: rdma_connect failed addr=%s errno=%d\n", addr.c_str(), errno); + CRB_LOG_WARN("rdma connect: rdma_connect failed addr={} errno={}", addr, errno); rdma_destroy_qp(id); ibv_destroy_cq(connection_cq); @@ -577,8 +609,7 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int return nullptr; } if (!wait_cm_event(cm_channel_, &event, 5000) || event == nullptr || event->event != RDMA_CM_EVENT_ESTABLISHED) { - std::fprintf(stderr, "rdma connect: ESTABLISHED missing event=%d status=%d errno=%d\n", - event == nullptr ? -1 : event->event, event == nullptr ? 0 : event->status, errno); + CRB_LOG_WARN("rdma connect: ESTABLISHED missing addr={} event={} status={} errno={}", addr, event == nullptr ? -1 : event->event, event == nullptr ? 0 : event->status, errno); if (event != nullptr) @@ -589,7 +620,7 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int return nullptr; } rdma_ack_cm_event(event); - auto pools = std::make_unique(); + auto pools = std::make_unique(); pools->send = std::make_unique(id->pd, 64U << 20, 1024); pools->recv = std::make_unique(id->pd, 64U << 20, 1024); if (pools->send == nullptr || pools->recv == nullptr) { @@ -599,12 +630,12 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int rdma_destroy_id(id); return nullptr; } - auto connection = std::make_shared(reinterpret_cast(id), addr, pools->recv.get()); + auto connection = std::make_shared(reinterpret_cast(id), addr, pools->recv.get()); connection->user_data = pools.get(); connection->set_path(ConnectionPath::Rdma); connection->transport_handle = reinterpret_cast(id->qp); if (!prime_receive(connection)) { - std::fprintf(stderr, "rdma connect: prime_receive failed after ESTABLISHED errno=%d\n", errno); + rdma_destroy_qp(id); rdma_destroy_id(id); return nullptr; @@ -612,6 +643,7 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int { std::scoped_lock lock(cm_mu_); cm_ids_.push_back(id); + cm_connections_.emplace(id, connection); cm_cqs_.emplace(id, connection_cq); cm_channels_.emplace(id, connection_channel); cm_pools_.emplace(id, std::move(pools)); @@ -669,34 +701,39 @@ std::shared_ptr RdmaTransport::accept() } rdma_cm_id *id = event->id; rdma_ack_cm_event(event); - std::fprintf(stderr, "rdma accept: creating connection CQ verbs=%p\n", static_cast(id->verbs)); + ibv_comp_channel *connection_channel = ibv_create_comp_channel(id->verbs); if (connection_channel != nullptr) make_nonblocking(connection_channel->fd); - ibv_cq *connection_cq = connection_channel == nullptr - ? nullptr - : ibv_create_cq(id->verbs, 2048, nullptr, connection_channel, 0); - std::fprintf(stderr, "rdma accept: connection CQ=%p errno=%d\n", static_cast(connection_cq), errno); + ibv_cq *connection_cq = + connection_channel == nullptr ? nullptr : ibv_create_cq(id->verbs, 2048, nullptr, connection_channel, 0); + if (connection_cq == nullptr) { - std::fprintf(stderr, "rdma accept: create CM CQ failed errno=%d\n", errno); + if (connection_channel != nullptr) ibv_destroy_comp_channel(connection_channel); rdma_destroy_id(id); return nullptr; } + if (ibv_req_notify_cq(connection_cq, 0) != 0) { + CRB_LOG_WARN("rdma accept: arm connection CQ notifications failed errno={}", errno); + ibv_destroy_cq(connection_cq); + ibv_destroy_comp_channel(connection_channel); + rdma_destroy_id(id); + return nullptr; + } ibv_qp_init_attr qp_attr{}; - qp_attr.send_cq = connection_cq; - qp_attr.recv_cq = connection_cq; - qp_attr.qp_type = IBV_QPT_RC; - qp_attr.cap.max_send_wr = 1024; - qp_attr.cap.max_recv_wr = 1024; - qp_attr.cap.max_send_sge = 2; - qp_attr.cap.max_recv_sge = 2; + qp_attr.send_cq = connection_cq; + qp_attr.recv_cq = connection_cq; + qp_attr.qp_type = IBV_QPT_RC; + qp_attr.cap.max_send_wr = 1024; + qp_attr.cap.max_recv_wr = 1024; + qp_attr.cap.max_send_sge = 2; + qp_attr.cap.max_recv_sge = 2; qp_attr.cap.max_inline_data = 0; - qp_attr.sq_sig_all = 1; - qp_attr.qp_context = id; - std::fprintf(stderr, "rdma accept: creating QP id=%p pd=%p cq=%p\n", static_cast(id), - static_cast(id->pd), static_cast(connection_cq)); + qp_attr.sq_sig_all = 1; + qp_attr.qp_context = id; + if (rdma_create_qp(id, id->pd, &qp_attr) != 0) { ibv_destroy_cq(connection_cq); ibv_destroy_comp_channel(connection_channel); @@ -708,8 +745,8 @@ std::shared_ptr RdmaTransport::accept() params.responder_resources = 1; params.retry_count = 7; params.rnr_retry_count = 7; - const int accept_rc = rdma_accept(id, ¶ms); - std::fprintf(stderr, "rdma accept: rdma_accept rc=%d errno=%d\n", accept_rc, errno); + const int accept_rc = rdma_accept(id, ¶ms); + if (accept_rc != 0) { rdma_destroy_qp(id); ibv_destroy_cq(connection_cq); @@ -717,12 +754,10 @@ std::shared_ptr RdmaTransport::accept() rdma_destroy_id(id); return nullptr; } - event = nullptr; + event = nullptr; const int event_rc = wait_cm_event(cm_channel_, &event, 5000) ? 0 : -1; - std::fprintf(stderr, "rdma accept: completion event rc=%d event=%d status=%d errno=%d\n", event_rc, - event == nullptr ? -1 : event->event, event == nullptr ? 0 : event->status, errno); - if (event_rc != 0 || event == nullptr || - event->event != RDMA_CM_EVENT_ESTABLISHED) { + + if (event_rc != 0 || event == nullptr || event->event != RDMA_CM_EVENT_ESTABLISHED) { if (event != nullptr) rdma_ack_cm_event(event); rdma_destroy_qp(id); @@ -732,8 +767,8 @@ std::shared_ptr RdmaTransport::accept() return nullptr; } rdma_ack_cm_event(event); - std::fprintf(stderr, "rdma accept: established, priming receive\n"); - auto pools = std::make_unique(); + + auto pools = std::make_unique(); pools->send = std::make_unique(id->pd, 64U << 20, 1024); pools->recv = std::make_unique(id->pd, 64U << 20, 1024); if (pools->send == nullptr || pools->recv == nullptr) { @@ -743,7 +778,7 @@ std::shared_ptr RdmaTransport::accept() rdma_destroy_id(id); return nullptr; } - auto connection = std::make_shared(reinterpret_cast(id), "rdma-peer", pools->recv.get()); + auto connection = std::make_shared(reinterpret_cast(id), "rdma-peer", pools->recv.get()); connection->user_data = pools.get(); connection->set_path(ConnectionPath::Rdma); connection->transport_handle = reinterpret_cast(id->qp); @@ -754,10 +789,11 @@ std::shared_ptr RdmaTransport::accept() rdma_destroy_id(id); return nullptr; } - std::fprintf(stderr, "rdma accept: receive primed\n"); + { std::scoped_lock lock(cm_mu_); cm_ids_.push_back(id); + cm_connections_.emplace(id, connection); cm_cqs_.emplace(id, connection_cq); cm_channels_.emplace(id, connection_channel); cm_pools_.emplace(id, std::move(pools)); @@ -789,11 +825,9 @@ int RdmaTransport::cm_event_fd() const int RdmaTransport::drain_cq(uint32_t max_completions) { - if (!running_.load(std::memory_order_acquire)) - return 0; uint32_t drained = 0; std::array completions{}; - auto poll_one = [&](ibv_cq *cq) { + auto poll_one = [&](ibv_cq *cq) { while (drained < max_completions && cq != nullptr) { const int batch = ibv_poll_cq(cq, static_cast(std::min(32, max_completions - drained)), completions.data()); @@ -801,16 +835,12 @@ int RdmaTransport::drain_cq(uint32_t max_completions) break; for (int i = 0; i < batch; ++i) { const auto opcode = completions[static_cast(i)].opcode; - const auto wr_id = completions[static_cast(i)].wr_id; + const auto wr_id = completions[static_cast(i)].wr_id; if ((wr_id & uintptr_t{1}) != 0 || opcode == IBV_WC_RECV || opcode == IBV_WC_RECV_RDMA_WITH_IMM) release_completed_receive(completions[static_cast(i)]); else if (opcode == IBV_WC_SEND) release_completed_frame(completions[static_cast(i)]); else { - std::fprintf(stderr, "rdma completion: unknown opcode=%u status=%u wr_id=%#lx\n", - static_cast(opcode), - static_cast(completions[static_cast(i)].status), - static_cast(completions[static_cast(i)].wr_id)); } } drained += static_cast(batch); @@ -870,7 +900,7 @@ int RdmaTransport::handle_external_event(uint64_t tag, uint32_t max_completions) for (const auto &[id, channel] : cm_channels_) { (void)id; ibv_cq *event_cq = nullptr; - void *context = nullptr; + void *context = nullptr; if (ibv_get_cq_event(channel, &event_cq, &context) == 0) { ibv_ack_cq_events(event_cq, 1); (void)ibv_req_notify_cq(event_cq, 0); @@ -884,19 +914,17 @@ int RdmaTransport::handle_external_event(uint64_t tag, uint32_t max_completions) bool RdmaTransport::prime_receive(const std::shared_ptr &connection) { - if (connection == nullptr || connection->pool() == nullptr) - { - std::fprintf(stderr, "rdma receive: missing connection/pool\\n"); + if (connection == nullptr || connection->pool() == nullptr) { + return false; } constexpr uint32_t kInitialReceiveDepth = 8; - BufferPool *pool = connection->pool(); + BufferPool *pool = connection->pool(); for (uint32_t i = 0; i < kInitialReceiveDepth; ++i) { Buffer *control = pool->alloc(RDMA_CONTROL_CAPACITY); Buffer *data = pool->alloc(1U << 20); if (control == nullptr || data == nullptr) { - std::fprintf(stderr, "rdma receive: buffer allocation failed depth=%u control=%p data=%p\\n", i, - static_cast(control), static_cast(data)); + if (control != nullptr) control->release(); if (data != nullptr) @@ -904,7 +932,7 @@ bool RdmaTransport::prime_receive(const std::shared_ptr &connection) return false; } if (!post_receive(connection.get(), control, data)) { - std::fprintf(stderr, "rdma receive: initial ibv_post_recv failed depth=%u errno=%d\\n", i, errno); + control->release(); data->release(); return false; @@ -919,9 +947,7 @@ bool RdmaTransport::prime_receive(const std::shared_ptr &connection) void RdmaTransport::release_completed_receive(ibv_wc &wc) { - std::fprintf(stderr, "rdma receive completion: wr_id=%#lx status=%u opcode=%u byte_len=%u\n", - static_cast(wc.wr_id), static_cast(wc.status), - static_cast(wc.opcode), wc.byte_len); + auto *work = reinterpret_cast(wc.wr_id & ~uintptr_t{1}); if (work == nullptr) return; @@ -948,6 +974,7 @@ void RdmaTransport::release_completed_receive(ibv_wc &wc) } const uint32_t control_bytes = work->control->capacity; if (control_bytes < HEADER_SIZE) { + CRB_LOG_WARN("rdma receive: control capacity too small capacity={} header={}", control_bytes, HEADER_SIZE); quarantine_and_release(); return; } @@ -957,6 +984,10 @@ void RdmaTransport::release_completed_receive(ibv_wc &wc) if (header.magic != MAGIC || header.msg_offset < HEADER_SIZE || static_cast(header.msg_offset) + header.msg_size > control_bytes || expected_bytes > wc.byte_len || (header.data_size > 0 && (work->data == nullptr || header.data_size > work->data->capacity))) { + CRB_LOG_WARN( + "rdma receive: invalid frame magic={} type={} msg_size={} data_size={} offset={} capacity={} byte_len={}", + header.magic, header.msg_type, header.msg_size, header.data_size, header.msg_offset, control_bytes, + wc.byte_len); quarantine_and_release(); return; } @@ -1026,9 +1057,7 @@ void RdmaTransport::release_completed_frame(ibv_wc &wc) auto *frame = reinterpret_cast(wc.wr_id); if (frame == nullptr) return; - std::fprintf(stderr, "rdma send completion: wr_id=%#lx status=%u frame=%p control=%p\n", - static_cast(wc.wr_id), static_cast(wc.status), - static_cast(frame), static_cast(frame->control)); + if (wc.status != IBV_WC_SUCCESS) { stats_.cq_errors.fetch_add(1, std::memory_order_relaxed); // A failed SEND makes the stream uncertain; do not allow a TCP diff --git a/lib/crowdb-rpc/src/transport/socket_transport.cpp b/lib/crowdb-rpc/src/transport/socket_transport.cpp index 15099d927..400ba3ab5 100644 --- a/lib/crowdb-rpc/src/transport/socket_transport.cpp +++ b/lib/crowdb-rpc/src/transport/socket_transport.cpp @@ -594,6 +594,10 @@ SocketTransport::SocketTransport(uint32_t num_workers, BufferPool *pool) : Socke SocketTransport::~SocketTransport() { stop(); +#ifdef CROWDB_RPC_HAVE_RDMA + if (rdma_transport_ != nullptr) + rdma_transport_->shutdown(); +#endif } void SocketTransport::start() @@ -931,8 +935,7 @@ std::shared_ptr SocketTransport::connect(const std::string &addr, in } void SocketTransport::attach_rdma_connection(const std::shared_ptr &conn, - Connection::OnFrameCallback on_frame, - Connection::OnCloseCallback on_close) + Connection::OnFrameCallback on_frame, Connection::OnCloseCallback on_close) { if (conn == nullptr) return; @@ -940,7 +943,7 @@ void SocketTransport::attach_rdma_connection(const std::shared_ptr & conn->set_on_close(std::move(on_close)); register_conn(conn); if (!workers_.empty()) { - Worker *worker = get_worker(); + Worker *worker = get_worker(); conn->io_worker = worker; conn->io_engine = worker == nullptr ? nullptr : worker->engine(); } diff --git a/lib/crowdb-rpc/tests/rdma_test.cpp b/lib/crowdb-rpc/tests/rdma_test.cpp index 04d255d8b..a81d24795 100644 --- a/lib/crowdb-rpc/tests/rdma_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_test.cpp @@ -1,33 +1,33 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -#include "crowdb-rpc/transport/rdma/rdma_transport.h" #include "crowdb-rpc/client/client.h" #include "crowdb-rpc/server/message.h" #include "crowdb-rpc/server/server.h" +#include "crowdb-rpc/transport/rdma/rdma_transport.h" #include "crowdb-rpc/transport/socket_transport.h" #ifdef CROWDB_RPC_HAVE_RDMA +# include # include +# include # include -# include -# include -# include # include # include +# include # include -# include +# include # include -# include +# include namespace { void rdma_test_crash_handler(int signal_number) { - void *frames[64]; - const int count = ::backtrace(frames, 64); + void *frames[64]; + const int count = ::backtrace(frames, 64); const char prefix[] = "R239 RDMA crash backtrace:\n"; (void)::write(STDERR_FILENO, prefix, sizeof(prefix) - 1); ::backtrace_symbols_fd(frames, count, STDERR_FILENO); @@ -42,7 +42,7 @@ void install_rdma_test_crash_handler() std::signal(SIGABRT, rdma_test_crash_handler); }); } -} +} // namespace struct WorkerCompletion { @@ -51,7 +51,7 @@ struct WorkerCompletion }; extern "C" void rdma_worker_completion(uint64_t, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, - crowdb_rpc_status status, void *opaque) + crowdb_rpc_status status, void *opaque) { auto *state = static_cast(opaque); (status == CROWDB_RPC_OK ? state->completed : state->failed).fetch_add(1, std::memory_order_relaxed); @@ -66,15 +66,15 @@ using crowdb::rpc::BufferOwnership; using crowdb::rpc::Connection; using crowdb::rpc::Frame; using crowdb::rpc::Header; +using crowdb::rpc::HEADER_SIZE; using crowdb::rpc::OutFrame; using crowdb::rpc::RdmaBufferPool; using crowdb::rpc::RdmaTransport; using crowdb::rpc::RpcClient; using crowdb::rpc::RpcServer; +using crowdb::rpc::serialize_header; using crowdb::rpc::SocketTransport; using crowdb::rpc::SystemBufferPool; -using crowdb::rpc::HEADER_SIZE; -using crowdb::rpc::serialize_header; TEST(RdmaBufferPoolTest, SlicesOneRegisteredRegion) { @@ -134,16 +134,15 @@ TEST(RdmaBufferPoolTest, TransportCopiesSystemBufferIntoRegisteredPool) TEST(RdmaHardwareTest, CompleteCmQpSendReceiveFlow) { - constexpr uint64_t kFrameCount = 8; + constexpr uint64_t kFrameCount = 8; constexpr int kTimeoutIterations = 500; if (std::getenv("CROWDB_RDMA_E2E") == nullptr) GTEST_SKIP() << "set CROWDB_RDMA_E2E=1 to run against a live HCA"; install_rdma_test_crash_handler(); - const char *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + const char *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); const char *connect_address = std::getenv("CROWDB_RDMA_E2E_CONNECT_ADDR"); - const int port = std::getenv("CROWDB_RDMA_E2E_PORT") == nullptr - ? 19100 - : std::atoi(std::getenv("CROWDB_RDMA_E2E_PORT")); + const int port = + std::getenv("CROWDB_RDMA_E2E_PORT") == nullptr ? 19100 : std::atoi(std::getenv("CROWDB_RDMA_E2E_PORT")); ASSERT_NE(address, nullptr); if (connect_address == nullptr) connect_address = address; @@ -193,11 +192,11 @@ TEST(RdmaHardwareTest, CompleteCmQpSendReceiveFlow) Header header; header.msg_type = static_cast(1 + sequence); serialize_header(raw->data, header); - raw->len = HEADER_SIZE; + raw->len = HEADER_SIZE; Buffer *control = client.register_buffer(raw); ASSERT_NE(control, nullptr); - auto *frame = new OutFrame; - frame->header = header; + auto *frame = new OutFrame; + frame->header = header; frame->control = control; ASSERT_TRUE(client.submit(outgoing.get(), frame)); raw->release(); @@ -217,14 +216,14 @@ TEST(RdmaHardwareTest, CompleteCmQpSendReceiveFlow) client.start(); std::shared_ptr accepted; - std::thread accept_thread([&] { + std::thread accept_thread([&] { for (int i = 0; i < 200 && accepted == nullptr; ++i) { accepted = server.accept(); if (accepted == nullptr) std::this_thread::sleep_for(std::chrono::milliseconds(10)); } }); - auto outgoing = client.connect(connect_address, port); + auto outgoing = client.connect(connect_address, port); accept_thread.join(); ASSERT_NE(outgoing, nullptr); ASSERT_NE(accepted, nullptr); @@ -238,12 +237,12 @@ TEST(RdmaHardwareTest, CompleteCmQpSendReceiveFlow) Header header; header.msg_type = 1; serialize_header(raw->data, header); - raw->len = HEADER_SIZE; + raw->len = HEADER_SIZE; Buffer *control = client.register_buffer(raw); ASSERT_NE(control, nullptr); - auto *frame = new OutFrame; - frame->header = header; - frame->control = control; + auto *frame = new OutFrame; + frame->header = header; + frame->control = control; ASSERT_TRUE(client.submit(outgoing.get(), frame)); raw->release(); @@ -264,21 +263,27 @@ TEST(RdmaHardwareTest, RpcServerClientWorkerRoundTrip) GTEST_SKIP() << "set CROWDB_RDMA_RPC_E2E=1 to run the worker flow"; install_rdma_test_crash_handler(); const char *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); - const char *role = std::getenv("CROWDB_RDMA_E2E_ROLE"); + const char *role = std::getenv("CROWDB_RDMA_E2E_ROLE"); ASSERT_NE(address, nullptr); - const int port = std::getenv("CROWDB_RDMA_E2E_PORT") == nullptr ? 19101 : - std::atoi(std::getenv("CROWDB_RDMA_E2E_PORT")); - constexpr uint32_t kRequests = 8; + const int port = + std::getenv("CROWDB_RDMA_E2E_PORT") == nullptr ? 19101 : std::atoi(std::getenv("CROWDB_RDMA_E2E_PORT")); + constexpr uint32_t kRequests = 8; constexpr uint16_t kMessageType = 239; if (role != nullptr && std::strcmp(role, "server") == 0) { RpcServer server; ASSERT_TRUE(server.listen(address, port)); std::atomic received{0}; - server.register_handler(kMessageType, [&received](Frame *frame, Connection *) { + server.register_handler(kMessageType, [&received, &server](Frame *frame, Connection *) { received.fetch_add(1, std::memory_order_relaxed); + OutFrame *response = new OutFrame; + response->request_id = frame->request_id; + response->header.msg_type = static_cast(kMessageType); + response->control = + crowdb::rpc::build_ping_response(server.pool(), frame->request_id, frame->rpc_create_nano, 0); + response->header.msg_size = response->control == nullptr ? 0 : response->control->len; delete frame; - return static_cast(nullptr); + return response; }); server.start(); for (int i = 0; i < 100 && received.load(std::memory_order_relaxed) < kRequests; ++i) diff --git a/tools/rdma/run-local-loop.sh b/tools/rdma/run-local-loop.sh index d86582875..53a456ed9 100755 --- a/tools/rdma/run-local-loop.sh +++ b/tools/rdma/run-local-loop.sh @@ -7,7 +7,8 @@ root_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) pixi_bin=${PIXI_BIN:-/home/cj/.pixi/bin/pixi} test_bin=${CROWDB_RDMA_TEST_BIN:-$root_dir/lib/crowdb-rpc/build/crowdb_rpc_tests} mode=${1:-local} -port=${CROWDB_RDMA_E2E_PORT:-19101} +port_offset=${CROWDB_PORT_OFFSET:-0} +port=${CROWDB_RDMA_E2E_PORT:-$((19101 + port_offset))} filter=${CROWDB_RDMA_E2E_FILTER:-RdmaHardwareTest.CompleteCmQpSendReceiveFlow} server_pid="" diff --git a/tools/rdma/run-oci-profile.sh b/tools/rdma/run-oci-profile.sh index 83f2f0810..5a7907fb3 100755 --- a/tools/rdma/run-oci-profile.sh +++ b/tools/rdma/run-oci-profile.sh @@ -7,13 +7,15 @@ profile="${1:-}" image="${CROWDB_OCI_IMAGE:-crowdb:test}" startup_mode="${CROWDB_STARTUP_MODE:-single}" management_interface="${CROWDB_MANAGEMENT_INTERFACE:-eth0}" +port_offset="${CROWDB_PORT_OFFSET:-0}" +rdma_cm_port=$((19101 + port_offset)) if [[ -z "$profile" || "$profile" == "-h" || "$profile" == "--help" ]]; then cat >&2 <<'USAGE' -usage: run-oci-profile.sh tcp-only|two-hca|shared-hca [image] +usage: run-oci-profile.sh tcp-only|shared-hca [image] tcp-only starts one container without RDMA devices or RDMA capability. -two-hca starts one privileged host-network container pinned to CROWDB_RDMA_DEVICE. -shared-hca starts one privileged host-network container using the shared HCA. +shared-hca starts one privileged host-network container using the shared HCA; + invoke it once per logical container on the physical host. USAGE exit 2 fi @@ -23,24 +25,17 @@ case "$profile" in tcp-only) exec docker run --rm --network=host \ -e CROWDB_RDMA_ENABLED=0 \ + -e CROWDB_PORT_OFFSET="$port_offset" \ -e CROWDB_STARTUP_MODE="$startup_mode" \ -e CROWDB_MANAGEMENT_INTERFACE="$management_interface" \ "$image" ;; - two-hca) - device="${CROWDB_RDMA_DEVICE:-mlx5_0}" - exec docker run --rm --network=host --privileged --ulimit memlock=-1 \ - -e CROWDB_RDMA_ENABLED=1 \ - -e CROWDB_STARTUP_MODE="$startup_mode" \ - -e CROWDB_MANAGEMENT_INTERFACE="$management_interface" \ - -e CROWDB_RDMA_DEVICE="$device" \ - -e CROWDB_RDMA_PORT="${CROWDB_RDMA_PORT:-1}" \ - "$image" - ;; shared-hca) exec docker run --rm --network=host --privileged --ulimit memlock=-1 \ -e CROWDB_RDMA_ENABLED=1 \ -e CROWDB_RDMA_SHARED_HCA=1 \ + -e CROWDB_PORT_OFFSET="$port_offset" \ + -e CROWDB_RDMA_E2E_PORT="$rdma_cm_port" \ -e CROWDB_STARTUP_MODE="$startup_mode" \ -e CROWDB_MANAGEMENT_INTERFACE="$management_interface" \ -e CROWDB_RDMA_DEVICE="${CROWDB_RDMA_DEVICE:-mlx5_0}" \ From 7a1958d061a129a260de637a08cf9a7bf6bf885a Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 18 Oct 2026 16:21:25 +0800 Subject: [PATCH 003/258] Finish RDMA runtime port and build integration --- lib/crowdb-rpc/ffi/build.rs | 24 ++++++++++--- lib/crowdb-rpc/src/server/server.cpp | 35 +++++++++++++++++-- .../src/transport/rdma/rdma_transport.cpp | 22 +++++++++++- .../src/transport/socket_transport.cpp | 12 ++++++- tools/rdma/run-oci-profile.sh | 7 ++++ 5 files changed, 91 insertions(+), 9 deletions(-) diff --git a/lib/crowdb-rpc/ffi/build.rs b/lib/crowdb-rpc/ffi/build.rs index c31e9a56a..685ba2b5b 100644 --- a/lib/crowdb-rpc/ffi/build.rs +++ b/lib/crowdb-rpc/ffi/build.rs @@ -76,6 +76,14 @@ fn main() -> Result<(), Box> { let target = std::env::var("TARGET").unwrap_or_default(); let is_linux = target.contains("linux"); let is_macos = target.contains("darwin"); + let have_rdma = is_linux + && conda_prefix.as_ref().is_some_and(|prefix| { + prefix.join("include/infiniband/verbs.h").is_file() + && prefix.join("include/rdma/rdma_cma.h").is_file() + && (prefix.join("lib/libibverbs.so").is_file() + || prefix.join("lib/libibverbs.so.1").is_file()) + && (prefix.join("lib/librdmacm.so").is_file() || prefix.join("lib/librdmacm.so.1").is_file()) + }); sources.retain(|p| { let name = p.file_name().and_then(|s| s.to_str()).unwrap_or(""); @@ -85,10 +93,9 @@ fn main() -> Result<(), Box> { if name == "kqueue_engine.cpp" && !is_macos { return false; } - // RDMA stubs — excluded unless CROWDB_RPC_HAVE_RDMA is set. - if (name == "rdma_transport.cpp" || name == "rdma_buffer_pool.cpp") - && std::env::var("CROWDB_RPC_HAVE_RDMA").is_err() - { + // RDMA sources are included only when the Linux build environment + // provides both verbs and rdmacm headers/libraries. + if (name == "rdma_transport.cpp" || name == "rdma_buffer_pool.cpp") && !have_rdma { return false; } true @@ -116,6 +123,15 @@ fn main() -> Result<(), Box> { if have_spdlog { build.define("CROWDB_HAVE_SPDLOG", "1"); } + if have_rdma { + build.define("CROWDB_RPC_HAVE_RDMA", "1"); + if let Some(prefix) = &conda_prefix { + build.include(prefix.join("include")); + println!("cargo:rustc-link-search=native={}", prefix.join("lib").display()); + } + println!("cargo:rustc-link-lib=dylib=ibverbs"); + println!("cargo:rustc-link-lib=dylib=rdmacm"); + } // AddressSanitizer + LeakSanitizer (gated by CROWDB_ASAN=1). // Adds -fsanitize=address to C++ compile. libasan is linked diff --git a/lib/crowdb-rpc/src/server/server.cpp b/lib/crowdb-rpc/src/server/server.cpp index c0a5a1ee4..f889f45d5 100644 --- a/lib/crowdb-rpc/src/server/server.cpp +++ b/lib/crowdb-rpc/src/server/server.cpp @@ -183,15 +183,44 @@ void RpcServer::start() enabled != nullptr && std::strcmp(enabled, "1") == 0) { transport_->set_rdma_accept_handler([this](const std::shared_ptr &conn) { transport_->attach_rdma_connection( - conn, - [this](Frame *frame, Connection *c) { dispatch(frame, c); }, + conn, [this](Frame *frame, Connection *c) { dispatch(frame, c); }, [this](Connection *c) { if (request_client_ != nullptr) request_client_->fail_all(c, RpcError::ConnectionClosed); }); transport_->send_rdma_handshake(conn); }); - if (!transport_->listen_rdma(listen_addr_, listen_port_)) + int rdma_port = 0; + if (const char *rdma_port_env = std::getenv("CROWDB_RDMA_E2E_PORT"); + rdma_port_env != nullptr && *rdma_port_env != '\0') { + char *end = nullptr; + errno = 0; + const long configured_port = std::strtol(rdma_port_env, &end, 10); + if (errno == 0 && end != rdma_port_env && *end == '\0' && configured_port > 0 && configured_port <= 65535) { + rdma_port = static_cast(configured_port); + } + else { + CRB_LOG_WARN("rpc server: invalid RDMA CM port value={}", rdma_port_env); + } + } + else { + rdma_port = listen_port_; + if (const char *offset_env = std::getenv("CROWDB_PORT_OFFSET"); + offset_env != nullptr && *offset_env != '\0') { + char *end = nullptr; + errno = 0; + const long offset = std::strtol(offset_env, &end, 10); + if (errno != 0 || end == offset_env || *end != '\0' || offset < 0 || offset > 65535 - rdma_port) { + CRB_LOG_WARN("rpc server: invalid RDMA port offset value={}", offset_env); + rdma_port = 0; + } + else { + rdma_port += static_cast(offset); + } + } + } + const char *rdma_addr = std::getenv("CROWDB_RDMA_SOURCE_ADDR"); + if (rdma_port == 0 || !transport_->listen_rdma(rdma_addr == nullptr ? listen_addr_ : rdma_addr, rdma_port)) CRB_LOG_WARN("rpc server: RDMA listener unavailable; TCP remains active"); } transport_->start(); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index 1febd7ea5..11db2d3e0 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -172,6 +172,8 @@ RdmaTransport::RdmaTransport() ibv_device **devices = ibv_get_device_list(&device_count); if (devices == nullptr || device_count == 0) { + CRB_LOG_WARN("rdma init: no verbs devices count={} errno={}", device_count, errno); + if (devices != nullptr) { ibv_free_device_list(devices); } @@ -194,6 +196,8 @@ RdmaTransport::RdmaTransport() } if (selected == nullptr) { + CRB_LOG_WARN("rdma init: requested device not found name={} count={}", requested_device, device_count); + ibv_free_device_list(devices); return; } @@ -202,11 +206,17 @@ RdmaTransport::RdmaTransport() ibv_free_device_list(devices); if (context_ == nullptr) { + CRB_LOG_WARN("rdma init: ibv_open_device failed name={} errno={}", + requested_device == nullptr ? "default" : requested_device, errno); + return; } ibv_port_attr port_attributes{}; if (ibv_query_port(context_, port_, &port_attributes) != 0 || port_attributes.state != IBV_PORT_ACTIVE) { + CRB_LOG_WARN("rdma init: port unavailable port={} state={} errno={}", port_, + static_cast(port_attributes.state), errno); + ibv_close_device(context_); context_ = nullptr; return; @@ -214,6 +224,8 @@ RdmaTransport::RdmaTransport() pd_ = ibv_alloc_pd(context_); if (pd_ == nullptr) { + CRB_LOG_WARN("rdma init: ibv_alloc_pd failed errno={}", errno); + ibv_close_device(context_); context_ = nullptr; return; @@ -221,6 +233,8 @@ RdmaTransport::RdmaTransport() comp_channel_ = ibv_create_comp_channel(context_); if (comp_channel_ == nullptr) { + CRB_LOG_WARN("rdma init: ibv_create_comp_channel failed errno={}", errno); + ibv_dealloc_pd(pd_); pd_ = nullptr; ibv_close_device(context_); @@ -233,6 +247,8 @@ RdmaTransport::RdmaTransport() cq_ = ibv_create_cq(context_, 2048, this, comp_channel_, 0); if (cq_ == nullptr || ibv_req_notify_cq(cq_, 0) != 0) { + CRB_LOG_WARN("rdma init: CQ setup failed cq={} errno={}", static_cast(cq_), errno); + if (cq_ != nullptr) ibv_destroy_cq(cq_); cq_ = nullptr; @@ -655,8 +671,12 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int bool RdmaTransport::listen(const std::string &addr, int port) { - if (pd_ == nullptr || addr.empty() || port <= 0 || port > 65535 || listen_id_ != nullptr) + if (pd_ == nullptr || addr.empty() || port <= 0 || port > 65535 || listen_id_ != nullptr) { + + CRB_LOG_WARN("rdma listen: transport unavailable pd={} addr={} port={} existing_listener={}", + static_cast(pd_), addr, port, listen_id_ != nullptr); return false; + } if (cm_channel_ == nullptr) cm_channel_ = rdma_create_event_channel(); if (cm_channel_ == nullptr || rdma_create_id(cm_channel_, &listen_id_, this, RDMA_PS_TCP) != 0) { diff --git a/lib/crowdb-rpc/src/transport/socket_transport.cpp b/lib/crowdb-rpc/src/transport/socket_transport.cpp index 400ba3ab5..d7668b5aa 100644 --- a/lib/crowdb-rpc/src/transport/socket_transport.cpp +++ b/lib/crowdb-rpc/src/transport/socket_transport.cpp @@ -874,7 +874,17 @@ std::shared_ptr SocketTransport::connect(const std::string &addr, in { #ifdef CROWDB_RPC_HAVE_RDMA if (rdma_transport_ != nullptr) { - if (auto rdma_conn = rdma_transport_->connect(addr, port); rdma_conn != nullptr) { + int rdma_port = port; + if (const char *offset_env = std::getenv("CROWDB_PORT_OFFSET"); offset_env != nullptr && *offset_env != '\0') { + char *end = nullptr; + errno = 0; + const long offset = std::strtol(offset_env, &end, 10); + if (errno == 0 && end != offset_env && *end == '\0' && offset >= 0 && offset <= 65535 - port) + rdma_port += static_cast(offset); + else + CRB_LOG_WARN("connect: invalid RDMA port offset value={}", offset_env); + } + if (auto rdma_conn = rdma_transport_->connect(addr, rdma_port); rdma_conn != nullptr) { for (auto &engine : engines_) rdma_transport_->register_events(engine.get()); register_conn(rdma_conn); diff --git a/tools/rdma/run-oci-profile.sh b/tools/rdma/run-oci-profile.sh index 5a7907fb3..000c87ec5 100755 --- a/tools/rdma/run-oci-profile.sh +++ b/tools/rdma/run-oci-profile.sh @@ -9,6 +9,12 @@ startup_mode="${CROWDB_STARTUP_MODE:-single}" management_interface="${CROWDB_MANAGEMENT_INTERFACE:-eth0}" port_offset="${CROWDB_PORT_OFFSET:-0}" rdma_cm_port=$((19101 + port_offset)) +rdma_source_addr="${CROWDB_RDMA_SOURCE_ADDR:-}" +rdma_source_env=() +if [[ -n "$rdma_source_addr" ]]; then + rdma_source_env=(-e "CROWDB_RDMA_SOURCE_ADDR=$rdma_source_addr") +fi + if [[ -z "$profile" || "$profile" == "-h" || "$profile" == "--help" ]]; then cat >&2 <<'USAGE' usage: run-oci-profile.sh tcp-only|shared-hca [image] @@ -36,6 +42,7 @@ case "$profile" in -e CROWDB_RDMA_SHARED_HCA=1 \ -e CROWDB_PORT_OFFSET="$port_offset" \ -e CROWDB_RDMA_E2E_PORT="$rdma_cm_port" \ + "${rdma_source_env[@]}" \ -e CROWDB_STARTUP_MODE="$startup_mode" \ -e CROWDB_MANAGEMENT_INTERFACE="$management_interface" \ -e CROWDB_RDMA_DEVICE="${CROWDB_RDMA_DEVICE:-mlx5_0}" \ From 45df779728cad096684b55192933fb6980079f18 Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 18 Oct 2026 16:51:19 +0800 Subject: [PATCH 004/258] Allocate independent RDMA CM service ports --- container/single-node-container/profile.toml | 8 +++++++- doc/working/plan-rdma-rpc-transport.md | 7 ++++--- lib/crowdb-rpc/src/server/server.cpp | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/container/single-node-container/profile.toml b/container/single-node-container/profile.toml index b586286bb..0b5cbcba0 100644 --- a/container/single-node-container/profile.toml +++ b/container/single-node-container/profile.toml @@ -87,6 +87,7 @@ port = 9090 [[services]] id = "kv" +env = { CROWDB_RDMA_SERVICE_SLOT = "1" } program = "/opt/crowdb/bin/crowdb-kv-server" args = ["--root", "/opt/crowdb/data/kv/node-1", "--config", "/opt/crowdb/run/config/kv.toml", "--management-port", "10000", "--ports", "10100,10101", "--instance-id", "1", "--node-id", "1", "--binding-monitor-interval", "1", "--log-dir", "/opt/crowdb/data/log/kv", "--log"] dependencies = [] @@ -104,6 +105,7 @@ backoff_max_ms = 5000 [[services]] id = "diskdb" +env = { CROWDB_RDMA_SERVICE_SLOT = "2" } program = "/opt/crowdb/bin/crowdb-diskdb" args = ["--config", "/opt/crowdb/run/config/diskdb.toml", "--log-dir", "/opt/crowdb/data/log/diskdb", "--log"] dependencies = ["kv"] @@ -121,6 +123,7 @@ backoff_max_ms = 5000 [[services]] id = "diskio" +env = { CROWDB_RDMA_SERVICE_SLOT = "3" } program = "/opt/crowdb/bin/crowdb-diskio" args = ["--config", "/opt/crowdb/run/config/diskio.toml"] dependencies = ["kv", "diskdb"] @@ -138,6 +141,7 @@ backoff_max_ms = 5000 [[services]] id = "chunkdb" +env = { CROWDB_RDMA_SERVICE_SLOT = "4" } program = "/opt/crowdb/bin/crowdb-chunkdb" args = ["--config", "/opt/crowdb/run/config/chunkdb.toml", "--log-dir", "/opt/crowdb/data/log/chunkdb", "--log"] dependencies = ["kv", "diskdb", "diskio"] @@ -155,6 +159,7 @@ backoff_max_ms = 5000 [[services]] id = "chunk-kv" +env = { CROWDB_RDMA_SERVICE_SLOT = "5" } program = "/opt/crowdb/bin/crowdb-chunk-kv-server" args = ["--config", "/opt/crowdb/run/config/chunk-kv.toml", "--log-dir", "/opt/crowdb/data/log/chunk-kv", "--log"] dependencies = ["kv", "chunkdb", "diskio"] @@ -174,7 +179,7 @@ backoff_max_ms = 5000 id = "access" program = "/opt/crowdb/bin/crowdb-access-server" args = ["--config", "/opt/crowdb/run/config/access.toml"] -env = { CROWDB_S3_PUBLIC_URI = "http://localhost:9091", CROWDB_S3_REGION = "us-east-1", CROWDB_ICEBERG_PUBLIC_URI = "http://localhost:9092", CROWDB_DATASET_PUBLIC_URI = "http://localhost:9093", CROWDB_ICEBERG_GC_ENABLED = "0", CROWDB_MANAGEMENT_SEEDS = "http://127.0.0.1:10000", CROWDB_ACCESS_HEALTH_LISTEN = "127.0.0.1:9094", CROWDB_ACCESS_LOG_DIR = "/opt/crowdb/data/log/access" } +env = { CROWDB_S3_PUBLIC_URI = "http://localhost:9091", CROWDB_S3_REGION = "us-east-1", CROWDB_ICEBERG_PUBLIC_URI = "http://localhost:9092", CROWDB_DATASET_PUBLIC_URI = "http://localhost:9093", CROWDB_ICEBERG_GC_ENABLED = "0", CROWDB_MANAGEMENT_SEEDS = "http://127.0.0.1:10000", CROWDB_ACCESS_HEALTH_LISTEN = "127.0.0.1:9094", CROWDB_ACCESS_LOG_DIR = "/opt/crowdb/data/log/access", CROWDB_RDMA_SERVICE_SLOT = "6" } dependencies = ["kv", "chunk-kv", "chunkdb", "diskio"] fence_listeners = ["127.0.0.1:9092", "127.0.0.1:9091", "127.0.0.1:9093", "127.0.0.1:9094"] config_template = "/opt/crowdb/etc/templates/access.toml" @@ -196,6 +201,7 @@ backoff_max_ms = 5000 [[services]] id = "web" +env = { CROWDB_RDMA_SERVICE_SLOT = "7" } program = "/opt/crowdb/bin/crowdb-web" args = ["--config", "/opt/crowdb/run/config/crowdb-web.toml", "--node-monitor", "http://127.0.0.1:9095"] dependencies = ["access"] diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 7e34b8db8..a0645a256 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -221,9 +221,10 @@ advertises compatible capability. `CROWDB_RDMA_DEVICE`, `CROWDB_RDMA_PORT`, and `CROWDB_RDMA_SOURCE_ADDR` are deployment overrides, not required API calls. For host-network multi-container runs, deployment also supplies one `CROWDB_PORT_OFFSET` per container. The library and service launchers apply the -same offset to every default TCP/RPC and RDMA CM service port; the HCA physical -port selected by `CROWDB_RDMA_PORT` is unchanged. The default RDMA CM service -port is `19101`. +same offset to every default TCP/RPC and RDMA CM service port; each colocated +RPC service adds its stable `CROWDB_RDMA_SERVICE_SLOT` to the CM port so +services do not contend for one listener. The HCA physical port selected by +`CROWDB_RDMA_PORT` is unchanged. The default RDMA CM service port is `19101`. When any local capability, CM, QP, or pre-payload setup step is unavailable, the library reports the reason and uses TCP without requiring application code changes. After payload submission, the connection fails as one stream and does diff --git a/lib/crowdb-rpc/src/server/server.cpp b/lib/crowdb-rpc/src/server/server.cpp index f889f45d5..234e9ce8b 100644 --- a/lib/crowdb-rpc/src/server/server.cpp +++ b/lib/crowdb-rpc/src/server/server.cpp @@ -219,6 +219,21 @@ void RpcServer::start() } } } + if (rdma_port != 0) { + if (const char *slot_env = std::getenv("CROWDB_RDMA_SERVICE_SLOT"); + slot_env != nullptr && *slot_env != '\0') { + char *end = nullptr; + errno = 0; + const long slot = std::strtol(slot_env, &end, 10); + if (errno != 0 || end == slot_env || *end != '\0' || slot < 0 || slot > 65535 - rdma_port) { + CRB_LOG_WARN("rpc server: invalid RDMA service slot value={}", slot_env); + rdma_port = 0; + } + else { + rdma_port += static_cast(slot); + } + } + } const char *rdma_addr = std::getenv("CROWDB_RDMA_SOURCE_ADDR"); if (rdma_port == 0 || !transport_->listen_rdma(rdma_addr == nullptr ? listen_addr_ : rdma_addr, rdma_port)) CRB_LOG_WARN("rpc server: RDMA listener unavailable; TCP remains active"); From 22928fb50b3362450387c28e4c6a81056058892c Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 18 Oct 2026 17:16:04 +0800 Subject: [PATCH 005/258] Add RDMA RPC regression benchmark --- doc/backlog/R239-rdma-rpc-transport.md | 5 + lib/crowdb-rpc/examples/fb_server.cpp | 5 +- lib/crowdb-rpc/src/server/server.cpp | 3 +- tools/benchmark/bench-rdma-rpc-regression.sh | 99 ++++++++++++++++++++ tools/rpc-test/src/args.rs | 3 + tools/rpc-test/src/workload.rs | 6 +- 6 files changed, 115 insertions(+), 6 deletions(-) create mode 100755 tools/benchmark/bench-rdma-rpc-regression.sh diff --git a/doc/backlog/R239-rdma-rpc-transport.md b/doc/backlog/R239-rdma-rpc-transport.md index 8475ff75b..04cffb4fb 100644 --- a/doc/backlog/R239-rdma-rpc-transport.md +++ b/doc/backlog/R239-rdma-rpc-transport.md @@ -83,6 +83,11 @@ without changing request semantics. completes CM/QP/send/receive/repost without invoking `sudo`, loading modules, moving links, or entering host namespaces. Invariant: privileged host setup is separated from unprivileged application execution. E2E test. +- Given the RDMA benchmark runner uses the same echo workload as the TCP + regression benchmark, when it runs on a prepared HCA, it reports throughput, + latency percentiles, and errors for matching concurrency cases. Invariant: + transport regressions and large flow bugs are visible before downstream use. + Benchmark test. - Given an application linked against `crowdb-rpc`, when deployment provides verbs/rdmacm libraries, HCA device access, memlock, and peer reachability, the application uses the ordinary RPC API and the library selects RDMA or diff --git a/lib/crowdb-rpc/examples/fb_server.cpp b/lib/crowdb-rpc/examples/fb_server.cpp index fe14bb602..20124be73 100644 --- a/lib/crowdb-rpc/examples/fb_server.cpp +++ b/lib/crowdb-rpc/examples/fb_server.cpp @@ -7,7 +7,7 @@ // until SIGTERM/SIGINT. On shutdown, prints transport stats to stdout // as key=value lines (parsed by the CLI bench runner). // -// Usage: crowdb-rpc-fb-server [--port=18080] [--io-engines=1] [--io-workers=1] +// Usage: crowdb-rpc-fb-server [--host=127.0.0.1] [--port=18080] [--io-engines=1] [--io-workers=1] // [--enable-nagle] [--logdir=] [--metrics-interval=5] // Short aliases: -p -e -w -n -l -m (e.g. -p=18080 -e=2 -w=4). // The default log directory is below the workspace runtime root. @@ -36,6 +36,7 @@ static void on_signal(int /*signo*/) // gflags has no native alias support, so short forms (-p, -e, etc.) are // separate flags forwarded to the long ones after parse (see main). DEFINE_int32(port, 18080, "Listen port"); +DEFINE_string(host, "127.0.0.1", "Listen address"); DEFINE_int32(p, 18080, "Alias for --port"); DEFINE_uint32(io_engines, 1, "Independent epoll instances (round-robin). >1 parallelizes event queues."); DEFINE_uint32(e, 1, "Alias for --io_engines"); @@ -109,7 +110,7 @@ int main(int argc, char *argv[]) crowdb_rpc_server_set_event_write(server, FLAGS_event_write ? 1 : 0); crowdb_rpc_server_set_send_queue_capacity(server, FLAGS_send_queue_capacity); - if (crowdb_rpc_server_listen(server, "127.0.0.1", FLAGS_port) != CROWDB_RPC_OK) { + if (crowdb_rpc_server_listen(server, FLAGS_host.c_str(), FLAGS_port) != CROWDB_RPC_OK) { std::fprintf(stderr, "error: failed to listen on port %d\n", FLAGS_port); crowdb_rpc_server_destroy(server); return 1; diff --git a/lib/crowdb-rpc/src/server/server.cpp b/lib/crowdb-rpc/src/server/server.cpp index 234e9ce8b..0efbd217a 100644 --- a/lib/crowdb-rpc/src/server/server.cpp +++ b/lib/crowdb-rpc/src/server/server.cpp @@ -180,7 +180,8 @@ void RpcServer::start() } CRB_LOG_INFO("rpc server: starting"); if (const char *enabled = std::getenv("CROWDB_RDMA_ENABLED"); - enabled != nullptr && std::strcmp(enabled, "1") == 0) { + enabled != nullptr && std::strcmp(enabled, "1") == 0 && + !(std::getenv("CROWDB_RDMA_LISTEN") != nullptr && std::strcmp(std::getenv("CROWDB_RDMA_LISTEN"), "0") == 0)) { transport_->set_rdma_accept_handler([this](const std::shared_ptr &conn) { transport_->attach_rdma_connection( conn, [this](Frame *frame, Connection *c) { dispatch(frame, c); }, diff --git a/tools/benchmark/bench-rdma-rpc-regression.sh b/tools/benchmark/bench-rdma-rpc-regression.sh new file mode 100755 index 000000000..b6bc0ddea --- /dev/null +++ b/tools/benchmark/bench-rdma-rpc-regression.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# RDMA RPC echo benchmark. Mirrors bench-rpc-regression.sh with an HCA-backed +# connection so transport throughput and latency can be compared directly. +set -euo pipefail +cd "$(dirname "$0")/../.." + +RUN_STAMP=$(date +%Y%m%d-%H%M%S) +LOG_ROOT="${RDMA_BENCH_LOG_ROOT:-${CROWDB_RUNTIME_ROOT:-$(pwd)/.crowdb-runtime}/artifacts/bench/rdma-rpc-$RUN_STAMP}" +RESULTS_FILE="${RDMA_BENCH_RESULTS:-$LOG_ROOT/results.tsv}" +REGRESSION_LOG_ROOT="$LOG_ROOT" +source tools/benchmark/bench-regression-common.sh +export CROWDB_LOG_ROOT="$LOG_ROOT" +regression_init + +DURATION="${RDMA_BENCH_DURATION:-10}" +VALUE_SIZE="${RDMA_BENCH_VALUE_SIZE:-128}" +CASES="${RDMA_BENCH_CASES:-}" +ADDR="${CROWDB_RDMA_SOURCE_ADDR:-10.200.1.1}" +DEVICE="${CROWDB_RDMA_DEVICE:-rocep35s0f0}" +HCA_PORT="${CROWDB_RDMA_PORT:-1}" +PORT_OFFSET="${CROWDB_PORT_OFFSET:-0}" +PORT=$((19101 + PORT_OFFSET)) +SERVER_PID="" +SERVER_BIN="${RDMA_BENCH_SERVER_BIN:-lib/crowdb-rpc/build/crowdb-rpc-fb-server}" +CLI_BIN="${RDMA_BENCH_CLI_BIN:-target/release/crowdb-cli}" + +start_server() { + local workers="$1" + echo " [server] RDMA host=$ADDR port=$PORT device=$DEVICE workers=$workers" >&2 + CROWDB_RDMA_ENABLED=1 CROWDB_RDMA_E2E_PORT="$PORT" \ + CROWDB_RDMA_DEVICE="$DEVICE" CROWDB_RDMA_PORT="$HCA_PORT" \ + CROWDB_RDMA_SOURCE_ADDR="$ADDR" \ + "$SERVER_BIN" --host "$ADDR" --port "$PORT" \ + --io-engines 1 --io-workers "$workers" --logdir "$LOG_ROOT/server" \ + >"$LOG_ROOT/server.stdout" 2>&1 & + SERVER_PID=$! + for _ in $(seq 1 100); do + if rg -q '^listening port=' "$LOG_ROOT/server.stdout"; then return 0; fi + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + cat "$LOG_ROOT/server.stdout" >&2 + return 1 + fi + sleep 0.1 + done + echo " [server] timeout waiting for readiness" >&2 + return 1 +} + +stop_server() { + if [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then + kill -TERM "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + SERVER_PID="" +} + +run_bench() { + local loaders="$1" connections="$2" label="$3" workers="$4" + if [[ -n "$CASES" && " $CASES " != *" $label "* ]]; then return; fi + echo ">>> $label (RDMA, workers=$workers) ..." + stop_server + mkdir -p "$LOG_ROOT/server" + start_server "$workers" + local output json + output=$(CROWDB_RDMA_ENABLED=1 CROWDB_RDMA_LISTEN=0 CROWDB_RDMA_E2E_PORT="$PORT" \ + CROWDB_RDMA_DEVICE="$DEVICE" CROWDB_RDMA_PORT="$HCA_PORT" \ + CROWDB_RDMA_SOURCE_ADDR="$ADDR" CROWDB_PORT_OFFSET="$PORT_OFFSET" \ + pixi run -- "$CLI_BIN" bench rpc \ + --duration-secs "$DURATION" --loader-num "$loaders" --connections "$connections" \ + --value-size "$VALUE_SIZE" --io-engines 1 --io-workers "$workers" \ + --server-host "$ADDR" --server-port "$PORT" 2>&1) || true + json=$(echo "$output" | sed -n '/^{/,/^}/p') + if [[ -z "$json" ]]; then + echo "$output" | tail -20 >&2 + printf '%s\t%s\t0\t0\t0\t0\t0\t1\n' "$label" "$workers" >>"$RESULTS_FILE" + else + local ops avg p50 p99 p999 errors + ops=$(echo "$json" | jq -r '.total_ops * 1000 / .duration_ms' | awk '{printf "%.0f", $1}') + avg=$(echo "$json" | jq -r '.by_op.write.latency_us.avg_us') + p50=$(echo "$json" | jq -r '.by_op.write.latency_us.p50_us') + p99=$(echo "$json" | jq -r '.by_op.write.latency_us.p99_us') + p999=$(echo "$json" | jq -r '.by_op.write.latency_us.p999_us') + errors=$(echo "$json" | jq -r '.total_errors') + echo " ops/s=$ops avg=${avg}us p50=${p50}us p99=${p99}us p999=${p999}us err=$errors" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$label" "$workers" "$ops" "$avg" "$p50" "$p99" "$p999" "$errors" >>"$RESULTS_FILE" + fi + stop_server +} + +trap stop_server EXIT +mkdir -p "$LOG_ROOT" +printf 'label\twkr\tops_s\tavg_us\tp50_us\tp99_us\tp999_us\terrors\n' >"$RESULTS_FILE" +echo "=== RDMA RPC echo ($VALUE_SIZE B, ${DURATION}s, HCA $DEVICE:$HCA_PORT) ===" +run_bench 1 1 rdma_1w_1l_1c 1 +run_bench 64 4 rdma_4w_64l_4c 4 +run_bench 512 8 rdma_8w_512l_8c 8 +run_bench 1000 32 rdma_16w_1000l_32c 16 +echo "=== DONE ===" +echo "Results in $RESULTS_FILE" diff --git a/tools/rpc-test/src/args.rs b/tools/rpc-test/src/args.rs index cd558c192..8f2c08598 100644 --- a/tools/rpc-test/src/args.rs +++ b/tools/rpc-test/src/args.rs @@ -34,6 +34,9 @@ pub struct RpcArgs { /// fb-server port to connect to. #[arg(long)] pub server_port: u16, + /// Address used for the TCP handshake and RDMA CM connection. + #[arg(long, default_value = "127.0.0.1")] + pub server_host: String, /// Enable TCP Nagle (coalesce small frames per writev). #[arg(long, default_value_t = false)] pub enable_nagle: bool, diff --git a/tools/rpc-test/src/workload.rs b/tools/rpc-test/src/workload.rs index 565a9c3bd..5c220a86c 100644 --- a/tools/rpc-test/src/workload.rs +++ b/tools/rpc-test/src/workload.rs @@ -54,12 +54,12 @@ pub async fn run(log_dir: &Path, args: RpcArgs) -> ExitCode { // Connect `connections` connections to the fb-server. let mut conns: Vec = Vec::with_capacity(args.connections.max(1)); for _ in 0..args.connections.max(1) { - match server.connect("127.0.0.1", i32::from(args.server_port)) { + match server.connect(&args.server_host, i32::from(args.server_port)) { Ok(c) => conns.push(c), Err(e) => { eprintln!( - "error: connect to fb-server 127.0.0.1:{} failed: {e}", - args.server_port + "error: connect to fb-server {}:{} failed: {e}", + args.server_host, args.server_port ); server.stop(); return ExitCode::from(2); From 0d5598906db8157fac82fc2c19d0d59128f5c5cd Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 18 Oct 2026 17:55:41 +0800 Subject: [PATCH 006/258] Register RDMA payload buffers before send --- .../src/transport/rdma/rdma_transport.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index 11db2d3e0..d063bb978 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -338,6 +338,19 @@ bool RdmaTransport::submit(Connection *conn, OutFrame *frame) frame->data->release(); frame->data = nullptr; } + if (frame->data != nullptr && frame->data->type != BufferType::Registered) { + Buffer *source_data = frame->data; + Buffer *registered = register_buffer(source_data); + if (registered == nullptr) { + frame->control->release(); + frame->control = nullptr; + source_data->release(); + frame->data = nullptr; + return false; + } + frame->data = registered; + source_data->release(); + } RdmaFrameSges frame_sges; if (!build_rdma_frame_sges(frame->control, frame->data, &frame_sges)) { @@ -768,6 +781,7 @@ std::shared_ptr RdmaTransport::accept() const int accept_rc = rdma_accept(id, ¶ms); if (accept_rc != 0) { + CRB_LOG_WARN("rdma accept: rdma_accept failed errno={}", errno); rdma_destroy_qp(id); ibv_destroy_cq(connection_cq); ibv_destroy_comp_channel(connection_channel); @@ -778,6 +792,8 @@ std::shared_ptr RdmaTransport::accept() const int event_rc = wait_cm_event(cm_channel_, &event, 5000) ? 0 : -1; if (event_rc != 0 || event == nullptr || event->event != RDMA_CM_EVENT_ESTABLISHED) { + CRB_LOG_WARN("rdma accept: ESTABLISHED missing event={} status={} errno={}", + event == nullptr ? -1 : event->event, event == nullptr ? 0 : event->status, errno); if (event != nullptr) rdma_ack_cm_event(event); rdma_destroy_qp(id); From 324111f9c88b36d71921b20f56423d25a5d70c5c Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 18 Oct 2026 18:24:09 +0800 Subject: [PATCH 007/258] Harden RDMA benchmark payload path --- .../src/transport/rdma/rdma_transport.cpp | 3 ++ .../src/transport/socket_transport.cpp | 36 ++++++++++++++----- tools/rpc-test/src/workload.rs | 2 +- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index d063bb978..63f78a6aa 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -999,6 +999,9 @@ void RdmaTransport::release_completed_receive(ibv_wc &wc) delete work; }; if (wc.status != IBV_WC_SUCCESS) { + CRB_LOG_WARN("rdma receive: completion failed status={} vendor_err={} opcode={}", + static_cast(wc.status), static_cast(wc.vendor_err), + static_cast(wc.opcode)); quarantine_and_release(); return; } diff --git a/lib/crowdb-rpc/src/transport/socket_transport.cpp b/lib/crowdb-rpc/src/transport/socket_transport.cpp index d7668b5aa..525f77f17 100644 --- a/lib/crowdb-rpc/src/transport/socket_transport.cpp +++ b/lib/crowdb-rpc/src/transport/socket_transport.cpp @@ -874,15 +874,33 @@ std::shared_ptr SocketTransport::connect(const std::string &addr, in { #ifdef CROWDB_RPC_HAVE_RDMA if (rdma_transport_ != nullptr) { - int rdma_port = port; - if (const char *offset_env = std::getenv("CROWDB_PORT_OFFSET"); offset_env != nullptr && *offset_env != '\0') { - char *end = nullptr; - errno = 0; - const long offset = std::strtol(offset_env, &end, 10); - if (errno == 0 && end != offset_env && *end == '\0' && offset >= 0 && offset <= 65535 - port) - rdma_port += static_cast(offset); - else - CRB_LOG_WARN("connect: invalid RDMA port offset value={}", offset_env); + int rdma_port = port; + bool explicit_rdma_port = false; + if (const char *port_env = std::getenv("CROWDB_RDMA_E2E_PORT"); port_env != nullptr && *port_env != '\0') { + char *end = nullptr; + errno = 0; + const long configured = std::strtol(port_env, &end, 10); + if (errno == 0 && end != port_env && *end == '\0' && configured > 0 && configured <= 65535) { + rdma_port = static_cast(configured); + explicit_rdma_port = true; + } + else { + CRB_LOG_WARN("connect: invalid RDMA CM port value={}", port_env); + } + } + if (!explicit_rdma_port) { + const char *offset_env = std::getenv("CROWDB_PORT_OFFSET"); + if (offset_env == nullptr || *offset_env == '\0') + offset_env = nullptr; + if (offset_env != nullptr) { + char *end = nullptr; + errno = 0; + const long offset = std::strtol(offset_env, &end, 10); + if (errno == 0 && end != offset_env && *end == '\0' && offset >= 0 && offset <= 65535 - port) + rdma_port += static_cast(offset); + else + CRB_LOG_WARN("connect: invalid RDMA port offset value={}", offset_env); + } } if (auto rdma_conn = rdma_transport_->connect(addr, rdma_port); rdma_conn != nullptr) { for (auto &engine : engines_) diff --git a/tools/rpc-test/src/workload.rs b/tools/rpc-test/src/workload.rs index 5c220a86c..e28bd7161 100644 --- a/tools/rpc-test/src/workload.rs +++ b/tools/rpc-test/src/workload.rs @@ -40,7 +40,7 @@ use std::path::Path; const ECHO_MSG_TYPE: u16 = 100; pub async fn run(log_dir: &Path, args: RpcArgs) -> ExitCode { - let data_bytes = vec![0xAB; args.value_size.max(1)]; + let data_bytes = if args.value_size == 0 { Vec::new() } else { vec![0xAB; args.value_size] }; // Client-side server for connection management. let server = Arc::new(RpcServer::with_engines(None, args.io_engines, args.io_workers)); From 94f84d5f427ecac9aaecd53194ed90c1ed65a408 Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 18 Oct 2026 21:13:58 +0800 Subject: [PATCH 008/258] Use contiguous RDMA wire slots --- doc/backlog/R239-rdma-rpc-transport.md | 9 +- doc/design/rpc/design-crowdb-rpc-rdma.md | 96 ++++--- doc/working/plan-rdma-rpc-transport.md | 19 +- lib/crowdb-rpc/include/crowdb-rpc/buffer.h | 38 ++- .../transport/rdma/rdma_capability.h | 15 - .../transport/rdma/rdma_transport.h | 10 +- lib/crowdb-rpc/src/c_api.cpp | 12 +- lib/crowdb-rpc/src/c_api/handlers.cpp | 2 +- .../src/transport/rdma/rdma_capability.cpp | 26 -- .../src/transport/rdma/rdma_transport.cpp | 264 +++++++----------- lib/crowdb-rpc/tests/rdma_capability_test.cpp | 23 -- 11 files changed, 214 insertions(+), 300 deletions(-) diff --git a/doc/backlog/R239-rdma-rpc-transport.md b/doc/backlog/R239-rdma-rpc-transport.md index 04cffb4fb..4260e762f 100644 --- a/doc/backlog/R239-rdma-rpc-transport.md +++ b/doc/backlog/R239-rdma-rpc-transport.md @@ -14,8 +14,10 @@ without changing request semantics. ## Solution 1. Add an RC SEND/RECV transport behind the common RPC `Transport` interface. - Data messages use one 1 KiB control SGE and one selected data SGE; no CPU - concatenation is introduced. + Each message is copied into one fixed-size registered wire slot and sent + with one SGE. Receive slots are pre-posted from a registered arena and are + reused only after Frame ownership is released. The complete wire message + must fit the slot capacity. 2. Use service discovery for coarse RDMA capability and TCP handshake for the current CM endpoint, device/GID identity, capability generation, protocol version, and supported data classes. Local configuration and probing can @@ -57,7 +59,8 @@ without changing request semantics. ## Acceptance - Given both peers advertise RDMA, when the TCP handshake completes, the - client establishes the negotiated RC path and sends a valid two-SGE frame. + client establishes the negotiated RC path and sends a valid one-SGE wire + slot. Invariant: shared RPC framing. Integration test. - Given no RDMA capability or local RDMA configuration, when a connection is created, TCP is selected without an RDMA attempt. Invariant: optional RDMA. diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index ade470eba..2a85758cd 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -15,8 +15,8 @@ established TCP connection. - [1. Scope and goals](#1-scope-and-goals) - [2. Endpoint discovery and path selection](#2-endpoint-discovery-and-path-selection) -- [3. Message and aggregation model](#3-message-and-aggregation-model) -- [4. Registered buffers and ownership](#4-registered-buffers-and-ownership) +- [3. Message and wire-slot model](#3-message-and-wire-slot-model) +- [4. Registered arenas and slot ownership](#4-registered-arenas-and-slot-ownership) - [5. Event and completion model](#5-event-and-completion-model) - [6. Failure and fallback](#6-failure-and-fallback) - [7. Service interactions](#7-service-interactions) @@ -53,49 +53,59 @@ one-way fallback. The path is selected before the first payload. TCP is chosen when RDMA setup fails before payload submission; an established TCP connection is never upgraded to RDMA. -## 3. Message and aggregation model +## 3. Message and wire-slot model ChunkIO and Chunk-KV retain ownership of aggregation and flush policy. RDMA receives an already aggregated buffer and transmits it without additional -waiting or coalescing. A data message has: +waiting or coalescing. The RDMA wire representation is contiguous inside one +fixed-size wire slot: ```text -SGE 0: 1 KiB control block -SGE 1: selected data buffer +[common RPC header][control metadata][payload][unused slot bytes] ``` -The control block contains the common RPC header and control metadata. The -selected data capacity is 64 KiB for metadata, 256 KiB for aggregated -Chunk-KV results, or 1 MiB for the S3/HTTP/DiskIO datapath. The actual payload -length is carried in the header. A logical result larger than the selected -capacity is an ordered sequence identified by request ID, stream ID, sequence, -logical offset, and START/DATA/END/ERROR flags. +The common header carries the control and payload lengths. The complete wire +length must not exceed the slot capacity. A logical result larger than the +selected capacity is an ordered sequence identified by request ID, stream ID, +sequence, logical offset, and START/DATA/END/ERROR flags. -The two-SGE layout is intentional. It keeps control and data ownership clear -and avoids CPU concatenation. A sender may inline the small control block when -the device permits it; receive buffers retain the two-SGE layout. +Each SEND uses one SGE pointing at the slot and the actual wire length. Each +RECV is posted with one SGE pointing at a complete receive slot and its full +capacity. The sender and receiver do not share addresses and do not need equal +ring counts; the receive slot is a bounded window for incoming messages. One +SGE on each side keeps the wire layout independent of sender SGE boundaries. -## 4. Registered buffers and ownership +The transport may copy caller-owned control and payload buffers into the slot +before posting SEND. This copy is the transport boundary needed to make the +wire message contiguous; it is not an additional application-level message or +aggregation step. -Each process has a bounded registered host-memory budget. Data classes and the -control pool are shared by streams; a stream does not reserve a private set of -buffers for its lifetime. A buffer moves through these states: +## 4. Registered arenas and slot ownership + +Each connection owns a bounded registered arena. The arena is registered once +and split into fixed-size slots; individual slots do not call `ibv_reg_mr` or +`ibv_dereg_mr`. Send and receive rings contain descriptors for slots in their +respective arenas. A stream does not reserve a private set of slots for its +lifetime. A slot moves through these states: ```text -available -> posted_receive -> receive_in_flight -> caller_owned -available -> send_in_flight -> available -receive_in_flight -> disk_owned -> available +send_free -> send_in_flight -> send_free +receive_free -> posted_receive -> receive_in_flight -> caller_owned +caller_owned -> receive_free any uncertain state -> quarantined ``` -The owner that receives a completed SDK buffer must explicitly release its -lease. DiskIO reposts a receive buffer only after its asynchronous disk write -completes. An uncertain completion is quarantined until connection quiescence. -Registered memory is not reused merely because a deadline expired. +The SEND completion returns a send slot to the send ring. A receive completion +transfers the slot to the Frame owner after header and length validation. The +Frame owner explicitly releases the slot; only then is it posted again to the +receive ring. DiskIO reposts a receive slot only after its asynchronous disk +write completes. An uncertain completion quarantines the slot until connection +quiescence. Registered memory is not reused merely because a deadline expired. -Posted receives are the implicit data credits. Buffer exhaustion, a full SDK -lease budget, or a full DiskIO write queue applies backpressure to the existing -producer or aggregation layer. RDMA adds no application-level credit message. +Posted receive slots are the implicit data credits. Buffer exhaustion, a full +SDK lease budget, or a full DiskIO write queue applies backpressure to the +existing producer or aggregation layer. RDMA adds no application-level credit +message. A slot is never posted twice concurrently. ## 5. Event and completion model @@ -146,33 +156,35 @@ changing the lease contract. The transport configuration supplies defaults when the transport is created: ```toml -stream_buffer_size = "1MiB" -chunk_kv_metadata_buffer_size = "64KiB" -chunk_kv_aggregate_buffer_size = "256KiB" -rpc_control_block_size = "1KiB" -rdma_registered_bytes = "64MiB" -rdma_max_inflight_bytes = "32MiB" +rdma_wire_slot_size = "1MiB" +rdma_send_slot_count = 64 +rdma_receive_slot_count = 64 +rdma_registered_bytes = "128MiB" ``` -The effective pool counts are bounded by registered bytes, receive depth, send -depth, DiskIO queue bytes, and SDK-held bytes. A configured class is a capacity -limit, not a required wire length. +The effective slot counts are bounded by registered bytes, QP send/receive +depth, DiskIO queue bytes, and SDK-held slots. A configured slot size is a +capacity limit, not a required wire length; SEND uses the actual message +length while RECV posts the full slot capacity. Services may select a smaller +slot class only when the complete wire message fits that class. ## 9. Correctness invariants - **I1:** A message is published only after its control header and payload - length pass validation for the negotiated class. -- **I2:** A buffer has exactly one owner and is not reused before transport, + length pass validation and the complete wire length fits one slot. +- **I2:** A slot has exactly one owner and is not reused before transport, disk, and caller access have completed. - **I3:** Posted receive capacity is the bounded receive window; no unbounded - allocation replaces a missing buffer. + allocation or second post replaces a missing slot. - **I4:** A logical stream publishes messages in sequence order and never mixes bytes from TCP and a partially completed RDMA stream. - **I5:** The shared request ID and generation prevent an ambiguous retry from being mistaken for a distinct operation. - **I6:** FFI completion never transfers a buffer whose ownership or length is unresolved. -- **I7:** Unsupported RDMA leaves the TCP service usable and does not require +- **I7:** A SEND uses one contiguous registered slot and a RECV uses one + complete posted slot; sender SGE boundaries never define receiver parsing. +- **I8:** Unsupported RDMA leaves the TCP service usable and does not require RDMA libraries or devices at runtime. ## 10. Scope boundary diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index a0645a256..36daada48 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -16,7 +16,7 @@ The transport foundation now includes validated endpoint capability data (protocol, generation, device, GID, port, and supported capacity classes), explicit RDMA path states, and a pre-payload-only TCP fallback predicate. Implemented since this checkpoint: active/passive RDMA-CM setup, RC QP -creation, two-SGE SEND construction, bounded receive posting, completion-channel +creation, one-MR arena-backed wire slots, one-SGE SEND/RECV posting, bounded receive posting, completion-channel polling with CQ acknowledgement, error quarantine, validated receive-frame dispatch into `Connection::on_frame`, and epoll external-fd registration. The CQ path is now drained by an epoll worker through `completion_fd()` and @@ -52,6 +52,8 @@ Completed: status, and clean client/server exit; - actual `RpcServer`/`RpcClient` worker-path request/response round trip with sequential frames and clean teardown; +- contiguous one-SGE wire-slot send/receive flow with 128B, 1KiB, and 4KiB + payload benchmark smoke tests reporting zero errors; - non-root runner selection and the library consumer/deployment contract. Remaining: @@ -105,11 +107,12 @@ Remaining: ## Phase 4: frame submission and completion -- [x] **Build two-SGE WRs**: map the 1 KiB control buffer and selected data - buffer to one SEND WR. Inline control only when the configured device limit - allows; never concatenate control and data in the CPU. -- [x] **Post receives**: prepost control/data receive buffers, associate each - with a stable `wr_id`, and replenish only after the owner releases it. +- [x] **Build one-SGE wire slots**: allocate one fixed-size registered slot + from the send ring, copy header/control/payload contiguously, and post one + SEND SGE with the actual wire length. Reject messages larger than the slot. +- [x] **Post one-slot receives**: prepost one full registered receive slot per + receive ring entry, associate each with a stable `wr_id`, and replenish only + after the owner releases it. - [ ] **Resolve WR identity**: `wr_id` must identify connection, request ID, stream ID, sequence, logical offset, buffers, and ownership state. Handle success, flush, local protection, retry, and remote access errors distinctly. @@ -132,7 +135,7 @@ Unit tests: - endpoint capability and generation validation; - connection-state transitions and illegal TCP-to-RDMA upgrade; -- frame class/length validation and two-SGE construction; +- frame class/length validation and one-SGE wire-slot construction; - `wr_id` mapping and completion state transitions; - fallback boundary and request-ID reuse rules. @@ -141,7 +144,7 @@ Integration tests: - RDMA-disabled build and host selects TCP without CM calls; - mocked CM/CQ setup, reconnect, CQ error, flush completion, and shutdown; - epoll external event wakes worker and drains all current CQ completions; -- send/receive sequence preserves exact control/data lengths; +- send/receive sequence preserves exact control/data lengths inside one slot; - pre-payload failure retries idempotent request on TCP; - partial payload failure never appends TCP data. diff --git a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h index cf1e772bf..ae59eeb0e 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h @@ -57,15 +57,37 @@ struct Buffer class BufferPool *pool = nullptr; std::atomic *ref = nullptr; // shared refcount (pool-allocated or standalone) std::atomic ownership{BufferOwnership::Available}; - BufferCapacityClass capacity_class = BufferCapacityClass::Control; - uint64_t request_id = 0; - uint64_t stream_id = 0; - uint64_t sequence = 0; - uint64_t logical_offset = 0; - uint32_t registration_id = 0; - uint32_t registration_key = 0; + BufferCapacityClass capacity_class = BufferCapacityClass::Control; + uint64_t request_id = 0; + uint64_t stream_id = 0; + uint64_t sequence = 0; + uint64_t logical_offset = 0; + uint32_t registration_id = 0; + uint32_t registration_key = 0; uintptr_t registration_handle = 0; - Buffer *next_pool = nullptr; + uint8_t *allocation_data = nullptr; + uint32_t allocation_capacity = 0; + uint32_t view_offset = 0; + Buffer *next_pool = nullptr; + + void reset_view() + { + if (allocation_data != nullptr) { + data = allocation_data; + capacity = allocation_capacity; + view_offset = 0; + } + } + + [[nodiscard]] uint8_t *payload_data() + { + return data == nullptr ? nullptr : data + view_offset; + } + + [[nodiscard]] const uint8_t *payload_data() const + { + return data == nullptr ? nullptr : data + view_offset; + } // External free callback: when set, release() calls free_cb(free_ctx) // instead of freeing `data` directly. Used for zero-copy buffers that diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h index 68edc368f..e2044dcba 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h @@ -43,21 +43,6 @@ serialize_rdma_handshake(const RdmaEndpointCapability &capability); [[nodiscard]] bool parse_rdma_handshake(std::span bytes, RdmaEndpointCapability *out); -struct RdmaSge -{ - uintptr_t address = 0; - uint32_t length = 0; - uint32_t lkey = 0; -}; - -struct RdmaFrameSges -{ - std::array entries{}; - uint8_t count = 0; -}; - -[[nodiscard]] bool build_rdma_frame_sges(const Buffer *control, const Buffer *data, RdmaFrameSges *out); - [[nodiscard]] bool validate_rdma_capability(const RdmaEndpointCapability &capability, uint16_t expected_protocol, uint64_t expected_generation, const std::string &expected_device); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index 256cf63f4..c869e9eae 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -27,6 +27,8 @@ namespace crowdb::rpc class SocketEngine; +constexpr uint32_t RDMA_WIRE_SLOT_CAPACITY = RDMA_DATAPATH_CAPACITY; + // RdmaBufferPool: allocates buffers from ibv_mr-registered memory. // The pool pre-registers a large memory region and carves out buffers // from it, avoiding per-buffer ibv_reg_mr calls (which are expensive). @@ -81,7 +83,7 @@ class RdmaTransport : public Transport ~RdmaTransport() override; bool submit(Connection *conn, OutFrame *frame) override; - bool post_receive(Connection *conn, Buffer *control, Buffer *data); + bool post_receive(Connection *conn, Buffer *slot); Buffer *register_buffer(Buffer *buf) override; void shutdown() override; @@ -153,16 +155,14 @@ class RdmaTransport : public Transport struct ReceiveWork { Connection *conn; - Buffer *control; - Buffer *data; + Buffer *slot; }; struct RepostContext { RdmaTransport *transport; Connection *connection; - Buffer *control; - Buffer *data; + Buffer *slot; }; struct ConnectionPools diff --git a/lib/crowdb-rpc/src/c_api.cpp b/lib/crowdb-rpc/src/c_api.cpp index 64f0ca746..14ff9616e 100644 --- a/lib/crowdb-rpc/src/c_api.cpp +++ b/lib/crowdb-rpc/src/c_api.cpp @@ -90,7 +90,7 @@ const uint8_t *crowdb_rpc_buffer_data(crowdb_rpc_buffer_t buf) if (buf == nullptr || buf->buf == nullptr) { return nullptr; } - return buf->buf->data; + return buf->buf->payload_data(); } catch (...) { return nullptr; @@ -140,8 +140,7 @@ uint32_t crowdb_rpc_buffer_registration_id(crowdb_rpc_buffer_t buf) return (buf != nullptr && buf->buf != nullptr) ? buf->buf->registration_id : 0; } -void crowdb_rpc_buffer_set_metadata(crowdb_rpc_buffer_t buf, uint64_t request_id, - uint64_t stream_id, uint64_t sequence, +void crowdb_rpc_buffer_set_metadata(crowdb_rpc_buffer_t buf, uint64_t request_id, uint64_t stream_id, uint64_t sequence, uint64_t logical_offset, uint32_t registration_id) { if (buf != nullptr && buf->buf != nullptr) { @@ -156,8 +155,8 @@ int crowdb_rpc_buffer_transition(crowdb_rpc_buffer_t buf, int from, int to) } return buf->buf->transition(static_cast(from), static_cast(to)) - ? 1 - : 0; + ? 1 + : 0; } int crowdb_rpc_buffer_quarantine(crowdb_rpc_buffer_t buf) @@ -904,7 +903,7 @@ static crowdb::rpc::OutFrame *echo_handler(crowdb::rpc::Frame *request, crowdb:: if (request->data_buf != nullptr && request->data_buf->len > 0) { resp_data = pool->alloc(request->data_buf->len); if (resp_data != nullptr) { - std::memcpy(resp_data->data, request->data_buf->data, request->data_buf->len); + std::memcpy(resp_data->data, request->data_buf->payload_data(), request->data_buf->len); resp_data->write(resp_data->data, request->data_buf->len); } } @@ -928,6 +927,7 @@ void crowdb_rpc_server_register_echo_handler(crowdb_rpc_server_t server, uint16_ } } + // ── Logging ─────────────────────────────────────────────────────── void crowdb_rpc_init_logging(const char *log_dir, const char *level, size_t max_file_mb, size_t max_files, diff --git a/lib/crowdb-rpc/src/c_api/handlers.cpp b/lib/crowdb-rpc/src/c_api/handlers.cpp index b2ad34e79..41f189078 100644 --- a/lib/crowdb-rpc/src/c_api/handlers.cpp +++ b/lib/crowdb-rpc/src/c_api/handlers.cpp @@ -40,7 +40,7 @@ void crowdb::rpc::invoke_c_handler(crowdb_rpc_handler_fn callback, void *user_da const uint8_t *data_ptr = nullptr; uint32_t data_len = 0; if (request->data_buf != nullptr && request->data_buf->len > 0) { - data_ptr = request->data_buf->data; + data_ptr = request->data_buf->payload_data(); data_len = request->data_buf->len; } void *conn_handle = static_cast(conn); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp index 0a24b8aa4..3f306fce5 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp @@ -70,32 +70,6 @@ bool parse_rdma_handshake(std::span bytes, RdmaEndpointCapability return true; } -bool build_rdma_frame_sges(const Buffer *control, const Buffer *data, RdmaFrameSges *out) -{ - if (control == nullptr || out == nullptr || control->data == nullptr || control->len > RDMA_CONTROL_CAPACITY || - control->len > control->capacity) { - return false; - } - out->entries[0] = {.address = reinterpret_cast(control->data), - .length = control->len, - .lkey = control->registration_key}; - out->count = 1; - if (data == nullptr || data->len == 0) - return true; - if (data->data == nullptr || data->len > data->capacity) - return false; - BufferCapacityClass cls; - uint32_t capacity; - if (!classify_rdma_capacity(data->capacity, &cls, &capacity)) - return false; - (void)cls; - (void)capacity; - out->entries[1] = { - .address = reinterpret_cast(data->data), .length = data->len, .lkey = data->registration_key}; - out->count = 2; - return true; -} - bool validate_rdma_capability(const RdmaEndpointCapability &capability, uint16_t expected_protocol, uint64_t expected_generation, const std::string &expected_device) { diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index 63f78a6aa..e20020147 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -113,6 +113,8 @@ Buffer *RdmaBufferPool::alloc_fresh(uint32_t capacity) buf->data = static_cast(region_) + aligned_offset; buf->len = 0; + buf->allocation_data = buf->data; + buf->allocation_capacity = capacity; buf->registration_handle = reinterpret_cast(mr_); buf->registration_key = mr_->lkey; next_offset_ = aligned_offset + capacity; @@ -133,6 +135,7 @@ Buffer *RdmaBufferPool::alloc(uint32_t capacity) if (it != free_list_.end() && !it->second.empty()) { Buffer *buf = it->second.back(); it->second.pop_back(); + buf->reset_view(); buf->len = 0; buf->ref->store(1, std::memory_order_relaxed); buf->capacity_class = cls; @@ -152,6 +155,7 @@ void RdmaBufferPool::recycle(Buffer *buf) { { std::scoped_lock lock(mu_); + buf->reset_view(); if (buf->ownership.load(std::memory_order_acquire) == BufferOwnership::Quarantined) { quarantine_.push_back(buf); } @@ -302,9 +306,9 @@ bool RdmaTransport::submit(Connection *conn, OutFrame *frame) return false; } - // SEND carries the RPC wire header and control message in one registered - // control SGE. TCP serializes the header in writev; RDMA must materialize - // the same bytes before posting the WR while keeping data in its own SGE. + // TCP serializes the header and buffers independently. RDMA materializes + // the complete wire message in one registered slot so the receiver can + // post and recycle one contiguous buffer. Buffer *source_control = frame->control; const uint32_t control_len = source_control->len; if (source_control->data == nullptr || control_len > source_control->capacity || @@ -313,116 +317,90 @@ bool RdmaTransport::submit(Connection *conn, OutFrame *frame) source_control->capacity); return false; } - Buffer *wire_control = pools->send->alloc(RDMA_CONTROL_CAPACITY); - if (wire_control == nullptr) + Buffer *slot = pools->send->alloc(RDMA_WIRE_SLOT_CAPACITY); + if (slot == nullptr) return false; - frame->header.msg_size = control_len; - frame->header.data_size = frame->data == nullptr ? 0 : frame->data->len; - serialize_header(wire_control->data, frame->header); - if (control_len != 0) - std::memcpy(wire_control->data + HEADER_SIZE, source_control->data, control_len); - wire_control->len = HEADER_SIZE + control_len; - wire_control->request_id = source_control->request_id; - wire_control->stream_id = source_control->stream_id; - wire_control->sequence = source_control->sequence; - wire_control->logical_offset = source_control->logical_offset; - frame->control = wire_control; - source_control->release(); - - if (frame->data != nullptr && frame->data->len > frame->data->capacity) { - frame->control->release(); - frame->control = nullptr; + const uint32_t data_len = frame->data == nullptr ? 0 : frame->data->len; + const uint64_t wire_len = static_cast(HEADER_SIZE) + control_len + data_len; + if (wire_len > slot->capacity) { + slot->release(); + CRB_LOG_WARN("rdma send: message exceeds wire slot control_len={} data_len={} capacity={}", control_len, + data_len, slot->capacity); return false; } - if (frame->data != nullptr && frame->data->len == 0) { - frame->data->release(); - frame->data = nullptr; - } - if (frame->data != nullptr && frame->data->type != BufferType::Registered) { - Buffer *source_data = frame->data; - Buffer *registered = register_buffer(source_data); - if (registered == nullptr) { - frame->control->release(); - frame->control = nullptr; - source_data->release(); - frame->data = nullptr; + frame->header.msg_size = control_len; + frame->header.data_size = data_len; + serialize_header(slot->data, frame->header); + if (control_len != 0) + std::memcpy(slot->data + HEADER_SIZE, source_control->data, control_len); + if (data_len != 0) { + if (frame->data->payload_data() == nullptr || data_len > frame->data->capacity) { + slot->release(); + CRB_LOG_WARN("rdma send: data buffer exceeds capacity len={} capacity={}", data_len, frame->data->capacity); return false; } - frame->data = registered; - source_data->release(); - } - - RdmaFrameSges frame_sges; - if (!build_rdma_frame_sges(frame->control, frame->data, &frame_sges)) { - return false; - } - ibv_sge sges[2]{}; - for (uint8_t i = 0; i < frame_sges.count; ++i) { - sges[i].addr = frame_sges.entries[i].address; - sges[i].length = frame_sges.entries[i].length; - sges[i].lkey = frame_sges.entries[i].lkey; - } + std::memcpy(slot->data + HEADER_SIZE + control_len, frame->data->payload_data(), data_len); + } + slot->len = static_cast(wire_len); + slot->request_id = source_control->request_id; + slot->stream_id = source_control->stream_id; + slot->sequence = source_control->sequence; + slot->logical_offset = source_control->logical_offset; + + ibv_sge sge{}; + sge.addr = reinterpret_cast(slot->data); + sge.length = slot->len; + sge.lkey = slot->registration_key; ibv_send_wr wr{}; wr.wr_id = reinterpret_cast(frame); - wr.sg_list = sges; - wr.num_sge = frame_sges.count; + wr.sg_list = &sge; + wr.num_sge = 1; wr.opcode = IBV_WR_SEND; wr.send_flags = IBV_SEND_SIGNALED; ibv_send_wr *bad = nullptr; if (ibv_post_send(reinterpret_cast(conn->transport_handle), &wr, &bad) != 0) { CRB_LOG_WARN("rdma send: post failed qp={} errno={}", reinterpret_cast(conn->transport_handle), errno); + slot->release(); return false; } + source_control->release(); + if (frame->data != nullptr) + frame->data->release(); + frame->control = slot; + frame->data = nullptr; + stats_.sends.fetch_add(1, std::memory_order_relaxed); return true; } -bool RdmaTransport::post_receive(Connection *conn, Buffer *control, Buffer *data) +bool RdmaTransport::post_receive(Connection *conn, Buffer *slot) { - if (conn == nullptr || control == nullptr || control->data == nullptr || cq_ == nullptr || - conn->transport_handle == 0 || control->capacity < RDMA_CONTROL_CAPACITY) { + if (conn == nullptr || slot == nullptr || slot->data == nullptr || cq_ == nullptr || conn->transport_handle == 0 || + slot->capacity < RDMA_WIRE_SLOT_CAPACITY) { return false; } - ibv_sge sges[2]{}; - sges[0].addr = reinterpret_cast(control->data); - sges[0].length = control->capacity; - sges[0].lkey = control->registration_key; - int num_sge = 1; - if (data != nullptr) { - if (data->data == nullptr || data->capacity == 0) - return false; - sges[1].addr = reinterpret_cast(data->data); - sges[1].length = data->capacity; - sges[1].lkey = data->registration_key; - num_sge = 2; - } - auto work = std::make_unique(ReceiveWork{conn, control, data}); - control->ref_clone(); - if (data != nullptr) - data->ref_clone(); + ibv_sge sge{}; + sge.addr = reinterpret_cast(slot->data); + sge.length = slot->capacity; + sge.lkey = slot->registration_key; + auto work = std::make_unique(ReceiveWork{conn, slot}); + slot->ref_clone(); ibv_recv_wr wr{}; - wr.wr_id = reinterpret_cast(work.get()) | 1U; - wr.sg_list = sges; - wr.num_sge = num_sge; - ibv_recv_wr *bad = nullptr; - const bool control_posted = control->transition(BufferOwnership::Available, BufferOwnership::PostedReceive); - const bool data_posted = data == nullptr || - data->transition(BufferOwnership::Available, BufferOwnership::PostedReceive) || - data->transition(BufferOwnership::CallerOwned, BufferOwnership::PostedReceive); - int post_rc = -1; - if (control_posted && data_posted) + wr.wr_id = reinterpret_cast(work.get()) | 1U; + wr.sg_list = &sge; + wr.num_sge = 1; + ibv_recv_wr *bad = nullptr; + const bool slot_posted = slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive) || + slot->transition(BufferOwnership::CallerOwned, BufferOwnership::PostedReceive); + int post_rc = -1; + if (slot_posted) post_rc = ibv_post_recv(reinterpret_cast(conn->transport_handle), &wr, &bad); - if (!control_posted || !data_posted || post_rc != 0) { - CRB_LOG_WARN("rdma receive: post failed rc={} control_state={} data_state={}", post_rc, - static_cast(control->ownership.load(std::memory_order_relaxed)), - data == nullptr ? 0U : static_cast(data->ownership.load(std::memory_order_relaxed))); - control->ownership.store(BufferOwnership::Available, std::memory_order_release); - if (data != nullptr) - data->ownership.store(BufferOwnership::Available, std::memory_order_release); - control->release(); - if (data != nullptr) - data->release(); + if (!slot_posted || post_rc != 0) { + CRB_LOG_WARN("rdma receive: post failed rc={} slot_state={}", post_rc, + static_cast(slot->ownership.load(std::memory_order_relaxed))); + slot->ownership.store(BufferOwnership::Available, std::memory_order_release); + slot->release(); return false; } @@ -443,7 +421,7 @@ Buffer *RdmaTransport::register_buffer(Buffer *buf) if (registered == nullptr) { return nullptr; } - std::memcpy(registered->data, buf->data, buf->len); + std::memcpy(registered->data, buf->payload_data(), buf->len); registered->len = buf->len; registered->request_id = buf->request_id; registered->stream_id = buf->stream_id; @@ -957,26 +935,17 @@ bool RdmaTransport::prime_receive(const std::shared_ptr &connection) constexpr uint32_t kInitialReceiveDepth = 8; BufferPool *pool = connection->pool(); for (uint32_t i = 0; i < kInitialReceiveDepth; ++i) { - Buffer *control = pool->alloc(RDMA_CONTROL_CAPACITY); - Buffer *data = pool->alloc(1U << 20); - if (control == nullptr || data == nullptr) { - - if (control != nullptr) - control->release(); - if (data != nullptr) - data->release(); + Buffer *slot = pool->alloc(RDMA_WIRE_SLOT_CAPACITY); + if (slot == nullptr) { return false; } - if (!post_receive(connection.get(), control, data)) { - - control->release(); - data->release(); + if (!post_receive(connection.get(), slot)) { + slot->release(); return false; } - // post_receive retains the work-request references; release the caller - // references so the completion owns the pair until it is consumed. - control->release(); - data->release(); + // post_receive retains the work-request reference; the completion + // owns the slot after the caller reference is released. + slot->release(); } return true; } @@ -990,12 +959,8 @@ void RdmaTransport::release_completed_receive(ibv_wc &wc) auto quarantine_and_release = [&] { stats_.cq_errors.fetch_add(1, std::memory_order_relaxed); work->conn->close(); - (void)work->control->quarantine(); - if (work->data != nullptr) - (void)work->data->quarantine(); - work->control->release(); - if (work->data != nullptr) - work->data->release(); + (void)work->slot->quarantine(); + work->slot->release(); delete work; }; if (wc.status != IBV_WC_SUCCESS) { @@ -1005,89 +970,62 @@ void RdmaTransport::release_completed_receive(ibv_wc &wc) quarantine_and_release(); return; } - if (!work->control->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight) || - (work->data != nullptr && - !work->data->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight))) { + if (!work->slot->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight)) { quarantine_and_release(); return; } - const uint32_t control_bytes = work->control->capacity; - if (control_bytes < HEADER_SIZE) { - CRB_LOG_WARN("rdma receive: control capacity too small capacity={} header={}", control_bytes, HEADER_SIZE); + const uint32_t slot_capacity = work->slot->capacity; + if (slot_capacity < HEADER_SIZE) { + CRB_LOG_WARN("rdma receive: slot capacity too small capacity={} header={}", slot_capacity, HEADER_SIZE); quarantine_and_release(); return; } - const Header header = parse_header(work->control->data); + const Header header = parse_header(work->slot->data); const uint32_t control_end = static_cast(HEADER_SIZE) + header.msg_size; const uint64_t expected_bytes = static_cast(control_end) + header.data_size; if (header.magic != MAGIC || header.msg_offset < HEADER_SIZE || - static_cast(header.msg_offset) + header.msg_size > control_bytes || expected_bytes > wc.byte_len || - (header.data_size > 0 && (work->data == nullptr || header.data_size > work->data->capacity))) { + static_cast(header.msg_offset) + header.msg_size > slot_capacity || expected_bytes > wc.byte_len || + expected_bytes > slot_capacity) { CRB_LOG_WARN( "rdma receive: invalid frame magic={} type={} msg_size={} data_size={} offset={} capacity={} byte_len={}", - header.magic, header.msg_type, header.msg_size, header.data_size, header.msg_offset, control_bytes, + header.magic, header.msg_type, header.msg_size, header.data_size, header.msg_offset, slot_capacity, wc.byte_len); quarantine_and_release(); return; } - work->control->len = control_end; - if (work->data != nullptr) - work->data->len = header.data_size; auto *frame = new Frame; frame->header = header; if (header.msg_size != 0) { - frame->control.assign(work->control->data + header.msg_offset, - work->control->data + header.msg_offset + header.msg_size); - extract_control_fields(work->control->data + header.msg_offset, header.msg_size, frame->request_id, + frame->control.assign(work->slot->data + header.msg_offset, + work->slot->data + header.msg_offset + header.msg_size); + extract_control_fields(work->slot->data + header.msg_offset, header.msg_size, frame->request_id, frame->rpc_create_nano, frame->response_create_nano); } - if (work->data != nullptr && header.data_size != 0) { - if (!work->data->transition(BufferOwnership::ReceiveInFlight, BufferOwnership::CallerOwned)) { - delete frame; - quarantine_and_release(); - return; - } - frame->data_buf = work->data; - work->data = nullptr; - } - Buffer *repost_control = work->control->ref_clone(); - Buffer *repost_data = (header.data_size == 0 && work->data != nullptr) ? work->data->ref_clone() : nullptr; - auto *repost = new RepostContext{this, work->conn, repost_control, repost_data}; + work->slot->view_offset = control_end; + work->slot->capacity = slot_capacity - control_end; + work->slot->len = header.data_size; + work->slot->ownership.store(BufferOwnership::CallerOwned, std::memory_order_release); + frame->data_buf = work->slot; + Buffer *repost_slot = work->slot->ref_clone(); + auto *repost = new RepostContext{this, work->conn, repost_slot}; frame->release_callback = &RdmaTransport::repost_frame_release; frame->release_context = repost; - work->control->ownership.store(BufferOwnership::Available, std::memory_order_release); - if (work->data != nullptr && header.data_size == 0) - work->data->ownership.store(BufferOwnership::Available, std::memory_order_release); - work->control->release(); - if (work->data != nullptr && header.data_size != 0) - work->data = nullptr; - if (work->data != nullptr) - work->data->release(); work->conn->on_frame(frame); stats_.receives.fetch_add(1, std::memory_order_relaxed); delete work; } -void RdmaTransport::repost_frame_release(void *context, Buffer *data) +void RdmaTransport::repost_frame_release(void *context, Buffer *) { auto *repost = static_cast(context); if (repost == nullptr) return; - Buffer *receive_data = data != nullptr ? data : repost->data; - if (repost->transport == nullptr || - !repost->transport->post_receive(repost->connection, repost->control, receive_data)) { - repost->control->release(); - if (receive_data != nullptr && receive_data != data) - receive_data->release(); - } - else { - // The callback owns the caller references while post_receive owns - // the WR references. - repost->control->release(); - if (receive_data != nullptr && receive_data != data) - receive_data->release(); - } + repost->slot->reset_view(); + repost->slot->ownership.store(BufferOwnership::Available, std::memory_order_release); + if (repost->transport != nullptr) + (void)repost->transport->post_receive(repost->connection, repost->slot); + repost->slot->release(); delete repost; } diff --git a/lib/crowdb-rpc/tests/rdma_capability_test.cpp b/lib/crowdb-rpc/tests/rdma_capability_test.cpp index 78137af9b..02466b5b3 100644 --- a/lib/crowdb-rpc/tests/rdma_capability_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_capability_test.cpp @@ -79,26 +79,3 @@ TEST(RdmaCapabilityTest, SetupFailureCanFallbackBeforePayload) EXPECT_TRUE(state.fallback_to_tcp()); EXPECT_EQ(state.state(), RdmaPathState::TcpFallback); } - -TEST(RdmaCapabilityTest, BuildsControlAndDataSgesWithoutConcatenation) -{ - SystemBufferPool pool(2); - auto *control = pool.alloc(1024); - auto *data = pool.alloc(64U << 10); - ASSERT_NE(control, nullptr); - ASSERT_NE(data, nullptr); - const uint8_t control_bytes[] = {1, 2}; - const uint8_t data_bytes[] = {3, 4, 5}; - control->write(control_bytes, sizeof(control_bytes)); - data->write(data_bytes, sizeof(data_bytes)); - - crowdb::rpc::RdmaFrameSges sges; - ASSERT_TRUE(crowdb::rpc::build_rdma_frame_sges(control, data, &sges)); - EXPECT_EQ(sges.count, 2); - EXPECT_EQ(sges.entries[0].address, reinterpret_cast(control->data)); - EXPECT_EQ(sges.entries[0].length, sizeof(control_bytes)); - EXPECT_EQ(sges.entries[1].address, reinterpret_cast(data->data)); - EXPECT_EQ(sges.entries[1].length, sizeof(data_bytes)); - data->release(); - control->release(); -} From 999609abc6840d58f7cdeb350cacc687ddf8f774 Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 18 Oct 2026 21:55:22 +0800 Subject: [PATCH 009/258] Define shared CQ polling model --- doc/design/rpc/design-crowdb-rpc-rdma.md | 46 ++++++++++++++++++------ doc/working/plan-rdma-rpc-transport.md | 22 ++++++++---- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index 2a85758cd..f916586af 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -107,20 +107,38 @@ SDK lease budget, or a full DiskIO write queue applies backpressure to the existing producer or aggregation layer. RDMA adds no application-level credit message. A slot is never posted twice concurrently. -## 5. Event and completion model +## 5. Shared CQ and completion model -The RDMA completion-channel fd and RDMA-CM event fd are registered with the -existing epoll engine. Epoll provides wakeups; the RPC worker then: +Connections share a bounded CQ pool. Each CQ owns one completion channel and +is assigned to a polling worker. Every QP uses the same shared CQ for SEND and +RECV completions. The CQ maps `qp_num` to the owning RPC connection; a +completion never requires scanning every connection CQ. -1. acknowledges and re-arms the channel event; -2. drains CM events and updates connection state; -3. polls bounded batches from the CQ; -4. resolves each work request through its `wr_id` record; -5. dispatches frame completion or buffer ownership transfer. +The polling worker is the primary completion path: + +1. polls the assigned CQ in a bounded busy-poll loop; +2. dispatches all currently available completions; +3. after the configured empty-poll budget, arms CQ notification with + `ibv_req_notify_cq`; +4. polls once more to close the notification race; +5. registers the CQ completion fd with epoll only while idle. + +Epoll is a wakeup path, not the completion processor. When the CQ completion +fd becomes readable, the worker acknowledges the event, removes the idle +state, and returns to polling. The RDMA-CM event fd follows the same worker +ownership rule but remains a connection-management event source. + +The worker processes a CQ batch as follows: + +1. resolves the completion's `qp_num` to a connection; +2. resolves the work request through its stable `wr_id` record; +3. dispatches SEND or RECV completion; +4. returns or quarantines the slot according to completion status. The RPC library controls polling. FFI receives only a completion callback or a -buffer lease after C++ has validated the frame and resolved ownership. There is -no independent RDMA polling thread in the default path. +buffer lease after C++ has validated the frame and resolved ownership. A +connection does not own a CQ or completion channel; it owns only its QP and +slot rings. ## 6. Failure and fallback @@ -159,6 +177,8 @@ The transport configuration supplies defaults when the transport is created: rdma_wire_slot_size = "1MiB" rdma_send_slot_count = 64 rdma_receive_slot_count = 64 +rdma_cq_count = 4 +rdma_poll_empty_budget = 50000 rdma_registered_bytes = "128MiB" ``` @@ -184,7 +204,11 @@ slot class only when the complete wire message fits that class. unresolved. - **I7:** A SEND uses one contiguous registered slot and a RECV uses one complete posted slot; sender SGE boundaries never define receiver parsing. -- **I8:** Unsupported RDMA leaves the TCP service usable and does not require +- **I8:** Every QP is assigned to exactly one shared CQ, and every shared CQ is + polled by exactly one worker at a time. +- **I9:** Epoll notification only wakes an idle CQ polling worker; completion + dispatch occurs in the polling worker. +- **I10:** Unsupported RDMA leaves the TCP service usable and does not require RDMA libraries or devices at runtime. ## 10. Scope boundary diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 36daada48..20c010ade 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -16,7 +16,7 @@ The transport foundation now includes validated endpoint capability data (protocol, generation, device, GID, port, and supported capacity classes), explicit RDMA path states, and a pre-payload-only TCP fallback predicate. Implemented since this checkpoint: active/passive RDMA-CM setup, RC QP -creation, one-MR arena-backed wire slots, one-SGE SEND/RECV posting, bounded receive posting, completion-channel +creation, one-MR arena-backed wire slots, one-SGE SEND/RECV posting, bounded receive posting, and the initial completion-channel polling with CQ acknowledgement, error quarantine, validated receive-frame dispatch into `Connection::on_frame`, and epoll external-fd registration. The CQ path is now drained by an epoll worker through `completion_fd()` and @@ -42,12 +42,11 @@ multi-node validation is deferred to [the separate follow-up plan](plan-rdma-mul Completed: - active and passive RDMA-CM address, route, connect, and accept flow; -- connection-level PD, CQ, completion channel, and registered send/receive - pools; +- connection-level PD and registered send/receive slot pools; - initial receive posting, CQ completion dispatch, owner-release repost, and RNR retry configuration; - RDMA-to-TCP capability negotiation and pre-payload fallback wiring; -- epoll registration for CM, transport CQ, and connection CQ events; +- epoll registration for CM and CQ wakeup events; - live HCA local-loop CM/QP/send/receive E2E with all frames received and clean CQ status, and clean client/server exit; - actual `RpcServer`/`RpcClient` worker-path request/response round trip with @@ -63,6 +62,10 @@ Remaining: - run the single-node HCA local-loop multi-container OCI profile against the built image; - remove temporary hardware diagnostics, normalize failure logging, and run the full C++ format/tree-lint gates. +- replace per-connection CQs with a shared CQ pool and polling workers; +- make epoll CQ notifications wake idle polling workers instead of dispatching + completions directly; +- rerun all RDMA benchmark concurrency and payload cases after the CQ change. ## Phase 1: common transport seam @@ -99,9 +102,12 @@ Remaining: `lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp`. - [ ] **Drain CM events**: acknowledge all received CM events, update the connection state, and re-arm the CM channel where required. -- [x] **Drain CQ events**: acknowledge and re-arm completion notifications, then - poll bounded batches until the current CQ work is drained or the worker - budget is reached. No independent polling thread. +- [ ] **Create shared CQ pool**: allocate a bounded CQ/channel set and assign + each QP to one CQ by round-robin worker ownership. +- [ ] **Add CQ polling workers**: busy-poll each shared CQ within an empty-poll + budget, then arm CQ notification and hand the fd to epoll while idle. +- [ ] **Make epoll a CQ wakeup**: acknowledge the CQ event and resume polling; + epoll must not scan or dispatch every connection CQ. - [ ] **Make stop wakeable**: stop must wake epoll, stop new submissions, drain known work, and join without waiting forever on an idle channel. @@ -143,6 +149,8 @@ Integration tests: - RDMA-disabled build and host selects TCP without CM calls; - mocked CM/CQ setup, reconnect, CQ error, flush completion, and shutdown; +- multiple connections sharing one CQ pool preserve completion ownership and + request correlation; - epoll external event wakes worker and drains all current CQ completions; - send/receive sequence preserves exact control/data lengths inside one slot; - pre-payload failure retries idempotent request on TCP; From abb9bbe96f602adfca5ef64f5a888973ba258a16 Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 18 Oct 2026 22:11:12 +0800 Subject: [PATCH 010/258] Share RDMA CQ across connections --- doc/working/plan-rdma-rpc-transport.md | 27 ++-- .../transport/rdma/rdma_transport.h | 6 +- .../src/transport/rdma/rdma_transport.cpp | 131 +++--------------- 3 files changed, 39 insertions(+), 125 deletions(-) diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 20c010ade..51a17ddfd 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -19,8 +19,11 @@ Implemented since this checkpoint: active/passive RDMA-CM setup, RC QP creation, one-MR arena-backed wire slots, one-SGE SEND/RECV posting, bounded receive posting, and the initial completion-channel polling with CQ acknowledgement, error quarantine, validated receive-frame dispatch into `Connection::on_frame`, and epoll external-fd registration. -The CQ path is now drained by an epoll worker through `completion_fd()` and -`drain_cq()` rather than a private polling thread. Capability handshake bytes +The CQ path now uses one transport-wide shared CQ for all QPs. Its completion +channel is registered with one epoll engine; the worker drains completions with +a bounded busy-poll budget and rearms notification after the queue is idle. +Epoll is therefore only the wakeup path, while completion dispatch remains in +the CQ drain. Capability handshake bytes have a fixed wire representation with round-trip tests. Negotiated capability exchange through the existing TCP handshake, common transport path selection/fallback, receive repost after owner release, and two-peer OCI E2E @@ -62,10 +65,12 @@ Remaining: - run the single-node HCA local-loop multi-container OCI profile against the built image; - remove temporary hardware diagnostics, normalize failure logging, and run the full C++ format/tree-lint gates. -- replace per-connection CQs with a shared CQ pool and polling workers; -- make epoll CQ notifications wake idle polling workers instead of dispatching - completions directly; +- add a configurable multi-CQ pool when one shared CQ becomes a bottleneck; +- move CQ draining to a dedicated polling worker if engine worker contention + becomes measurable; - rerun all RDMA benchmark concurrency and payload cases after the CQ change. + Completed on the local HCA at 128B with 0 errors for 1, 4, 8, and 32 + connections; the 4-connection case reached about 442k ops/s. ## Phase 1: common transport seam @@ -102,12 +107,12 @@ Remaining: `lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp`. - [ ] **Drain CM events**: acknowledge all received CM events, update the connection state, and re-arm the CM channel where required. -- [ ] **Create shared CQ pool**: allocate a bounded CQ/channel set and assign - each QP to one CQ by round-robin worker ownership. -- [ ] **Add CQ polling workers**: busy-poll each shared CQ within an empty-poll - budget, then arm CQ notification and hand the fd to epoll while idle. -- [ ] **Make epoll a CQ wakeup**: acknowledge the CQ event and resume polling; - epoll must not scan or dispatch every connection CQ. +- [x] **Create shared CQ**: all QPs use the transport-wide CQ and completion + channel, so multiple connections share one completion stream. +- [x] **Add bounded CQ polling**: drain the shared CQ with an empty-poll + budget, then arm CQ notification for the next epoll wakeup. +- [x] **Make epoll a CQ wakeup**: acknowledge the CQ event and resume polling; + epoll does not scan or dispatch per-connection CQs. - [ ] **Make stop wakeable**: stop must wake epoll, stop new submissions, drain known work, and join without waiting forever on an idle channel. diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index c869e9eae..eda0026fd 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -104,9 +104,9 @@ class RdmaTransport : public Transport void stop(); [[nodiscard]] int completion_fd() const; [[nodiscard]] int cm_event_fd() const; - int drain_cq(uint32_t max_completions = 32); + int drain_cq(uint32_t max_completions = 256); void register_events(SocketEngine *engine); - int handle_external_event(uint64_t tag, uint32_t max_completions = 32); + int handle_external_event(uint64_t tag, uint32_t max_completions = 256); void set_connection_handler(ConnectionHandler handler) { @@ -138,8 +138,6 @@ class RdmaTransport : public Transport uint8_t port_ = 1; std::vector cm_ids_; std::unordered_map> cm_connections_; - std::unordered_map cm_cqs_; - std::unordered_map cm_channels_; SocketEngine *event_engine_ = nullptr; std::mutex cm_mu_; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index e20020147..4fa8ff3df 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -446,16 +446,6 @@ void RdmaTransport::shutdown() for (rdma_cm_id *id : cm_ids_) { if (id->qp != nullptr) rdma_destroy_qp(id); - if (auto it = cm_cqs_.find(id); it != cm_cqs_.end()) { - ibv_destroy_cq(it->second); - cm_cqs_.erase(it); - } - if (auto it = cm_channels_.find(id); it != cm_channels_.end()) { - if (event_engine_ != nullptr) - event_engine_->remove_external_fd(it->second->fd); - ibv_destroy_comp_channel(it->second); - cm_channels_.erase(it); - } rdma_destroy_id(id); } cm_ids_.clear(); @@ -554,37 +544,14 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int } rdma_ack_cm_event(event); - ibv_comp_channel *connection_channel = ibv_create_comp_channel(id->verbs); - if (connection_channel != nullptr) - make_nonblocking(connection_channel->fd); - ibv_cq *connection_cq = - connection_channel == nullptr ? nullptr : ibv_create_cq(id->verbs, 2048, nullptr, connection_channel, 0); - - if (connection_cq == nullptr) { - - if (connection_channel != nullptr) - ibv_destroy_comp_channel(connection_channel); - rdma_destroy_id(id); - return nullptr; - } - if (ibv_req_notify_cq(connection_cq, 0) != 0) { - CRB_LOG_WARN("rdma connect: arm connection CQ notifications failed errno={}", errno); - ibv_destroy_cq(connection_cq); - ibv_destroy_comp_channel(connection_channel); - rdma_destroy_id(id); - return nullptr; - } ibv_device_attr device_attributes{}; if (ibv_query_device(id->verbs, &device_attributes) != 0) { - - ibv_destroy_cq(connection_cq); - ibv_destroy_comp_channel(connection_channel); rdma_destroy_id(id); return nullptr; } ibv_qp_init_attr qp_attr{}; - qp_attr.send_cq = connection_cq; - qp_attr.recv_cq = connection_cq; + qp_attr.send_cq = cq_; + qp_attr.recv_cq = cq_; qp_attr.qp_type = IBV_QPT_RC; qp_attr.cap.max_send_wr = std::min(1024, device_attributes.max_qp_wr); qp_attr.cap.max_recv_wr = std::min(1024, device_attributes.max_qp_wr); @@ -596,8 +563,6 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int if (rdma_create_qp(id, id->pd, &qp_attr) != 0) { - ibv_destroy_cq(connection_cq); - ibv_destroy_comp_channel(connection_channel); rdma_destroy_id(id); return nullptr; } @@ -610,8 +575,6 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int CRB_LOG_WARN("rdma connect: rdma_connect failed addr={} errno={}", addr, errno); rdma_destroy_qp(id); - ibv_destroy_cq(connection_cq); - ibv_destroy_comp_channel(connection_channel); rdma_destroy_id(id); return nullptr; } @@ -622,7 +585,6 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int if (event != nullptr) rdma_ack_cm_event(event); rdma_destroy_qp(id); - ibv_destroy_cq(connection_cq); rdma_destroy_id(id); return nullptr; } @@ -632,8 +594,6 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int pools->recv = std::make_unique(id->pd, 64U << 20, 1024); if (pools->send == nullptr || pools->recv == nullptr) { rdma_destroy_qp(id); - ibv_destroy_cq(connection_cq); - ibv_destroy_comp_channel(connection_channel); rdma_destroy_id(id); return nullptr; } @@ -651,11 +611,7 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int std::scoped_lock lock(cm_mu_); cm_ids_.push_back(id); cm_connections_.emplace(id, connection); - cm_cqs_.emplace(id, connection_cq); - cm_channels_.emplace(id, connection_channel); cm_pools_.emplace(id, std::move(pools)); - if (event_engine_ != nullptr) - event_engine_->add_external_fd(connection_channel->fd, 3); } return connection; } @@ -713,29 +669,9 @@ std::shared_ptr RdmaTransport::accept() rdma_cm_id *id = event->id; rdma_ack_cm_event(event); - ibv_comp_channel *connection_channel = ibv_create_comp_channel(id->verbs); - if (connection_channel != nullptr) - make_nonblocking(connection_channel->fd); - ibv_cq *connection_cq = - connection_channel == nullptr ? nullptr : ibv_create_cq(id->verbs, 2048, nullptr, connection_channel, 0); - - if (connection_cq == nullptr) { - - if (connection_channel != nullptr) - ibv_destroy_comp_channel(connection_channel); - rdma_destroy_id(id); - return nullptr; - } - if (ibv_req_notify_cq(connection_cq, 0) != 0) { - CRB_LOG_WARN("rdma accept: arm connection CQ notifications failed errno={}", errno); - ibv_destroy_cq(connection_cq); - ibv_destroy_comp_channel(connection_channel); - rdma_destroy_id(id); - return nullptr; - } ibv_qp_init_attr qp_attr{}; - qp_attr.send_cq = connection_cq; - qp_attr.recv_cq = connection_cq; + qp_attr.send_cq = cq_; + qp_attr.recv_cq = cq_; qp_attr.qp_type = IBV_QPT_RC; qp_attr.cap.max_send_wr = 1024; qp_attr.cap.max_recv_wr = 1024; @@ -746,8 +682,6 @@ std::shared_ptr RdmaTransport::accept() qp_attr.qp_context = id; if (rdma_create_qp(id, id->pd, &qp_attr) != 0) { - ibv_destroy_cq(connection_cq); - ibv_destroy_comp_channel(connection_channel); rdma_destroy_id(id); return nullptr; } @@ -761,8 +695,6 @@ std::shared_ptr RdmaTransport::accept() if (accept_rc != 0) { CRB_LOG_WARN("rdma accept: rdma_accept failed errno={}", errno); rdma_destroy_qp(id); - ibv_destroy_cq(connection_cq); - ibv_destroy_comp_channel(connection_channel); rdma_destroy_id(id); return nullptr; } @@ -775,8 +707,6 @@ std::shared_ptr RdmaTransport::accept() if (event != nullptr) rdma_ack_cm_event(event); rdma_destroy_qp(id); - ibv_destroy_cq(connection_cq); - ibv_destroy_comp_channel(connection_channel); rdma_destroy_id(id); return nullptr; } @@ -787,8 +717,6 @@ std::shared_ptr RdmaTransport::accept() pools->recv = std::make_unique(id->pd, 64U << 20, 1024); if (pools->send == nullptr || pools->recv == nullptr) { rdma_destroy_qp(id); - ibv_destroy_cq(connection_cq); - ibv_destroy_comp_channel(connection_channel); rdma_destroy_id(id); return nullptr; } @@ -798,8 +726,6 @@ std::shared_ptr RdmaTransport::accept() connection->transport_handle = reinterpret_cast(id->qp); if (!prime_receive(connection)) { rdma_destroy_qp(id); - ibv_destroy_cq(connection_cq); - ibv_destroy_comp_channel(connection_channel); rdma_destroy_id(id); return nullptr; } @@ -808,11 +734,7 @@ std::shared_ptr RdmaTransport::accept() std::scoped_lock lock(cm_mu_); cm_ids_.push_back(id); cm_connections_.emplace(id, connection); - cm_cqs_.emplace(id, connection_cq); - cm_channels_.emplace(id, connection_channel); cm_pools_.emplace(id, std::move(pools)); - if (event_engine_ != nullptr) - event_engine_->add_external_fd(connection_channel->fd, 3); } return connection; } @@ -839,14 +761,22 @@ int RdmaTransport::cm_event_fd() const int RdmaTransport::drain_cq(uint32_t max_completions) { - uint32_t drained = 0; + uint32_t drained = 0; + uint32_t empty_polls = 0; + constexpr uint32_t kEmptyPollBudget = 50000; std::array completions{}; auto poll_one = [&](ibv_cq *cq) { while (drained < max_completions && cq != nullptr) { const int batch = ibv_poll_cq(cq, static_cast(std::min(32, max_completions - drained)), completions.data()); - if (batch <= 0) + if (batch < 0) break; + if (batch == 0) { + if (++empty_polls >= kEmptyPollBudget) + break; + continue; + } + empty_polls = 0; for (int i = 0; i < batch; ++i) { const auto opcode = completions[static_cast(i)].opcode; const auto wr_id = completions[static_cast(i)].wr_id; @@ -865,18 +795,9 @@ int RdmaTransport::drain_cq(uint32_t max_completions) void *context = nullptr; if (ibv_get_cq_event(comp_channel_, &event_cq, &context) == 0) { ibv_ack_cq_events(event_cq, 1); - ibv_req_notify_cq(event_cq, 0); - poll_one(event_cq); - } - } - { - std::scoped_lock lock(cm_mu_); - for (const auto &[id, cq] : cm_cqs_) { - (void)id; - poll_one(cq); - if (drained >= max_completions) - break; } + poll_one(cq_); + (void)ibv_req_notify_cq(cq_, 0); } return static_cast(drained); } @@ -885,6 +806,11 @@ void RdmaTransport::register_events(SocketEngine *engine) { if (engine == nullptr) return; + // A shared CQ has one completion channel. Register that channel with one + // epoll engine only; registering the same fd in every engine causes + // duplicate wakeups and lets different workers race the CQ event. + if (event_engine_ != nullptr && event_engine_ != engine) + return; event_engine_ = engine; if (completion_fd() >= 0) engine->add_external_fd(completion_fd(), 1); @@ -908,21 +834,6 @@ int RdmaTransport::handle_external_event(uint64_t tag, uint32_t max_completions) } return accepted; } - if (tag == 3) { - { - std::scoped_lock lock(cm_mu_); - for (const auto &[id, channel] : cm_channels_) { - (void)id; - ibv_cq *event_cq = nullptr; - void *context = nullptr; - if (ibv_get_cq_event(channel, &event_cq, &context) == 0) { - ibv_ack_cq_events(event_cq, 1); - (void)ibv_req_notify_cq(event_cq, 0); - } - } - } - return drain_cq(max_completions); - } return 0; } From 08b16065f83e5b1ebaeddb8e0b2a57d0e1d2e443 Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 18 Oct 2026 22:12:08 +0800 Subject: [PATCH 011/258] Record RDMA CQ benchmark sweep --- doc/working/plan-rdma-rpc-transport.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 51a17ddfd..8f33eccc6 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -69,8 +69,9 @@ Remaining: - move CQ draining to a dedicated polling worker if engine worker contention becomes measurable; - rerun all RDMA benchmark concurrency and payload cases after the CQ change. - Completed on the local HCA at 128B with 0 errors for 1, 4, 8, and 32 - connections; the 4-connection case reached about 442k ops/s. + Completed on the local HCA at 128B, 1KiB, and 4KiB with 0 errors for 1, 4, + 8, and 32 connections. The 4-connection cases reached about 442k ops/s + (128B), 414k ops/s (1KiB), and 376k ops/s (4KiB). ## Phase 1: common transport seam From f0a77f977609e9da2bd59e7f69b47352efd05f7f Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 18 Oct 2026 22:17:13 +0800 Subject: [PATCH 012/258] Record RDMA benchmark baseline --- tools/benchmark/bench-rdma-rpc-regression.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tools/benchmark/bench-rdma-rpc-regression.sh b/tools/benchmark/bench-rdma-rpc-regression.sh index b6bc0ddea..6ec1a1225 100755 --- a/tools/benchmark/bench-rdma-rpc-regression.sh +++ b/tools/benchmark/bench-rdma-rpc-regression.sh @@ -1,6 +1,23 @@ #!/usr/bin/env bash # RDMA RPC echo benchmark. Mirrors bench-rpc-regression.sh with an HCA-backed # connection so transport throughput and latency can be compared directly. +# +# Baseline (2026-10-18, HCA rocep35s0f0:1, 3s per case, 0 errors): +# payload workers/connections ops/s avg-us p50-us p99-us +# 128 B 1/1 43,161 22.1 65 65 +# 128 B 4/4 441,779 143.6 124 215 +# 128 B 8/8 732,084 695.0 675 917 +# 128 B 16/32 1,186,876 832.9 419 5,439 +# 1 KiB 1/1 43,003 22.2 65 65 +# 1 KiB 4/4 414,466 153.2 136 221 +# 1 KiB 8/8 707,931 718.3 704 954 +# 1 KiB 16/32 1,120,659 882.5 448 6,193 +# 4 KiB 1/1 39,625 24.0 65 65 +# 4 KiB 4/4 376,107 168.8 149 266 +# 4 KiB 8/8 582,940 872.2 811 1,810 +# 4 KiB 16/32 834,076 1,184.5 606 6,881 +# These are reference values for the shared-CQ implementation; reruns should +# record the exact HCA, duration, payload, and worker/connection configuration. set -euo pipefail cd "$(dirname "$0")/../.." From d2103e6f681f278624d688b0c39b2d6ba9b8059c Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 18 Oct 2026 22:25:04 +0800 Subject: [PATCH 013/258] Harden RDMA connection teardown --- doc/working/plan-rdma-rpc-transport.md | 20 +++++++++-------- .../transport/rdma/rdma_transport.h | 4 ++++ .../src/transport/rdma/rdma_transport.cpp | 22 +++++++++++++++++++ 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 8f33eccc6..794c54bcb 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -23,7 +23,9 @@ The CQ path now uses one transport-wide shared CQ for all QPs. Its completion channel is registered with one epoll engine; the worker drains completions with a bounded busy-poll budget and rearms notification after the queue is idle. Epoll is therefore only the wakeup path, while completion dispatch remains in -the CQ drain. Capability handshake bytes +the CQ drain. Stop now wakes the owning engine, CQ polling counters are exposed, +and CM disconnect events close and retire their connection pools before QP/ID +teardown. Capability handshake bytes have a fixed wire representation with round-trip tests. Negotiated capability exchange through the existing TCP handshake, common transport path selection/fallback, receive repost after owner release, and two-peer OCI E2E @@ -96,9 +98,9 @@ Remaining: - [x] **Implement active/passive CM flow**: create IDs, bind/listen or resolve address/route, create QP, connect/accept, and publish ready only after initial receive buffers are posted. Files: RDMA transport sources. -- [ ] **Implement connection teardown**: map CM disconnect, QP error, and CQ - error to one drain/fail transition. Stop new work, resolve known completions, - quarantine uncertain buffers, and wake blocked submitters. +- [x] **Implement connection teardown**: map CM disconnect and CQ completion + errors to one close/fail transition. Retire the connection pools before QP/ID + destruction, quarantine uncertain buffers, and wake the owning engine on stop. ## Phase 3: event/poll loop @@ -114,8 +116,8 @@ Remaining: budget, then arm CQ notification for the next epoll wakeup. - [x] **Make epoll a CQ wakeup**: acknowledge the CQ event and resume polling; epoll does not scan or dispatch per-connection CQs. -- [ ] **Make stop wakeable**: stop must wake epoll, stop new submissions, drain - known work, and join without waiting forever on an idle channel. +- [x] **Make stop wakeable**: stop wakes the owning epoll engine, then shutdown + drains known CQ work before destroying IDs and pools. ## Phase 4: frame submission and completion @@ -137,9 +139,9 @@ Remaining: fails before payload, select TCP with the same request identity. - [ ] **Reject mixed completion**: after any payload submission, fail the stream on uncertainty and never enqueue a TCP suffix. -- [ ] **Add metrics**: expose RDMA capability, selected device, QP state, CQ - depth, posted receives, RNR/retry errors, fallback count, and quarantined - bytes. +- [ ] **Add metrics**: posted receives, fallback count, quarantined bytes, CQ + poll/empty-poll counts, and CM disconnects are exposed; QP state, CQ depth, + and RNR/retry counters remain. ## Tests diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index eda0026fd..4d38dbe0d 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -77,6 +77,9 @@ class RdmaTransport : public Transport std::atomic fallback_count{0}; std::atomic posted_receives{0}; std::atomic quarantined_bytes{0}; + std::atomic cq_polls{0}; + std::atomic cq_empty_polls{0}; + std::atomic cm_disconnects{0}; }; RdmaTransport(); @@ -170,6 +173,7 @@ class RdmaTransport : public Transport }; std::unordered_map> cm_pools_; + std::vector> retired_pools_; void release_completed_frame(struct ibv_wc &wc); void release_completed_receive(struct ibv_wc &wc); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index 4fa8ff3df..ee05cbd11 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -657,8 +657,26 @@ std::shared_ptr RdmaTransport::accept() if (rdma_get_cm_event(cm_channel_, &event) != 0 || event->event != RDMA_CM_EVENT_CONNECT_REQUEST) { if (event != nullptr) { rdma_cm_id *event_id = event->id; + const int event_type = event->event; rdma_ack_cm_event(event); if (event_id != nullptr && event_id != listen_id_) { + std::shared_ptr closed_connection; + { + std::scoped_lock lock(cm_mu_); + if (auto it = cm_connections_.find(event_id); it != cm_connections_.end()) { + closed_connection = it->second; + cm_connections_.erase(it); + } + if (auto it = cm_pools_.find(event_id); it != cm_pools_.end()) { + retired_pools_.push_back(std::move(it->second)); + cm_pools_.erase(it); + } + cm_ids_.erase(std::remove(cm_ids_.begin(), cm_ids_.end(), event_id), cm_ids_.end()); + } + if (closed_connection != nullptr) + closed_connection->close(); + if (event_type == RDMA_CM_EVENT_DISCONNECTED || event_type == RDMA_CM_EVENT_CONNECT_ERROR) + stats_.cm_disconnects.fetch_add(1, std::memory_order_relaxed); if (event_id->qp != nullptr) rdma_destroy_qp(event_id); rdma_destroy_id(event_id); @@ -747,6 +765,8 @@ void RdmaTransport::start() void RdmaTransport::stop() { running_.store(false, std::memory_order_release); + if (event_engine_ != nullptr) + event_engine_->notify_worker(); } int RdmaTransport::completion_fd() const @@ -761,6 +781,7 @@ int RdmaTransport::cm_event_fd() const int RdmaTransport::drain_cq(uint32_t max_completions) { + stats_.cq_polls.fetch_add(1, std::memory_order_relaxed); uint32_t drained = 0; uint32_t empty_polls = 0; constexpr uint32_t kEmptyPollBudget = 50000; @@ -772,6 +793,7 @@ int RdmaTransport::drain_cq(uint32_t max_completions) if (batch < 0) break; if (batch == 0) { + stats_.cq_empty_polls.fetch_add(1, std::memory_order_relaxed); if (++empty_polls >= kEmptyPollBudget) break; continue; From 7daa0bf35b542c61e9e2a0d151df2967d1dbda6f Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 18 Oct 2026 22:25:17 +0800 Subject: [PATCH 014/258] Update R239 local-loop status --- doc/working/plan-rdma.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index 67b6fb0a9..f5918a730 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -121,9 +121,9 @@ ownership remains unresolved. correlation, fallback, and shutdown. - [ ] **Mock verification**: cover CQ success/error/flush, CM transitions, receive exhaustion, fallback before payload, and no mixed TCP/RDMA stream. -- [ ] **Local-loop verification**: run `setup-rdma.sh local` followed by - `run-local-loop.sh local`; verify both CM roles reach `ESTABLISHED`, all - frames are received, and exact lengths and sequence values are preserved. +- [x] **Local-loop verification**: run `setup-rdma.sh local` followed by + `run-local-loop.sh local`; both CM roles, framed SEND/RECV, receive repost, + and exact payload lengths have passed on the prepared HCA. - [ ] **Container local-loop verification**: start multiple containers on one node after the local setup; verify each container opens independent QPs over the same HCA port and that request correlation and bounded buffers remain From 28140ea7f8727484021f41b4bf121bf87c29c55d Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 18 Oct 2026 22:29:24 +0800 Subject: [PATCH 015/258] Complete RDMA error metrics and OCI profile --- doc/working/plan-rdma-rpc-transport.md | 12 +++++---- .../transport/rdma/rdma_transport.h | 5 ++++ .../src/transport/rdma/rdma_transport.cpp | 25 +++++++++++++++++-- tools/rdma/run-oci-profile.sh | 20 ++++++++++++++- 4 files changed, 54 insertions(+), 8 deletions(-) diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 794c54bcb..5eeefd2c2 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -23,9 +23,11 @@ The CQ path now uses one transport-wide shared CQ for all QPs. Its completion channel is registered with one epoll engine; the worker drains completions with a bounded busy-poll budget and rearms notification after the queue is idle. Epoll is therefore only the wakeup path, while completion dispatch remains in -the CQ drain. Stop now wakes the owning engine, CQ polling counters are exposed, -and CM disconnect events close and retire their connection pools before QP/ID -teardown. Capability handshake bytes +the CQ drain. Stop now wakes the owning engine, CQ polling and completion-error +counters are exposed, and CM disconnect events close and retire their connection +pools before QP/ID teardown. The OCI runner now also has an unprivileged +shared-HCA profile with explicit verbs/CM devices, IPC_LOCK, and bounded +memlock. Capability handshake bytes have a fixed wire representation with round-trip tests. Negotiated capability exchange through the existing TCP handshake, common transport path selection/fallback, receive repost after owner release, and two-peer OCI E2E @@ -140,8 +142,8 @@ Remaining: - [ ] **Reject mixed completion**: after any payload submission, fail the stream on uncertainty and never enqueue a TCP suffix. - [ ] **Add metrics**: posted receives, fallback count, quarantined bytes, CQ - poll/empty-poll counts, and CM disconnects are exposed; QP state, CQ depth, - and RNR/retry counters remain. + poll/empty-poll counts, CM disconnects, flush, local-protection, retry, and + remote-access errors are exposed; QP state, CQ depth, and RNR counters remain. ## Tests diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index 4d38dbe0d..4b81fb7f7 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -80,6 +80,10 @@ class RdmaTransport : public Transport std::atomic cq_polls{0}; std::atomic cq_empty_polls{0}; std::atomic cm_disconnects{0}; + std::atomic flush_errors{0}; + std::atomic local_protection_errors{0}; + std::atomic retry_exceeded_errors{0}; + std::atomic remote_access_errors{0}; }; RdmaTransport(); @@ -177,6 +181,7 @@ class RdmaTransport : public Transport void release_completed_frame(struct ibv_wc &wc); void release_completed_receive(struct ibv_wc &wc); + void record_completion_error(const struct ibv_wc &wc); static void repost_frame_release(void *context, Buffer *data); }; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index ee05cbd11..d70a1a492 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -883,6 +883,27 @@ bool RdmaTransport::prime_receive(const std::shared_ptr &connection) return true; } +void RdmaTransport::record_completion_error(const ibv_wc &wc) +{ + stats_.cq_errors.fetch_add(1, std::memory_order_relaxed); + switch (wc.status) { + case IBV_WC_WR_FLUSH_ERR: + stats_.flush_errors.fetch_add(1, std::memory_order_relaxed); + break; + case IBV_WC_LOC_PROT_ERR: + stats_.local_protection_errors.fetch_add(1, std::memory_order_relaxed); + break; + case IBV_WC_RETRY_EXC_ERR: + stats_.retry_exceeded_errors.fetch_add(1, std::memory_order_relaxed); + break; + case IBV_WC_REM_ACCESS_ERR: + stats_.remote_access_errors.fetch_add(1, std::memory_order_relaxed); + break; + default: + break; + } +} + void RdmaTransport::release_completed_receive(ibv_wc &wc) { @@ -890,7 +911,7 @@ void RdmaTransport::release_completed_receive(ibv_wc &wc) if (work == nullptr) return; auto quarantine_and_release = [&] { - stats_.cq_errors.fetch_add(1, std::memory_order_relaxed); + record_completion_error(wc); work->conn->close(); (void)work->slot->quarantine(); work->slot->release(); @@ -969,7 +990,7 @@ void RdmaTransport::release_completed_frame(ibv_wc &wc) return; if (wc.status != IBV_WC_SUCCESS) { - stats_.cq_errors.fetch_add(1, std::memory_order_relaxed); + record_completion_error(wc); // A failed SEND makes the stream uncertain; do not allow a TCP // suffix to be appended to this RDMA connection. if (frame->control != nullptr) diff --git a/tools/rdma/run-oci-profile.sh b/tools/rdma/run-oci-profile.sh index 000c87ec5..9bdb0a9f0 100755 --- a/tools/rdma/run-oci-profile.sh +++ b/tools/rdma/run-oci-profile.sh @@ -17,11 +17,13 @@ fi if [[ -z "$profile" || "$profile" == "-h" || "$profile" == "--help" ]]; then cat >&2 <<'USAGE' -usage: run-oci-profile.sh tcp-only|shared-hca [image] +usage: run-oci-profile.sh tcp-only|shared-hca|shared-hca-unprivileged [image] tcp-only starts one container without RDMA devices or RDMA capability. shared-hca starts one privileged host-network container using the shared HCA; invoke it once per logical container on the physical host. +shared-hca-unprivileged starts without --privileged and grants only the + verbs/CM devices, IPC_LOCK, and a bounded memlock limit. USAGE exit 2 fi @@ -49,6 +51,22 @@ case "$profile" in -e CROWDB_RDMA_PORT="${CROWDB_RDMA_PORT:-1}" \ "$image" ;; + shared-hca-unprivileged) + exec docker run --rm --network=host \ + --device /dev/infiniband/uverbs0 \ + --device /dev/infiniband/rdma_cm \ + --cap-add IPC_LOCK --ulimit memlock=67108864:67108864 \ + -e CROWDB_RDMA_ENABLED=1 \ + -e CROWDB_RDMA_SHARED_HCA=1 \ + -e CROWDB_PORT_OFFSET="$port_offset" \ + -e CROWDB_RDMA_E2E_PORT="$rdma_cm_port" \ + "${rdma_source_env[@]}" \ + -e CROWDB_STARTUP_MODE="$startup_mode" \ + -e CROWDB_MANAGEMENT_INTERFACE="$management_interface" \ + -e CROWDB_RDMA_DEVICE="${CROWDB_RDMA_DEVICE:-mlx5_0}" \ + -e CROWDB_RDMA_PORT="${CROWDB_RDMA_PORT:-1}" \ + "$image" + ;; *) echo "unknown RDMA profile: $profile" >&2 exit 2 From f289c7764a1cbd1775270b912856652d8eb843af Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 18 Oct 2026 22:39:44 +0800 Subject: [PATCH 016/258] Record R239 verification status --- doc/working/plan-rdma-rpc-transport.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 5eeefd2c2..3309aa35b 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -44,6 +44,11 @@ validation includes executing the single-node multi-container OCI profiles against a built `crowdb:test` image. Physical multi-node validation is deferred to [the separate follow-up plan](plan-rdma-multi-node.md). +The C++ unit/integration gate and Rust FFI tests pass after the teardown and +completion-error changes; `tree-lint` completes with the repository's existing +clang-tidy warning set. The OCI profiles remain environment-dependent because +the image requires a prepared data volume and discovery profile. + ## R239 status checkpoint Completed: From 6194187dcc831138181206fab939da28e6597613 Mon Sep 17 00:00:00 2001 From: Gian Date: Mon, 19 Oct 2026 10:56:42 +0800 Subject: [PATCH 017/258] Complete lazy RDMA transport setup and local loop smoke --- container/single-node-container/entrypoint.sh | 21 ++ doc/working/plan-rdma-rpc-transport.md | 8 +- .../transport/rdma/rdma_transport.h | 30 ++- .../crowdb-rpc/transport/socket_transport.h | 10 +- .../src/transport/rdma/rdma_transport.cpp | 222 +++++++++--------- .../src/transport/socket_transport.cpp | 9 +- tools/rdma/run-connectivity-smoke.sh | 42 ++++ tools/rdma/run-container-loop.sh | 56 +++++ tools/rdma/run-local-loop.sh | 7 +- tools/rdma/run-oci-profile.sh | 21 +- 10 files changed, 302 insertions(+), 124 deletions(-) create mode 100755 tools/rdma/run-connectivity-smoke.sh create mode 100755 tools/rdma/run-container-loop.sh diff --git a/container/single-node-container/entrypoint.sh b/container/single-node-container/entrypoint.sh index efc153a1e..46f84cb22 100644 --- a/container/single-node-container/entrypoint.sh +++ b/container/single-node-container/entrypoint.sh @@ -51,6 +51,27 @@ if test "$#" -gt 0; then fi profile=${CROWDB_PROFILE_PATH:-/opt/crowdb/etc/profile.toml} +port_offset=${CROWDB_PORT_OFFSET:-0} +case "$port_offset" in + ''|*[!0-9]*) echo 'CROWDB_PORT_OFFSET must be a non-negative integer' >&2; exit 1 ;; +esac +if test "$port_offset" -gt 0; then + offset_root=/opt/crowdb/run/port-offset + offset_templates=$offset_root/templates + mkdir -p "$offset_templates" + cp "$profile" "$offset_root/profile.toml" + cp /opt/crowdb/etc/templates/*.toml "$offset_templates/" + # The preview profile is intentionally self-contained. Shift every + # declared listener and endpoint together so host-network containers can + # coexist while retaining the same relative service layout. + for base in 9090 9091 9092 10000 10100 10101 11000 11100 11200 12100 12200 13000 15100 15200; do + next=$((base + port_offset)) + sed -i -E "s/(^|[^0-9])${base}([^0-9]|$)/\\1${next}\\2/g" \ + "$offset_root/profile.toml" "$offset_templates"/*.toml + done + sed -i "s#/opt/crowdb/etc/templates#$offset_templates#g" "$offset_root/profile.toml" + profile="$offset_root/profile.toml" +fi /opt/crowdb/bin/crowdb-monitor validate "$profile" if test -n "${CROWDB_STARTUP_MODE:-}"; then diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 3309aa35b..e4ebff242 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -46,8 +46,12 @@ multi-node validation is deferred to [the separate follow-up plan](plan-rdma-mul The C++ unit/integration gate and Rust FFI tests pass after the teardown and completion-error changes; `tree-lint` completes with the repository's existing -clang-tidy warning set. The OCI profiles remain environment-dependent because -the image requires a prepared data volume and discovery profile. +clang-tidy warning set. The OCI runner now mounts a named data volume and uses +the configured RDMA local interface by default. TCP-only and unprivileged +shared-HCA startup smoke tests both reached the container readiness message; +the unprivileged smoke correctly fell back to TCP because this container host +did not expose a usable HCA device. A privileged multi-container HCA run still +requires the prepared local-loop topology. ## R239 status checkpoint diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index 4b81fb7f7..540fdc011 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -29,6 +29,23 @@ class SocketEngine; constexpr uint32_t RDMA_WIRE_SLOT_CAPACITY = RDMA_DATAPATH_CAPACITY; +struct RdmaConfig +{ + std::string device; + uint8_t port = 1; + uint32_t cq_depth = 2048; + uint32_t buffer_region_bytes = 64U << 20; + uint32_t buffer_ring_depth = 1024; + uint32_t qp_max_send_wr = 1024; + uint32_t qp_max_recv_wr = 1024; + uint32_t qp_max_send_sge = 2; + uint32_t qp_max_recv_sge = 2; + uint32_t poll_batch = 256; + uint32_t poll_empty_budget = 256; + uint8_t retry_count = 7; + uint8_t rnr_retry_count = 7; +}; + // RdmaBufferPool: allocates buffers from ibv_mr-registered memory. // The pool pre-registers a large memory region and carves out buffers // from it, avoiding per-buffer ibv_reg_mr calls (which are expensive). @@ -86,7 +103,7 @@ class RdmaTransport : public Transport std::atomic remote_access_errors{0}; }; - RdmaTransport(); + explicit RdmaTransport(RdmaConfig config = {}); ~RdmaTransport() override; bool submit(Connection *conn, OutFrame *frame) override; @@ -111,9 +128,9 @@ class RdmaTransport : public Transport void stop(); [[nodiscard]] int completion_fd() const; [[nodiscard]] int cm_event_fd() const; - int drain_cq(uint32_t max_completions = 256); + int drain_cq(uint32_t max_completions = 0); void register_events(SocketEngine *engine); - int handle_external_event(uint64_t tag, uint32_t max_completions = 256); + int handle_external_event(uint64_t tag, uint32_t max_completions = 0); void set_connection_handler(ConnectionHandler handler) { @@ -132,16 +149,20 @@ class RdmaTransport : public Transport [[nodiscard]] bool ready() const { - return context_ != nullptr && pd_ != nullptr && cq_ != nullptr; + return capability_available_; } private: + RdmaConfig config_; + bool capability_available_ = false; struct ibv_context *context_ = nullptr; + bool owns_context_ = true; struct ibv_pd *pd_ = nullptr; struct ibv_comp_channel *comp_channel_ = nullptr; struct ibv_cq *cq_ = nullptr; struct rdma_event_channel *cm_channel_ = nullptr; struct rdma_cm_id *listen_id_ = nullptr; + std::string requested_device_; uint8_t port_ = 1; std::vector cm_ids_; std::unordered_map> cm_connections_; @@ -183,6 +204,7 @@ class RdmaTransport : public Transport void release_completed_receive(struct ibv_wc &wc); void record_completion_error(const struct ibv_wc &wc); static void repost_frame_release(void *context, Buffer *data); + bool adopt_cm_context(struct ibv_context *context); }; } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h index ef8d29a69..1ad220c0b 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h @@ -5,6 +5,9 @@ #include "crowdb-rpc/connection.h" #include "crowdb-rpc/transport.h" +#ifdef CROWDB_RPC_HAVE_RDMA +# include "crowdb-rpc/transport/rdma/rdma_transport.h" +#endif #include @@ -323,7 +326,11 @@ class SocketTransport : public Transport public: // Multi-engine ctor: io_engines independent epoll/kqueue instances, // with io_workers total workers (per-engine = io_workers / io_engines). - SocketTransport(uint32_t io_engines, uint32_t io_workers, BufferPool *pool = nullptr); + SocketTransport(uint32_t io_engines, uint32_t io_workers, BufferPool *pool = nullptr +#ifdef CROWDB_RPC_HAVE_RDMA + , RdmaConfig rdma_config = {} +#endif + ); // Deprecated alias: maps to (1, num_workers). Kept for backward compat. SocketTransport(uint32_t num_workers, BufferPool *pool); ~SocketTransport() override; @@ -507,6 +514,7 @@ class SocketTransport : public Transport BufferPool *pool_; #ifdef CROWDB_RPC_HAVE_RDMA std::unique_ptr rdma_transport_; + RdmaConfig rdma_config_; #endif std::vector> workers_; std::vector> engines_; // transport owns all engines diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index d70a1a492..cadee3a20 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -170,103 +170,43 @@ void RdmaBufferPool::recycle(Buffer *buf) // ── RdmaTransport ───────────────────────────────────────────────── -RdmaTransport::RdmaTransport() +RdmaTransport::RdmaTransport(RdmaConfig config) : config_(std::move(config)) { + if (config_.device.empty()) { + if (const char *requested_device = std::getenv("CROWDB_RDMA_DEVICE"); requested_device != nullptr) + config_.device = requested_device; + } + requested_device_ = config_.device; + if (const char *requested_port = std::getenv("CROWDB_RDMA_PORT"); requested_port != nullptr) { + const int value = std::atoi(requested_port); + if (value > 0 && value <= 255 && config.port == 1) + config_.port = static_cast(value); + } + port_ = config_.port; + int device_count = 0; ibv_device **devices = ibv_get_device_list(&device_count); if (devices == nullptr || device_count == 0) { - - CRB_LOG_WARN("rdma init: no verbs devices count={} errno={}", device_count, errno); - - if (devices != nullptr) { + if (devices != nullptr) ibv_free_device_list(devices); - } return; } - const char *requested_device = std::getenv("CROWDB_RDMA_DEVICE"); - if (const char *requested_port = std::getenv("CROWDB_RDMA_PORT"); requested_port != nullptr) { - const int value = std::atoi(requested_port); - if (value > 0 && value <= 255) - port_ = static_cast(value); - } - ibv_device *selected = devices[0]; - if (requested_device != nullptr) { - selected = nullptr; - for (int i = 0; i < device_count; ++i) { - if (std::strcmp(ibv_get_device_name(devices[i]), requested_device) == 0) { - selected = devices[i]; - break; - } + ibv_device *selected = nullptr; + for (int i = 0; i < device_count; ++i) { + if (requested_device_.empty() || requested_device_ == ibv_get_device_name(devices[i])) { + selected = devices[i]; + break; } - if (selected == nullptr) { - - CRB_LOG_WARN("rdma init: requested device not found name={} count={}", requested_device, device_count); - - ibv_free_device_list(devices); - return; + } + if (selected != nullptr) { + if (ibv_context *probe = ibv_open_device(selected); probe != nullptr) { + ibv_port_attr attributes{}; + capability_available_ = ibv_query_port(probe, port_, &attributes) == 0 && + attributes.state == IBV_PORT_ACTIVE; + ibv_close_device(probe); } } - context_ = ibv_open_device(selected); ibv_free_device_list(devices); - if (context_ == nullptr) { - - CRB_LOG_WARN("rdma init: ibv_open_device failed name={} errno={}", - requested_device == nullptr ? "default" : requested_device, errno); - - return; - } - ibv_port_attr port_attributes{}; - if (ibv_query_port(context_, port_, &port_attributes) != 0 || port_attributes.state != IBV_PORT_ACTIVE) { - - CRB_LOG_WARN("rdma init: port unavailable port={} state={} errno={}", port_, - static_cast(port_attributes.state), errno); - - ibv_close_device(context_); - context_ = nullptr; - return; - } - pd_ = ibv_alloc_pd(context_); - if (pd_ == nullptr) { - - CRB_LOG_WARN("rdma init: ibv_alloc_pd failed errno={}", errno); - - ibv_close_device(context_); - context_ = nullptr; - return; - } - comp_channel_ = ibv_create_comp_channel(context_); - if (comp_channel_ == nullptr) { - - CRB_LOG_WARN("rdma init: ibv_create_comp_channel failed errno={}", errno); - - ibv_dealloc_pd(pd_); - pd_ = nullptr; - ibv_close_device(context_); - context_ = nullptr; - return; - } - const int channel_flags = ::fcntl(comp_channel_->fd, F_GETFL, 0); - if (channel_flags >= 0) - ::fcntl(comp_channel_->fd, F_SETFL, channel_flags | O_NONBLOCK); - cq_ = ibv_create_cq(context_, 2048, this, comp_channel_, 0); - if (cq_ == nullptr || ibv_req_notify_cq(cq_, 0) != 0) { - - CRB_LOG_WARN("rdma init: CQ setup failed cq={} errno={}", static_cast(cq_), errno); - - if (cq_ != nullptr) - ibv_destroy_cq(cq_); - cq_ = nullptr; - ibv_destroy_comp_channel(comp_channel_); - comp_channel_ = nullptr; - ibv_dealloc_pd(pd_); - pd_ = nullptr; - ibv_close_device(context_); - context_ = nullptr; - return; - } - constexpr uint32_t pool_bytes = 64U << 20; - send_pool_ = std::make_unique(pd_, pool_bytes, 1024); - recv_pool_ = std::make_unique(pd_, pool_bytes, 1024); } RdmaTransport::~RdmaTransport() @@ -286,12 +226,66 @@ RdmaTransport::~RdmaTransport() ibv_dealloc_pd(pd_); pd_ = nullptr; } - if (context_ != nullptr) { + if (context_ != nullptr && owns_context_) { ibv_close_device(context_); context_ = nullptr; } } +bool RdmaTransport::adopt_cm_context(struct ibv_context *context) +{ + if (context == nullptr) + return false; + if (context_ == context) + return true; + + auto *new_pd = ibv_alloc_pd(context); + if (new_pd == nullptr) + return false; + auto *new_channel = ibv_create_comp_channel(context); + if (new_channel == nullptr) { + ibv_dealloc_pd(new_pd); + return false; + } + make_nonblocking(new_channel->fd); + auto *new_cq = ibv_create_cq(context, static_cast(config_.cq_depth), this, new_channel, 0); + if (new_cq == nullptr || ibv_req_notify_cq(new_cq, 0) != 0) { + if (new_cq != nullptr) + ibv_destroy_cq(new_cq); + ibv_destroy_comp_channel(new_channel); + ibv_dealloc_pd(new_pd); + return false; + } + auto new_send = std::make_unique(new_pd, config_.buffer_region_bytes, config_.buffer_ring_depth); + auto new_recv = std::make_unique(new_pd, config_.buffer_region_bytes, config_.buffer_ring_depth); + if (new_send == nullptr || new_recv == nullptr) { + ibv_destroy_cq(new_cq); + ibv_destroy_comp_channel(new_channel); + ibv_dealloc_pd(new_pd); + return false; + } + + send_pool_.reset(); + recv_pool_.reset(); + if (cq_ != nullptr) + ibv_destroy_cq(cq_); + if (comp_channel_ != nullptr) + ibv_destroy_comp_channel(comp_channel_); + if (pd_ != nullptr) + ibv_dealloc_pd(pd_); + if (context_ != nullptr && owns_context_) + ibv_close_device(context_); + + context_ = context; + owns_context_ = false; + pd_ = new_pd; + comp_channel_ = new_channel; + cq_ = new_cq; + send_pool_ = std::move(new_send); + recv_pool_ = std::move(new_recv); + return true; +} + bool RdmaTransport::submit(Connection *conn, OutFrame *frame) { if (conn == nullptr || frame == nullptr || cq_ == nullptr || frame->control == nullptr || @@ -435,10 +429,10 @@ void RdmaTransport::shutdown() // A QP may still own SEND or flushed receive WRs after the worker stops. // Drain bounded CQ batches before destroying the IDs so provider teardown // never observes live completion objects. - for (int attempt = 0; attempt < 100; ++attempt) { - const int drained = drain_cq(256); + for (int attempt = 0; attempt < 4; ++attempt) { + const int drained = drain_cq(config_.poll_batch); if (drained == 0) - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + break; } stop(); { @@ -466,7 +460,7 @@ void RdmaTransport::shutdown() std::shared_ptr RdmaTransport::connect(const std::string &addr, int port) { - if (pd_ == nullptr || cq_ == nullptr || addr.empty() || port <= 0 || port > 65535) { + if (addr.empty() || port <= 0 || port > 65535) { CRB_LOG_WARN("rdma connect: transport unavailable pd={} cq={} addr={} port={}", static_cast(pd_), static_cast(cq_), addr, port); @@ -544,6 +538,11 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int } rdma_ack_cm_event(event); + if (!adopt_cm_context(id->verbs)) { + rdma_destroy_id(id); + return nullptr; + } + ibv_device_attr device_attributes{}; if (ibv_query_device(id->verbs, &device_attributes) != 0) { rdma_destroy_id(id); @@ -553,15 +552,15 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int qp_attr.send_cq = cq_; qp_attr.recv_cq = cq_; qp_attr.qp_type = IBV_QPT_RC; - qp_attr.cap.max_send_wr = std::min(1024, device_attributes.max_qp_wr); - qp_attr.cap.max_recv_wr = std::min(1024, device_attributes.max_qp_wr); - qp_attr.cap.max_send_sge = std::min(2, device_attributes.max_sge); - qp_attr.cap.max_recv_sge = std::min(2, device_attributes.max_sge); + qp_attr.cap.max_send_wr = std::min(config_.qp_max_send_wr, static_cast(device_attributes.max_qp_wr)); + qp_attr.cap.max_recv_wr = std::min(config_.qp_max_recv_wr, static_cast(device_attributes.max_qp_wr)); + qp_attr.cap.max_send_sge = std::min(config_.qp_max_send_sge, static_cast(device_attributes.max_sge)); + qp_attr.cap.max_recv_sge = std::min(config_.qp_max_recv_sge, static_cast(device_attributes.max_sge)); qp_attr.cap.max_inline_data = 0; qp_attr.sq_sig_all = 1; qp_attr.qp_context = id; - if (rdma_create_qp(id, id->pd, &qp_attr) != 0) { + if (rdma_create_qp(id, pd_, &qp_attr) != 0) { rdma_destroy_id(id); return nullptr; @@ -569,8 +568,8 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int rdma_conn_param params{}; params.initiator_depth = 1; params.responder_resources = 1; - params.retry_count = 7; - params.rnr_retry_count = 7; + params.retry_count = config_.retry_count; + params.rnr_retry_count = config_.rnr_retry_count; if (rdma_connect(id, ¶ms) != 0) { CRB_LOG_WARN("rdma connect: rdma_connect failed addr={} errno={}", addr, errno); @@ -618,7 +617,7 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int bool RdmaTransport::listen(const std::string &addr, int port) { - if (pd_ == nullptr || addr.empty() || port <= 0 || port > 65535 || listen_id_ != nullptr) { + if (addr.empty() || port <= 0 || port > 65535 || listen_id_ != nullptr) { CRB_LOG_WARN("rdma listen: transport unavailable pd={} addr={} port={} existing_listener={}", static_cast(pd_), addr, port, listen_id_ != nullptr); @@ -651,7 +650,7 @@ bool RdmaTransport::listen(const std::string &addr, int port) std::shared_ptr RdmaTransport::accept() { - if (listen_id_ == nullptr || cm_channel_ == nullptr || cq_ == nullptr) + if (listen_id_ == nullptr || cm_channel_ == nullptr) return nullptr; rdma_cm_event *event = nullptr; if (rdma_get_cm_event(cm_channel_, &event) != 0 || event->event != RDMA_CM_EVENT_CONNECT_REQUEST) { @@ -687,27 +686,32 @@ std::shared_ptr RdmaTransport::accept() rdma_cm_id *id = event->id; rdma_ack_cm_event(event); + if (!adopt_cm_context(id->verbs)) { + rdma_destroy_id(id); + return nullptr; + } + ibv_qp_init_attr qp_attr{}; qp_attr.send_cq = cq_; qp_attr.recv_cq = cq_; qp_attr.qp_type = IBV_QPT_RC; - qp_attr.cap.max_send_wr = 1024; - qp_attr.cap.max_recv_wr = 1024; - qp_attr.cap.max_send_sge = 2; - qp_attr.cap.max_recv_sge = 2; + qp_attr.cap.max_send_wr = config_.qp_max_send_wr; + qp_attr.cap.max_recv_wr = config_.qp_max_recv_wr; + qp_attr.cap.max_send_sge = config_.qp_max_send_sge; + qp_attr.cap.max_recv_sge = config_.qp_max_recv_sge; qp_attr.cap.max_inline_data = 0; qp_attr.sq_sig_all = 1; qp_attr.qp_context = id; - if (rdma_create_qp(id, id->pd, &qp_attr) != 0) { + if (rdma_create_qp(id, pd_, &qp_attr) != 0) { rdma_destroy_id(id); return nullptr; } rdma_conn_param params{}; params.initiator_depth = 1; params.responder_resources = 1; - params.retry_count = 7; - params.rnr_retry_count = 7; + params.retry_count = config_.retry_count; + params.rnr_retry_count = config_.rnr_retry_count; const int accept_rc = rdma_accept(id, ¶ms); if (accept_rc != 0) { @@ -781,10 +785,12 @@ int RdmaTransport::cm_event_fd() const int RdmaTransport::drain_cq(uint32_t max_completions) { + if (max_completions == 0) + max_completions = config_.poll_batch; stats_.cq_polls.fetch_add(1, std::memory_order_relaxed); uint32_t drained = 0; uint32_t empty_polls = 0; - constexpr uint32_t kEmptyPollBudget = 50000; + const uint32_t kEmptyPollBudget = config_.poll_empty_budget; std::array completions{}; auto poll_one = [&](ibv_cq *cq) { while (drained < max_completions && cq != nullptr) { diff --git a/lib/crowdb-rpc/src/transport/socket_transport.cpp b/lib/crowdb-rpc/src/transport/socket_transport.cpp index 525f77f17..5e9b3ccc3 100644 --- a/lib/crowdb-rpc/src/transport/socket_transport.cpp +++ b/lib/crowdb-rpc/src/transport/socket_transport.cpp @@ -539,7 +539,11 @@ static bool on_writable_impl(Connection *conn, int fd, TransportStats *stats) // ── SocketTransport ─────────────────────────────────────────────── -SocketTransport::SocketTransport(uint32_t io_engines, uint32_t io_workers, BufferPool *pool) : pool_(pool) +SocketTransport::SocketTransport(uint32_t io_engines, uint32_t io_workers, BufferPool *pool +#ifdef CROWDB_RPC_HAVE_RDMA + , RdmaConfig rdma_config +#endif + ) : pool_(pool) { assert(io_engines >= 1 && io_workers >= 1); assert(io_workers % io_engines == 0); @@ -547,9 +551,10 @@ SocketTransport::SocketTransport(uint32_t io_engines, uint32_t io_workers, Buffe pool_ = new SystemBufferPool(); // own a default pool } #ifdef CROWDB_RPC_HAVE_RDMA + rdma_config_ = std::move(rdma_config); if (const char *enabled = std::getenv("CROWDB_RDMA_ENABLED"); enabled != nullptr && std::strcmp(enabled, "1") == 0) { - rdma_transport_ = std::make_unique(); + rdma_transport_ = std::make_unique(rdma_config_); } #endif // Create N independent engines, each with M workers. When M>1, the diff --git a/tools/rdma/run-connectivity-smoke.sh b/tools/rdma/run-connectivity-smoke.sh new file mode 100755 index 000000000..457b37018 --- /dev/null +++ b/tools/rdma/run-connectivity-smoke.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Verify the host RDMA data path before running container or benchmark tests. +set -Eeuo pipefail + +root_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +mode=${1:-local} +device=${CROWDB_RDMA_DEVICE:-} +port=${CROWDB_RDMA_PORT:-1} +netdev=${CROWDB_RDMA_LOCAL_IF:-} +address=${CROWDB_RDMA_LOCAL_ADDR:-} + +[[ "$mode" == local || "$mode" == namespace ]] || { echo 'usage: run-connectivity-smoke.sh [local|namespace]' >&2; exit 2; } +command -v ibv_devinfo >/dev/null || { echo 'missing ibv_devinfo' >&2; exit 2; } +command -v rdma >/dev/null || { echo 'missing rdma' >&2; exit 2; } + +if [[ "$mode" == namespace ]]; then + exec "$root_dir/tools/rdma/run-local-loop.sh" namespace +fi + +if [[ -z "$device" ]]; then + device=$(rdma link show | awk 'NR == 1 {sub("/1$", "", $2); print $2}') +fi +if [[ -z "$netdev" ]]; then + netdev=$(rdma link show | awk -v d="$device" '$2 ~ ("^" d "(/|$)") && / netdev / {for (i = 1; i < NF; ++i) if ($i == "netdev") {print $(i + 1); exit}}') +fi +if [[ -z "$address" && -n "$netdev" ]]; then + address=$(ip -4 -o addr show dev "$netdev" scope global | awk 'NR == 1 {split($4, p, "/"); print p[1]}') +fi +[[ -n "$device" && -n "$netdev" && -n "$address" ]] || { + echo "unable to resolve RDMA device/netdev/address" >&2 + exit 1 +} + +echo "RDMA device=$device port=$port netdev=$netdev address=$address" +rdma link show +ibv_devinfo -d "$device" -i "$port" + +export CROWDB_RDMA_DEVICE="$device" +export CROWDB_RDMA_PORT="$port" +export CROWDB_RDMA_LOCAL_IF="$netdev" +export CROWDB_RDMA_LOCAL_ADDR="$address" +exec "$root_dir/tools/rdma/run-local-loop.sh" local diff --git a/tools/rdma/run-container-loop.sh b/tools/rdma/run-container-loop.sh new file mode 100755 index 000000000..84c7e6d53 --- /dev/null +++ b/tools/rdma/run-container-loop.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Run the RDMA CM/QP SEND/RECV smoke across two host-network containers using +# one local HCA address. This intentionally bypasses the multi-service preview +# profile so the test exercises only the transport path. +set -Eeuo pipefail + +root_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +image=${CROWDB_RDMA_TEST_IMAGE:-crowdb-node:dev} +device=${CROWDB_RDMA_DEVICE:-rocep35s0f0} +port=${CROWDB_RDMA_E2E_PORT:-19161} +client_address=${CROWDB_RDMA_LOCAL_ADDR:-10.200.1.1} +server_address=${CROWDB_RDMA_LOCAL_SERVER_ADDR:-$client_address} +test_bin=/tmp/crowdb_rpc_tests +server_name=crowdb-r239-rdma-server +client_name=crowdb-r239-rdma-client + +cleanup() { + docker rm -f "$server_name" "$client_name" >/dev/null 2>&1 || true +} +trap cleanup EXIT INT TERM + +[[ -x "$root_dir/lib/crowdb-rpc/build/crowdb_rpc_tests" ]] || { + echo "missing RDMA test binary; run pixi run build-tests" >&2 + exit 2 +} +[[ -d "$root_dir/.pixi/envs/default/lib" ]] || { + echo "missing Pixi runtime libraries" >&2 + exit 2 +} +command -v docker >/dev/null || { echo 'missing docker' >&2; exit 2; } + +cleanup +common=(--network=host --device /dev/infiniband/uverbs0 --device /dev/infiniband/rdma_cm + --cap-add IPC_LOCK --ulimit memlock=67108864:67108864 + --entrypoint "$test_bin" + -v "$root_dir/lib/crowdb-rpc/build/crowdb_rpc_tests:$test_bin:ro" + -v "$root_dir/.pixi/envs/default/lib:/tmp/crowdb-libs:ro" + -v "$root_dir/.pixi/envs/default/etc/libibverbs.d:/tmp/crowdb-ibverbs.d:ro" + -e LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/opt/crowdb/lib:/tmp/crowdb-libs + -e LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libibverbs.so.1:/usr/lib/x86_64-linux-gnu/librdmacm.so.1 + -e RDMAV_DRIVERS_DIR=/opt/crowdb/lib/libibverbs + -e IBV_CONFIG_DIR=/tmp/crowdb-ibverbs.d + -e CROWDB_RDMA_E2E=1 -e CROWDB_RDMA_E2E_PORT="$port" + -e CROWDB_RDMA_DEVICE="$device") + +echo "container RDMA loop: image=$image device=$device client=$client_address server=$server_address port=$port" +docker run -d --name "$server_name" "${common[@]}" \ + -e CROWDB_RDMA_E2E_ROLE=server -e CROWDB_RDMA_E2E_LISTEN_ADDR="$server_address" "$image" \ + --gtest_filter=RdmaHardwareTest.CompleteCmQpSendReceiveFlow >/dev/null +sleep 1 +docker run --rm --name "$client_name" "${common[@]}" \ + -e CROWDB_RDMA_E2E_ROLE=client -e CROWDB_RDMA_E2E_CONNECT_ADDR="$server_address" \ + -e CROWDB_RDMA_SOURCE_ADDR="$client_address" -e CROWDB_RDMA_E2E_LISTEN_ADDR="$server_address" "$image" \ + --gtest_filter=RdmaHardwareTest.CompleteCmQpSendReceiveFlow +docker wait "$server_name" >/dev/null +echo 'container RDMA loop passed.' diff --git a/tools/rdma/run-local-loop.sh b/tools/rdma/run-local-loop.sh index 53a456ed9..37277b392 100755 --- a/tools/rdma/run-local-loop.sh +++ b/tools/rdma/run-local-loop.sh @@ -108,16 +108,15 @@ if [[ "$mode" == local ]]; then device=${CROWDB_RDMA_DEVICE:-$(first_device)} netdev=${CROWDB_RDMA_LOCAL_IF:-$(device_netdev "$device")} netdev=${netdev:-$(root_netdev_for_device "$device")} - address=${CROWDB_RDMA_LOCAL_ADDR:-$(addr_for root "$netdev")} + client_addr=${CROWDB_RDMA_LOCAL_ADDR:-$(addr_for root "$netdev")} + server_addr=${CROWDB_RDMA_LOCAL_SERVER_ADDR:-$client_addr} [[ -n "$device" ]] || die "could not discover an RDMA device" [[ -n "$netdev" ]] || die "could not find the local Ethernet netdev" - [[ -n "$address" ]] || die "could not find an IPv4 address on $netdev" + [[ -n "$client_addr" && -n "$server_addr" ]] || die "could not find local RDMA addresses" client_ns=root server_ns=root client_device=$device server_device=$device - client_addr=$address - server_addr=$address client_if=$netdev server_if=$netdev else diff --git a/tools/rdma/run-oci-profile.sh b/tools/rdma/run-oci-profile.sh index 9bdb0a9f0..58e80c8a5 100755 --- a/tools/rdma/run-oci-profile.sh +++ b/tools/rdma/run-oci-profile.sh @@ -6,10 +6,20 @@ set -Eeuo pipefail profile="${1:-}" image="${CROWDB_OCI_IMAGE:-crowdb:test}" startup_mode="${CROWDB_STARTUP_MODE:-single}" -management_interface="${CROWDB_MANAGEMENT_INTERFACE:-eth0}" +management_interface="${CROWDB_MANAGEMENT_INTERFACE:-${CROWDB_RDMA_LOCAL_IF:-enp35s0f0np0}}" port_offset="${CROWDB_PORT_OFFSET:-0}" rdma_cm_port=$((19101 + port_offset)) +data_volume="${CROWDB_CONTAINER_DATA_VOLUME:-crowdb-rdma-${profile:-profile}}" +rdma_device="${CROWDB_RDMA_DEVICE:-}" +if [[ -z "$rdma_device" ]] && command -v rdma >/dev/null 2>&1; then + rdma_device=$(rdma link show | awk -v netdev="$management_interface" \ + '$0 ~ (" netdev " netdev " ") {sub("/1$", "", $2); print $2; exit}') +fi rdma_source_addr="${CROWDB_RDMA_SOURCE_ADDR:-}" +if [[ -z "$rdma_source_addr" ]] && command -v ip >/dev/null 2>&1; then + rdma_source_addr=$(ip -4 -o addr show dev "$management_interface" scope global | + awk 'NR == 1 {split($4, parts, "/"); print parts[1]}') +fi rdma_source_env=() if [[ -n "$rdma_source_addr" ]]; then rdma_source_env=(-e "CROWDB_RDMA_SOURCE_ADDR=$rdma_source_addr") @@ -32,6 +42,7 @@ if [[ $# -ge 2 ]]; then image="$2"; fi case "$profile" in tcp-only) exec docker run --rm --network=host \ + --mount "type=volume,source=$data_volume,target=/opt/crowdb/data" \ -e CROWDB_RDMA_ENABLED=0 \ -e CROWDB_PORT_OFFSET="$port_offset" \ -e CROWDB_STARTUP_MODE="$startup_mode" \ @@ -39,7 +50,9 @@ case "$profile" in "$image" ;; shared-hca) + [[ -n "$rdma_device" ]] || { echo "could not resolve RDMA device for $management_interface" >&2; exit 1; } exec docker run --rm --network=host --privileged --ulimit memlock=-1 \ + --mount "type=volume,source=$data_volume,target=/opt/crowdb/data" \ -e CROWDB_RDMA_ENABLED=1 \ -e CROWDB_RDMA_SHARED_HCA=1 \ -e CROWDB_PORT_OFFSET="$port_offset" \ @@ -47,12 +60,14 @@ case "$profile" in "${rdma_source_env[@]}" \ -e CROWDB_STARTUP_MODE="$startup_mode" \ -e CROWDB_MANAGEMENT_INTERFACE="$management_interface" \ - -e CROWDB_RDMA_DEVICE="${CROWDB_RDMA_DEVICE:-mlx5_0}" \ + -e CROWDB_RDMA_DEVICE="$rdma_device" \ -e CROWDB_RDMA_PORT="${CROWDB_RDMA_PORT:-1}" \ "$image" ;; shared-hca-unprivileged) + [[ -n "$rdma_device" ]] || { echo "could not resolve RDMA device for $management_interface" >&2; exit 1; } exec docker run --rm --network=host \ + --mount "type=volume,source=$data_volume,target=/opt/crowdb/data" \ --device /dev/infiniband/uverbs0 \ --device /dev/infiniband/rdma_cm \ --cap-add IPC_LOCK --ulimit memlock=67108864:67108864 \ @@ -63,7 +78,7 @@ case "$profile" in "${rdma_source_env[@]}" \ -e CROWDB_STARTUP_MODE="$startup_mode" \ -e CROWDB_MANAGEMENT_INTERFACE="$management_interface" \ - -e CROWDB_RDMA_DEVICE="${CROWDB_RDMA_DEVICE:-mlx5_0}" \ + -e CROWDB_RDMA_DEVICE="$rdma_device" \ -e CROWDB_RDMA_PORT="${CROWDB_RDMA_PORT:-1}" \ "$image" ;; From 37e310b8d14cb6d2e164d3116fdc954ed36e9a2e Mon Sep 17 00:00:00 2001 From: Gian Date: Wed, 21 Oct 2026 09:27:13 +0800 Subject: [PATCH 018/258] Complete RDMA registered buffer foundation --- doc/working/plan-rdma-registered-buffers.md | 59 +++--- doc/working/plan-rdma-rpc-transport.md | 40 +++-- doc/working/plan-rdma.md | 49 +++-- lib/crowdb-rpc/ffi/src/buffer.rs | 38 ++++ lib/crowdb-rpc/ffi/src/lib.rs | 2 +- lib/crowdb-rpc/ffi/src/sys.rs | 1 + lib/crowdb-rpc/ffi/tests/buffer_test.rs | 29 ++- lib/crowdb-rpc/include/crowdb-rpc/buffer.h | 2 + lib/crowdb-rpc/include/crowdb-rpc/c_api.h | 1 + .../transport/rdma/rdma_transport.h | 32 +++- lib/crowdb-rpc/src/buffer.cpp | 41 ++++- lib/crowdb-rpc/src/c_api.cpp | 10 +- .../src/transport/rdma/rdma_transport.cpp | 168 +++++++++++++----- lib/crowdb-rpc/tests/buffer_test.cpp | 39 ++++ lib/crowdb-rpc/tests/rdma_test.cpp | 19 ++ 15 files changed, 413 insertions(+), 117 deletions(-) diff --git a/doc/working/plan-rdma-registered-buffers.md b/doc/working/plan-rdma-registered-buffers.md index dcd766673..a8826797c 100644 --- a/doc/working/plan-rdma-registered-buffers.md +++ b/doc/working/plan-rdma-registered-buffers.md @@ -16,7 +16,10 @@ Capacity classes, ownership states, lease metadata, bounded atomic admission, lock-free freelists, quarantine exclusion, and C/Rust metadata accessors are implemented in `buffer.h`, `buffer.cpp`, `c_api.h`, and the FFI buffer wrapper. The pool is device-independent so TCP-only builds retain the bounded contract. -Verbs MR attachment, transport handoff, and shutdown integration remain open. +The verbs MR arena and basic send/receive handoff are complete. Its allocation +and recycle path uses bounded atomic class freelists and an atomic arena bump +pointer; owner transitions, premature-release quarantine, inflight shutdown +quarantine, and FFI ownership accessors now use the common contract. The verbs pool now pre-registers one region and carves aligned capacity-class slices from it. Each lease records the region MR handle and lkey, and @@ -25,50 +28,50 @@ shutdown. ## Phase 1: inventory and contract -- [ ] **Map existing buffer ownership**: document the current `Buffer`, +- [x] **Map existing buffer ownership**: document the current `Buffer`, `BufferPool`, `OutFrame`, callback, and FFI ownership transitions before adding registration. Files: `lib/crowdb-rpc/include/crowdb-rpc/buffer.h`, `lib/crowdb-rpc/include/crowdb-rpc/transport.h`, `lib/crowdb-rpc/include/crowdb-rpc/connection.h`, `lib/crowdb-rpc/ffi/src/sys.rs`. -- [ ] **Define capacity classes**: add named constants/configuration for 64 KiB, +- [x] **Define capacity classes**: add named constants/configuration for 64 KiB, 256 KiB, and 1 MiB data capacities plus the 1 KiB control capacity. Reject zero, overflowed, and unsupported capacities before pool admission. Files: `lib/crowdb-rpc/include/crowdb-rpc/buffer.h`, configuration headers, and corresponding Rust FFI types. -- [ ] **Define lease metadata**: specify request ID, stream ID, sequence, +- [x] **Define lease metadata**: specify request ID, stream ID, sequence, logical offset, actual length, capacity, registration identity, and release state. Keep the handle opaque across FFI. Files: buffer and FFI headers, `lib/crowdb-rpc/ffi/src/sys.rs`. ## Phase 2: registered pool -- [ ] **Implement device registration owner**: create and destroy a pool for a - protection domain, registered region, lkey, and device identity. Registration - failure must leave the TCP pool usable. Files: - `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_buffer_pool.h`, - `lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp`. -- [ ] **Implement class freelists**: carve buffers from bounded registered - regions instead of registering every allocation. Keep control and data pools - separate, enforce total bytes and class counts, and avoid an unbounded - fallback allocation. Files: same pool files and pool tests. -- [ ] **Implement ownership transitions**: make acquire, post, complete, disk - handoff, caller release, and quarantine explicit operations. Reject a second - release or recycle before completion. Files: pool/lease implementation. -- [ ] **Implement shutdown and quarantine**: drain known operations, mark - uncertain buffers quarantined, and deregister only after no device access can - remain. Files: pool, connection shutdown, and test fixtures. -- [ ] **Review hot-path synchronization**: keep allocation/recycle compatible - with the repository's lock-free rule; isolate lifecycle-only coordination and - record any unavoidable synchronization boundary. Files: pool implementation. +- [x] **Unify device registration owner**: make the verbs-backed arena use the + common registration identity and failure contract while retaining one MR per + arena. Registration failure must leave TCP allocation usable. Files: + `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h` and + `lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp`. +- [x] **Unify class freelists**: carve all four classes from bounded registered + regions, preserve the byte and slot budgets, and avoid a separate pool path + with different recycle semantics. Files: same pool files and pool tests. +- [x] **Complete ownership transitions**: make acquire, post, completion, disk + handoff, caller release, and quarantine explicit at every async boundary; + reject a second release or recycle before completion. Files: pool, + transport completion paths, and ownership tests. +- [x] **Complete shutdown and quarantine**: drain known operations, keep + uncertain buffers out of reuse until connection quiescence, and deregister + only after device access ends. Files: pool, connection shutdown, and tests. +- [x] **Review hot-path synchronization**: keep allocation/recycle lock-free; + lifecycle destruction is separate from the atomic freelists. Files: pool + implementation. ## Phase 3: FFI lease surface -- [ ] **Expose lease accessors**: return opaque handle, data pointer, actual +- [x] **Expose lease accessors**: return opaque handle, data pointer, actual length, capacity, sequence, and logical offset without exposing verbs structs. Files: `lib/crowdb-rpc/ffi/src/sys.rs`, `client.rs`, `server.rs`, and C ABI declarations. -- [ ] **Enforce release ordering**: caller release returns a buffer only after +- [x] **Enforce release ordering**: caller release returns a buffer only after the C++ owner confirms completion; Rust must not retain a dereferenceable view after release. Files: FFI wrapper and ownership tests. @@ -99,8 +102,8 @@ E2E tests: ## Consolidated files - `lib/crowdb-rpc/include/crowdb-rpc/buffer.h` -- `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_buffer_pool.h` -- `lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp` +- `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h` +- `lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp` - `lib/crowdb-rpc/ffi/src/sys.rs` - `lib/crowdb-rpc/ffi/src/client.rs` - `lib/crowdb-rpc/ffi/src/server.rs` @@ -116,5 +119,5 @@ pixi run tree-lint ``` Completion requires all ownership, budget, shutdown, and TCP-only tests to -pass. Do not begin R239 transport work while the pool can recycle an uncertain -buffer. +pass. R238 is complete once the recorded gates below pass; R239 transport work +may consume the stable lease contract. diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index e4ebff242..5c5122a89 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -12,7 +12,7 @@ composition and one-way RDMA-to-TCP fallback. ## Progress -The transport foundation now includes validated endpoint capability data +The transport implementation now includes validated endpoint capability data (protocol, generation, device, GID, port, and supported capacity classes), explicit RDMA path states, and a pre-payload-only TCP fallback predicate. Implemented since this checkpoint: active/passive RDMA-CM setup, RC QP @@ -40,8 +40,9 @@ receive repost and connection-level CQ handling. The actual `RpcServer`/ `RpcClient` worker path now passes a live request/response round trip with multiple sequential frames on one QP, including CQ wakeup, frame dispatch, request correlation, response completion, and clean teardown. Remaining -validation includes executing the single-node multi-container OCI profiles -against a built `crowdb:test` image. Physical +acceptance includes failure-injection coverage, pre-payload TCP fallback proof, +stable completion ownership mapping, and executing the single-node +multi-container OCI profiles against a built `crowdb:test` image. Physical multi-node validation is deferred to [the separate follow-up plan](plan-rdma-multi-node.md). The C++ unit/integration gate and Rust FFI tests pass after the teardown and @@ -71,32 +72,35 @@ Completed: payload benchmark smoke tests reporting zero errors; - non-root runner selection and the library consumer/deployment contract. -Remaining: +Remaining implementation and acceptance work: - add mock/integration coverage for CM disconnect, QP error, CQ flush/error, - epoll wakeup, and pre-payload TCP fallback; -- run the single-node HCA local-loop multi-container OCI profile against the built image; + epoll wakeup, receive exhaustion, and pre-payload TCP fallback; +- make each completion's `wr_id` resolve a stable connection/request/slot + owner and test success, flush, protection, retry, and remote-access errors; +- run the single-node HCA local-loop multi-container OCI profile against the + built image; - remove temporary hardware diagnostics, normalize failure logging, and run - the full C++ format/tree-lint gates. -- add a configurable multi-CQ pool when one shared CQ becomes a bottleneck; -- move CQ draining to a dedicated polling worker if engine worker contention - becomes measurable; -- rerun all RDMA benchmark concurrency and payload cases after the CQ change. + the full C++ format/tree-lint gates; +- keep multi-CQ and a dedicated polling worker as follow-up performance work; + they are not R239 blockers unless the acceptance profile shows contention; +- rerun the RDMA benchmark concurrency and payload cases after the acceptance + changes. Completed on the local HCA at 128B, 1KiB, and 4KiB with 0 errors for 1, 4, 8, and 32 connections. The 4-connection cases reached about 442k ops/s (128B), 414k ops/s (1KiB), and 376k ops/s (4KiB). ## Phase 1: common transport seam -- [ ] **Trace TCP submission**: map `Transport::submit`, `Connection`, +- [x] **Trace TCP submission**: map `Transport::submit`, `Connection`, `OutFrame`, parser dispatch, completion callbacks, and connection shutdown. Files: `lib/crowdb-rpc/include/crowdb-rpc/transport.h`, `connection.h`, `out_frame.h`, TCP transport and worker sources. -- [ ] **Add RDMA capability state**: define unavailable, probing, connecting, +- [x] **Add RDMA capability state**: define unavailable, probing, connecting, ready, draining, failed, and TCP-fallback states. Keep path state per logical connection and prevent TCP-to-RDMA upgrade. Files: transport/connection headers and state tests. -- [ ] **Define endpoint handshake data**: represent coarse discovery result, +- [x] **Define endpoint handshake data**: represent coarse discovery result, CM address/port, GID/device identity, capability generation, protocol version, and supported data classes. Validate stale generation and peer identity before CM setup. Files: RPC control schema and endpoint config. @@ -138,7 +142,7 @@ Remaining: - [x] **Post one-slot receives**: prepost one full registered receive slot per receive ring entry, associate each with a stable `wr_id`, and replenish only after the owner releases it. -- [ ] **Resolve WR identity**: `wr_id` must identify connection, request ID, +- [~] **Resolve WR identity**: `wr_id` must identify connection, request ID, stream ID, sequence, logical offset, buffers, and ownership state. Handle success, flush, local protection, retry, and remote access errors distinctly. - [x] **Dispatch common callbacks**: validate lengths and class before parser @@ -146,11 +150,11 @@ Remaining: ## Phase 5: fallback and observability -- [ ] **Implement pre-payload fallback**: if CM/QP setup or first submission +- [~] **Implement pre-payload fallback**: if CM/QP setup or first submission fails before payload, select TCP with the same request identity. -- [ ] **Reject mixed completion**: after any payload submission, fail the stream +- [x] **Reject mixed completion**: after any payload submission, fail the stream on uncertainty and never enqueue a TCP suffix. -- [ ] **Add metrics**: posted receives, fallback count, quarantined bytes, CQ +- [x] **Add metrics**: posted receives, fallback count, quarantined bytes, CQ poll/empty-poll counts, CM disconnects, flush, local-protection, retry, and remote-access errors are exposed; QP state, CQ depth, and RNR counters remain. diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index f5918a730..472e5021f 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -18,12 +18,14 @@ single-node HCA local-loop with multiple containers. ## Current implementation status -R238 foundation work has started. The common RPC buffer layer now has the four -capacity classes, bounded registered-buffer admission, explicit ownership -transitions, quarantine handling, and C/Rust lease metadata accessors. The -host setup and local-loop smoke test are now explicit: `setup-rdma.sh local` -prepares one HCA port, and `run-local-loop.sh local` drives both roles on that -port. Namespace setup is outside the supported test model. +R238 is complete. The common RPC buffer layer and verbs-backed per-connection +arenas share the four capacity classes, bounded admission, explicit owner +transitions, safe release ordering, quarantine handling, shutdown quarantine, +and C/Rust lease metadata accessors. R239's transport path is implemented and +passes the prepared HCA +local-loop plus the real `RpcServer`/`RpcClient` worker round trip. Its +requirement is still open because the fault-injection, pre-payload fallback, +multi-container, and final gate cases have not all passed. This is the root plan. The five child plans contain implementation details; only this plan controls phase order, cross-backlog decisions, and final @@ -74,6 +76,17 @@ datapath Chunk-KV/access R242 fat client SDK and FFI lease ``` +## Immediate implementation order + +1. Finish the R238 foundation contract and its gates. Do not add new R239 + transport behavior until the common and verbs-backed pools agree on class, + ownership, quarantine, and shutdown semantics. +2. Close R239 acceptance gaps: injectable CM/CQ failure coverage, stable + `wr_id` ownership mapping, pre-payload retry on TCP, partial-transfer + isolation, and the single-node multi-container OCI profile. +3. Run the R238/R239 handoff gates and then start R240 and R241 in parallel. + R242 remains downstream of R241 and the stable FFI lease surface. + R240 and R241 may run in parallel after R239 passes its transport acceptance. R242 starts only after R241 establishes the access semantics and after the FFI lease contract from R238 is stable. @@ -104,11 +117,17 @@ TCP-only profile and build/mocked tests. ## Phase 1: R238 registered buffers -- [ ] **Execute R238 child plan**: implement classes, budgets, lease metadata, - ownership transitions, quarantine, and shutdown. -- [ ] **Run R238 gates**: unit and integration tests pass with RDMA disabled - and enabled build configurations. -- [ ] **Review handoff**: verify that R239 can obtain control/data buffers and +- [x] **Close R238 pool contract**: align `RegisteredBufferPool` and the + verbs-backed `RdmaBufferPool` on the four classes, lease metadata, owner + transitions, quarantine accounting, and bounded shutdown. Keep registration + optional and do not introduce a new lock on the allocation/recycle path. + Files: `lib/crowdb-rpc/include/crowdb-rpc/buffer.h`, + `lib/crowdb-rpc/src/buffer.cpp`, and + `lib/crowdb-rpc/src/transport/rdma/rdma_transport.*`. +- [x] **Run R238 gates**: unit and integration tests pass with RDMA disabled + and enabled build configurations, including registration failure and + uncertain completion cases. +- [x] **Review handoff**: verify that R239 can obtain control/data buffers and that every asynchronous owner has a release transition. Exit condition: no buffer can be recycled while transport, disk, or caller @@ -116,11 +135,9 @@ ownership remains unresolved. ## Phase 2: R239 common RDMA RPC -- [ ] **Execute R239 child plan**: implement endpoint capability exchange, CM - and QP lifecycle, two-SGE SEND/RECV, epoll/CQ event composition, request - correlation, fallback, and shutdown. -- [ ] **Mock verification**: cover CQ success/error/flush, CM transitions, - receive exhaustion, fallback before payload, and no mixed TCP/RDMA stream. +- [ ] **Close R239 acceptance**: add mock/integration coverage for CQ + success/error/flush, CM disconnect/QP error, receive exhaustion, stable + `wr_id` ownership, pre-payload fallback, and no mixed TCP/RDMA stream. - [x] **Local-loop verification**: run `setup-rdma.sh local` followed by `run-local-loop.sh local`; both CM roles, framed SEND/RECV, receive repost, and exact payload lengths have passed on the prepared HCA. diff --git a/lib/crowdb-rpc/ffi/src/buffer.rs b/lib/crowdb-rpc/ffi/src/buffer.rs index 26ffb52fa..966658ed2 100644 --- a/lib/crowdb-rpc/ffi/src/buffer.rs +++ b/lib/crowdb-rpc/ffi/src/buffer.rs @@ -21,6 +21,34 @@ pub struct BufferLeaseMetadata { pub logical_offset: u64, pub capacity: u32, pub registration_id: u32, + pub ownership: BufferOwnership, +} + +#[repr(i32)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum BufferOwnership { + #[default] + Available = 0, + PostedReceive = 1, + ReceiveInFlight = 2, + SendInFlight = 3, + DiskOwned = 4, + CallerOwned = 5, + Quarantined = 6, +} + +impl BufferOwnership { + fn from_raw(value: i32) -> Self { + match value { + 0 => Self::Available, + 1 => Self::PostedReceive, + 2 => Self::ReceiveInFlight, + 3 => Self::SendInFlight, + 4 => Self::DiskOwned, + 5 => Self::CallerOwned, + _ => Self::Quarantined, + } + } } /// An immutable, bounded scatter/gather data payload. @@ -186,10 +214,20 @@ impl Buffer { logical_offset: sys::crowdb_rpc_buffer_logical_offset(self.handle), capacity: sys::crowdb_rpc_buffer_capacity(self.handle), registration_id: sys::crowdb_rpc_buffer_registration_id(self.handle), + ownership: BufferOwnership::from_raw(sys::crowdb_rpc_buffer_ownership(self.handle)), } } } + pub fn ownership(&self) -> BufferOwnership { + BufferOwnership::from_raw(unsafe { sys::crowdb_rpc_buffer_ownership(self.handle) }) + } + + pub fn transition(&mut self, from: BufferOwnership, to: BufferOwnership) -> bool { + !self.handle.is_null() + && unsafe { sys::crowdb_rpc_buffer_transition(self.handle, from as i32, to as i32) != 0 } + } + pub fn set_lease_metadata(&mut self, metadata: BufferLeaseMetadata, registration_id: u32) { unsafe { sys::crowdb_rpc_buffer_set_metadata( diff --git a/lib/crowdb-rpc/ffi/src/lib.rs b/lib/crowdb-rpc/ffi/src/lib.rs index f0ff137b9..41534cb5d 100644 --- a/lib/crowdb-rpc/ffi/src/lib.rs +++ b/lib/crowdb-rpc/ffi/src/lib.rs @@ -21,7 +21,7 @@ mod owned_route; mod server; pub mod sys; -pub use buffer::{Buffer, BufferChain, BufferChainError, BufferLeaseMetadata, BufferPool}; +pub use buffer::{Buffer, BufferChain, BufferChainError, BufferLeaseMetadata, BufferOwnership, BufferPool}; pub use client::{noop_completion, CallFuture, ClientRequest, Response, RpcClient}; pub use connection_pool::{ConnectionPoolError, ConnectionPoolIndex, SelectedConnection}; pub use cpp_global_metrics::{cpp_global_metrics_max_name_len, flush_cpp_global_metrics}; diff --git a/lib/crowdb-rpc/ffi/src/sys.rs b/lib/crowdb-rpc/ffi/src/sys.rs index 735637037..bb32f4ec8 100644 --- a/lib/crowdb-rpc/ffi/src/sys.rs +++ b/lib/crowdb-rpc/ffi/src/sys.rs @@ -130,6 +130,7 @@ extern "C" { pub fn crowdb_rpc_buffer_sequence(buf: crowdb_rpc_buffer_t) -> u64; pub fn crowdb_rpc_buffer_logical_offset(buf: crowdb_rpc_buffer_t) -> u64; pub fn crowdb_rpc_buffer_registration_id(buf: crowdb_rpc_buffer_t) -> u32; + pub fn crowdb_rpc_buffer_ownership(buf: crowdb_rpc_buffer_t) -> i32; pub fn crowdb_rpc_buffer_set_metadata( buf: crowdb_rpc_buffer_t, request_id: u64, diff --git a/lib/crowdb-rpc/ffi/tests/buffer_test.rs b/lib/crowdb-rpc/ffi/tests/buffer_test.rs index 2f6cb8b2f..139c054d6 100644 --- a/lib/crowdb-rpc/ffi/tests/buffer_test.rs +++ b/lib/crowdb-rpc/ffi/tests/buffer_test.rs @@ -2,7 +2,34 @@ // Licensed under the Apache License, Version 2.0. use bytes::Bytes; -use crowdb_rpc_ffi::{Buffer, BufferChain, BufferChainError, BufferPool}; +use crowdb_rpc_ffi::{Buffer, BufferChain, BufferChainError, BufferOwnership, BufferPool}; + +#[test] +fn lease_metadata_includes_registration_and_owner_state() { + let pool = BufferPool::new(1); + let mut buffer = pool.alloc_buffer(32).expect("pool allocation"); + buffer.set_lease_metadata( + crowdb_rpc_ffi::BufferLeaseMetadata { + request_id: 7, + stream_id: 9, + sequence: 11, + logical_offset: 13, + capacity: 0, + registration_id: 0, + ownership: BufferOwnership::Available, + }, + 42, + ); + let metadata = buffer.lease_metadata(); + assert_eq!(metadata.request_id, 7); + assert_eq!(metadata.stream_id, 9); + assert_eq!(metadata.sequence, 11); + assert_eq!(metadata.logical_offset, 13); + assert_eq!(metadata.registration_id, 42); + assert_eq!(metadata.ownership, BufferOwnership::Available); + assert!(buffer.transition(BufferOwnership::Available, BufferOwnership::CallerOwned)); + assert_eq!(buffer.ownership(), BufferOwnership::CallerOwned); +} #[test] fn detached_read_buffer_survives_pool_and_keeps_pointer() { diff --git a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h index ae59eeb0e..26fad9c5d 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h @@ -69,6 +69,7 @@ struct Buffer uint32_t allocation_capacity = 0; uint32_t view_offset = 0; Buffer *next_pool = nullptr; + Buffer *next_all = nullptr; void reset_view() { @@ -166,6 +167,7 @@ class RegisteredBufferPool : public BufferPool Buffer *alloc(uint32_t capacity) override; void recycle(Buffer *buf) override; + bool detach(Buffer *buf) override; [[nodiscard]] uint64_t committed_bytes() const { diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h index 85b138205..091a9e418 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h @@ -43,6 +43,7 @@ uint64_t crowdb_rpc_buffer_stream_id(crowdb_rpc_buffer_t buf); uint64_t crowdb_rpc_buffer_sequence(crowdb_rpc_buffer_t buf); uint64_t crowdb_rpc_buffer_logical_offset(crowdb_rpc_buffer_t buf); uint32_t crowdb_rpc_buffer_registration_id(crowdb_rpc_buffer_t buf); +int crowdb_rpc_buffer_ownership(crowdb_rpc_buffer_t buf); void crowdb_rpc_buffer_set_metadata(crowdb_rpc_buffer_t buf, uint64_t request_id, uint64_t stream_id, uint64_t sequence, uint64_t logical_offset, uint32_t registration_id); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index 540fdc011..01d64f167 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -50,7 +50,9 @@ struct RdmaConfig // The pool pre-registers a large memory region and carves out buffers // from it, avoiding per-buffer ibv_reg_mr calls (which are expensive). // Every buffer is a slice of the single MR, so registration is bounded by the -// pool region and deregistration happens exactly once at pool shutdown. +// pool region and deregistration happens exactly once at pool shutdown. Class +// freelists and arena admission are atomic; shutdown is the only lifecycle +// operation that walks the full allocation list. class RdmaBufferPool : public BufferPool { public: @@ -61,21 +63,35 @@ class RdmaBufferPool : public BufferPool Buffer *alloc(uint32_t capacity) override; void recycle(Buffer *buf) override; + [[nodiscard]] uint64_t quarantined_bytes() const + { + return quarantined_bytes_.load(std::memory_order_acquire); + } + + [[nodiscard]] bool quiescent() const + { + return outstanding_.load(std::memory_order_acquire) == 0; + } + + void quarantine_inflight(); + private: struct ibv_pd *pd_; struct ibv_mr *mr_ = nullptr; void *region_ = nullptr; uint32_t mr_size_; uint32_t max_buffers_; - uint32_t outstanding_ = 0; - uint32_t next_offset_ = 0; - - std::mutex mu_; - std::unordered_map> free_list_; - std::vector quarantine_; - std::vector all_buffers_; + std::atomic outstanding_{0}; + std::atomic next_offset_{0}; + std::array, 4> free_heads_{}; + std::atomic quarantine_head_{nullptr}; + std::atomic all_head_{nullptr}; + std::atomic quarantined_bytes_{0}; Buffer *alloc_fresh(uint32_t capacity); + Buffer *pop(std::atomic &head); + void push(std::atomic &head, Buffer *buf); + bool reserve_outstanding(); }; // RdmaTransport: RDMA transport using libibverbs + librdmacm. diff --git a/lib/crowdb-rpc/src/buffer.cpp b/lib/crowdb-rpc/src/buffer.cpp index 3fa6d35c8..8e3de1b4b 100644 --- a/lib/crowdb-rpc/src/buffer.cpp +++ b/lib/crowdb-rpc/src/buffer.cpp @@ -49,13 +49,28 @@ void Buffer::write(const void *src, uint32_t n) Buffer *Buffer::ref_clone() { assert(ref != nullptr && "Buffer::ref_clone on released buffer"); - ref->fetch_add(1, std::memory_order_relaxed); + int32_t count = ref->load(std::memory_order_acquire); + while (count > 0 && + !ref->compare_exchange_weak(count, count + 1, std::memory_order_acq_rel, + std::memory_order_acquire)) { + } + if (count <= 0) + return nullptr; return this; } void Buffer::release() { - if (ref->fetch_sub(1, std::memory_order_acq_rel) == 1) { + if (ref == nullptr) + return; + int32_t count = ref->load(std::memory_order_acquire); + while (count > 0 && + !ref->compare_exchange_weak(count, count - 1, std::memory_order_acq_rel, + std::memory_order_acquire)) { + } + if (count == 0 || count < 0) + return; + if (count == 1) { // Last reference — recycle to pool, call external free, or free // standalone. if (free_cb != nullptr) { @@ -153,7 +168,10 @@ void SystemBufferPool::recycle(Buffer *buf) bool SystemBufferPool::detach(Buffer *buf) { - if (buf == nullptr || buf->pool != this || buf->ref->load(std::memory_order_acquire) != 1) { + const BufferOwnership state = buf == nullptr ? BufferOwnership::Quarantined + : buf->ownership.load(std::memory_order_acquire); + if (buf == nullptr || buf->pool != this || buf->ref->load(std::memory_order_acquire) != 1 || + (state != BufferOwnership::Available && state != BufferOwnership::CallerOwned)) { return false; } buf->pool = nullptr; @@ -253,6 +271,10 @@ Buffer *RegisteredBufferPool::alloc(uint32_t requested) void RegisteredBufferPool::recycle(Buffer *buf) { outstanding_.fetch_sub(1, std::memory_order_acq_rel); + const BufferOwnership state = buf->ownership.load(std::memory_order_acquire); + if (state != BufferOwnership::Available && state != BufferOwnership::Quarantined) { + (void)buf->quarantine(); + } if (buf->ownership.load(std::memory_order_acquire) == BufferOwnership::Quarantined) { quarantined_bytes_.fetch_add(buf->capacity, std::memory_order_acq_rel); push(quarantine_head_, buf); @@ -263,4 +285,17 @@ void RegisteredBufferPool::recycle(Buffer *buf) push(free_heads_[index(buf->capacity_class)], buf); } +bool RegisteredBufferPool::detach(Buffer *buf) +{ + const BufferOwnership state = buf == nullptr ? BufferOwnership::Quarantined + : buf->ownership.load(std::memory_order_acquire); + if (buf == nullptr || buf->pool != this || buf->ref == nullptr || + buf->ref->load(std::memory_order_acquire) != 1 || + (state != BufferOwnership::Available && state != BufferOwnership::CallerOwned)) + return false; + buf->pool = nullptr; + outstanding_.fetch_sub(1, std::memory_order_acq_rel); + return true; +} + } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/c_api.cpp b/lib/crowdb-rpc/src/c_api.cpp index 14ff9616e..ac72e0725 100644 --- a/lib/crowdb-rpc/src/c_api.cpp +++ b/lib/crowdb-rpc/src/c_api.cpp @@ -140,6 +140,13 @@ uint32_t crowdb_rpc_buffer_registration_id(crowdb_rpc_buffer_t buf) return (buf != nullptr && buf->buf != nullptr) ? buf->buf->registration_id : 0; } +int crowdb_rpc_buffer_ownership(crowdb_rpc_buffer_t buf) +{ + return (buf != nullptr && buf->buf != nullptr) + ? static_cast(buf->buf->ownership.load(std::memory_order_acquire)) + : -1; +} + void crowdb_rpc_buffer_set_metadata(crowdb_rpc_buffer_t buf, uint64_t request_id, uint64_t stream_id, uint64_t sequence, uint64_t logical_offset, uint32_t registration_id) { @@ -184,8 +191,7 @@ crowdb_rpc_buffer_t crowdb_rpc_buffer_ref(crowdb_rpc_buffer_t buf) if (buf == nullptr || buf->buf == nullptr) { return nullptr; } - buf->buf->ref_clone(); - return buf; + return buf->buf->ref_clone() == nullptr ? nullptr : buf; } catch (...) { return nullptr; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index cadee3a20..cb0c57ed6 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -53,6 +53,8 @@ RdmaBufferPool::RdmaBufferPool(struct ibv_pd *pd, uint32_t mr_size, uint32_t max mr_size_(mr_size), max_buffers_(max_buffers) { + for (auto &head : free_heads_) + head.store(nullptr, std::memory_order_relaxed); if (pd_ == nullptr || mr_size_ == 0 || max_buffers_ == 0) { return; } @@ -72,19 +74,18 @@ RdmaBufferPool::RdmaBufferPool(struct ibv_pd *pd, uint32_t mr_size, uint32_t max RdmaBufferPool::~RdmaBufferPool() { - std::scoped_lock lock(mu_); - if (outstanding_ != 0) { + if (outstanding_.load(std::memory_order_acquire) != 0) { // Keep the MR registered while a caller, disk operation, or device // completion still owns a slice. Process teardown reclaims it. return; } - for (Buffer *buf : all_buffers_) { + Buffer *buf = all_head_.exchange(nullptr, std::memory_order_acq_rel); + while (buf != nullptr) { + Buffer *next = buf->next_all; delete buf->ref; delete buf; + buf = next; } - all_buffers_.clear(); - free_list_.clear(); - quarantine_.clear(); if (mr_ != nullptr) { ibv_dereg_mr(mr_); mr_ = nullptr; @@ -97,12 +98,20 @@ RdmaBufferPool::~RdmaBufferPool() Buffer *RdmaBufferPool::alloc_fresh(uint32_t capacity) { - if (outstanding_ >= max_buffers_ || mr_ == nullptr || region_ == nullptr) { + if (mr_ == nullptr || region_ == nullptr) { return nullptr; } - const uint32_t aligned_offset = (next_offset_ + 63U) & ~63U; - if (capacity > mr_size_ - aligned_offset) { - return nullptr; + uint32_t offset = next_offset_.load(std::memory_order_relaxed); + uint32_t aligned_offset; + while (true) { + if (offset > mr_size_ || offset > UINT32_MAX - 63U) + return nullptr; + aligned_offset = (offset + 63U) & ~63U; + if (aligned_offset > mr_size_ || capacity > mr_size_ - aligned_offset) + return nullptr; + if (next_offset_.compare_exchange_weak(offset, aligned_offset + capacity, std::memory_order_acq_rel, + std::memory_order_relaxed)) + break; } auto *buf = new Buffer; @@ -117,12 +126,54 @@ Buffer *RdmaBufferPool::alloc_fresh(uint32_t capacity) buf->allocation_capacity = capacity; buf->registration_handle = reinterpret_cast(mr_); buf->registration_key = mr_->lkey; - next_offset_ = aligned_offset + capacity; - ++outstanding_; - all_buffers_.push_back(buf); + Buffer *all = all_head_.load(std::memory_order_acquire); + do { + buf->next_all = all; + } while (!all_head_.compare_exchange_weak(all, buf, std::memory_order_acq_rel, + std::memory_order_acquire)); return buf; } +Buffer *RdmaBufferPool::pop(std::atomic &head) +{ + Buffer *current = head.load(std::memory_order_acquire); + while (current != nullptr && + !head.compare_exchange_weak(current, current->next_pool, std::memory_order_acq_rel, + std::memory_order_acquire)) { + } + if (current != nullptr) + current->next_pool = nullptr; + return current; +} + +void RdmaBufferPool::push(std::atomic &head, Buffer *buf) +{ + Buffer *current = head.load(std::memory_order_acquire); + do { + buf->next_pool = current; + } while (!head.compare_exchange_weak(current, buf, std::memory_order_acq_rel, + std::memory_order_acquire)); +} + +bool RdmaBufferPool::reserve_outstanding() +{ + uint32_t count = outstanding_.load(std::memory_order_relaxed); + while (count < max_buffers_ && + !outstanding_.compare_exchange_weak(count, count + 1, std::memory_order_acq_rel, + std::memory_order_relaxed)) { + } + return count < max_buffers_; +} + +void RdmaBufferPool::quarantine_inflight() +{ + for (Buffer *buf = all_head_.load(std::memory_order_acquire); buf != nullptr; buf = buf->next_all) { + const BufferOwnership state = buf->ownership.load(std::memory_order_acquire); + if (state != BufferOwnership::Available && state != BufferOwnership::Quarantined) + (void)buf->quarantine(); + } +} + Buffer *RdmaBufferPool::alloc(uint32_t capacity) { BufferCapacityClass cls; @@ -130,42 +181,46 @@ Buffer *RdmaBufferPool::alloc(uint32_t capacity) if (!classify_rdma_capacity(capacity, &cls, &bucket)) { return nullptr; } - std::scoped_lock lock(mu_); - auto it = free_list_.find(bucket); - if (it != free_list_.end() && !it->second.empty()) { - Buffer *buf = it->second.back(); - it->second.pop_back(); + Buffer *buf = pop(free_heads_[static_cast(cls)]); + if (buf != nullptr) { buf->reset_view(); buf->len = 0; buf->ref->store(1, std::memory_order_relaxed); buf->capacity_class = cls; buf->ownership.store(BufferOwnership::Available, std::memory_order_release); - ++outstanding_; + if (!reserve_outstanding()) { + push(free_heads_[static_cast(cls)], buf); + return nullptr; + } return buf; } - Buffer *buf = alloc_fresh(bucket); + if (!reserve_outstanding()) + return nullptr; + buf = alloc_fresh(bucket); if (buf != nullptr) { buf->capacity_class = cls; buf->registration_id = mr_ == nullptr ? 0 : mr_->lkey; } + else + outstanding_.fetch_sub(1, std::memory_order_acq_rel); return buf; } void RdmaBufferPool::recycle(Buffer *buf) { - { - std::scoped_lock lock(mu_); - buf->reset_view(); - if (buf->ownership.load(std::memory_order_acquire) == BufferOwnership::Quarantined) { - quarantine_.push_back(buf); - } - else { - buf->len = 0; - buf->ownership.store(BufferOwnership::Available, std::memory_order_release); - free_list_[buf->capacity].push_back(buf); - } + outstanding_.fetch_sub(1, std::memory_order_acq_rel); + buf->reset_view(); + const BufferOwnership state = buf->ownership.load(std::memory_order_acquire); + if (state != BufferOwnership::Available && state != BufferOwnership::Quarantined) + (void)buf->quarantine(); + if (buf->ownership.load(std::memory_order_acquire) == BufferOwnership::Quarantined) { + quarantined_bytes_.fetch_add(buf->capacity, std::memory_order_acq_rel); + push(quarantine_head_, buf); + return; } - --outstanding_; + buf->len = 0; + buf->ownership.store(BufferOwnership::Available, std::memory_order_release); + push(free_heads_[static_cast(buf->capacity_class)], buf); } // ── RdmaTransport ───────────────────────────────────────────────── @@ -314,9 +369,14 @@ bool RdmaTransport::submit(Connection *conn, OutFrame *frame) Buffer *slot = pools->send->alloc(RDMA_WIRE_SLOT_CAPACITY); if (slot == nullptr) return false; + if (!slot->transition(BufferOwnership::Available, BufferOwnership::SendInFlight)) { + slot->release(); + return false; + } const uint32_t data_len = frame->data == nullptr ? 0 : frame->data->len; const uint64_t wire_len = static_cast(HEADER_SIZE) + control_len + data_len; if (wire_len > slot->capacity) { + (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); slot->release(); CRB_LOG_WARN("rdma send: message exceeds wire slot control_len={} data_len={} capacity={}", control_len, data_len, slot->capacity); @@ -329,6 +389,7 @@ bool RdmaTransport::submit(Connection *conn, OutFrame *frame) std::memcpy(slot->data + HEADER_SIZE, source_control->data, control_len); if (data_len != 0) { if (frame->data->payload_data() == nullptr || data_len > frame->data->capacity) { + (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); slot->release(); CRB_LOG_WARN("rdma send: data buffer exceeds capacity len={} capacity={}", data_len, frame->data->capacity); return false; @@ -354,6 +415,7 @@ bool RdmaTransport::submit(Connection *conn, OutFrame *frame) ibv_send_wr *bad = nullptr; if (ibv_post_send(reinterpret_cast(conn->transport_handle), &wr, &bad) != 0) { CRB_LOG_WARN("rdma send: post failed qp={} errno={}", reinterpret_cast(conn->transport_handle), errno); + (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); slot->release(); return false; } @@ -379,21 +441,27 @@ bool RdmaTransport::post_receive(Connection *conn, Buffer *slot) sge.length = slot->capacity; sge.lkey = slot->registration_key; auto work = std::make_unique(ReceiveWork{conn, slot}); - slot->ref_clone(); + const bool slot_posted = slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive) || + slot->transition(BufferOwnership::CallerOwned, BufferOwnership::PostedReceive); + if (!slot_posted) + return false; + if (slot->ref_clone() == nullptr) { + (void)slot->transition(BufferOwnership::PostedReceive, BufferOwnership::Available); + return false; + } ibv_recv_wr wr{}; wr.wr_id = reinterpret_cast(work.get()) | 1U; wr.sg_list = &sge; wr.num_sge = 1; ibv_recv_wr *bad = nullptr; - const bool slot_posted = slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive) || - slot->transition(BufferOwnership::CallerOwned, BufferOwnership::PostedReceive); int post_rc = -1; if (slot_posted) post_rc = ibv_post_recv(reinterpret_cast(conn->transport_handle), &wr, &bad); if (!slot_posted || post_rc != 0) { CRB_LOG_WARN("rdma receive: post failed rc={} slot_state={}", post_rc, static_cast(slot->ownership.load(std::memory_order_relaxed))); - slot->ownership.store(BufferOwnership::Available, std::memory_order_release); + if (!slot->transition(BufferOwnership::PostedReceive, BufferOwnership::Available)) + (void)slot->quarantine(); slot->release(); return false; } @@ -438,13 +506,18 @@ void RdmaTransport::shutdown() { std::scoped_lock lock(cm_mu_); for (rdma_cm_id *id : cm_ids_) { + if (auto it = cm_pools_.find(id); it != cm_pools_.end()) { + it->second->send->quarantine_inflight(); + it->second->recv->quarantine_inflight(); + retired_pools_.push_back(std::move(it->second)); + cm_pools_.erase(it); + } if (id->qp != nullptr) rdma_destroy_qp(id); rdma_destroy_id(id); } cm_ids_.clear(); cm_connections_.clear(); - cm_pools_.clear(); } if (listen_id_ != nullptr) { if (listen_id_->qp != nullptr) @@ -965,9 +1038,18 @@ void RdmaTransport::release_completed_receive(ibv_wc &wc) work->slot->view_offset = control_end; work->slot->capacity = slot_capacity - control_end; work->slot->len = header.data_size; - work->slot->ownership.store(BufferOwnership::CallerOwned, std::memory_order_release); + if (!work->slot->transition(BufferOwnership::ReceiveInFlight, BufferOwnership::CallerOwned)) { + quarantine_and_release(); + delete frame; + return; + } frame->data_buf = work->slot; Buffer *repost_slot = work->slot->ref_clone(); + if (repost_slot == nullptr) { + quarantine_and_release(); + delete frame; + return; + } auto *repost = new RepostContext{this, work->conn, repost_slot}; frame->release_callback = &RdmaTransport::repost_frame_release; frame->release_context = repost; @@ -982,8 +1064,10 @@ void RdmaTransport::repost_frame_release(void *context, Buffer *) if (repost == nullptr) return; repost->slot->reset_view(); - repost->slot->ownership.store(BufferOwnership::Available, std::memory_order_release); - if (repost->transport != nullptr) + if (!repost->slot->transition(BufferOwnership::CallerOwned, BufferOwnership::Available)) + (void)repost->slot->quarantine(); + if (repost->transport != nullptr && + repost->slot->ownership.load(std::memory_order_acquire) == BufferOwnership::Available) (void)repost->transport->post_receive(repost->connection, repost->slot); repost->slot->release(); delete repost; @@ -1008,6 +1092,10 @@ void RdmaTransport::release_completed_frame(ibv_wc &wc) for (uint8_t i = 0; i < frame->data_views(); ++i) (void)frame->data_view(i)->quarantine(); } + else if (frame->control != nullptr && + !frame->control->transition(BufferOwnership::SendInFlight, BufferOwnership::Available)) { + (void)frame->control->quarantine(); + } if (frame->control != nullptr) frame->control->release(); frame->release_data(); diff --git a/lib/crowdb-rpc/tests/buffer_test.cpp b/lib/crowdb-rpc/tests/buffer_test.cpp index f33c2a9fb..51a86a748 100644 --- a/lib/crowdb-rpc/tests/buffer_test.cpp +++ b/lib/crowdb-rpc/tests/buffer_test.cpp @@ -150,3 +150,42 @@ TEST(BufferTest, RegisteredCapacityRejectsUnsupportedValues) EXPECT_EQ(pool.alloc(crowdb::rpc::RDMA_DATAPATH_CAPACITY + 1), nullptr); EXPECT_EQ(pool.alloc(0), nullptr); } + +TEST(BufferTest, RegisteredPoolRejectsPrematureRecycle) +{ + RegisteredBufferPool pool(2 * crowdb::rpc::RDMA_CONTROL_CAPACITY, 2); + Buffer *buf = pool.alloc(1); + ASSERT_NE(buf, nullptr); + ASSERT_TRUE(buf->transition(BufferOwnership::Available, BufferOwnership::DiskOwned)); + + buf->release(); + EXPECT_EQ(pool.quarantined_bytes(), crowdb::rpc::RDMA_CONTROL_CAPACITY); + + Buffer *replacement = pool.alloc(1); + ASSERT_NE(replacement, nullptr); + EXPECT_NE(replacement, buf); + replacement->release(); +} + +TEST(BufferTest, RegisteredPoolDetachTransfersReleaseToCaller) +{ + RegisteredBufferPool pool(2 * crowdb::rpc::RDMA_CONTROL_CAPACITY, 2); + Buffer *buf = pool.alloc(1); + ASSERT_NE(buf, nullptr); + ASSERT_TRUE(pool.detach(buf)); + EXPECT_EQ(buf->pool, nullptr); + EXPECT_FALSE(pool.detach(buf)); + buf->release(); +} + +TEST(BufferTest, ReleasedRegisteredBufferCannotBeReleasedTwice) +{ + RegisteredBufferPool pool(crowdb::rpc::RDMA_CONTROL_CAPACITY, 1); + Buffer *buf = pool.alloc(1); + ASSERT_NE(buf, nullptr); + buf->release(); + buf->release(); + Buffer *reused = pool.alloc(1); + ASSERT_EQ(reused, buf); + reused->release(); +} diff --git a/lib/crowdb-rpc/tests/rdma_test.cpp b/lib/crowdb-rpc/tests/rdma_test.cpp index a81d24795..cb293f31c 100644 --- a/lib/crowdb-rpc/tests/rdma_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_test.cpp @@ -68,6 +68,7 @@ using crowdb::rpc::Frame; using crowdb::rpc::Header; using crowdb::rpc::HEADER_SIZE; using crowdb::rpc::OutFrame; +using crowdb::rpc::RdmaConfig; using crowdb::rpc::RdmaBufferPool; using crowdb::rpc::RdmaTransport; using crowdb::rpc::RpcClient; @@ -132,6 +133,24 @@ TEST(RdmaBufferPoolTest, TransportCopiesSystemBufferIntoRegisteredPool) source->release(); } +TEST(RdmaBufferPoolTest, RegistrationFailureLeavesSystemPoolUsable) +{ + RdmaConfig config; + config.device = "crowdb-device-that-does-not-exist"; + RdmaTransport transport(config); + SystemBufferPool system_pool(1); + Buffer *source = system_pool.alloc(32); + ASSERT_NE(source, nullptr); + source->write("tcp", 3); + EXPECT_EQ(transport.register_buffer(source), nullptr); + EXPECT_EQ(source->len, 3U); + EXPECT_EQ(std::memcmp(source->data, "tcp", 3), 0); + source->release(); + Buffer *reused = system_pool.alloc(32); + ASSERT_NE(reused, nullptr); + reused->release(); +} + TEST(RdmaHardwareTest, CompleteCmQpSendReceiveFlow) { constexpr uint64_t kFrameCount = 8; From 1f40d1a08c38abab16b18e516d597b560e9f537c Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 22 Oct 2026 18:21:52 +0800 Subject: [PATCH 019/258] Stabilize RDMA completion ownership and timed load testing --- doc/working/plan-rdma-rpc-transport.md | 120 ++- doc/working/plan-rdma.md | 22 +- lib/crowdb-rpc/CMakeLists.txt | 4 + lib/crowdb-rpc/include/crowdb-rpc/buffer.h | 5 + .../transport/rdma/rdma_transport.h | 152 ++- .../crowdb-rpc/transport/socket_transport.h | 16 +- lib/crowdb-rpc/src/buffer.cpp | 28 +- lib/crowdb-rpc/src/server/server.cpp | 1 + .../src/transport/epoll/epoll_engine.cpp | 5 +- .../src/transport/rdma/rdma_buffer_pool.cpp | 190 +++- .../src/transport/rdma/rdma_channel.h | 34 + .../src/transport/rdma/rdma_completion.cpp | 284 ++++++ .../src/transport/rdma/rdma_config.cpp | 56 ++ .../src/transport/rdma/rdma_connection.cpp | 399 ++++++++ .../src/transport/rdma/rdma_transport.cpp | 874 ++---------------- .../src/transport/rdma/rdma_work_request.cpp | 94 ++ .../src/transport/socket_transport.cpp | 12 +- .../tests/common/test_rdma_load.cpp | 188 ++++ lib/crowdb-rpc/tests/common/test_rdma_load.h | 94 ++ .../tests/common/test_rdma_transport.h | 112 +++ lib/crowdb-rpc/tests/rdma_completion_test.cpp | 275 ++++++ lib/crowdb-rpc/tests/rdma_config_test.cpp | 47 + lib/crowdb-rpc/tests/rdma_test.cpp | 81 +- lib/crowdb-rpc/tests/rdma_timed_load_test.cpp | 203 ++++ lib/crowdb-rpc/tests/rdma_worker_test.cpp | 243 +++++ lib/crowdb-rpc/tests/transport_test.cpp | 1 + tools/pixi-tasks/build-cpp.sh | 10 +- tools/rdma/run-local-loop.sh | 29 +- tools/rdma/run-timed-load.py | 97 ++ tools/rpc-test/src/workload.rs | 6 +- 30 files changed, 2737 insertions(+), 945 deletions(-) create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_channel.h create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp create mode 100644 lib/crowdb-rpc/tests/common/test_rdma_load.cpp create mode 100644 lib/crowdb-rpc/tests/common/test_rdma_load.h create mode 100644 lib/crowdb-rpc/tests/common/test_rdma_transport.h create mode 100644 lib/crowdb-rpc/tests/rdma_completion_test.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_config_test.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_timed_load_test.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_worker_test.cpp create mode 100644 tools/rdma/run-timed-load.py diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 5c5122a89..6db0031bc 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -45,6 +45,119 @@ stable completion ownership mapping, and executing the single-node multi-container OCI profiles against a built `crowdb:test` image. Physical multi-node validation is deferred to [the separate follow-up plan](plan-rdma-multi-node.md). +The completion ownership mapping now uses a bounded, generation-tagged +`WorkRequest` table for SEND and RECV completions. It retains the connection, +request, stream, sequence, and logical offset. Error dispatch uses the retained +work kind rather than an undefined completion opcode. A receive lease is +reposted by `Buffer::last_release_cb` only after the final C++/FFI owner drops +it; held caller leases survive frame deletion and transport shutdown. + +The current code also preposts receives before `rdma_connect`/`rdma_accept`, +uses a short configurable RNR timer, drains acknowledged CQ work before CM +close, and fixes CQ registration and buffer-pool teardown ordering. The earlier +64-request burst exposed RNR stalls and premature test-server teardown. The +worker test now has a client-finished barrier, configurable workload size, +worker count, QP count, bounded in-flight window, per-request payload validation, +and TPS/P50/P95/P99 output. + +## Current continuation checkpoint + +- [x] **Use standard queue terminology**: `sq_depth`, `rq_depth`, + `posted_receive_target`, `cq_depth`, `wire_slot_capacity`, `send_arena_bytes`, + and `receive_arena_bytes` are the configuration variables. SQ/RQ map to + verbs `max_send_wr`/`max_recv_wr`; byte budgets use 64-bit values so a 4096 + entry RQ with 1 MiB slots can represent its full 4 GiB receive arena. +- [x] **Verify queue/buffer regressions**: the 32-way C++ build and 26 focused + configuration, buffer, work ownership, CQ/CM, and epoll tests passed. +- [x] **Verify the small-message functional baseline**: 1000 requests, + 128-byte payload, one worker, one QP, one in-flight request, 1 KiB wire + slots, `sq_depth = rq_depth = posted_receive_target = 1024`, and 1 MiB send + and receive arenas passed at 71,682 TPS, P50 13.08 us, P95 17.99 us, + P99 20.69 us, and zero RPC errors. Posted receives flushed at normal peer + shutdown are reported separately by the transport counters. These short + fixed-count results are functional evidence only, not sustained load evidence. +- [x] **Complete staged small-message load acceptance**: discuss these actual code variables + with the user, then run each profile for one second of warmup and ten + seconds of measured traffic. Count successful completions inside that + interval; exclude warmup and post-window draining. Increase + `producer_count`, `io_workers`, and `connection_count` through 1, 2, 4, 8, + 16, and 32, with a separate one-in-flight serial baseline and 32 in flight + per producer for pipelined profiles. Validate 1024 and 4096 RQ depths; + use small wire slots for small-message tests and larger slots/arenas for + later realistic payloads. `RdmaHardwareTest.RpcServerClientTimedLoad` and + `tools/rdma/run-timed-load.py` retain complete stage logs and CSV results + below `.crowdb-runtime/artifacts/rdma/`. Completion correlation, payload + contents, duplicate completion, accepted-versus-completed counts, and + server request totals must agree at every stage. +- [x] **Rerun code gates**: `pixi run test-cpp`, the 26 Rust RPC FFI tests, + `cargo fmt --all -- --check`, `rs-lint`, changed C++ formatting, and + `tree-lint` passed on this diff. Tree-lint exits zero with style warnings. + The first full C++ run hit the existing 128-byte temporary-path limit; + rerunning under `CROWDB_RUNTIME_ROOT=.crowdb-runtime/rdma-check` passed. + Full gate artifacts: + `.crowdb-runtime/artifacts/unit/cpp-3ab318e0c5a04f27a8328be78ca1cab6/`. +- [~] **Complete remaining transport and OCI acceptance**: negotiated TCP + capability preflight, real registered-lease PD/MR lifetime verification, + and multi-container profiles remain open. Do not treat the historical + gate record below as verification of these unfinished contracts. + +### Timed load evidence + +- `tools/rdma/run-timed-load.py` now runs one-second warmup followed by a + ten-second completion window for every profile. It validates unique QPs, + RDMA path selection, request correlation, 128-byte payload contents, drain + counts, and server totals. Metrics retain bounded per-producer histograms + and in-flight slots rather than one entry per request. +- The 16-worker run exposed competing CM consumers taking passive setup + events. Passive setup now uses a private CM channel until ESTABLISHED, + and shared CM event handling has one atomic consumer. +- The 32-worker run exposed a lost continuation: a competing worker could + clear `continuation_fd_` before acquiring the CQ. Only the CQ consumer now + clears it. The bounded epoll continuation test reproduces that contention + deterministically; all 26 focused tests pass after the fix. +- Every profile in the corrected matrix completed its full ten seconds with + zero RPC, correlation, or payload errors. The single-in-flight baseline + completed 626,983 requests (62,698 TPS, P99 26.24 us). +- All rows below use `producer_count = connection_count`, + `in_flight_per_producer = 32`, `wire_slot_capacity = 1024`, + `sq_depth = rq_depth = posted_receive_target = 1024`. Each I/O worker + count is **per peer**, with both peers on one physical host/HCA. + +| Producers / QPs | TPS: I/O workers = producers | TPS: I/O workers = 1 | +|-----------------|------------------------------|---------------------| +| 1 | 182,431 | 183,340 | +| 2 | 292,921 | 322,559 | +| 4 | 327,623 | 356,282 | +| 8 | 354,189 | 396,683 | +| 16 | 206,622 | 392,360 | +| 32 | 83,218 | 292,424 | + +- Complete corrected matrix: + `.crowdb-runtime/artifacts/rdma/20261022T100944Z/results.csv`. +- Fixed-I/O-worker control: + `.crowdb-runtime/artifacts/rdma/20261022T101119Z/results.csv`. +- The `rq_depth = posted_receive_target = 4096` control, with + `receive_arena_bytes = 4 MiB` per QP and `io_workers = 1`, passed at + 369,719 TPS for 8 producers/QPs (P99 761.86 us) and 278,266 TPS for 32 + (P99 4653.06 us), both with zero errors. Complete logs: + `.crowdb-runtime/artifacts/rdma/20261022T101244Z/results.csv`. +- The control supports investigating shared-CQ worker contention and CPU + scheduling before increasing receive depth. It does not establish the HCA + hardware throughput limit. CPU affinity and frequency are not fixed. +- Remaining contract work is separate from this sustained small-message + acceptance: complete TCP capability preflight before CM probing and prove + fallback, retain real PD/MR lifetimes through held leases, and run the OCI + profiles. Existing capability serialization and state predicates alone do + not prove that every connection follows the negotiated setup contract. + +Provider references for the implementation choices: + +- [libibverbs QP capacity terminology](https://github.com/linux-rdma/rdma-core/blob/master/libibverbs/man/ibv_create_qp.3). +- [RDMA-CM receive posting before connection readiness](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/man/rdma_create_qp.3). +- [RDMA-CM RNR behavior and default delay](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/man/rdma_accept.3). + +## Historical verification record + The C++ unit/integration gate and Rust FFI tests pass after the teardown and completion-error changes; `tree-lint` completes with the repository's existing clang-tidy warning set. The OCI runner now mounts a named data volume and uses @@ -142,9 +255,10 @@ Remaining implementation and acceptance work: - [x] **Post one-slot receives**: prepost one full registered receive slot per receive ring entry, associate each with a stable `wr_id`, and replenish only after the owner releases it. -- [~] **Resolve WR identity**: `wr_id` must identify connection, request ID, - stream ID, sequence, logical offset, buffers, and ownership state. Handle - success, flush, local protection, retry, and remote access errors distinctly. +- [x] **Resolve WR identity**: `wr_id` identifies connection, request ID, + stream ID, sequence, logical offset, buffers, and ownership state. Success, + flush, local protection, retry, and remote access errors are accounted for; + injected provider-error coverage remains in the acceptance checklist. - [x] **Dispatch common callbacks**: validate lengths and class before parser publication, then invoke the existing C++ completion path and FFI callback. diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index 472e5021f..87a61b316 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -18,14 +18,28 @@ single-node HCA local-loop with multiple containers. ## Current implementation status -R238 is complete. The common RPC buffer layer and verbs-backed per-connection +R238's foundation implementation is committed. The common RPC buffer layer and verbs-backed per-connection arenas share the four capacity classes, bounded admission, explicit owner transitions, safe release ordering, quarantine handling, shutdown quarantine, and C/Rust lease metadata accessors. R239's transport path is implemented and passes the prepared HCA local-loop plus the real `RpcServer`/`RpcClient` worker round trip. Its -requirement is still open because the fault-injection, pre-payload fallback, -multi-container, and final gate cases have not all passed. +requirement is still open because negotiated capability preflight, +pre-payload fallback, real PD/MR lifetime with held caller leases, and +multi-container acceptance have not all passed. + +The latest transport handoff closes the completion ownership gap: SEND and +RECV work requests now carry stable connection/request/stream metadata, receive +leases remain caller-owned until their final C++/FFI reference is released +before repost, and CQ send failures close the uncertain RDMA stream after quarantining +its buffers. The worker test checks a non-empty response payload as well as +request correlation. The current local-loop runtime is prepared. The single-worker baseline with +1024 receive WRs and small wire slots passes. The staged 1/2/4/8/16/32 +producer/QP profiles now each run for ten measured seconds after warmup, +with zero errors after fixing CM event ownership and CQ continuation races. +The 1024 and 4096 RQ profiles, full C++ gate, Rust RPC FFI suite, formatting, +clippy, and tree-lint pass. The transport plan records exact counts, latency, +and the fixed-I/O-worker control; protocol and OCI acceptance remain open. This is the root plan. The five child plans contain implementation details; only this plan controls phase order, cross-backlog decisions, and final @@ -35,7 +49,7 @@ the formal design first. ## Fixed design decisions - Use one common RPC abstraction for TCP and RDMA. -- Use RC SEND/RECV, one 1 KiB control SGE and one selected data SGE. +- Use RC SEND/RECV, one contiguous wire slot and one SGE per message. - Use service discovery for coarse RDMA capability and TCP handshake for the current RDMA endpoint, capability generation, and data classes. - Select RDMA before the first payload; fallback is RDMA to TCP only. diff --git a/lib/crowdb-rpc/CMakeLists.txt b/lib/crowdb-rpc/CMakeLists.txt index abd8e55a7..110604303 100644 --- a/lib/crowdb-rpc/CMakeLists.txt +++ b/lib/crowdb-rpc/CMakeLists.txt @@ -85,6 +85,10 @@ if(NOT TARGET crowdbcommon) endif() add_library(crowdb-rpc STATIC ${CROWDB_RPC_SRC}) +option(CROWDB_RPC_TEST_UTIL "Enable private RDMA test access" OFF) +if(CROWDB_RPC_TEST_UTIL) + target_compile_definitions(crowdb-rpc PUBLIC CROWDB_RPC_TEST_UTIL=1) +endif() target_include_directories(crowdb-rpc PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/../crowdb-protocol/cpp/include diff --git a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h index 26fad9c5d..ec2945235 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h @@ -96,6 +96,11 @@ struct Buffer void (*free_cb)(void *) = nullptr; void *free_ctx = nullptr; + // Transfer the final reference to an asynchronous receive recycler. + // The callback consumes that reference, including any repost attempt. + void (*last_release_cb)(void *, Buffer *) = nullptr; + void *last_release_ctx = nullptr; + // Copy src into data, set len. Called once per buffer. void write(const void *src, uint32_t n); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index 01d64f167..deea3cb06 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -33,17 +33,24 @@ struct RdmaConfig { std::string device; uint8_t port = 1; - uint32_t cq_depth = 2048; - uint32_t buffer_region_bytes = 64U << 20; - uint32_t buffer_ring_depth = 1024; - uint32_t qp_max_send_wr = 1024; - uint32_t qp_max_recv_wr = 1024; - uint32_t qp_max_send_sge = 2; - uint32_t qp_max_recv_sge = 2; - uint32_t poll_batch = 256; - uint32_t poll_empty_budget = 256; + uint32_t cq_depth = 8192; + uint32_t wire_slot_capacity = RDMA_WIRE_SLOT_CAPACITY; + uint64_t send_arena_bytes = 64ULL << 20; + uint64_t receive_arena_bytes = 1ULL << 30; + uint32_t sq_depth = 1024; + uint32_t rq_depth = 1024; + uint32_t qp_max_send_sge = 2; + uint32_t qp_max_recv_sge = 2; + uint32_t poll_batch = 256; + uint32_t poll_empty_budget = 256; + uint32_t shutdown_drain_ms = 3000; + uint32_t posted_receive_target = 1024; + uint8_t min_rnr_timer = 1; uint8_t retry_count = 7; uint8_t rnr_retry_count = 7; + + [[nodiscard]] bool apply_environment(); + [[nodiscard]] bool valid() const; }; // RdmaBufferPool: allocates buffers from ibv_mr-registered memory. @@ -57,7 +64,7 @@ class RdmaBufferPool : public BufferPool { public: // mr_size: total registered memory size. max_buffers: pool capacity. - RdmaBufferPool(struct ibv_pd *pd, uint32_t mr_size = 1 << 20, uint32_t max_buffers = 1024); + RdmaBufferPool(struct ibv_pd *pd, uint64_t mr_size = 1 << 20, uint32_t max_buffers = 1024); ~RdmaBufferPool() override; Buffer *alloc(uint32_t capacity) override; @@ -76,17 +83,17 @@ class RdmaBufferPool : public BufferPool void quarantine_inflight(); private: - struct ibv_pd *pd_; - struct ibv_mr *mr_ = nullptr; - void *region_ = nullptr; - uint32_t mr_size_; - uint32_t max_buffers_; + struct ibv_pd *pd_; + struct ibv_mr *mr_ = nullptr; + void *region_ = nullptr; + uint64_t mr_size_; + uint32_t max_buffers_; std::atomic outstanding_{0}; - std::atomic next_offset_{0}; + std::atomic next_offset_{0}; std::array, 4> free_heads_{}; std::atomic quarantine_head_{nullptr}; std::atomic all_head_{nullptr}; - std::atomic quarantined_bytes_{0}; + std::atomic quarantined_bytes_{0}; Buffer *alloc_fresh(uint32_t capacity); Buffer *pop(std::atomic &head); @@ -169,17 +176,20 @@ class RdmaTransport : public Transport } private: +# ifdef CROWDB_RPC_TEST_UTIL + friend struct TestRdmaTransport; +# endif RdmaConfig config_; - bool capability_available_ = false; - struct ibv_context *context_ = nullptr; - bool owns_context_ = true; - struct ibv_pd *pd_ = nullptr; - struct ibv_comp_channel *comp_channel_ = nullptr; - struct ibv_cq *cq_ = nullptr; - struct rdma_event_channel *cm_channel_ = nullptr; - struct rdma_cm_id *listen_id_ = nullptr; - std::string requested_device_; - uint8_t port_ = 1; + bool capability_available_ = false; + struct ibv_context *context_ = nullptr; + bool owns_context_ = true; + struct ibv_pd *pd_ = nullptr; + struct ibv_comp_channel *comp_channel_ = nullptr; + struct ibv_cq *cq_ = nullptr; + struct rdma_event_channel *cm_channel_ = nullptr; + struct rdma_cm_id *listen_id_ = nullptr; + std::string requested_device_; + uint8_t port_ = 1; std::vector cm_ids_; std::unordered_map> cm_connections_; SocketEngine *event_engine_ = nullptr; @@ -190,34 +200,98 @@ class RdmaTransport : public Transport ConnectionHandler connection_handler_; std::atomic running_{false}; + std::atomic_flag cq_polling_ = ATOMIC_FLAG_INIT; + std::atomic_flag cm_polling_ = ATOMIC_FLAG_INIT; + int continuation_fd_ = -1; bool prime_receive(const std::shared_ptr &connection); Stats stats_; - struct ReceiveWork + enum class WorkKind : uint8_t { Send, Receive }; + + struct WorkRequest { - Connection *conn; - Buffer *slot; + std::atomic token{0}; + WorkKind kind; + std::shared_ptr conn; + Buffer *slot = nullptr; + OutFrame *frame = nullptr; + uint64_t request_id = 0; + uint64_t stream_id = 0; + uint64_t sequence = 0; + uint64_t logical_offset = 0; }; - struct RepostContext + static constexpr uint64_t WORK_RESERVED = UINT64_MAX; + std::unique_ptr work_requests_; + std::atomic next_work_{0}; + std::atomic work_generation_{1}; + + WorkRequest *acquire_work(WorkKind kind, Connection *conn, Buffer *slot, OutFrame *frame = nullptr); + WorkRequest *claim_work(uint64_t token); + void release_work(WorkRequest *work); + void dispatch_completion(const struct ibv_wc &wc); + void cancel_work(Connection *connection = nullptr); + void close_after_drain(Connection *connection); + + struct RepostState { - RdmaTransport *transport; - Connection *connection; - Buffer *slot; + static constexpr uint64_t STOPPED = uint64_t{1} << 63; + std::atomic users{0}; + RdmaTransport *transport = nullptr; }; - struct ConnectionPools + class Operation + { + public: + explicit Operation(std::shared_ptr state) : state_(std::move(state)) + { + uint64_t users = state_->users.load(std::memory_order_acquire); + while ((users & RepostState::STOPPED) == 0 && + !state_->users.compare_exchange_weak(users, users + 1, std::memory_order_acq_rel)) { + } + entered_ = (users & RepostState::STOPPED) == 0; + } + + ~Operation() + { + if (entered_) + state_->users.fetch_sub(1, std::memory_order_release); + } + + explicit operator bool() const + { + return entered_; + } + + Operation(const Operation &) = delete; + Operation &operator=(const Operation &) = delete; + + private: + std::shared_ptr state_; + bool entered_ = false; + }; + + std::shared_ptr repost_state_; + + struct ConnectionPools : std::enable_shared_from_this { std::unique_ptr send; std::unique_ptr recv; }; - std::unordered_map> cm_pools_; - std::vector> retired_pools_; + struct RepostContext + { + std::shared_ptr state; + std::shared_ptr connection; + std::shared_ptr pools; + }; + + std::unordered_map> cm_pools_; + std::vector> retired_pools_; - void release_completed_frame(struct ibv_wc &wc); - void release_completed_receive(struct ibv_wc &wc); + void release_completed_frame(WorkRequest *work, const struct ibv_wc &wc); + void release_completed_receive(WorkRequest *work, const struct ibv_wc &wc); void record_completion_error(const struct ibv_wc &wc); static void repost_frame_release(void *context, Buffer *data); bool adopt_cm_context(struct ibv_context *context); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h index 1ad220c0b..42635c7b3 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h @@ -328,7 +328,8 @@ class SocketTransport : public Transport // with io_workers total workers (per-engine = io_workers / io_engines). SocketTransport(uint32_t io_engines, uint32_t io_workers, BufferPool *pool = nullptr #ifdef CROWDB_RPC_HAVE_RDMA - , RdmaConfig rdma_config = {} + , + RdmaConfig rdma_config = {} #endif ); // Deprecated alias: maps to (1, num_workers). Kept for backward compat. @@ -380,8 +381,7 @@ class SocketTransport : public Transport std::shared_ptr connect(const std::string &addr, int port); // Attach an RDMA connection to the common callback/registry lifecycle. - void attach_rdma_connection(const std::shared_ptr &conn, - Connection::OnFrameCallback on_frame = {}, + void attach_rdma_connection(const std::shared_ptr &conn, Connection::OnFrameCallback on_frame = {}, Connection::OnCloseCallback on_close = {}); using RdmaAcceptHandler = std::function &)>; @@ -510,11 +510,19 @@ class SocketTransport : public Transport bool send_rdma_handshake(const std::shared_ptr &conn); bool listen_rdma(const std::string &addr, int port); +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) + const RdmaTransport::Stats *rdma_stats_for_tests() const + { + return rdma_transport_ == nullptr ? nullptr : &rdma_transport_->stats(); + } + +#endif + private: BufferPool *pool_; #ifdef CROWDB_RPC_HAVE_RDMA std::unique_ptr rdma_transport_; - RdmaConfig rdma_config_; + RdmaConfig rdma_config_; #endif std::vector> workers_; std::vector> engines_; // transport owns all engines diff --git a/lib/crowdb-rpc/src/buffer.cpp b/lib/crowdb-rpc/src/buffer.cpp index 8e3de1b4b..2dd3560d9 100644 --- a/lib/crowdb-rpc/src/buffer.cpp +++ b/lib/crowdb-rpc/src/buffer.cpp @@ -51,8 +51,7 @@ Buffer *Buffer::ref_clone() assert(ref != nullptr && "Buffer::ref_clone on released buffer"); int32_t count = ref->load(std::memory_order_acquire); while (count > 0 && - !ref->compare_exchange_weak(count, count + 1, std::memory_order_acq_rel, - std::memory_order_acquire)) { + !ref->compare_exchange_weak(count, count + 1, std::memory_order_acq_rel, std::memory_order_acquire)) { } if (count <= 0) return nullptr; @@ -65,12 +64,22 @@ void Buffer::release() return; int32_t count = ref->load(std::memory_order_acquire); while (count > 0 && - !ref->compare_exchange_weak(count, count - 1, std::memory_order_acq_rel, - std::memory_order_acquire)) { + !ref->compare_exchange_weak(count, count - 1, std::memory_order_acq_rel, std::memory_order_acquire)) { } if (count == 0 || count < 0) return; if (count == 1) { + if (last_release_cb != nullptr) { + auto callback = last_release_cb; + void *context = last_release_ctx; + last_release_cb = nullptr; + last_release_ctx = nullptr; + // No other owner remains. Give the recycler one reference so it + // can repost or release using the ordinary ownership contract. + ref->store(1, std::memory_order_release); + callback(context, this); + return; + } // Last reference — recycle to pool, call external free, or free // standalone. if (free_cb != nullptr) { @@ -168,8 +177,8 @@ void SystemBufferPool::recycle(Buffer *buf) bool SystemBufferPool::detach(Buffer *buf) { - const BufferOwnership state = buf == nullptr ? BufferOwnership::Quarantined - : buf->ownership.load(std::memory_order_acquire); + const BufferOwnership state = + buf == nullptr ? BufferOwnership::Quarantined : buf->ownership.load(std::memory_order_acquire); if (buf == nullptr || buf->pool != this || buf->ref->load(std::memory_order_acquire) != 1 || (state != BufferOwnership::Available && state != BufferOwnership::CallerOwned)) { return false; @@ -287,10 +296,9 @@ void RegisteredBufferPool::recycle(Buffer *buf) bool RegisteredBufferPool::detach(Buffer *buf) { - const BufferOwnership state = buf == nullptr ? BufferOwnership::Quarantined - : buf->ownership.load(std::memory_order_acquire); - if (buf == nullptr || buf->pool != this || buf->ref == nullptr || - buf->ref->load(std::memory_order_acquire) != 1 || + const BufferOwnership state = + buf == nullptr ? BufferOwnership::Quarantined : buf->ownership.load(std::memory_order_acquire); + if (buf == nullptr || buf->pool != this || buf->ref == nullptr || buf->ref->load(std::memory_order_acquire) != 1 || (state != BufferOwnership::Available && state != BufferOwnership::CallerOwned)) return false; buf->pool = nullptr; diff --git a/lib/crowdb-rpc/src/server/server.cpp b/lib/crowdb-rpc/src/server/server.cpp index 0efbd217a..44fd6d9e0 100644 --- a/lib/crowdb-rpc/src/server/server.cpp +++ b/lib/crowdb-rpc/src/server/server.cpp @@ -56,6 +56,7 @@ RpcServer::~RpcServer() // callback gauge (e.g. conn_count_gauge) that captures a raw // transport pointer returns 0 instead of accessing freed memory. alive_->store(false, std::memory_order_release); + transport_.reset(); if (owns_pool_) { delete pool_; } diff --git a/lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp b/lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp index 3c0d253fd..3000a262a 100644 --- a/lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp +++ b/lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp @@ -129,7 +129,10 @@ void EpollEngine::add_external_fd(int fd, uint64_t tag) std::memset(&ev, 0, sizeof(ev)); ev.events = EPOLLIN; ev.data.u64 = EXTERNAL_EVENT | static_cast(fd); - if (::epoll_ctl(epoll_fd_, EPOLL_CTL_ADD, fd, &ev) < 0) { + int result = ::epoll_ctl(epoll_fd_, EPOLL_CTL_ADD, fd, &ev); + if (result < 0 && errno == EEXIST) + result = ::epoll_ctl(epoll_fd_, EPOLL_CTL_MOD, fd, &ev); + if (result < 0) { std::scoped_lock lock(conn_mu_); external_fds_.erase(fd); } diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp index 2b8aa06d1..8690c6662 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp @@ -1,14 +1,188 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -// RdmaBufferPool implementation is in rdma_transport.cpp (same compilation -// unit for the RDMA gate). This file exists so the CMakeLists.txt -// rdma_buffer_pool.cpp entry has a source to compile when RDMA is enabled. -// On non-RDMA builds, both files are excluded from the source list. +#include "crowdb-rpc/transport/rdma/rdma_transport.h" #ifdef CROWDB_RPC_HAVE_RDMA -// The actual RdmaBufferPool methods are defined in rdma_transport.cpp. -// This file is intentionally empty — it exists only for the CMake source -// list symmetry. When the RDMA implementation is completed, the buffer -// pool code may be split into this file. + +# include + +namespace crowdb::rpc +{ + +// ── RdmaBufferPool ──────────────────────────────────────────────── + +RdmaBufferPool::RdmaBufferPool(struct ibv_pd *pd, uint64_t mr_size, uint32_t max_buffers) + : pd_(pd), + mr_size_(mr_size), + max_buffers_(max_buffers) +{ + for (auto &head : free_heads_) + head.store(nullptr, std::memory_order_relaxed); + if (pd_ == nullptr || mr_size_ == 0 || max_buffers_ == 0) { + return; + } + if (posix_memalign(®ion_, 64, mr_size_) != 0) { + region_ = nullptr; + return; + } + mr_ = ibv_reg_mr(pd_, region_, mr_size_, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | + IBV_ACCESS_REMOTE_ATOMIC); + if (mr_ == nullptr) { + std::free(region_); + region_ = nullptr; + return; + } +} + +RdmaBufferPool::~RdmaBufferPool() +{ + if (outstanding_.load(std::memory_order_acquire) != 0) { + // Keep the MR registered while a caller, disk operation, or device + // completion still owns a slice. Process teardown reclaims it. + return; + } + Buffer *buf = all_head_.exchange(nullptr, std::memory_order_acq_rel); + while (buf != nullptr) { + Buffer *next = buf->next_all; + delete buf->ref; + delete buf; + buf = next; + } + if (mr_ != nullptr) { + ibv_dereg_mr(mr_); + mr_ = nullptr; + } + if (region_ != nullptr) { + std::free(region_); + region_ = nullptr; + } +} + +Buffer *RdmaBufferPool::alloc_fresh(uint32_t capacity) +{ + if (mr_ == nullptr || region_ == nullptr) { + return nullptr; + } + uint64_t offset = next_offset_.load(std::memory_order_relaxed); + uint64_t aligned_offset; + while (true) { + if (offset > mr_size_ || offset > UINT64_MAX - 63U) + return nullptr; + aligned_offset = (offset + 63U) & ~uint64_t{63}; + if (aligned_offset > mr_size_ || capacity > mr_size_ - aligned_offset) + return nullptr; + if (next_offset_.compare_exchange_weak(offset, aligned_offset + capacity, std::memory_order_acq_rel, + std::memory_order_relaxed)) + break; + } + + auto *buf = new Buffer; + buf->capacity = capacity; + buf->type = BufferType::Registered; + buf->pool = this; + buf->ref = new std::atomic(1); + + buf->data = static_cast(region_) + aligned_offset; + buf->len = 0; + buf->allocation_data = buf->data; + buf->allocation_capacity = capacity; + buf->registration_handle = reinterpret_cast(mr_); + buf->registration_key = mr_->lkey; + Buffer *all = all_head_.load(std::memory_order_acquire); + do { + buf->next_all = all; + } while (!all_head_.compare_exchange_weak(all, buf, std::memory_order_acq_rel, std::memory_order_acquire)); + return buf; +} + +Buffer *RdmaBufferPool::pop(std::atomic &head) +{ + Buffer *current = head.load(std::memory_order_acquire); + while (current != nullptr && !head.compare_exchange_weak(current, current->next_pool, std::memory_order_acq_rel, + std::memory_order_acquire)) { + } + if (current != nullptr) + current->next_pool = nullptr; + return current; +} + +void RdmaBufferPool::push(std::atomic &head, Buffer *buf) +{ + Buffer *current = head.load(std::memory_order_acquire); + do { + buf->next_pool = current; + } while (!head.compare_exchange_weak(current, buf, std::memory_order_acq_rel, std::memory_order_acquire)); +} + +bool RdmaBufferPool::reserve_outstanding() +{ + uint32_t count = outstanding_.load(std::memory_order_relaxed); + while (count < max_buffers_ && !outstanding_.compare_exchange_weak(count, count + 1, std::memory_order_acq_rel, + std::memory_order_relaxed)) { + } + return count < max_buffers_; +} + +void RdmaBufferPool::quarantine_inflight() +{ + for (Buffer *buf = all_head_.load(std::memory_order_acquire); buf != nullptr; buf = buf->next_all) { + const BufferOwnership state = buf->ownership.load(std::memory_order_acquire); + if (state != BufferOwnership::Available && state != BufferOwnership::Quarantined) + (void)buf->quarantine(); + } +} + +Buffer *RdmaBufferPool::alloc(uint32_t capacity) +{ + BufferCapacityClass cls; + uint32_t bucket; + if (!classify_rdma_capacity(capacity, &cls, &bucket)) { + return nullptr; + } + Buffer *buf = pop(free_heads_[static_cast(cls)]); + if (buf != nullptr) { + buf->reset_view(); + buf->len = 0; + buf->ref->store(1, std::memory_order_relaxed); + buf->capacity_class = cls; + buf->ownership.store(BufferOwnership::Available, std::memory_order_release); + if (!reserve_outstanding()) { + push(free_heads_[static_cast(cls)], buf); + return nullptr; + } + return buf; + } + if (!reserve_outstanding()) + return nullptr; + buf = alloc_fresh(bucket); + if (buf != nullptr) { + buf->capacity_class = cls; + buf->registration_id = mr_ == nullptr ? 0 : mr_->lkey; + } + else + outstanding_.fetch_sub(1, std::memory_order_acq_rel); + return buf; +} + +void RdmaBufferPool::recycle(Buffer *buf) +{ + outstanding_.fetch_sub(1, std::memory_order_acq_rel); + buf->reset_view(); + const BufferOwnership state = buf->ownership.load(std::memory_order_acquire); + if (state != BufferOwnership::Available && state != BufferOwnership::Quarantined) + (void)buf->quarantine(); + if (buf->ownership.load(std::memory_order_acquire) == BufferOwnership::Quarantined) { + quarantined_bytes_.fetch_add(buf->capacity, std::memory_order_acq_rel); + push(quarantine_head_, buf); + return; + } + buf->len = 0; + buf->ownership.store(BufferOwnership::Available, std::memory_order_release); + push(free_heads_[static_cast(buf->capacity_class)], buf); +} + +} // namespace crowdb::rpc + #endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_channel.h b/lib/crowdb-rpc/src/transport/rdma/rdma_channel.h new file mode 100644 index 000000000..2962edc8a --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_channel.h @@ -0,0 +1,34 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#ifdef CROWDB_RPC_HAVE_RDMA + +# include +# include +# include + +namespace crowdb::rpc::rdma_detail +{ +inline void make_nonblocking(int fd) +{ + if (fd < 0) + return; + const int flags = ::fcntl(fd, F_GETFL, 0); + if (flags >= 0) + (void)::fcntl(fd, F_SETFL, flags | O_NONBLOCK); +} + +inline bool wait_cm_event(rdma_event_channel *channel, rdma_cm_event **event, int timeout_ms) +{ + if (channel == nullptr || event == nullptr) + return false; + pollfd descriptor{channel->fd, POLLIN, 0}; + if (::poll(&descriptor, 1, timeout_ms) <= 0 || (descriptor.revents & POLLIN) == 0) + return false; + return rdma_get_cm_event(channel, event) == 0; +} +} // namespace crowdb::rpc::rdma_detail + +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp new file mode 100644 index 000000000..31920a6f3 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp @@ -0,0 +1,284 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-common/log.h" +#include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "crowdb-rpc/transport/socket_transport.h" + +#ifdef CROWDB_RPC_HAVE_RDMA + +# include + +# include + +namespace crowdb::rpc +{ + +int RdmaTransport::drain_cq(uint32_t max_completions) +{ + if (cq_ == nullptr || cq_polling_.test_and_set(std::memory_order_acquire)) + return 0; + // Only the CQ consumer may clear its continuation. A competing worker + // must leave the wakeup readable when it cannot acquire the CQ. + if (continuation_fd_ >= 0) { + uint64_t value = 0; + (void)::read(continuation_fd_, &value, sizeof(value)); + } + if (max_completions == 0) + max_completions = std::max(1U, config_.poll_batch); + stats_.cq_polls.fetch_add(1, std::memory_order_relaxed); + uint32_t drained = 0; + uint32_t empty_polls = 0; + const uint32_t kEmptyPollBudget = config_.poll_empty_budget; + std::array completions{}; + auto poll_one = [&](ibv_cq *cq) { + while (drained < max_completions && cq != nullptr) { + const int batch = ibv_poll_cq(cq, static_cast(std::min(32, max_completions - drained)), + completions.data()); + if (batch < 0) + break; + if (batch == 0) { + stats_.cq_empty_polls.fetch_add(1, std::memory_order_relaxed); + if (++empty_polls >= kEmptyPollBudget) + break; + continue; + } + empty_polls = 0; + for (int i = 0; i < batch; ++i) { + dispatch_completion(completions[static_cast(i)]); + } + drained += static_cast(batch); + } + }; + if (comp_channel_ != nullptr) { + ibv_cq *event_cq = nullptr; + void *context = nullptr; + while (ibv_get_cq_event(comp_channel_, &event_cq, &context) == 0) { + ibv_ack_cq_events(event_cq, 1); + } + } + poll_one(cq_); + (void)ibv_req_notify_cq(cq_, 0); + // Completions arriving between the empty poll and notification arming + // need one final poll. A full batch gets an explicit epoll continuation. + empty_polls = 0; + poll_one(cq_); + if (drained == max_completions && continuation_fd_ >= 0) { + const uint64_t value = 1; + (void)::write(continuation_fd_, &value, sizeof(value)); + } + cq_polling_.clear(std::memory_order_release); + return static_cast(drained); +} + +void RdmaTransport::register_events(SocketEngine *engine) +{ + if (engine == nullptr) + return; + // A shared CQ has one completion channel. Register that channel with one + // epoll engine only; registering the same fd in every engine causes + // duplicate wakeups and lets different workers race the CQ event. + if (event_engine_ != nullptr && event_engine_ != engine) + return; + event_engine_ = engine; + if (completion_fd() >= 0) + engine->add_external_fd(completion_fd(), 1); + if (cm_event_fd() >= 0) + engine->add_external_fd(cm_event_fd(), 2); + if (continuation_fd_ >= 0) + engine->add_external_fd(continuation_fd_, 3); +} + +int RdmaTransport::handle_external_event(uint64_t tag, uint32_t max_completions) +{ + if (tag == 1 || tag == 3) { + return drain_cq(max_completions); + } + if (tag == 2) { + // The shared CM channel has one consumer while passive setup waits + // for ESTABLISHED. Another worker must not consume that event. + if (cm_polling_.test_and_set(std::memory_order_acquire)) + return 0; + auto release = [](std::atomic_flag *flag) { flag->clear(std::memory_order_release); }; + const std::unique_ptr owner(&cm_polling_, release); + int accepted = 0; + while (true) { + auto connection = accept(); + if (connection == nullptr) + break; + ++accepted; + if (connection_handler_) + connection_handler_(connection); + } + return accepted; + } + return 0; +} + +bool RdmaTransport::prime_receive(const std::shared_ptr &connection) +{ + if (connection == nullptr || connection->pool() == nullptr) { + + return false; + } + const uint32_t depth = config_.posted_receive_target; + if (depth == 0 || depth > config_.rq_depth || + static_cast(depth) * config_.wire_slot_capacity > config_.receive_arena_bytes) + return false; + BufferPool *pool = connection->pool(); + for (uint32_t i = 0; i < depth; ++i) { + Buffer *slot = pool->alloc(config_.wire_slot_capacity); + if (slot == nullptr) { + return false; + } + if (!post_receive(connection.get(), slot)) { + slot->release(); + return false; + } + // post_receive retains the work-request reference; the completion + // owns the slot after the caller reference is released. + slot->release(); + } + return true; +} + +void RdmaTransport::record_completion_error(const ibv_wc &wc) +{ + stats_.cq_errors.fetch_add(1, std::memory_order_relaxed); + switch (wc.status) { + case IBV_WC_WR_FLUSH_ERR: + stats_.flush_errors.fetch_add(1, std::memory_order_relaxed); + break; + case IBV_WC_LOC_PROT_ERR: + stats_.local_protection_errors.fetch_add(1, std::memory_order_relaxed); + break; + case IBV_WC_RETRY_EXC_ERR: + stats_.retry_exceeded_errors.fetch_add(1, std::memory_order_relaxed); + break; + case IBV_WC_REM_ACCESS_ERR: + stats_.remote_access_errors.fetch_add(1, std::memory_order_relaxed); + break; + default: + break; + } +} + +void RdmaTransport::release_completed_receive(WorkRequest *work, const ibv_wc &wc) +{ + + auto quarantine_and_release = [&] { + record_completion_error(wc); + work->conn->close(); + (void)work->slot->quarantine(); + work->slot->release(); + release_work(work); + }; + if (wc.status != IBV_WC_SUCCESS) { + CRB_LOG_WARN("rdma receive: completion failed status={} vendor_err={} opcode={}", + static_cast(wc.status), static_cast(wc.vendor_err), + static_cast(wc.opcode)); + quarantine_and_release(); + return; + } + if (!work->slot->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight)) { + quarantine_and_release(); + return; + } + const uint32_t slot_capacity = work->slot->capacity; + if (slot_capacity < HEADER_SIZE || wc.byte_len < HEADER_SIZE || !work->conn->is_open()) { + CRB_LOG_WARN("rdma receive: slot capacity too small capacity={} header={}", slot_capacity, HEADER_SIZE); + quarantine_and_release(); + return; + } + const Header header = parse_header(work->slot->data); + const uint64_t data_offset = static_cast(header.msg_offset) + header.msg_size; + const uint64_t expected_bytes = data_offset + header.data_size; + if (header.magic != MAGIC || header.msg_offset != HEADER_SIZE || data_offset > RDMA_CONTROL_CAPACITY || + expected_bytes != wc.byte_len || expected_bytes > slot_capacity) { + CRB_LOG_WARN( + "rdma receive: invalid frame magic={} type={} msg_size={} data_size={} offset={} capacity={} byte_len={}", + header.magic, header.msg_type, header.msg_size, header.data_size, header.msg_offset, slot_capacity, + wc.byte_len); + quarantine_and_release(); + return; + } + + auto *frame = new Frame; + frame->header = header; + if (header.msg_size != 0) { + frame->control.assign(work->slot->data + header.msg_offset, + work->slot->data + header.msg_offset + header.msg_size); + extract_control_fields(work->slot->data + header.msg_offset, header.msg_size, frame->request_id, + frame->rpc_create_nano, frame->response_create_nano); + } + work->request_id = frame->request_id; + work->stream_id = work->slot->stream_id; + work->sequence = work->slot->sequence; + work->logical_offset = work->slot->logical_offset; + work->slot->view_offset = static_cast(data_offset); + work->slot->capacity = slot_capacity - static_cast(data_offset); + work->slot->len = header.data_size; + if (!work->slot->transition(BufferOwnership::ReceiveInFlight, BufferOwnership::CallerOwned)) { + quarantine_and_release(); + delete frame; + return; + } + auto pools = work->conn->user_data == nullptr + ? nullptr + : static_cast(work->conn->user_data)->shared_from_this(); + auto *repost = new RepostContext{repost_state_, work->conn, std::move(pools)}; + frame->data_buf = work->slot; + work->slot->last_release_cb = &RdmaTransport::repost_frame_release; + work->slot->last_release_ctx = repost; + work->conn->on_frame(frame); + stats_.receives.fetch_add(1, std::memory_order_relaxed); + release_work(work); +} + +void RdmaTransport::repost_frame_release(void *context, Buffer *slot) +{ + auto repost = std::unique_ptr(static_cast(context)); + slot->reset_view(); + if (!slot->transition(BufferOwnership::CallerOwned, BufferOwnership::Available)) + (void)slot->quarantine(); + { + Operation operation(repost->state); + if (operation && slot->ownership.load(std::memory_order_acquire) == BufferOwnership::Available) + (void)repost->state->transport->post_receive(repost->connection.get(), slot); + } + slot->release(); +} + +void RdmaTransport::release_completed_frame(WorkRequest *work, const ibv_wc &wc) +{ + OutFrame *frame = work->frame; + + if (wc.status != IBV_WC_SUCCESS) { + record_completion_error(wc); + // A failed SEND makes the stream uncertain; do not allow a TCP + // suffix to be appended to this RDMA connection. + if (work->slot != nullptr) + (void)work->slot->quarantine(); + for (uint8_t i = 0; i < frame->data_views(); ++i) + (void)frame->data_view(i)->quarantine(); + if (work->conn != nullptr) + work->conn->close(); + } + else if (work->slot != nullptr && + !work->slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available)) { + (void)work->slot->quarantine(); + if (work->conn != nullptr) + work->conn->close(); + } + if (work->slot != nullptr) + work->slot->release(); + if (frame->control != nullptr) + frame->control->release(); + frame->release_data(); + delete frame; + release_work(work); +} + +} // namespace crowdb::rpc + +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp new file mode 100644 index 000000000..a80c97492 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp @@ -0,0 +1,56 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/transport/rdma/rdma_transport.h" + +#ifdef CROWDB_RPC_HAVE_RDMA + +# include "crowdb-common/log.h" + +# include +# include +# include +# include + +namespace crowdb::rpc +{ + +bool RdmaConfig::apply_environment() +{ + auto read = [](const char *name, T &destination) { + const char *value = std::getenv(name); + if (value == nullptr) + return true; + const std::string_view input(value); + uint64_t parsed = 0; + const auto result = std::from_chars(input.data(), input.data() + input.size(), parsed); + if (result.ec != std::errc{} || result.ptr != input.data() + input.size() || + parsed > std::numeric_limits::max()) { + CRB_LOG_WARN("rdma configuration: invalid unsigned value name={} value={}", name, value); + return false; + } + destination = static_cast(parsed); + return true; + }; + return read("CROWDB_RDMA_SQ_DEPTH", sq_depth) && read("CROWDB_RDMA_RQ_DEPTH", rq_depth) && + read("CROWDB_RDMA_POSTED_RECEIVE_TARGET", posted_receive_target) && + read("CROWDB_RDMA_WIRE_SLOT_CAPACITY", wire_slot_capacity) && + read("CROWDB_RDMA_SEND_ARENA_BYTES", send_arena_bytes) && + read("CROWDB_RDMA_RECEIVE_ARENA_BYTES", receive_arena_bytes) && read("CROWDB_RDMA_CQ_DEPTH", cq_depth); +} + +bool RdmaConfig::valid() const +{ + BufferCapacityClass capacity_class; + uint32_t capacity = 0; + return classify_rdma_capacity(wire_slot_capacity, &capacity_class, &capacity) && capacity == wire_slot_capacity && + sq_depth > 0 && rq_depth > 0 && posted_receive_target > 0 && posted_receive_target <= rq_depth && + cq_depth > posted_receive_target && cq_depth <= static_cast(INT32_MAX) && + send_arena_bytes >= wire_slot_capacity && + receive_arena_bytes >= static_cast(posted_receive_target) * wire_slot_capacity && + min_rnr_timer <= 31 && retry_count <= 7 && rnr_retry_count <= 7; +} + +} // namespace crowdb::rpc + +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp new file mode 100644 index 000000000..cee33286b --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp @@ -0,0 +1,399 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-common/log.h" +#include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "rdma_channel.h" + +#ifdef CROWDB_RPC_HAVE_RDMA + +# include + +# include +# include +# include + +namespace crowdb::rpc +{ + +std::shared_ptr RdmaTransport::connect(const std::string &addr, int port) +{ + if (!ready() || addr.empty() || port <= 0 || port > 65535) { + + CRB_LOG_WARN("rdma connect: transport unavailable pd={} cq={} addr={} port={}", static_cast(pd_), + static_cast(cq_), addr, port); + return nullptr; + } + { + std::scoped_lock lock(cm_mu_); + if (cm_channel_ == nullptr) + cm_channel_ = rdma_create_event_channel(); + if (cm_channel_ == nullptr) + return nullptr; + rdma_detail::make_nonblocking(cm_channel_->fd); + } + // Active setup has a private channel. The epoll worker may drain existing + // connections without consuming this attempt's address/route events. + auto setup_channel = std::unique_ptr( + rdma_create_event_channel(), &rdma_destroy_event_channel); + if (setup_channel == nullptr) + return nullptr; + rdma_detail::make_nonblocking(setup_channel->fd); + rdma_cm_id *id = nullptr; + if (rdma_create_id(setup_channel.get(), &id, this, RDMA_PS_TCP) != 0) + return nullptr; + sockaddr_in destination{}; + destination.sin_family = AF_INET; + destination.sin_port = htons(static_cast(port)); + if (::inet_pton(AF_INET, addr.c_str(), &destination.sin_addr) != 1) { + CRB_LOG_WARN("rdma connect: invalid destination {}", addr); + rdma_destroy_id(id); + return nullptr; + } + sockaddr_in source{}; + sockaddr *source_address = nullptr; + if (const char *source_env = std::getenv("CROWDB_RDMA_SOURCE_ADDR"); source_env != nullptr && *source_env != '\0') { + source.sin_family = AF_INET; + if (::inet_pton(AF_INET, source_env, &source.sin_addr) != 1) { + CRB_LOG_WARN("rdma connect: invalid source address {}", source_env); + rdma_destroy_id(id); + return nullptr; + } + source_address = reinterpret_cast(&source); + if (rdma_bind_addr(id, source_address) != 0) { + + CRB_LOG_WARN("rdma connect: bind source failed source={} errno={}", source_env, errno); + rdma_destroy_id(id); + return nullptr; + } + } + // When a source address was explicitly bound above, librdmacm expects a + // null source here; passing it twice can fail immediately with EINVAL. + if (rdma_resolve_addr(id, nullptr, reinterpret_cast(&destination), 2000) != 0) { + + CRB_LOG_WARN("rdma connect: resolve_addr failed addr={} port={} errno={}", addr, port, errno); + rdma_destroy_id(id); + return nullptr; + } + rdma_cm_event *event = nullptr; + if (!rdma_detail::wait_cm_event(setup_channel.get(), &event, 3000) || event == nullptr || + event->event != RDMA_CM_EVENT_ADDR_RESOLVED) { + + CRB_LOG_WARN("rdma connect: ADDR_RESOLVED missing addr={} event={} errno={}", addr, + event == nullptr ? -1 : event->event, errno); + if (event != nullptr) + rdma_ack_cm_event(event); + rdma_destroy_id(id); + return nullptr; + } + rdma_ack_cm_event(event); + event = nullptr; + if (rdma_resolve_route(id, 2000) != 0) { + CRB_LOG_WARN("rdma connect: resolve_route failed addr={} errno={}", addr, errno); + rdma_destroy_id(id); + return nullptr; + } + if (!rdma_detail::wait_cm_event(setup_channel.get(), &event, 3000) || event == nullptr || + event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { + + CRB_LOG_WARN("rdma connect: ROUTE_RESOLVED missing addr={} event={} errno={}", addr, + event == nullptr ? -1 : event->event, errno); + if (event != nullptr) + rdma_ack_cm_event(event); + rdma_destroy_id(id); + return nullptr; + } + rdma_ack_cm_event(event); + event = nullptr; + + if (!adopt_cm_context(id->verbs)) { + rdma_destroy_id(id); + return nullptr; + } + + ibv_device_attr device_attributes{}; + if (ibv_query_device(id->verbs, &device_attributes) != 0) { + rdma_destroy_id(id); + return nullptr; + } + ibv_qp_init_attr qp_attr{}; + qp_attr.send_cq = cq_; + qp_attr.recv_cq = cq_; + qp_attr.qp_type = IBV_QPT_RC; + qp_attr.cap.max_send_wr = std::min(config_.sq_depth, static_cast(device_attributes.max_qp_wr)); + qp_attr.cap.max_recv_wr = std::min(config_.rq_depth, static_cast(device_attributes.max_qp_wr)); + qp_attr.cap.max_send_sge = std::min(config_.qp_max_send_sge, static_cast(device_attributes.max_sge)); + qp_attr.cap.max_recv_sge = std::min(config_.qp_max_recv_sge, static_cast(device_attributes.max_sge)); + qp_attr.cap.max_inline_data = 0; + qp_attr.sq_sig_all = 1; + qp_attr.qp_context = id; + + if (rdma_create_qp(id, pd_, &qp_attr) != 0) { + + rdma_destroy_id(id); + return nullptr; + } + auto pools = std::make_shared(); + pools->send = std::make_unique(id->pd, config_.send_arena_bytes, config_.sq_depth); + pools->recv = std::make_unique(id->pd, config_.receive_arena_bytes, config_.rq_depth); + if (pools->send == nullptr || pools->recv == nullptr) { + rdma_destroy_qp(id); + rdma_destroy_id(id); + return nullptr; + } + auto connection = std::make_shared(reinterpret_cast(id), addr, pools->recv.get()); + connection->user_data = pools.get(); + connection->set_path(ConnectionPath::Rdma); + connection->transport_handle = reinterpret_cast(id->qp); + if (!prime_receive(connection)) { + connection->close(); + rdma_destroy_qp(id); + cancel_work(connection.get()); + rdma_destroy_id(id); + return nullptr; + } + rdma_conn_param params{}; + params.initiator_depth = 1; + params.responder_resources = 1; + params.retry_count = config_.retry_count; + params.rnr_retry_count = config_.rnr_retry_count; + if (rdma_connect(id, ¶ms) != 0) { + + CRB_LOG_WARN("rdma connect: rdma_connect failed addr={} errno={}", addr, errno); + connection->close(); + rdma_destroy_qp(id); + cancel_work(connection.get()); + rdma_destroy_id(id); + return nullptr; + } + if (!rdma_detail::wait_cm_event(setup_channel.get(), &event, 5000) || event == nullptr || + event->event != RDMA_CM_EVENT_ESTABLISHED) { + + CRB_LOG_WARN("rdma connect: ESTABLISHED missing addr={} event={} status={} errno={}", addr, + event == nullptr ? -1 : event->event, event == nullptr ? 0 : event->status, errno); + if (event != nullptr) + rdma_ack_cm_event(event); + connection->close(); + rdma_destroy_qp(id); + cancel_work(connection.get()); + rdma_destroy_id(id); + return nullptr; + } + rdma_ack_cm_event(event); + if (rdma_migrate_id(id, cm_channel_) != 0) { + connection->close(); + rdma_destroy_qp(id); + cancel_work(connection.get()); + rdma_destroy_id(id); + return nullptr; + } + ibv_qp_attr receive_attributes{}; + receive_attributes.min_rnr_timer = config_.min_rnr_timer; + if (ibv_modify_qp(id->qp, &receive_attributes, IBV_QP_MIN_RNR_TIMER) != 0) { + connection->close(); + rdma_destroy_qp(id); + cancel_work(connection.get()); + rdma_destroy_id(id); + return nullptr; + } + { + std::scoped_lock lock(cm_mu_); + cm_ids_.push_back(id); + cm_connections_.emplace(id, connection); + cm_pools_.emplace(id, std::move(pools)); + } + return connection; +} + +bool RdmaTransport::listen(const std::string &addr, int port) +{ + if (addr.empty() || port <= 0 || port > 65535 || listen_id_ != nullptr) { + + CRB_LOG_WARN("rdma listen: transport unavailable pd={} addr={} port={} existing_listener={}", + static_cast(pd_), addr, port, listen_id_ != nullptr); + return false; + } + if (cm_channel_ == nullptr) + cm_channel_ = rdma_create_event_channel(); + if (cm_channel_ == nullptr || rdma_create_id(cm_channel_, &listen_id_, this, RDMA_PS_TCP) != 0) { + CRB_LOG_WARN("rdma listen: create CM id failed addr={} port={} errno={}", addr, port, errno); + return false; + } + rdma_detail::make_nonblocking(cm_channel_->fd); + sockaddr_in local{}; + local.sin_family = AF_INET; + local.sin_port = htons(static_cast(port)); + if (::inet_pton(AF_INET, addr.c_str(), &local.sin_addr) != 1) { + CRB_LOG_WARN("rdma listen: invalid address {}", addr); + rdma_destroy_id(listen_id_); + listen_id_ = nullptr; + return false; + } + if (rdma_bind_addr(listen_id_, reinterpret_cast(&local)) != 0 || rdma_listen(listen_id_, 16) != 0) { + CRB_LOG_WARN("rdma listen: bind/listen failed addr={} port={} errno={}", addr, port, errno); + rdma_destroy_id(listen_id_); + listen_id_ = nullptr; + return false; + } + return true; +} + +std::shared_ptr RdmaTransport::accept() +{ + if (cm_channel_ == nullptr) + return nullptr; + rdma_cm_event *event = nullptr; + while (rdma_get_cm_event(cm_channel_, &event) == 0) { + if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST && listen_id_ != nullptr) + break; + if (event != nullptr) { + rdma_cm_id *event_id = event->id; + const int event_type = event->event; + rdma_ack_cm_event(event); + if (event_id != nullptr && event_id != listen_id_ && + (event_type == RDMA_CM_EVENT_DISCONNECTED || event_type == RDMA_CM_EVENT_CONNECT_ERROR || + event_type == RDMA_CM_EVENT_DEVICE_REMOVAL || event_type == RDMA_CM_EVENT_ADDR_CHANGE)) { + std::shared_ptr closed_connection; + { + std::scoped_lock lock(cm_mu_); + if (auto it = cm_connections_.find(event_id); it != cm_connections_.end()) { + closed_connection = it->second; + cm_connections_.erase(it); + } + if (auto it = cm_pools_.find(event_id); it != cm_pools_.end()) { + retired_pools_.push_back(std::move(it->second)); + cm_pools_.erase(it); + } + cm_ids_.erase(std::remove(cm_ids_.begin(), cm_ids_.end(), event_id), cm_ids_.end()); + } + if (closed_connection != nullptr) + close_after_drain(closed_connection.get()); + if (event_type == RDMA_CM_EVENT_DISCONNECTED || event_type == RDMA_CM_EVENT_CONNECT_ERROR) + stats_.cm_disconnects.fetch_add(1, std::memory_order_relaxed); + if (event_id->qp != nullptr) + rdma_destroy_qp(event_id); + if (closed_connection != nullptr) { + closed_connection->transport_handle = 0; + cancel_work(closed_connection.get()); + } + rdma_destroy_id(event_id); + } + } + event = nullptr; + } + if (event == nullptr) + return nullptr; + rdma_cm_id *id = event->id; + rdma_ack_cm_event(event); + + // A new CONNECT_REQUEST may precede this connection's ESTABLISHED on + // the listener channel. Keep setup events private until readiness. + auto setup_channel = std::unique_ptr( + rdma_create_event_channel(), &rdma_destroy_event_channel); + if (setup_channel == nullptr || rdma_migrate_id(id, setup_channel.get()) != 0) { + rdma_destroy_id(id); + return nullptr; + } + rdma_detail::make_nonblocking(setup_channel->fd); + + if (!adopt_cm_context(id->verbs)) { + rdma_destroy_id(id); + return nullptr; + } + + ibv_qp_init_attr qp_attr{}; + qp_attr.send_cq = cq_; + qp_attr.recv_cq = cq_; + qp_attr.qp_type = IBV_QPT_RC; + qp_attr.cap.max_send_wr = config_.sq_depth; + qp_attr.cap.max_recv_wr = config_.rq_depth; + qp_attr.cap.max_send_sge = config_.qp_max_send_sge; + qp_attr.cap.max_recv_sge = config_.qp_max_recv_sge; + qp_attr.cap.max_inline_data = 0; + qp_attr.sq_sig_all = 1; + qp_attr.qp_context = id; + + if (rdma_create_qp(id, pd_, &qp_attr) != 0) { + rdma_destroy_id(id); + return nullptr; + } + auto pools = std::make_shared(); + pools->send = std::make_unique(id->pd, config_.send_arena_bytes, config_.sq_depth); + pools->recv = std::make_unique(id->pd, config_.receive_arena_bytes, config_.rq_depth); + if (pools->send == nullptr || pools->recv == nullptr) { + rdma_destroy_qp(id); + rdma_destroy_id(id); + return nullptr; + } + auto connection = std::make_shared(reinterpret_cast(id), "rdma-peer", pools->recv.get()); + connection->user_data = pools.get(); + connection->set_path(ConnectionPath::Rdma); + connection->transport_handle = reinterpret_cast(id->qp); + if (!prime_receive(connection)) { + connection->close(); + rdma_destroy_qp(id); + cancel_work(connection.get()); + rdma_destroy_id(id); + return nullptr; + } + + rdma_conn_param params{}; + params.initiator_depth = 1; + params.responder_resources = 1; + params.retry_count = config_.retry_count; + params.rnr_retry_count = config_.rnr_retry_count; + const int accept_rc = rdma_accept(id, ¶ms); + + if (accept_rc != 0) { + CRB_LOG_WARN("rdma accept: rdma_accept failed errno={}", errno); + connection->close(); + rdma_destroy_qp(id); + cancel_work(connection.get()); + rdma_destroy_id(id); + return nullptr; + } + event = nullptr; + const int event_rc = rdma_detail::wait_cm_event(setup_channel.get(), &event, 5000) ? 0 : -1; + + if (event_rc != 0 || event == nullptr || event->event != RDMA_CM_EVENT_ESTABLISHED) { + CRB_LOG_WARN("rdma accept: ESTABLISHED missing event={} status={} errno={}", + event == nullptr ? -1 : event->event, event == nullptr ? 0 : event->status, errno); + if (event != nullptr) + rdma_ack_cm_event(event); + connection->close(); + rdma_destroy_qp(id); + cancel_work(connection.get()); + rdma_destroy_id(id); + return nullptr; + } + rdma_ack_cm_event(event); + + if (rdma_migrate_id(id, cm_channel_) != 0) { + connection->close(); + rdma_destroy_qp(id); + cancel_work(connection.get()); + rdma_destroy_id(id); + return nullptr; + } + + ibv_qp_attr receive_attributes{}; + receive_attributes.min_rnr_timer = config_.min_rnr_timer; + if (ibv_modify_qp(id->qp, &receive_attributes, IBV_QP_MIN_RNR_TIMER) != 0) { + connection->close(); + rdma_destroy_qp(id); + cancel_work(connection.get()); + rdma_destroy_id(id); + return nullptr; + } + { + std::scoped_lock lock(cm_mu_); + cm_ids_.push_back(id); + cm_connections_.emplace(id, connection); + cm_pools_.emplace(id, std::move(pools)); + } + return connection; +} + +} // namespace crowdb::rpc + +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index cb0c57ed6..ec10539d5 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -5,6 +5,7 @@ #include "crowdb-common/log.h" #include "crowdb-rpc/transport/socket_transport.h" +#include "rdma_channel.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -12,6 +13,7 @@ # include # include # include +# include # include # include @@ -24,209 +26,21 @@ namespace crowdb::rpc { -namespace -{ -void make_nonblocking(int fd) -{ - if (fd < 0) - return; - const int flags = ::fcntl(fd, F_GETFL, 0); - if (flags >= 0) - (void)::fcntl(fd, F_SETFL, flags | O_NONBLOCK); -} - -bool wait_cm_event(rdma_event_channel *channel, rdma_cm_event **event, int timeout_ms) -{ - if (channel == nullptr || event == nullptr) - return false; - pollfd descriptor{channel->fd, POLLIN, 0}; - if (::poll(&descriptor, 1, timeout_ms) <= 0 || (descriptor.revents & POLLIN) == 0) - return false; - return rdma_get_cm_event(channel, event) == 0; -} -} // namespace - -// ── RdmaBufferPool ──────────────────────────────────────────────── - -RdmaBufferPool::RdmaBufferPool(struct ibv_pd *pd, uint32_t mr_size, uint32_t max_buffers) - : pd_(pd), - mr_size_(mr_size), - max_buffers_(max_buffers) -{ - for (auto &head : free_heads_) - head.store(nullptr, std::memory_order_relaxed); - if (pd_ == nullptr || mr_size_ == 0 || max_buffers_ == 0) { - return; - } - if (posix_memalign(®ion_, 64, mr_size_) != 0) { - region_ = nullptr; - return; - } - mr_ = ibv_reg_mr(pd_, region_, mr_size_, - IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | - IBV_ACCESS_REMOTE_ATOMIC); - if (mr_ == nullptr) { - std::free(region_); - region_ = nullptr; - return; - } -} - -RdmaBufferPool::~RdmaBufferPool() -{ - if (outstanding_.load(std::memory_order_acquire) != 0) { - // Keep the MR registered while a caller, disk operation, or device - // completion still owns a slice. Process teardown reclaims it. - return; - } - Buffer *buf = all_head_.exchange(nullptr, std::memory_order_acq_rel); - while (buf != nullptr) { - Buffer *next = buf->next_all; - delete buf->ref; - delete buf; - buf = next; - } - if (mr_ != nullptr) { - ibv_dereg_mr(mr_); - mr_ = nullptr; - } - if (region_ != nullptr) { - std::free(region_); - region_ = nullptr; - } -} - -Buffer *RdmaBufferPool::alloc_fresh(uint32_t capacity) -{ - if (mr_ == nullptr || region_ == nullptr) { - return nullptr; - } - uint32_t offset = next_offset_.load(std::memory_order_relaxed); - uint32_t aligned_offset; - while (true) { - if (offset > mr_size_ || offset > UINT32_MAX - 63U) - return nullptr; - aligned_offset = (offset + 63U) & ~63U; - if (aligned_offset > mr_size_ || capacity > mr_size_ - aligned_offset) - return nullptr; - if (next_offset_.compare_exchange_weak(offset, aligned_offset + capacity, std::memory_order_acq_rel, - std::memory_order_relaxed)) - break; - } - - auto *buf = new Buffer; - buf->capacity = capacity; - buf->type = BufferType::Registered; - buf->pool = this; - buf->ref = new std::atomic(1); - - buf->data = static_cast(region_) + aligned_offset; - buf->len = 0; - buf->allocation_data = buf->data; - buf->allocation_capacity = capacity; - buf->registration_handle = reinterpret_cast(mr_); - buf->registration_key = mr_->lkey; - Buffer *all = all_head_.load(std::memory_order_acquire); - do { - buf->next_all = all; - } while (!all_head_.compare_exchange_weak(all, buf, std::memory_order_acq_rel, - std::memory_order_acquire)); - return buf; -} - -Buffer *RdmaBufferPool::pop(std::atomic &head) -{ - Buffer *current = head.load(std::memory_order_acquire); - while (current != nullptr && - !head.compare_exchange_weak(current, current->next_pool, std::memory_order_acq_rel, - std::memory_order_acquire)) { - } - if (current != nullptr) - current->next_pool = nullptr; - return current; -} - -void RdmaBufferPool::push(std::atomic &head, Buffer *buf) -{ - Buffer *current = head.load(std::memory_order_acquire); - do { - buf->next_pool = current; - } while (!head.compare_exchange_weak(current, buf, std::memory_order_acq_rel, - std::memory_order_acquire)); -} - -bool RdmaBufferPool::reserve_outstanding() -{ - uint32_t count = outstanding_.load(std::memory_order_relaxed); - while (count < max_buffers_ && - !outstanding_.compare_exchange_weak(count, count + 1, std::memory_order_acq_rel, - std::memory_order_relaxed)) { - } - return count < max_buffers_; -} - -void RdmaBufferPool::quarantine_inflight() -{ - for (Buffer *buf = all_head_.load(std::memory_order_acquire); buf != nullptr; buf = buf->next_all) { - const BufferOwnership state = buf->ownership.load(std::memory_order_acquire); - if (state != BufferOwnership::Available && state != BufferOwnership::Quarantined) - (void)buf->quarantine(); - } -} - -Buffer *RdmaBufferPool::alloc(uint32_t capacity) -{ - BufferCapacityClass cls; - uint32_t bucket; - if (!classify_rdma_capacity(capacity, &cls, &bucket)) { - return nullptr; - } - Buffer *buf = pop(free_heads_[static_cast(cls)]); - if (buf != nullptr) { - buf->reset_view(); - buf->len = 0; - buf->ref->store(1, std::memory_order_relaxed); - buf->capacity_class = cls; - buf->ownership.store(BufferOwnership::Available, std::memory_order_release); - if (!reserve_outstanding()) { - push(free_heads_[static_cast(cls)], buf); - return nullptr; - } - return buf; - } - if (!reserve_outstanding()) - return nullptr; - buf = alloc_fresh(bucket); - if (buf != nullptr) { - buf->capacity_class = cls; - buf->registration_id = mr_ == nullptr ? 0 : mr_->lkey; - } - else - outstanding_.fetch_sub(1, std::memory_order_acq_rel); - return buf; -} - -void RdmaBufferPool::recycle(Buffer *buf) -{ - outstanding_.fetch_sub(1, std::memory_order_acq_rel); - buf->reset_view(); - const BufferOwnership state = buf->ownership.load(std::memory_order_acquire); - if (state != BufferOwnership::Available && state != BufferOwnership::Quarantined) - (void)buf->quarantine(); - if (buf->ownership.load(std::memory_order_acquire) == BufferOwnership::Quarantined) { - quarantined_bytes_.fetch_add(buf->capacity, std::memory_order_acq_rel); - push(quarantine_head_, buf); - return; - } - buf->len = 0; - buf->ownership.store(BufferOwnership::Available, std::memory_order_release); - push(free_heads_[static_cast(buf->capacity_class)], buf); -} - // ── RdmaTransport ───────────────────────────────────────────────── RdmaTransport::RdmaTransport(RdmaConfig config) : config_(std::move(config)) { + const bool valid_config = config_.apply_environment() && config_.valid(); + if (!valid_config) + config_.cq_depth = 1; + work_requests_ = std::make_unique(config_.cq_depth); + continuation_fd_ = ::eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + repost_state_ = std::make_shared(); + repost_state_->transport = this; + if (!valid_config) { + CRB_LOG_WARN("rdma configuration: inconsistent queue depths, slot capacity, or arena budget; using TCP"); + return; + } if (config_.device.empty()) { if (const char *requested_device = std::getenv("CROWDB_RDMA_DEVICE"); requested_device != nullptr) config_.device = requested_device; @@ -256,8 +70,8 @@ RdmaTransport::RdmaTransport(RdmaConfig config) : config_(std::move(config)) if (selected != nullptr) { if (ibv_context *probe = ibv_open_device(selected); probe != nullptr) { ibv_port_attr attributes{}; - capability_available_ = ibv_query_port(probe, port_, &attributes) == 0 && - attributes.state == IBV_PORT_ACTIVE; + capability_available_ = + ibv_query_port(probe, port_, &attributes) == 0 && attributes.state == IBV_PORT_ACTIVE; ibv_close_device(probe); } } @@ -267,6 +81,8 @@ RdmaTransport::RdmaTransport(RdmaConfig config) : config_(std::move(config)) RdmaTransport::~RdmaTransport() { RdmaTransport::shutdown(); + if (continuation_fd_ >= 0) + ::close(continuation_fd_); send_pool_.reset(); recv_pool_.reset(); if (cq_ != nullptr) { @@ -302,7 +118,7 @@ bool RdmaTransport::adopt_cm_context(struct ibv_context *context) ibv_dealloc_pd(new_pd); return false; } - make_nonblocking(new_channel->fd); + rdma_detail::make_nonblocking(new_channel->fd); auto *new_cq = ibv_create_cq(context, static_cast(config_.cq_depth), this, new_channel, 0); if (new_cq == nullptr || ibv_req_notify_cq(new_cq, 0) != 0) { if (new_cq != nullptr) @@ -311,8 +127,8 @@ bool RdmaTransport::adopt_cm_context(struct ibv_context *context) ibv_dealloc_pd(new_pd); return false; } - auto new_send = std::make_unique(new_pd, config_.buffer_region_bytes, config_.buffer_ring_depth); - auto new_recv = std::make_unique(new_pd, config_.buffer_region_bytes, config_.buffer_ring_depth); + auto new_send = std::make_unique(new_pd, config_.send_arena_bytes, config_.sq_depth); + auto new_recv = std::make_unique(new_pd, config_.send_arena_bytes, config_.sq_depth); if (new_send == nullptr || new_recv == nullptr) { ibv_destroy_cq(new_cq); ibv_destroy_comp_channel(new_channel); @@ -338,12 +154,16 @@ bool RdmaTransport::adopt_cm_context(struct ibv_context *context) cq_ = new_cq; send_pool_ = std::move(new_send); recv_pool_ = std::move(new_recv); + register_events(event_engine_); return true; } bool RdmaTransport::submit(Connection *conn, OutFrame *frame) { - if (conn == nullptr || frame == nullptr || cq_ == nullptr || frame->control == nullptr || + Operation operation(repost_state_); + if (!operation) + return false; + if (conn == nullptr || !conn->is_open() || frame == nullptr || cq_ == nullptr || frame->control == nullptr || conn->transport_handle == 0) { return false; } @@ -366,7 +186,7 @@ bool RdmaTransport::submit(Connection *conn, OutFrame *frame) source_control->capacity); return false; } - Buffer *slot = pools->send->alloc(RDMA_WIRE_SLOT_CAPACITY); + Buffer *slot = pools->send->alloc(config_.wire_slot_capacity); if (slot == nullptr) return false; if (!slot->transition(BufferOwnership::Available, BufferOwnership::SendInFlight)) { @@ -402,72 +222,86 @@ bool RdmaTransport::submit(Connection *conn, OutFrame *frame) slot->sequence = source_control->sequence; slot->logical_offset = source_control->logical_offset; + WorkRequest *work = acquire_work(WorkKind::Send, conn, slot, frame); + if (work == nullptr) { + (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); + slot->release(); + return false; + } + ibv_sge sge{}; sge.addr = reinterpret_cast(slot->data); sge.length = slot->len; sge.lkey = slot->registration_key; ibv_send_wr wr{}; - wr.wr_id = reinterpret_cast(frame); + wr.wr_id = work->token.load(std::memory_order_acquire); wr.sg_list = &sge; wr.num_sge = 1; wr.opcode = IBV_WR_SEND; wr.send_flags = IBV_SEND_SIGNALED; ibv_send_wr *bad = nullptr; if (ibv_post_send(reinterpret_cast(conn->transport_handle), &wr, &bad) != 0) { + release_work(work); CRB_LOG_WARN("rdma send: post failed qp={} errno={}", reinterpret_cast(conn->transport_handle), errno); (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); slot->release(); return false; } - source_control->release(); - if (frame->data != nullptr) - frame->data->release(); - frame->control = slot; - frame->data = nullptr; - + // All work and frame ownership is published before posting. A CQ worker + // may already have released both by the time ibv_post_send returns. stats_.sends.fetch_add(1, std::memory_order_relaxed); return true; } bool RdmaTransport::post_receive(Connection *conn, Buffer *slot) { - if (conn == nullptr || slot == nullptr || slot->data == nullptr || cq_ == nullptr || conn->transport_handle == 0 || - slot->capacity < RDMA_WIRE_SLOT_CAPACITY) { + Operation operation(repost_state_); + if (!operation) + return false; + if (conn == nullptr || !conn->is_open() || slot == nullptr || slot->data == nullptr || cq_ == nullptr || + conn->transport_handle == 0 || slot->capacity < config_.wire_slot_capacity) { return false; } ibv_sge sge{}; - sge.addr = reinterpret_cast(slot->data); - sge.length = slot->capacity; - sge.lkey = slot->registration_key; - auto work = std::make_unique(ReceiveWork{conn, slot}); + sge.addr = reinterpret_cast(slot->data); + sge.length = slot->capacity; + sge.lkey = slot->registration_key; const bool slot_posted = slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive) || slot->transition(BufferOwnership::CallerOwned, BufferOwnership::PostedReceive); - if (!slot_posted) + if (!slot_posted) { return false; - if (slot->ref_clone() == nullptr) { + } + const bool retained = slot->ref_clone() != nullptr; + if (!retained) { (void)slot->transition(BufferOwnership::PostedReceive, BufferOwnership::Available); return false; } + WorkRequest *work = acquire_work(WorkKind::Receive, conn, slot); + if (work == nullptr) { + (void)slot->transition(BufferOwnership::PostedReceive, BufferOwnership::Available); + slot->release(); + return false; + } ibv_recv_wr wr{}; - wr.wr_id = reinterpret_cast(work.get()) | 1U; - wr.sg_list = &sge; - wr.num_sge = 1; - ibv_recv_wr *bad = nullptr; - int post_rc = -1; + wr.wr_id = work->token.load(std::memory_order_acquire); + wr.sg_list = &sge; + wr.num_sge = 1; + ibv_recv_wr *bad = nullptr; + int post_rc = -1; if (slot_posted) post_rc = ibv_post_recv(reinterpret_cast(conn->transport_handle), &wr, &bad); if (!slot_posted || post_rc != 0) { + release_work(work); CRB_LOG_WARN("rdma receive: post failed rc={} slot_state={}", post_rc, static_cast(slot->ownership.load(std::memory_order_relaxed))); if (!slot->transition(BufferOwnership::PostedReceive, BufferOwnership::Available)) (void)slot->quarantine(); - slot->release(); + slot->release(); // drop the completion's retained reference return false; } stats_.posted_receives.fetch_add(1, std::memory_order_relaxed); - work.release(); return true; } @@ -494,15 +328,27 @@ Buffer *RdmaTransport::register_buffer(Buffer *buf) void RdmaTransport::shutdown() { - // A QP may still own SEND or flushed receive WRs after the worker stops. - // Drain bounded CQ batches before destroying the IDs so provider teardown - // never observes live completion objects. - for (int attempt = 0; attempt < 4; ++attempt) { - const int drained = drain_cq(config_.poll_batch); - if (drained == 0) + repost_state_->users.fetch_or(RepostState::STOPPED, std::memory_order_acq_rel); + while ((repost_state_->users.load(std::memory_order_acquire) & ~RepostState::STOPPED) != 0) + std::this_thread::yield(); + stop(); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(config_.shutdown_drain_ms); + while (true) { + bool pending_sends = false; + for (uint32_t index = 0; index < config_.cq_depth; ++index) { + const auto &work = work_requests_[index]; + const uint64_t token = work.token.load(std::memory_order_acquire); + pending_sends |= token != 0 && token != WORK_RESERVED && work.kind == WorkKind::Send; + } + if (!pending_sends || std::chrono::steady_clock::now() >= deadline) break; + (void)drain_cq(config_.poll_batch); + std::this_thread::yield(); + } + for (const auto &[id, connection] : cm_connections_) { + (void)id; + connection->close(); } - stop(); { std::scoped_lock lock(cm_mu_); for (rdma_cm_id *id : cm_ids_) { @@ -514,11 +360,16 @@ void RdmaTransport::shutdown() } if (id->qp != nullptr) rdma_destroy_qp(id); + if (auto it = cm_connections_.find(id); it != cm_connections_.end()) + it->second->transport_handle = 0; rdma_destroy_id(id); } cm_ids_.clear(); cm_connections_.clear(); } + while (drain_cq(config_.poll_batch) > 0) { + } + cancel_work(); if (listen_id_ != nullptr) { if (listen_id_->qp != nullptr) rdma_destroy_qp(listen_id_); @@ -531,311 +382,12 @@ void RdmaTransport::shutdown() } } -std::shared_ptr RdmaTransport::connect(const std::string &addr, int port) -{ - if (addr.empty() || port <= 0 || port > 65535) { - - CRB_LOG_WARN("rdma connect: transport unavailable pd={} cq={} addr={} port={}", static_cast(pd_), - static_cast(cq_), addr, port); - return nullptr; - } - if (cm_channel_ == nullptr) - cm_channel_ = rdma_create_event_channel(); - if (cm_channel_ == nullptr) { - - return nullptr; - } - make_nonblocking(cm_channel_->fd); - rdma_cm_id *id = nullptr; - if (rdma_create_id(cm_channel_, &id, this, RDMA_PS_TCP) != 0) { - - return nullptr; - } - sockaddr_in destination{}; - destination.sin_family = AF_INET; - destination.sin_port = htons(static_cast(port)); - if (::inet_pton(AF_INET, addr.c_str(), &destination.sin_addr) != 1) { - CRB_LOG_WARN("rdma connect: invalid destination {}", addr); - rdma_destroy_id(id); - return nullptr; - } - sockaddr_in source{}; - sockaddr *source_address = nullptr; - if (const char *source_env = std::getenv("CROWDB_RDMA_SOURCE_ADDR"); source_env != nullptr && *source_env != '\0') { - source.sin_family = AF_INET; - if (::inet_pton(AF_INET, source_env, &source.sin_addr) != 1) { - CRB_LOG_WARN("rdma connect: invalid source address {}", source_env); - rdma_destroy_id(id); - return nullptr; - } - source_address = reinterpret_cast(&source); - if (rdma_bind_addr(id, source_address) != 0) { - - CRB_LOG_WARN("rdma connect: bind source failed source={} errno={}", source_env, errno); - rdma_destroy_id(id); - return nullptr; - } - } - // When a source address was explicitly bound above, librdmacm expects a - // null source here; passing it twice can fail immediately with EINVAL. - if (rdma_resolve_addr(id, nullptr, reinterpret_cast(&destination), 2000) != 0) { - - CRB_LOG_WARN("rdma connect: resolve_addr failed addr={} port={} errno={}", addr, port, errno); - rdma_destroy_id(id); - return nullptr; - } - rdma_cm_event *event = nullptr; - if (!wait_cm_event(cm_channel_, &event, 3000) || event == nullptr || event->event != RDMA_CM_EVENT_ADDR_RESOLVED) { - - CRB_LOG_WARN("rdma connect: ADDR_RESOLVED missing addr={} event={} errno={}", addr, - event == nullptr ? -1 : event->event, errno); - if (event != nullptr) - rdma_ack_cm_event(event); - rdma_destroy_id(id); - return nullptr; - } - rdma_ack_cm_event(event); - if (rdma_resolve_route(id, 2000) != 0) { - CRB_LOG_WARN("rdma connect: resolve_route failed addr={} errno={}", addr, errno); - rdma_destroy_id(id); - return nullptr; - } - if (!wait_cm_event(cm_channel_, &event, 3000) || event == nullptr || event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { - - CRB_LOG_WARN("rdma connect: ROUTE_RESOLVED missing addr={} event={} errno={}", addr, - event == nullptr ? -1 : event->event, errno); - if (event != nullptr) - rdma_ack_cm_event(event); - rdma_destroy_id(id); - return nullptr; - } - rdma_ack_cm_event(event); - - if (!adopt_cm_context(id->verbs)) { - rdma_destroy_id(id); - return nullptr; - } - - ibv_device_attr device_attributes{}; - if (ibv_query_device(id->verbs, &device_attributes) != 0) { - rdma_destroy_id(id); - return nullptr; - } - ibv_qp_init_attr qp_attr{}; - qp_attr.send_cq = cq_; - qp_attr.recv_cq = cq_; - qp_attr.qp_type = IBV_QPT_RC; - qp_attr.cap.max_send_wr = std::min(config_.qp_max_send_wr, static_cast(device_attributes.max_qp_wr)); - qp_attr.cap.max_recv_wr = std::min(config_.qp_max_recv_wr, static_cast(device_attributes.max_qp_wr)); - qp_attr.cap.max_send_sge = std::min(config_.qp_max_send_sge, static_cast(device_attributes.max_sge)); - qp_attr.cap.max_recv_sge = std::min(config_.qp_max_recv_sge, static_cast(device_attributes.max_sge)); - qp_attr.cap.max_inline_data = 0; - qp_attr.sq_sig_all = 1; - qp_attr.qp_context = id; - - if (rdma_create_qp(id, pd_, &qp_attr) != 0) { - - rdma_destroy_id(id); - return nullptr; - } - rdma_conn_param params{}; - params.initiator_depth = 1; - params.responder_resources = 1; - params.retry_count = config_.retry_count; - params.rnr_retry_count = config_.rnr_retry_count; - if (rdma_connect(id, ¶ms) != 0) { - - CRB_LOG_WARN("rdma connect: rdma_connect failed addr={} errno={}", addr, errno); - rdma_destroy_qp(id); - rdma_destroy_id(id); - return nullptr; - } - if (!wait_cm_event(cm_channel_, &event, 5000) || event == nullptr || event->event != RDMA_CM_EVENT_ESTABLISHED) { - - CRB_LOG_WARN("rdma connect: ESTABLISHED missing addr={} event={} status={} errno={}", addr, - event == nullptr ? -1 : event->event, event == nullptr ? 0 : event->status, errno); - if (event != nullptr) - rdma_ack_cm_event(event); - rdma_destroy_qp(id); - rdma_destroy_id(id); - return nullptr; - } - rdma_ack_cm_event(event); - auto pools = std::make_unique(); - pools->send = std::make_unique(id->pd, 64U << 20, 1024); - pools->recv = std::make_unique(id->pd, 64U << 20, 1024); - if (pools->send == nullptr || pools->recv == nullptr) { - rdma_destroy_qp(id); - rdma_destroy_id(id); - return nullptr; - } - auto connection = std::make_shared(reinterpret_cast(id), addr, pools->recv.get()); - connection->user_data = pools.get(); - connection->set_path(ConnectionPath::Rdma); - connection->transport_handle = reinterpret_cast(id->qp); - if (!prime_receive(connection)) { - - rdma_destroy_qp(id); - rdma_destroy_id(id); - return nullptr; - } - { - std::scoped_lock lock(cm_mu_); - cm_ids_.push_back(id); - cm_connections_.emplace(id, connection); - cm_pools_.emplace(id, std::move(pools)); - } - return connection; -} - -bool RdmaTransport::listen(const std::string &addr, int port) -{ - if (addr.empty() || port <= 0 || port > 65535 || listen_id_ != nullptr) { - - CRB_LOG_WARN("rdma listen: transport unavailable pd={} addr={} port={} existing_listener={}", - static_cast(pd_), addr, port, listen_id_ != nullptr); - return false; - } - if (cm_channel_ == nullptr) - cm_channel_ = rdma_create_event_channel(); - if (cm_channel_ == nullptr || rdma_create_id(cm_channel_, &listen_id_, this, RDMA_PS_TCP) != 0) { - CRB_LOG_WARN("rdma listen: create CM id failed addr={} port={} errno={}", addr, port, errno); - return false; - } - make_nonblocking(cm_channel_->fd); - sockaddr_in local{}; - local.sin_family = AF_INET; - local.sin_port = htons(static_cast(port)); - if (::inet_pton(AF_INET, addr.c_str(), &local.sin_addr) != 1) { - CRB_LOG_WARN("rdma listen: invalid address {}", addr); - rdma_destroy_id(listen_id_); - listen_id_ = nullptr; - return false; - } - if (rdma_bind_addr(listen_id_, reinterpret_cast(&local)) != 0 || rdma_listen(listen_id_, 16) != 0) { - CRB_LOG_WARN("rdma listen: bind/listen failed addr={} port={} errno={}", addr, port, errno); - rdma_destroy_id(listen_id_); - listen_id_ = nullptr; - return false; - } - return true; -} - -std::shared_ptr RdmaTransport::accept() -{ - if (listen_id_ == nullptr || cm_channel_ == nullptr) - return nullptr; - rdma_cm_event *event = nullptr; - if (rdma_get_cm_event(cm_channel_, &event) != 0 || event->event != RDMA_CM_EVENT_CONNECT_REQUEST) { - if (event != nullptr) { - rdma_cm_id *event_id = event->id; - const int event_type = event->event; - rdma_ack_cm_event(event); - if (event_id != nullptr && event_id != listen_id_) { - std::shared_ptr closed_connection; - { - std::scoped_lock lock(cm_mu_); - if (auto it = cm_connections_.find(event_id); it != cm_connections_.end()) { - closed_connection = it->second; - cm_connections_.erase(it); - } - if (auto it = cm_pools_.find(event_id); it != cm_pools_.end()) { - retired_pools_.push_back(std::move(it->second)); - cm_pools_.erase(it); - } - cm_ids_.erase(std::remove(cm_ids_.begin(), cm_ids_.end(), event_id), cm_ids_.end()); - } - if (closed_connection != nullptr) - closed_connection->close(); - if (event_type == RDMA_CM_EVENT_DISCONNECTED || event_type == RDMA_CM_EVENT_CONNECT_ERROR) - stats_.cm_disconnects.fetch_add(1, std::memory_order_relaxed); - if (event_id->qp != nullptr) - rdma_destroy_qp(event_id); - rdma_destroy_id(event_id); - } - } - return nullptr; - } - rdma_cm_id *id = event->id; - rdma_ack_cm_event(event); - - if (!adopt_cm_context(id->verbs)) { - rdma_destroy_id(id); - return nullptr; - } - - ibv_qp_init_attr qp_attr{}; - qp_attr.send_cq = cq_; - qp_attr.recv_cq = cq_; - qp_attr.qp_type = IBV_QPT_RC; - qp_attr.cap.max_send_wr = config_.qp_max_send_wr; - qp_attr.cap.max_recv_wr = config_.qp_max_recv_wr; - qp_attr.cap.max_send_sge = config_.qp_max_send_sge; - qp_attr.cap.max_recv_sge = config_.qp_max_recv_sge; - qp_attr.cap.max_inline_data = 0; - qp_attr.sq_sig_all = 1; - qp_attr.qp_context = id; - - if (rdma_create_qp(id, pd_, &qp_attr) != 0) { - rdma_destroy_id(id); - return nullptr; - } - rdma_conn_param params{}; - params.initiator_depth = 1; - params.responder_resources = 1; - params.retry_count = config_.retry_count; - params.rnr_retry_count = config_.rnr_retry_count; - const int accept_rc = rdma_accept(id, ¶ms); - - if (accept_rc != 0) { - CRB_LOG_WARN("rdma accept: rdma_accept failed errno={}", errno); - rdma_destroy_qp(id); - rdma_destroy_id(id); - return nullptr; - } - event = nullptr; - const int event_rc = wait_cm_event(cm_channel_, &event, 5000) ? 0 : -1; - - if (event_rc != 0 || event == nullptr || event->event != RDMA_CM_EVENT_ESTABLISHED) { - CRB_LOG_WARN("rdma accept: ESTABLISHED missing event={} status={} errno={}", - event == nullptr ? -1 : event->event, event == nullptr ? 0 : event->status, errno); - if (event != nullptr) - rdma_ack_cm_event(event); - rdma_destroy_qp(id); - rdma_destroy_id(id); - return nullptr; - } - rdma_ack_cm_event(event); - - auto pools = std::make_unique(); - pools->send = std::make_unique(id->pd, 64U << 20, 1024); - pools->recv = std::make_unique(id->pd, 64U << 20, 1024); - if (pools->send == nullptr || pools->recv == nullptr) { - rdma_destroy_qp(id); - rdma_destroy_id(id); - return nullptr; - } - auto connection = std::make_shared(reinterpret_cast(id), "rdma-peer", pools->recv.get()); - connection->user_data = pools.get(); - connection->set_path(ConnectionPath::Rdma); - connection->transport_handle = reinterpret_cast(id->qp); - if (!prime_receive(connection)) { - rdma_destroy_qp(id); - rdma_destroy_id(id); - return nullptr; - } - - { - std::scoped_lock lock(cm_mu_); - cm_ids_.push_back(id); - cm_connections_.emplace(id, connection); - cm_pools_.emplace(id, std::move(pools)); - } - return connection; -} - void RdmaTransport::start() { + if ((repost_state_->users.load(std::memory_order_acquire) & RepostState::STOPPED) != 0) { + repost_state_ = std::make_shared(); + repost_state_->transport = this; + } running_.store(true, std::memory_order_relaxed); } @@ -856,252 +408,6 @@ int RdmaTransport::cm_event_fd() const return cm_channel_ == nullptr ? -1 : cm_channel_->fd; } -int RdmaTransport::drain_cq(uint32_t max_completions) -{ - if (max_completions == 0) - max_completions = config_.poll_batch; - stats_.cq_polls.fetch_add(1, std::memory_order_relaxed); - uint32_t drained = 0; - uint32_t empty_polls = 0; - const uint32_t kEmptyPollBudget = config_.poll_empty_budget; - std::array completions{}; - auto poll_one = [&](ibv_cq *cq) { - while (drained < max_completions && cq != nullptr) { - const int batch = ibv_poll_cq(cq, static_cast(std::min(32, max_completions - drained)), - completions.data()); - if (batch < 0) - break; - if (batch == 0) { - stats_.cq_empty_polls.fetch_add(1, std::memory_order_relaxed); - if (++empty_polls >= kEmptyPollBudget) - break; - continue; - } - empty_polls = 0; - for (int i = 0; i < batch; ++i) { - const auto opcode = completions[static_cast(i)].opcode; - const auto wr_id = completions[static_cast(i)].wr_id; - if ((wr_id & uintptr_t{1}) != 0 || opcode == IBV_WC_RECV || opcode == IBV_WC_RECV_RDMA_WITH_IMM) - release_completed_receive(completions[static_cast(i)]); - else if (opcode == IBV_WC_SEND) - release_completed_frame(completions[static_cast(i)]); - else { - } - } - drained += static_cast(batch); - } - }; - if (comp_channel_ != nullptr && cq_ != nullptr) { - ibv_cq *event_cq = nullptr; - void *context = nullptr; - if (ibv_get_cq_event(comp_channel_, &event_cq, &context) == 0) { - ibv_ack_cq_events(event_cq, 1); - } - poll_one(cq_); - (void)ibv_req_notify_cq(cq_, 0); - } - return static_cast(drained); -} - -void RdmaTransport::register_events(SocketEngine *engine) -{ - if (engine == nullptr) - return; - // A shared CQ has one completion channel. Register that channel with one - // epoll engine only; registering the same fd in every engine causes - // duplicate wakeups and lets different workers race the CQ event. - if (event_engine_ != nullptr && event_engine_ != engine) - return; - event_engine_ = engine; - if (completion_fd() >= 0) - engine->add_external_fd(completion_fd(), 1); - if (cm_event_fd() >= 0) - engine->add_external_fd(cm_event_fd(), 2); -} - -int RdmaTransport::handle_external_event(uint64_t tag, uint32_t max_completions) -{ - if (tag == 1) - return drain_cq(max_completions); - if (tag == 2) { - int accepted = 0; - while (true) { - auto connection = accept(); - if (connection == nullptr) - break; - ++accepted; - if (connection_handler_) - connection_handler_(connection); - } - return accepted; - } - return 0; -} - -bool RdmaTransport::prime_receive(const std::shared_ptr &connection) -{ - if (connection == nullptr || connection->pool() == nullptr) { - - return false; - } - constexpr uint32_t kInitialReceiveDepth = 8; - BufferPool *pool = connection->pool(); - for (uint32_t i = 0; i < kInitialReceiveDepth; ++i) { - Buffer *slot = pool->alloc(RDMA_WIRE_SLOT_CAPACITY); - if (slot == nullptr) { - return false; - } - if (!post_receive(connection.get(), slot)) { - slot->release(); - return false; - } - // post_receive retains the work-request reference; the completion - // owns the slot after the caller reference is released. - slot->release(); - } - return true; -} - -void RdmaTransport::record_completion_error(const ibv_wc &wc) -{ - stats_.cq_errors.fetch_add(1, std::memory_order_relaxed); - switch (wc.status) { - case IBV_WC_WR_FLUSH_ERR: - stats_.flush_errors.fetch_add(1, std::memory_order_relaxed); - break; - case IBV_WC_LOC_PROT_ERR: - stats_.local_protection_errors.fetch_add(1, std::memory_order_relaxed); - break; - case IBV_WC_RETRY_EXC_ERR: - stats_.retry_exceeded_errors.fetch_add(1, std::memory_order_relaxed); - break; - case IBV_WC_REM_ACCESS_ERR: - stats_.remote_access_errors.fetch_add(1, std::memory_order_relaxed); - break; - default: - break; - } -} - -void RdmaTransport::release_completed_receive(ibv_wc &wc) -{ - - auto *work = reinterpret_cast(wc.wr_id & ~uintptr_t{1}); - if (work == nullptr) - return; - auto quarantine_and_release = [&] { - record_completion_error(wc); - work->conn->close(); - (void)work->slot->quarantine(); - work->slot->release(); - delete work; - }; - if (wc.status != IBV_WC_SUCCESS) { - CRB_LOG_WARN("rdma receive: completion failed status={} vendor_err={} opcode={}", - static_cast(wc.status), static_cast(wc.vendor_err), - static_cast(wc.opcode)); - quarantine_and_release(); - return; - } - if (!work->slot->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight)) { - quarantine_and_release(); - return; - } - const uint32_t slot_capacity = work->slot->capacity; - if (slot_capacity < HEADER_SIZE) { - CRB_LOG_WARN("rdma receive: slot capacity too small capacity={} header={}", slot_capacity, HEADER_SIZE); - quarantine_and_release(); - return; - } - const Header header = parse_header(work->slot->data); - const uint32_t control_end = static_cast(HEADER_SIZE) + header.msg_size; - const uint64_t expected_bytes = static_cast(control_end) + header.data_size; - if (header.magic != MAGIC || header.msg_offset < HEADER_SIZE || - static_cast(header.msg_offset) + header.msg_size > slot_capacity || expected_bytes > wc.byte_len || - expected_bytes > slot_capacity) { - CRB_LOG_WARN( - "rdma receive: invalid frame magic={} type={} msg_size={} data_size={} offset={} capacity={} byte_len={}", - header.magic, header.msg_type, header.msg_size, header.data_size, header.msg_offset, slot_capacity, - wc.byte_len); - quarantine_and_release(); - return; - } - - auto *frame = new Frame; - frame->header = header; - if (header.msg_size != 0) { - frame->control.assign(work->slot->data + header.msg_offset, - work->slot->data + header.msg_offset + header.msg_size); - extract_control_fields(work->slot->data + header.msg_offset, header.msg_size, frame->request_id, - frame->rpc_create_nano, frame->response_create_nano); - } - work->slot->view_offset = control_end; - work->slot->capacity = slot_capacity - control_end; - work->slot->len = header.data_size; - if (!work->slot->transition(BufferOwnership::ReceiveInFlight, BufferOwnership::CallerOwned)) { - quarantine_and_release(); - delete frame; - return; - } - frame->data_buf = work->slot; - Buffer *repost_slot = work->slot->ref_clone(); - if (repost_slot == nullptr) { - quarantine_and_release(); - delete frame; - return; - } - auto *repost = new RepostContext{this, work->conn, repost_slot}; - frame->release_callback = &RdmaTransport::repost_frame_release; - frame->release_context = repost; - work->conn->on_frame(frame); - stats_.receives.fetch_add(1, std::memory_order_relaxed); - delete work; -} - -void RdmaTransport::repost_frame_release(void *context, Buffer *) -{ - auto *repost = static_cast(context); - if (repost == nullptr) - return; - repost->slot->reset_view(); - if (!repost->slot->transition(BufferOwnership::CallerOwned, BufferOwnership::Available)) - (void)repost->slot->quarantine(); - if (repost->transport != nullptr && - repost->slot->ownership.load(std::memory_order_acquire) == BufferOwnership::Available) - (void)repost->transport->post_receive(repost->connection, repost->slot); - repost->slot->release(); - delete repost; -} - -void RdmaTransport::release_completed_frame(ibv_wc &wc) -{ - auto *frame = reinterpret_cast(wc.wr_id); - if (frame == nullptr) - return; - - if (wc.status != IBV_WC_SUCCESS) { - record_completion_error(wc); - // A failed SEND makes the stream uncertain; do not allow a TCP - // suffix to be appended to this RDMA connection. - if (frame->control != nullptr) - (void)frame->control->quarantine(); - // The completion is no longer trustworthy; the pool keeps these - // slices out of reuse until transport shutdown. - if (frame->control != nullptr) - (void)frame->control->quarantine(); - for (uint8_t i = 0; i < frame->data_views(); ++i) - (void)frame->data_view(i)->quarantine(); - } - else if (frame->control != nullptr && - !frame->control->transition(BufferOwnership::SendInFlight, BufferOwnership::Available)) { - (void)frame->control->quarantine(); - } - if (frame->control != nullptr) - frame->control->release(); - frame->release_data(); - delete frame; -} - } // namespace crowdb::rpc #endif // CROWDB_RPC_HAVE_RDMA diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp new file mode 100644 index 000000000..a82e489d3 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp @@ -0,0 +1,94 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/transport/rdma/rdma_transport.h" + +#ifdef CROWDB_RPC_HAVE_RDMA + +namespace crowdb::rpc +{ + +RdmaTransport::WorkRequest *RdmaTransport::acquire_work(WorkKind kind, Connection *conn, Buffer *slot, OutFrame *frame) +{ + auto owner = conn->weak_from_this().lock(); + if (owner == nullptr) + return nullptr; + for (uint32_t attempt = 0; attempt < config_.cq_depth; ++attempt) { + const uint32_t index = next_work_.fetch_add(1, std::memory_order_relaxed) % config_.cq_depth; + auto &work = work_requests_[index]; + uint64_t empty = 0; + if (!work.token.compare_exchange_strong(empty, WORK_RESERVED, std::memory_order_acq_rel)) + continue; + work.kind = kind; + work.conn = std::move(owner); + work.slot = slot; + work.frame = frame; + work.request_id = frame == nullptr ? slot->request_id : frame->request_id; + work.stream_id = slot->stream_id; + work.sequence = slot->sequence; + work.logical_offset = slot->logical_offset; + const uint64_t generation = work_generation_.fetch_add(1, std::memory_order_relaxed); + work.token.store((generation << 32) | index, std::memory_order_release); + return &work; + } + return nullptr; +} + +RdmaTransport::WorkRequest *RdmaTransport::claim_work(uint64_t token) +{ + const auto index = static_cast(token); + if (token == 0 || token == WORK_RESERVED || index >= config_.cq_depth) + return nullptr; + auto &work = work_requests_[index]; + return work.token.compare_exchange_strong(token, WORK_RESERVED, std::memory_order_acq_rel) ? &work : nullptr; +} + +void RdmaTransport::release_work(WorkRequest *work) +{ + work->conn.reset(); + work->slot = nullptr; + work->frame = nullptr; + work->token.store(0, std::memory_order_release); +} + +void RdmaTransport::dispatch_completion(const ibv_wc &wc) +{ + WorkRequest *work = claim_work(wc.wr_id); + if (work == nullptr) + return; + // Error completions do not provide a trustworthy opcode. The retained + // work record owns dispatch, even after disconnect or QP destruction. + if (work->kind == WorkKind::Receive) + release_completed_receive(work, wc); + else + release_completed_frame(work, wc); +} + +void RdmaTransport::cancel_work(Connection *connection) +{ + // Called only after the affected QPs are destroyed and device access has + // stopped. Missing/uncertain completions must still release their owners. + for (uint32_t index = 0; index < config_.cq_depth; ++index) { + auto &work = work_requests_[index]; + const uint64_t token = work.token.load(std::memory_order_acquire); + if (token == 0 || token == WORK_RESERVED || (connection != nullptr && work.conn.get() != connection)) + continue; + ibv_wc completion{}; + completion.wr_id = token; + completion.status = IBV_WC_WR_FLUSH_ERR; + dispatch_completion(completion); + } +} + +void RdmaTransport::close_after_drain(Connection *connection) +{ + // A disconnect notification can race CQ readiness. Publish completions + // already acknowledged by the NIC before failing the remaining requests. + while (drain_cq(config_.poll_batch) > 0) { + } + connection->close(); +} + +} // namespace crowdb::rpc + +#endif diff --git a/lib/crowdb-rpc/src/transport/socket_transport.cpp b/lib/crowdb-rpc/src/transport/socket_transport.cpp index 5e9b3ccc3..7e343bdd4 100644 --- a/lib/crowdb-rpc/src/transport/socket_transport.cpp +++ b/lib/crowdb-rpc/src/transport/socket_transport.cpp @@ -541,9 +541,11 @@ static bool on_writable_impl(Connection *conn, int fd, TransportStats *stats) SocketTransport::SocketTransport(uint32_t io_engines, uint32_t io_workers, BufferPool *pool #ifdef CROWDB_RPC_HAVE_RDMA - , RdmaConfig rdma_config + , + RdmaConfig rdma_config #endif - ) : pool_(pool) + ) + : pool_(pool) { assert(io_engines >= 1 && io_workers >= 1); assert(io_workers % io_engines == 0); @@ -600,8 +602,8 @@ SocketTransport::~SocketTransport() { stop(); #ifdef CROWDB_RPC_HAVE_RDMA - if (rdma_transport_ != nullptr) - rdma_transport_->shutdown(); + // Finish CQ ownership before the engines and their wakeup fds disappear. + rdma_transport_.reset(); #endif } @@ -632,7 +634,7 @@ void SocketTransport::stop() } #ifdef CROWDB_RPC_HAVE_RDMA if (rdma_transport_ != nullptr) - rdma_transport_->stop(); + rdma_transport_->shutdown(); #endif } diff --git a/lib/crowdb-rpc/tests/common/test_rdma_load.cpp b/lib/crowdb-rpc/tests/common/test_rdma_load.cpp new file mode 100644 index 000000000..98536cd9f --- /dev/null +++ b/lib/crowdb-rpc/tests/common/test_rdma_load.cpp @@ -0,0 +1,188 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "test_rdma_load.h" + +#ifdef CROWDB_RPC_HAVE_RDMA + +# include "crowdb-rpc/server/message.h" + +# include +# include + +namespace crowdb::rpc +{ + +void TestRdmaLatency::observe(uint64_t ns) +{ + ns = std::max(uint64_t{1}, ns); + const unsigned exponent = std::bit_width(ns) - 1; + const unsigned shift = exponent > 7 ? exponent - 7 : 0; + const auto sub = static_cast((ns - (uint64_t{1} << exponent)) >> shift); + buckets[exponent * SUB_BUCKETS + sub].fetch_add(1, std::memory_order_relaxed); + sum_ns.fetch_add(ns, std::memory_order_relaxed); + uint64_t maximum = max_ns.load(std::memory_order_relaxed); + while (maximum < ns && !max_ns.compare_exchange_weak(maximum, ns, std::memory_order_relaxed)) { + } +} + +uint64_t TestRdmaLatency::percentile(const std::array &counts, uint64_t count, uint32_t percent) +{ + const uint64_t rank = std::max(uint64_t{1}, (count * percent + 99) / 100); + uint64_t seen = 0; + for (size_t index = 0; index < counts.size(); ++index) { + seen += counts[index]; + if (seen >= rank) { + const size_t exponent = index / SUB_BUCKETS; + const size_t shift = exponent > 7 ? exponent - 7 : 0; + return (uint64_t{1} << exponent) + ((index % SUB_BUCKETS + 1) << shift); + } + } + return 0; +} + +TestRdmaLoad::TestRdmaLoad(uint32_t count, uint32_t in_flight, uint32_t seconds) + : producer_count(count), + in_flight_per_producer(in_flight), + duration_seconds(seconds), + id_stride(std::bit_ceil(static_cast(count) * in_flight)) +{ + for (uint32_t index = 0; index < count; ++index) + producers.push_back(std::make_unique(*this, index)); +} + +TestRdmaLoad::~TestRdmaLoad() = default; + +void TestRdmaLoad::fail(const char *reason, uint64_t request_id, crowdb_rpc_status status) +{ + const auto previous = errors.fetch_add(1, std::memory_order_relaxed); + if (previous < 8) + std::fprintf(stderr, "RDMA timed load failure: %s request_id=%llu status=%d\n", reason, + static_cast(request_id), status); +} + +void TestRdmaLoad::complete(TestRdmaLoadSlot &slot, uint64_t request_id, crowdb_rpc_buffer_t data, + crowdb_rpc_status status) +{ + uint64_t expected = request_id; + if (!slot.expected_id.compare_exchange_strong(expected, UINT64_MAX, std::memory_order_acq_rel)) { + fail("duplicate or mismatched completion", request_id, status); + return; + } + const uint64_t now = test_rdma_clock_ns(); + const auto *bytes = data == nullptr ? nullptr : crowdb_rpc_buffer_data(data); + const uint32_t length = data == nullptr ? 0 : crowdb_rpc_buffer_len(data); + if (status != CROWDB_RPC_OK) + fail("RPC completion failed", request_id, status); + else if (length != PAYLOAD_BYTES || bytes == nullptr || + !std::all_of(bytes, bytes + length, + [request_id](uint8_t byte) { return byte == static_cast(request_id); })) + fail("response payload mismatch", request_id, status); + else if (slot.measured) { + if (now < measure_end_ns.load(std::memory_order_relaxed)) { + slot.producer->latency.observe(now - slot.started_ns); + slot.producer->window_completed.fetch_add(1, std::memory_order_relaxed); + } + else + slot.producer->drain_completed.fetch_add(1, std::memory_order_relaxed); + } + slot.producer->total_completed.fetch_add(1, std::memory_order_release); +} + +static void test_rdma_load_complete(uint64_t request_id, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, + crowdb_rpc_status status, void *opaque) +{ + auto &slot = *static_cast(opaque); + slot.producer->load.complete(slot, request_id, data, status); + if (control != nullptr) + crowdb_rpc_buffer_release(control); + if (data != nullptr) + crowdb_rpc_buffer_release(data); + if (slot.expected_id.load(std::memory_order_relaxed) == UINT64_MAX) + slot.expected_id.store(0, std::memory_order_release); +} + +TestRdmaProducer::TestRdmaProducer(TestRdmaLoad &owner, uint32_t producer_index) + : load(owner), + index(producer_index), + slots(std::make_unique(owner.in_flight_per_producer)) +{ + for (uint32_t entry = 0; entry < load.in_flight_per_producer; ++entry) + slots[entry].producer = this; +} + +void TestRdmaProducer::phase(RpcClient &client, SocketTransport &transport, Connection *connection, + uint64_t deadline_ns, bool measured) +{ + while (test_rdma_clock_ns() < deadline_ns && load.errors.load(std::memory_order_relaxed) == 0) { + bool progress = false; + for (uint32_t entry = 0; entry < load.in_flight_per_producer; ++entry) { + auto &slot = slots[entry]; + if (slot.expected_id.load(std::memory_order_acquire) != 0) + continue; + const uint64_t now = test_rdma_clock_ns(); + if (now >= deadline_ns) + break; + const uint64_t id = ++slot.generation * load.id_stride + index * load.in_flight_per_producer + entry; + Buffer *control = build_ping_request(transport.pool(), id, 0); + Buffer *data = transport.pool()->alloc(TestRdmaLoad::PAYLOAD_BYTES); + if (control == nullptr || data == nullptr) { + if (control != nullptr) + control->release(); + if (data != nullptr) + data->release(); + load.fail("request buffer admission failed", id, CROWDB_RPC_ERR_SEND_QUEUE); + return; + } + std::array payload{}; + payload.fill(static_cast(id)); + data->write(payload.data(), payload.size()); + slot.started_ns = now; + slot.measured = measured; + slot.expected_id.store(id, std::memory_order_release); + if (!client.send(&transport, connection, id, control, data, TestRdmaLoad::MESSAGE_TYPE, + test_rdma_load_complete, &slot)) { + slot.expected_id.store(0, std::memory_order_release); + load.fail("request submission rejected", id, CROWDB_RPC_ERR_SEND_QUEUE); + return; + } + ++total_submitted; + if (measured) + ++measured_submitted; + progress = true; + } + if (!progress) + std::this_thread::yield(); + } +} + +void TestRdmaProducer::drain() +{ + const uint64_t deadline = test_rdma_clock_ns() + 5'000'000'000ULL; + while (total_completed.load(std::memory_order_acquire) < total_submitted && + load.errors.load(std::memory_order_relaxed) == 0) { + if (test_rdma_clock_ns() >= deadline) { + load.fail("in-flight requests did not drain", 0, CROWDB_RPC_ERR_TIMEOUT); + break; + } + std::this_thread::yield(); + } +} + +void TestRdmaProducer::run(RpcClient &client, SocketTransport &transport, Connection *connection) +{ + load.ready_producers.fetch_add(1, std::memory_order_release); + while (load.warmup_end_ns.load(std::memory_order_acquire) == 0) + std::this_thread::yield(); + phase(client, transport, connection, load.warmup_end_ns.load(std::memory_order_relaxed), false); + drain(); + load.warmed_producers.fetch_add(1, std::memory_order_release); + while (load.measure_end_ns.load(std::memory_order_acquire) == 0) + std::this_thread::yield(); + phase(client, transport, connection, load.measure_end_ns.load(std::memory_order_relaxed), true); + drain(); +} + +} // namespace crowdb::rpc + +#endif diff --git a/lib/crowdb-rpc/tests/common/test_rdma_load.h b/lib/crowdb-rpc/tests/common/test_rdma_load.h new file mode 100644 index 000000000..9cd38fc99 --- /dev/null +++ b/lib/crowdb-rpc/tests/common/test_rdma_load.h @@ -0,0 +1,94 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/client/client.h" +#include "crowdb-rpc/transport/socket_transport.h" + +#ifdef CROWDB_RPC_HAVE_RDMA + +# include +# include +# include +# include +# include +# include + +namespace crowdb::rpc +{ + +inline uint64_t test_rdma_clock_ns() +{ + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) + .count()); +} + +struct TestRdmaLatency +{ + static constexpr size_t SUB_BUCKETS = 128; + static constexpr size_t BUCKETS = 64 * SUB_BUCKETS; + std::array, BUCKETS> buckets{}; + std::atomic sum_ns{0}; + std::atomic max_ns{0}; + + void observe(uint64_t ns); + static uint64_t percentile(const std::array &buckets, uint64_t count, uint32_t percent); +}; + +struct TestRdmaProducer; + +struct TestRdmaLoadSlot +{ + std::atomic expected_id{0}; + uint64_t started_ns = 0; + uint64_t generation = 0; + bool measured = false; + TestRdmaProducer *producer = nullptr; +}; + +struct TestRdmaLoad +{ + static constexpr uint16_t MESSAGE_TYPE = 239; + static constexpr uint16_t FINISHED_TYPE = 240; + static constexpr uint32_t PAYLOAD_BYTES = 128; + uint32_t producer_count; + uint32_t in_flight_per_producer; + uint32_t duration_seconds; + uint64_t id_stride; + std::atomic warmup_end_ns{0}; + std::atomic measure_end_ns{0}; + std::atomic ready_producers{0}; + std::atomic warmed_producers{0}; + std::atomic errors{0}; + std::vector> producers; + + TestRdmaLoad(uint32_t count, uint32_t in_flight, uint32_t seconds); + ~TestRdmaLoad(); + void complete(TestRdmaLoadSlot &slot, uint64_t request_id, crowdb_rpc_buffer_t data, crowdb_rpc_status status); + void fail(const char *reason, uint64_t request_id, crowdb_rpc_status status); +}; + +struct TestRdmaProducer +{ + TestRdmaLoad &load; + uint32_t index; + std::unique_ptr slots; + TestRdmaLatency latency; + uint64_t total_submitted = 0; + uint64_t measured_submitted = 0; + std::atomic total_completed{0}; + std::atomic window_completed{0}; + std::atomic drain_completed{0}; + + TestRdmaProducer(TestRdmaLoad &owner, uint32_t producer_index); + void run(RpcClient &client, SocketTransport &transport, Connection *connection); + void phase(RpcClient &client, SocketTransport &transport, Connection *connection, uint64_t deadline_ns, + bool measured); + void drain(); +}; + +} // namespace crowdb::rpc + +#endif diff --git a/lib/crowdb-rpc/tests/common/test_rdma_transport.h b/lib/crowdb-rpc/tests/common/test_rdma_transport.h new file mode 100644 index 000000000..fb8947f74 --- /dev/null +++ b/lib/crowdb-rpc/tests/common/test_rdma_transport.h @@ -0,0 +1,112 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/transport/rdma/rdma_transport.h" + +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) + +# include +# include + +namespace crowdb::rpc +{ + +struct TestRdmaTransport +{ + static uint64_t send(RdmaTransport &transport, const std::shared_ptr &connection, Buffer *slot, + OutFrame *frame) + { + auto *work = transport.acquire_work(RdmaTransport::WorkKind::Send, connection.get(), slot, frame); + return work == nullptr ? 0 : work->token.load(std::memory_order_acquire); + } + + static uint64_t receive(RdmaTransport &transport, const std::shared_ptr &connection, Buffer *slot) + { + auto *work = transport.acquire_work(RdmaTransport::WorkKind::Receive, connection.get(), slot); + return work == nullptr ? 0 : work->token.load(std::memory_order_acquire); + } + + static void complete(RdmaTransport &transport, uint64_t token, ibv_wc_status status = IBV_WC_SUCCESS, + uint32_t length = 0) + { + ibv_wc completion{}; + completion.wr_id = token; + completion.status = status; + completion.byte_len = length; + // Error providers may leave opcode unset or unrelated to the WR. + completion.opcode = IBV_WC_RDMA_WRITE; + transport.dispatch_completion(completion); + } + + static void cancel(RdmaTransport &transport) + { + transport.cancel_work(); + } + + static void disconnect(RdmaTransport &transport, Connection *connection) + { + transport.close_after_drain(connection); + } + + static void set_cq(RdmaTransport &transport, ibv_cq *cq) + { + transport.cq_ = cq; + if (cq == nullptr) + transport.event_engine_ = nullptr; + } + + static void set_cq_polling(RdmaTransport &transport, bool polling) + { + if (polling) + transport.cq_polling_.test_and_set(std::memory_order_acquire); + else + transport.cq_polling_.clear(std::memory_order_release); + } +}; + +struct TestCompletionQueue +{ + ibv_context context{}; + ibv_cq cq{}; + std::deque ready; + std::deque on_arm; + uint32_t arms = 0; + + TestCompletionQueue() + { + cq.context = &context; + cq.cq_context = this; + context.ops.poll_cq = [](ibv_cq *queue, int capacity, ibv_wc *output) { + auto &self = *static_cast(queue->cq_context); + const auto count = std::min(static_cast(capacity), self.ready.size()); + for (size_t index = 0; index < count; ++index) { + output[index] = self.ready.front(); + self.ready.pop_front(); + } + return static_cast(count); + }; + context.ops.req_notify_cq = [](ibv_cq *queue, int) { + auto &self = *static_cast(queue->cq_context); + ++self.arms; + while (!self.on_arm.empty()) { + self.ready.push_back(self.on_arm.front()); + self.on_arm.pop_front(); + } + return 0; + }; + } + + static ibv_wc completion(uint64_t token) + { + ibv_wc result{}; + result.wr_id = token; + result.status = IBV_WC_SUCCESS; + return result; + } +}; + +} // namespace crowdb::rpc + +#endif diff --git a/lib/crowdb-rpc/tests/rdma_completion_test.cpp b/lib/crowdb-rpc/tests/rdma_completion_test.cpp new file mode 100644 index 000000000..afe735595 --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_completion_test.cpp @@ -0,0 +1,275 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "common/test_rdma_transport.h" +#include "crowdb-rpc/transport/epoll/epoll_engine.h" + +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) + +# include + +using namespace crowdb::rpc; + +namespace +{ +RdmaConfig test_config() +{ + RdmaConfig config; + config.device = "test-no-device"; + config.cq_depth = 8; + config.rq_depth = 4; + config.posted_receive_target = 4; + config.poll_empty_budget = 1; + return config; +} + +class RdmaCompletionTest : public ::testing::Test +{ + protected: + RegisteredBufferPool slots{16U << 20, 32}; + SystemBufferPool buffers{32}; + TestCompletionQueue queue; + RdmaTransport transport{test_config()}; + std::shared_ptr connection = std::make_shared(1, "test", &slots); + + void TearDown() override + { + TestRdmaTransport::set_cq(transport, nullptr); + TestRdmaTransport::cancel(transport); + } + + uint64_t send(const std::shared_ptr &owner = nullptr) + { + Buffer *slot = slots.alloc(32); + if (slot == nullptr) + return 0; + (void)slot->transition(BufferOwnership::Available, BufferOwnership::SendInFlight); + auto *frame = new OutFrame; + frame->request_id = 239; + frame->control = buffers.alloc(1); + frame->data = buffers.alloc(16); + const auto token = TestRdmaTransport::send(transport, owner == nullptr ? connection : owner, slot, frame); + if (token == 0) { + (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); + slot->release(); + frame->control->release(); + frame->release_data(); + delete frame; + } + return token; + } +}; + +TEST_F(RdmaCompletionTest, ErrorsResolveOriginalOwnerWithoutTrustingOpcode) +{ + const std::array statuses{IBV_WC_WR_FLUSH_ERR, IBV_WC_LOC_PROT_ERR, IBV_WC_RETRY_EXC_ERR, IBV_WC_REM_ACCESS_ERR}; + for (const auto status : statuses) { + auto owner = std::make_shared(2, "error", &slots); + uint32_t closes = 0; + owner->set_on_close([&closes](Connection *) { ++closes; }); + const auto token = send(owner); + ASSERT_NE(token, 0U); + TestRdmaTransport::complete(transport, token, status); + TestRdmaTransport::complete(transport, token, status); + EXPECT_EQ(closes, 1U); + EXPECT_FALSE(owner->is_open()); + EXPECT_TRUE(connection->is_open()); + } + EXPECT_EQ(transport.stats().cq_errors.load(), 4U); + EXPECT_EQ(transport.stats().flush_errors.load(), 1U); + EXPECT_EQ(transport.stats().local_protection_errors.load(), 1U); + EXPECT_EQ(transport.stats().retry_exceeded_errors.load(), 1U); + EXPECT_EQ(transport.stats().remote_access_errors.load(), 1U); + EXPECT_EQ(slots.quarantined_bytes(), 4U * RDMA_CONTROL_CAPACITY); +} + +TEST_F(RdmaCompletionTest, BoundedWorkTableRejectsExhaustionAndStaleCompletions) +{ + std::array tokens{}; + for (auto &token : tokens) { + token = send(); + ASSERT_NE(token, 0U); + } + EXPECT_EQ(send(), 0U); + TestRdmaTransport::complete(transport, tokens.front()); + const auto replacement = send(); + ASSERT_NE(replacement, 0U); + EXPECT_NE(replacement, tokens.front()); + TestRdmaTransport::complete(transport, tokens.front(), IBV_WC_LOC_PROT_ERR); + EXPECT_TRUE(connection->is_open()); + EXPECT_EQ(transport.stats().cq_errors.load(), 0U); + TestRdmaTransport::complete(transport, replacement); + for (size_t index = 1; index < tokens.size(); ++index) + TestRdmaTransport::complete(transport, tokens[index]); +} + +TEST_F(RdmaCompletionTest, WorkRetainsConnectionUntilCompletion) +{ + auto owner = std::make_shared(3, "retained", &slots); + std::weak_ptr lifetime = owner; + const auto token = send(owner); + ASSERT_NE(token, 0U); + owner.reset(); + EXPECT_FALSE(lifetime.expired()); + TestRdmaTransport::complete(transport, token); + EXPECT_TRUE(lifetime.expired()); +} + +TEST_F(RdmaCompletionTest, ReceiveWaitsForFrameReleaseBeforeRecycling) +{ + Frame *received = nullptr; + connection->set_on_frame([&received](Frame *frame, Connection *) { received = frame; }); + Buffer *slot = slots.alloc(RDMA_WIRE_SLOT_CAPACITY); + ASSERT_NE(slot, nullptr); + slot->allocation_data = slot->data; + slot->allocation_capacity = slot->capacity; + Header header; + header.data_size = 4; + serialize_header(slot->data, header); + std::memcpy(slot->data + HEADER_SIZE, "data", 4); + ASSERT_TRUE(slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive)); + const auto token = TestRdmaTransport::receive(transport, connection, slot); + TestRdmaTransport::complete(transport, token, IBV_WC_SUCCESS, HEADER_SIZE + 4); + ASSERT_NE(received, nullptr); + EXPECT_EQ(received->data_buf->ownership.load(), BufferOwnership::CallerOwned); + EXPECT_EQ(std::memcmp(received->data_buf->payload_data(), "data", 4), 0); + EXPECT_EQ(slot->ref->load(), 1); + delete received; + EXPECT_EQ(slot->ref->load(), 0); + EXPECT_EQ(slot->ownership.load(), BufferOwnership::Available); +} + +TEST_F(RdmaCompletionTest, ReceiveErrorAndShutdownQuarantineUncertainBuffers) +{ + Buffer *slot = slots.alloc(RDMA_WIRE_SLOT_CAPACITY); + ASSERT_NE(slot, nullptr); + ASSERT_TRUE(slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive)); + const auto token = TestRdmaTransport::receive(transport, connection, slot); + TestRdmaTransport::complete(transport, token, IBV_WC_LOC_PROT_ERR); + EXPECT_EQ(slots.quarantined_bytes(), RDMA_WIRE_SLOT_CAPACITY); + EXPECT_FALSE(connection->is_open()); + auto owner = std::make_shared(4, "shutdown", &slots); + ASSERT_NE(send(owner), 0U); + TestRdmaTransport::cancel(transport); + EXPECT_FALSE(owner->is_open()); + EXPECT_EQ(slots.quarantined_bytes(), RDMA_WIRE_SLOT_CAPACITY + RDMA_CONTROL_CAPACITY); +} + +TEST_F(RdmaCompletionTest, CallerLeaseKeepsPayloadAfterFrameAndTransportRelease) +{ + auto source = std::make_unique(test_config()); + Frame *received = nullptr; + connection->set_on_frame([&received](Frame *frame, Connection *) { received = frame; }); + Buffer *slot = slots.alloc(RDMA_WIRE_SLOT_CAPACITY); + ASSERT_NE(slot, nullptr); + slot->allocation_data = slot->data; + slot->allocation_capacity = slot->capacity; + Header header; + header.data_size = 4; + serialize_header(slot->data, header); + std::memcpy(slot->data + HEADER_SIZE, "held", 4); + ASSERT_TRUE(slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive)); + const auto token = TestRdmaTransport::receive(*source, connection, slot); + TestRdmaTransport::complete(*source, token, IBV_WC_SUCCESS, HEADER_SIZE + 4); + ASSERT_NE(received, nullptr); + Buffer *lease = received->data_buf->ref_clone(); + ASSERT_NE(lease, nullptr); + delete received; + EXPECT_EQ(lease->ownership.load(), BufferOwnership::CallerOwned); + EXPECT_EQ(lease->view_offset, HEADER_SIZE); + source.reset(); + EXPECT_EQ(std::memcmp(lease->payload_data(), "held", 4), 0); + lease->release(); + EXPECT_EQ(slot->ref->load(), 0); +} + +TEST_F(RdmaCompletionTest, RejectsTruncatedOverflowedAndExtraWireBytes) +{ + for (int variant = 0; variant < 3; ++variant) { + auto owner = std::make_shared(5, "invalid", &slots); + uint32_t published = 0; + owner->set_on_frame([&published](Frame *frame, Connection *) { + ++published; + delete frame; + }); + Buffer *slot = slots.alloc(RDMA_WIRE_SLOT_CAPACITY); + ASSERT_NE(slot, nullptr); + Header header; + header.msg_size = variant == 1 ? UINT32_MAX : 0; + serialize_header(slot->data, header); + ASSERT_TRUE(slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive)); + const auto token = TestRdmaTransport::receive(transport, owner, slot); + const uint32_t bytes = variant == 0 ? HEADER_SIZE - 1 : HEADER_SIZE + 1; + TestRdmaTransport::complete(transport, token, IBV_WC_SUCCESS, bytes); + EXPECT_EQ(published, 0U); + EXPECT_FALSE(owner->is_open()); + } + EXPECT_EQ(slots.quarantined_bytes(), 3U * RDMA_WIRE_SLOT_CAPACITY); +} + +TEST_F(RdmaCompletionTest, PollAfterArmingClosesNotificationRace) +{ + const auto token = send(); + ASSERT_NE(token, 0U); + queue.on_arm.push_back(TestCompletionQueue::completion(token)); + TestRdmaTransport::set_cq(transport, &queue.cq); + EXPECT_EQ(transport.drain_cq(8), 1); + EXPECT_TRUE(queue.ready.empty()); + EXPECT_EQ(queue.arms, 1U); +} + +TEST_F(RdmaCompletionTest, CmDisconnectPublishesAlreadyCompletedResponseBeforeFailingRequests) +{ + bool received = false; + connection->set_on_frame([&received](Frame *frame, Connection *) { + received = true; + delete frame; + }); + connection->set_on_rpc_close([&received](Connection *) { EXPECT_TRUE(received); }); + Buffer *slot = slots.alloc(RDMA_WIRE_SLOT_CAPACITY); + ASSERT_NE(slot, nullptr); + slot->allocation_data = slot->data; + slot->allocation_capacity = slot->capacity; + serialize_header(slot->data, Header{}); + ASSERT_TRUE(slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive)); + const auto token = TestRdmaTransport::receive(transport, connection, slot); + auto completion = TestCompletionQueue::completion(token); + completion.byte_len = HEADER_SIZE; + queue.ready.push_back(completion); + TestRdmaTransport::set_cq(transport, &queue.cq); + TestRdmaTransport::disconnect(transport, connection.get()); + EXPECT_TRUE(received); + EXPECT_FALSE(connection->is_open()); +} + +TEST_F(RdmaCompletionTest, BoundedCqBatchContinuesThroughEpollUntilDrained) +{ + for (int index = 0; index < 8; ++index) { + const auto token = send(); + ASSERT_NE(token, 0U); + queue.ready.push_back(TestCompletionQueue::completion(token)); + } + TestRdmaTransport::set_cq(transport, &queue.cq); + EpollEngine engine; + ASSERT_EQ(engine.init(), 0); + transport.register_events(&engine); + EXPECT_EQ(transport.drain_cq(2), 2); + TestRdmaTransport::set_cq_polling(transport, true); + EXPECT_EQ(transport.handle_external_event(3, 2), 0); + TestRdmaTransport::set_cq_polling(transport, false); + int drained = 2; + for (int iteration = 0; iteration < 4 && !queue.ready.empty(); ++iteration) { + std::array events{}; + const int count = engine.wait(events.data(), events.size(), 100); + ASSERT_GT(count, 0); + for (int index = 0; index < count; ++index) + if (events[index].type == SocketEvent::External) + drained += transport.handle_external_event(events[index].external_tag, 2); + } + EXPECT_EQ(drained, 8); + EXPECT_TRUE(queue.ready.empty()); +} + +} // namespace + +#endif diff --git a/lib/crowdb-rpc/tests/rdma_config_test.cpp b/lib/crowdb-rpc/tests/rdma_config_test.cpp new file mode 100644 index 000000000..8909a295f --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_config_test.cpp @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/transport/rdma/rdma_transport.h" + +#ifdef CROWDB_RPC_HAVE_RDMA + +# include + +using crowdb::rpc::RdmaConfig; + +TEST(RdmaConfigTest, PostedReceiveTargetMustFitRqAndRegisteredBudget) +{ + RdmaConfig config; + EXPECT_TRUE(config.valid()); + config.posted_receive_target = config.rq_depth + 1; + EXPECT_FALSE(config.valid()); + config.posted_receive_target = config.rq_depth; + config.receive_arena_bytes = static_cast(config.posted_receive_target) * config.wire_slot_capacity - 1; + EXPECT_FALSE(config.valid()); +} + +TEST(RdmaConfigTest, SmallWireSlotsAllowDeepRqWithinSmallArena) +{ + RdmaConfig config; + config.wire_slot_capacity = 1024; + config.rq_depth = 4096; + config.posted_receive_target = 4096; + config.receive_arena_bytes = 4U << 20; + config.cq_depth = 8192; + EXPECT_TRUE(config.valid()); + config.wire_slot_capacity = 4096; + EXPECT_FALSE(config.valid()); +} + +TEST(RdmaConfigTest, LargeWireSlotsAnd4096ReceivesUse64BitArenaBudget) +{ + RdmaConfig config; + config.rq_depth = 4096; + config.posted_receive_target = 4096; + config.receive_arena_bytes = uint64_t{4} << 30; + config.cq_depth = 8192; + EXPECT_TRUE(config.valid()); + EXPECT_EQ(config.receive_arena_bytes, uint64_t{4096} * config.wire_slot_capacity); +} + +#endif diff --git a/lib/crowdb-rpc/tests/rdma_test.cpp b/lib/crowdb-rpc/tests/rdma_test.cpp index cb293f31c..25b8206ef 100644 --- a/lib/crowdb-rpc/tests/rdma_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_test.cpp @@ -14,6 +14,7 @@ # include # include +# include # include # include # include @@ -21,6 +22,8 @@ # include # include # include +# include +# include namespace { @@ -44,23 +47,6 @@ void install_rdma_test_crash_handler() } } // namespace -struct WorkerCompletion -{ - std::atomic completed{0}; - std::atomic failed{0}; -}; - -extern "C" void rdma_worker_completion(uint64_t, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, - crowdb_rpc_status status, void *opaque) -{ - auto *state = static_cast(opaque); - (status == CROWDB_RPC_OK ? state->completed : state->failed).fetch_add(1, std::memory_order_relaxed); - if (control != nullptr) - crowdb_rpc_buffer_release(control); - if (data != nullptr) - crowdb_rpc_buffer_release(data); -} - using crowdb::rpc::Buffer; using crowdb::rpc::BufferOwnership; using crowdb::rpc::Connection; @@ -68,8 +54,8 @@ using crowdb::rpc::Frame; using crowdb::rpc::Header; using crowdb::rpc::HEADER_SIZE; using crowdb::rpc::OutFrame; -using crowdb::rpc::RdmaConfig; using crowdb::rpc::RdmaBufferPool; +using crowdb::rpc::RdmaConfig; using crowdb::rpc::RdmaTransport; using crowdb::rpc::RpcClient; using crowdb::rpc::RpcServer; @@ -276,63 +262,4 @@ TEST(RdmaHardwareTest, CompleteCmQpSendReceiveFlow) server.stop(); } -TEST(RdmaHardwareTest, RpcServerClientWorkerRoundTrip) -{ - if (std::getenv("CROWDB_RDMA_RPC_E2E") == nullptr) - GTEST_SKIP() << "set CROWDB_RDMA_RPC_E2E=1 to run the worker flow"; - install_rdma_test_crash_handler(); - const char *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); - const char *role = std::getenv("CROWDB_RDMA_E2E_ROLE"); - ASSERT_NE(address, nullptr); - const int port = - std::getenv("CROWDB_RDMA_E2E_PORT") == nullptr ? 19101 : std::atoi(std::getenv("CROWDB_RDMA_E2E_PORT")); - constexpr uint32_t kRequests = 8; - constexpr uint16_t kMessageType = 239; - - if (role != nullptr && std::strcmp(role, "server") == 0) { - RpcServer server; - ASSERT_TRUE(server.listen(address, port)); - std::atomic received{0}; - server.register_handler(kMessageType, [&received, &server](Frame *frame, Connection *) { - received.fetch_add(1, std::memory_order_relaxed); - OutFrame *response = new OutFrame; - response->request_id = frame->request_id; - response->header.msg_type = static_cast(kMessageType); - response->control = - crowdb::rpc::build_ping_response(server.pool(), frame->request_id, frame->rpc_create_nano, 0); - response->header.msg_size = response->control == nullptr ? 0 : response->control->len; - delete frame; - return response; - }); - server.start(); - for (int i = 0; i < 100 && received.load(std::memory_order_relaxed) < kRequests; ++i) - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - EXPECT_EQ(received.load(std::memory_order_relaxed), kRequests); - server.stop(); - return; - } - - ASSERT_NE(role, nullptr); - ASSERT_STREQ(role, "client"); - SocketTransport transport(1, 1); - transport.start(); - auto connection = transport.connect(address, port); - ASSERT_NE(connection, nullptr); - RpcClient client; - client.set_completion_pool_size(kRequests); - client.attach(connection.get()); - WorkerCompletion completion; - for (uint32_t i = 0; i < kRequests; ++i) { - Buffer *control = crowdb::rpc::build_ping_request(transport.pool(), i + 1, 0); - ASSERT_NE(control, nullptr); - ASSERT_TRUE(client.send(&transport, connection.get(), i + 1, control, nullptr, kMessageType, - rdma_worker_completion, &completion)); - } - for (int i = 0; i < 200 && completion.completed.load(std::memory_order_relaxed) < kRequests; ++i) - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - EXPECT_EQ(completion.completed.load(std::memory_order_relaxed), kRequests); - EXPECT_EQ(completion.failed.load(std::memory_order_relaxed), 0U); - transport.stop(); -} - #endif diff --git a/lib/crowdb-rpc/tests/rdma_timed_load_test.cpp b/lib/crowdb-rpc/tests/rdma_timed_load_test.cpp new file mode 100644 index 000000000..4f45f775b --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_timed_load_test.cpp @@ -0,0 +1,203 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "common/test_rdma_load.h" + +#ifdef CROWDB_RPC_HAVE_RDMA + +# include "crowdb-rpc/server/message.h" +# include "crowdb-rpc/server/server.h" + +# include + +# include +# include +# include +# include + +using namespace crowdb::rpc; + +namespace +{ +uint32_t test_setting(const char *name, uint32_t fallback) +{ + const char *value = std::getenv(name); + return value == nullptr ? fallback : static_cast(std::strtoul(value, nullptr, 10)); +} + +struct TestLoadSettings +{ + uint32_t producer_count = test_setting("CROWDB_RDMA_TEST_PRODUCERS", 1); + uint32_t io_workers = test_setting("CROWDB_RDMA_TEST_WORKERS", 1); + uint32_t connection_count = test_setting("CROWDB_RDMA_TEST_CONNECTIONS", 1); + uint32_t in_flight_per_producer = test_setting("CROWDB_RDMA_TEST_WINDOW", 32); + uint32_t duration_seconds = test_setting("CROWDB_RDMA_TEST_SECONDS", 10); + uint32_t warmup_seconds = test_setting("CROWDB_RDMA_TEST_WARMUP_SECONDS", 1); + + bool valid() const + { + return producer_count > 0 && producer_count <= 32 && io_workers > 0 && io_workers <= 32 && + connection_count > 0 && connection_count <= 32 && in_flight_per_producer > 0 && + in_flight_per_producer <= 1024 && duration_seconds > 0 && duration_seconds <= 300 && + warmup_seconds > 0 && warmup_seconds <= 30; + } +}; + +struct TestLoadStop +{ + SocketTransport &transport; + + ~TestLoadStop() + { + transport.stop(); + } +}; + +OutFrame *test_echo(RpcServer &server, Frame *frame) +{ + auto *response = new OutFrame; + response->request_id = frame->request_id; + response->header.msg_type = TestRdmaLoad::MESSAGE_TYPE; + response->control = build_ping_response(server.pool(), frame->request_id, frame->rpc_create_nano, 0); + response->header.msg_size = response->control == nullptr ? 0 : response->control->len; + if (frame->data_buf != nullptr) { + response->data = server.pool()->alloc(frame->data_buf->len); + if (response->data != nullptr) + response->data->write(frame->data_buf->payload_data(), frame->data_buf->len); + } + delete frame; + return response; +} + +void test_timed_server(const char *address, int port, const TestLoadSettings &settings) +{ + RpcServer server(nullptr, 1, settings.io_workers); + ASSERT_TRUE(server.listen(address, port)); + std::atomic received{0}; + std::atomic expected{0}; + std::atomic finished{false}; + server.register_handler(TestRdmaLoad::MESSAGE_TYPE, [&](Frame *frame, Connection *) { + received.fetch_add(1, std::memory_order_relaxed); + return test_echo(server, frame); + }); + server.register_handler(TestRdmaLoad::FINISHED_TYPE, [&](Frame *frame, Connection *) { + expected.store(frame->request_id, std::memory_order_relaxed); + finished.store(true, std::memory_order_release); + delete frame; + return static_cast(nullptr); + }); + server.start(); + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(settings.warmup_seconds + settings.duration_seconds + 30); + while (!finished.load(std::memory_order_acquire) && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + server.stop(); + EXPECT_TRUE(finished.load(std::memory_order_acquire)); + EXPECT_GT(received.load(), 0U); + EXPECT_EQ(received.load(), expected.load()); +} + +void test_report_load(const TestLoadSettings &settings, TestRdmaLoad &load) +{ + std::array buckets{}; + uint64_t completed = 0, submitted = 0, drained = 0, sum_ns = 0, maximum_ns = 0; + for (const auto &producer : load.producers) { + completed += producer->window_completed.load(); + submitted += producer->measured_submitted; + drained += producer->drain_completed.load(); + sum_ns += producer->latency.sum_ns.load(); + maximum_ns = std::max(maximum_ns, producer->latency.max_ns.load()); + for (size_t index = 0; index < buckets.size(); ++index) + buckets[index] += producer->latency.buckets[index].load(); + } + auto percentile_us = [&](uint32_t percent) { + return static_cast(TestRdmaLatency::percentile(buckets, completed, percent)) / 1000.0; + }; + EXPECT_EQ(load.errors.load(), 0U); + EXPECT_GT(completed, 0U); + EXPECT_EQ(completed + drained, submitted); + std::fprintf(stderr, + "RDMA timed load: duration_s=%u warmup_s=%u producer_count=%u io_workers=%u connection_count=%u " + "in_flight_per_producer=%u payload_bytes=%u submitted=%llu completed=%llu drain_completed=%llu " + "TPS=%.0f p50_us=%.2f p95_us=%.2f p99_us=%.2f mean_us=%.2f max_us=%.2f errors=%llu\n", + settings.duration_seconds, settings.warmup_seconds, settings.producer_count, settings.io_workers, + settings.connection_count, settings.in_flight_per_producer, TestRdmaLoad::PAYLOAD_BYTES, + static_cast(submitted), static_cast(completed), + static_cast(drained), static_cast(completed) / settings.duration_seconds, + percentile_us(50), percentile_us(95), percentile_us(99), + completed == 0 ? 0.0 : static_cast(sum_ns) / completed / 1000.0, + static_cast(maximum_ns) / 1000.0, static_cast(load.errors.load())); +} + +void test_timed_client(const char *address, int port, const TestLoadSettings &settings) +{ + SocketTransport transport(1, settings.io_workers); + TestRdmaLoad load(settings.producer_count, settings.in_flight_per_producer, settings.duration_seconds); + RpcClient client; + TestLoadStop stop{transport}; + client.set_completion_pool_size(load.id_stride * 4); + transport.start(); + std::vector> connections; + std::unordered_set qp_numbers; + for (uint32_t index = 0; index < settings.connection_count; ++index) { + auto connection = transport.connect(address, port); + ASSERT_NE(connection, nullptr); + ASSERT_EQ(connection->path(), ConnectionPath::Rdma); + auto *qp = reinterpret_cast(connection->transport_handle); + ASSERT_NE(qp, nullptr); + qp_numbers.insert(qp->qp_num); + client.attach(connection.get()); + connections.push_back(std::move(connection)); + } + ASSERT_EQ(qp_numbers.size(), settings.connection_count); + std::fprintf(stderr, "RDMA topology: physical_hosts=1 shared_hca=%s independent_client_qps=%zu\n", + std::getenv("CROWDB_RDMA_DEVICE"), qp_numbers.size()); + std::vector threads; + for (uint32_t index = 0; index < settings.producer_count; ++index) + threads.emplace_back([&, index] { + load.producers[index]->run(client, transport, connections[index % connections.size()].get()); + }); + while (load.ready_producers.load(std::memory_order_acquire) != settings.producer_count) + std::this_thread::yield(); + load.warmup_end_ns.store(test_rdma_clock_ns() + settings.warmup_seconds * 1'000'000'000ULL, + std::memory_order_release); + while (load.warmed_producers.load(std::memory_order_acquire) != settings.producer_count) + std::this_thread::yield(); + load.measure_end_ns.store(test_rdma_clock_ns() + settings.duration_seconds * 1'000'000'000ULL, + std::memory_order_release); + for (auto &thread : threads) + thread.join(); + uint64_t total_submitted = 0; + for (const auto &producer : load.producers) + total_submitted += producer->total_submitted; + if (load.errors.load() == 0) { + auto *finished = build_ping_request(transport.pool(), total_submitted, 0); + ASSERT_NE(finished, nullptr); + EXPECT_TRUE(client.send_one_way(&transport, connections.front().get(), total_submitted, finished, nullptr, + TestRdmaLoad::FINISHED_TYPE)); + } + transport.stop(); + test_report_load(settings, load); +} +} // namespace + +TEST(RdmaHardwareTest, RpcServerClientTimedLoad) +{ + if (std::getenv("CROWDB_RDMA_RPC_E2E") == nullptr) + GTEST_SKIP() << "set CROWDB_RDMA_RPC_E2E=1 to run the timed worker load"; + const char *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + const char *role = std::getenv("CROWDB_RDMA_E2E_ROLE"); + const TestLoadSettings settings; + ASSERT_NE(address, nullptr); + ASSERT_NE(role, nullptr); + ASSERT_TRUE(settings.valid()); + const int port = static_cast(test_setting("CROWDB_RDMA_E2E_PORT", 19101)); + if (std::strcmp(role, "server") == 0) + test_timed_server(address, port, settings); + else { + ASSERT_STREQ(role, "client"); + test_timed_client(address, port, settings); + } +} + +#endif diff --git a/lib/crowdb-rpc/tests/rdma_worker_test.cpp b/lib/crowdb-rpc/tests/rdma_worker_test.cpp new file mode 100644 index 000000000..d570cd827 --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_worker_test.cpp @@ -0,0 +1,243 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/client/client.h" +#include "crowdb-rpc/server/message.h" +#include "crowdb-rpc/server/server.h" +#include "crowdb-rpc/transport/socket_transport.h" + +#ifdef CROWDB_RPC_HAVE_RDMA + +# include + +# include +# include +# include +# include +# include +# include +# include +# include + +using namespace crowdb::rpc; + +namespace +{ +uint64_t monotonic_ns() +{ + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) + .count()); +} + +uint32_t test_setting(const char *name, uint32_t fallback) +{ + const char *value = std::getenv(name); + return value == nullptr ? fallback : static_cast(std::strtoul(value, nullptr, 10)); +} +} // namespace + +struct TestTransportStop +{ + SocketTransport &transport; + + ~TestTransportStop() + { + transport.stop(); + } +}; + +struct WorkerCompletion +{ + std::atomic completed{0}; + std::atomic failed{0}; + std::atomic bad_payloads{0}; + std::vector submitted; + std::vector latencies; + std::unique_ptr[]> seen; + + explicit WorkerCompletion(uint32_t requests) + : submitted(requests + 1), + latencies(requests + 1), + seen(std::make_unique[]>(requests + 1)) + { + } +}; + +extern "C" void rdma_worker_completion(uint64_t request_id, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, + crowdb_rpc_status status, void *opaque) +{ + auto *state = static_cast(opaque); + if (request_id == 0 || request_id >= state->submitted.size() || + state->seen[request_id].exchange(1, std::memory_order_acq_rel) != 0) { + state->bad_payloads.fetch_add(1, std::memory_order_release); + } + else { + state->latencies[request_id] = monotonic_ns() - state->submitted[request_id]; + } + if (status != CROWDB_RPC_OK) + std::fprintf(stderr, "R239 worker completion failed status=%d request=%llu\n", status, + static_cast(request_id)); + if (status == CROWDB_RPC_OK) { + const uint32_t length = data == nullptr ? 0 : crowdb_rpc_buffer_len(data); + const auto *bytes = data == nullptr ? nullptr : crowdb_rpc_buffer_data(data); + if (length != 128 || bytes == nullptr || !std::all_of(bytes, bytes + length, [request_id](uint8_t value) { + return value == static_cast(request_id); + })) + state->bad_payloads.fetch_add(1, std::memory_order_release); + } + (status == CROWDB_RPC_OK ? state->completed : state->failed).fetch_add(1, std::memory_order_release); + if (control != nullptr) + crowdb_rpc_buffer_release(control); + if (data != nullptr) + crowdb_rpc_buffer_release(data); +} + +TEST(RdmaHardwareTest, RpcServerClientWorkerRoundTrip) +{ + if (std::getenv("CROWDB_RDMA_RPC_E2E") == nullptr) + GTEST_SKIP() << "set CROWDB_RDMA_RPC_E2E=1 to run the worker flow"; + const char *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + const char *role = std::getenv("CROWDB_RDMA_E2E_ROLE"); + ASSERT_NE(address, nullptr); + const int port = + std::getenv("CROWDB_RDMA_E2E_PORT") == nullptr ? 19101 : std::atoi(std::getenv("CROWDB_RDMA_E2E_PORT")); + const uint32_t kRequests = test_setting("CROWDB_RDMA_TEST_REQUESTS", 1000); + const uint32_t kConnections = test_setting("CROWDB_RDMA_TEST_CONNECTIONS", 1); + const uint32_t kWorkers = test_setting("CROWDB_RDMA_TEST_WORKERS", 1); + const uint32_t kWindow = test_setting("CROWDB_RDMA_TEST_WINDOW", 1); + ASSERT_GT(kRequests, 0U); + ASSERT_LE(kRequests, 1000000U); + ASSERT_GT(kConnections, 0U); + ASSERT_LE(kConnections, 32U); + ASSERT_GT(kWorkers, 0U); + ASSERT_LE(kWorkers, 32U); + ASSERT_GT(kWindow, 0U); + ASSERT_LE(kWindow, 1024U); + constexpr uint16_t kMessageType = 239; + constexpr uint16_t kFinishedType = 240; + + if (role != nullptr && std::strcmp(role, "server") == 0) { + RpcServer server(nullptr, 1, kWorkers); + ASSERT_TRUE(server.listen(address, port)); + std::atomic received{0}; + std::atomic client_finished{false}; + server.register_handler(kFinishedType, [&client_finished](Frame *frame, Connection *) { + client_finished.store(true, std::memory_order_release); + delete frame; + return static_cast(nullptr); + }); + server.register_handler(kMessageType, [&received, &server](Frame *frame, Connection *) { + received.fetch_add(1, std::memory_order_relaxed); + OutFrame *response = new OutFrame; + response->request_id = frame->request_id; + response->header.msg_type = static_cast(kMessageType); + response->control = + crowdb::rpc::build_ping_response(server.pool(), frame->request_id, frame->rpc_create_nano, 0); + response->header.msg_size = response->control == nullptr ? 0 : response->control->len; + if (frame->data_buf != nullptr && frame->data_buf->len != 0) { + response->data = server.pool()->alloc(frame->data_buf->len); + if (response->data != nullptr) + response->data->write(frame->data_buf->payload_data(), frame->data_buf->len); + } + delete frame; + return response; + }); + server.start(); + for (int i = 0; i < 100 && !client_finished.load(std::memory_order_acquire); ++i) + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + EXPECT_EQ(received.load(std::memory_order_relaxed), kRequests); + EXPECT_TRUE(client_finished.load(std::memory_order_acquire)); +# ifdef CROWDB_RPC_TEST_UTIL + if (const auto *stats = server.transport()->rdma_stats_for_tests(); stats != nullptr) + std::fprintf(stderr, "server RDMA: send=%llu recv=%llu cq_errors=%llu polls=%llu posted=%llu\n", + static_cast(stats->sends.load()), + static_cast(stats->receives.load()), + static_cast(stats->cq_errors.load()), + static_cast(stats->cq_polls.load()), + static_cast(stats->posted_receives.load())); +# endif + server.stop(); + return; + } + + ASSERT_NE(role, nullptr); + ASSERT_STREQ(role, "client"); + SocketTransport transport(1, kWorkers); + transport.start(); + std::vector> connections; + std::unordered_set qp_numbers; + WorkerCompletion completion(kRequests); + RpcClient client; + TestTransportStop stop{transport}; + client.set_completion_pool_size(kWindow * 2); + for (uint32_t index = 0; index < kConnections; ++index) { + auto connection = transport.connect(address, port); + ASSERT_NE(connection, nullptr); + ASSERT_EQ(connection->path(), crowdb::rpc::ConnectionPath::Rdma); + auto *qp = reinterpret_cast(connection->transport_handle); + ASSERT_NE(qp, nullptr); + qp_numbers.insert(qp->qp_num); + client.attach(connection.get()); + connections.push_back(std::move(connection)); + } + ASSERT_EQ(qp_numbers.size(), kConnections); + std::fprintf(stderr, "RDMA topology: physical_hosts=1 shared_hca=%s independent_client_qps=%zu\n", + std::getenv("CROWDB_RDMA_DEVICE"), qp_numbers.size()); + const auto started = std::chrono::steady_clock::now(); + const auto deadline = started + std::chrono::seconds(10); + for (uint32_t i = 0; i < kRequests; ++i) { + while (i - completion.completed.load(std::memory_order_acquire) >= kWindow && + completion.failed.load(std::memory_order_acquire) == 0 && std::chrono::steady_clock::now() < deadline) + std::this_thread::yield(); + ASSERT_EQ(completion.failed.load(std::memory_order_acquire), 0U); + ASSERT_LT(std::chrono::steady_clock::now(), deadline); + completion.submitted[i + 1] = monotonic_ns(); + Buffer *control = crowdb::rpc::build_ping_request(transport.pool(), i + 1, 0); + ASSERT_NE(control, nullptr); + Buffer *data = transport.pool()->alloc(128); + ASSERT_NE(data, nullptr); + std::array payload{}; + payload.fill(static_cast(i + 1)); + data->write(payload.data(), static_cast(payload.size())); + ASSERT_TRUE(client.send(&transport, connections[i % kConnections].get(), i + 1, control, data, kMessageType, + rdma_worker_completion, &completion)); + } + while (completion.completed.load(std::memory_order_acquire) < kRequests && + completion.failed.load(std::memory_order_acquire) == 0 && std::chrono::steady_clock::now() < deadline) + std::this_thread::yield(); + const auto finished_at = std::chrono::steady_clock::now(); + EXPECT_EQ(completion.completed.load(std::memory_order_relaxed), kRequests); + EXPECT_EQ(completion.failed.load(std::memory_order_relaxed), 0U); + EXPECT_EQ(completion.bad_payloads.load(std::memory_order_relaxed), 0U); + if (completion.completed.load(std::memory_order_acquire) == kRequests) { + Buffer *finished = crowdb::rpc::build_ping_request(transport.pool(), kRequests + 1, 0); + ASSERT_NE(finished, nullptr); + ASSERT_TRUE(client.send_one_way(&transport, connections.front().get(), kRequests + 1, finished, nullptr, + kFinishedType)); + } +# ifdef CROWDB_RPC_TEST_UTIL + if (const auto *stats = transport.rdma_stats_for_tests(); stats != nullptr) + std::fprintf(stderr, "client RDMA: send=%llu recv=%llu cq_errors=%llu polls=%llu posted=%llu\n", + static_cast(stats->sends.load()), + static_cast(stats->receives.load()), + static_cast(stats->cq_errors.load()), + static_cast(stats->cq_polls.load()), + static_cast(stats->posted_receives.load())); +# endif + transport.stop(); + const double seconds = std::chrono::duration(finished_at - started).count(); + auto latency = completion.latencies; + latency.erase(latency.begin()); + std::sort(latency.begin(), latency.end()); + auto percentile = [&](size_t percent) { + return static_cast(latency[(latency.size() - 1) * percent / 100]) / 1000.0; + }; + std::fprintf(stderr, + "RDMA load: requests=%u workers=%u connections=%u window=%u TPS=%.0f p50_us=%.2f p95_us=%.2f " + "p99_us=%.2f errors=%u\n", + kRequests, kWorkers, kConnections, kWindow, kRequests / seconds, percentile(50), percentile(95), + percentile(99), completion.failed.load()); +} + +#endif diff --git a/lib/crowdb-rpc/tests/transport_test.cpp b/lib/crowdb-rpc/tests/transport_test.cpp index e34b1e053..7809c1812 100644 --- a/lib/crowdb-rpc/tests/transport_test.cpp +++ b/lib/crowdb-rpc/tests/transport_test.cpp @@ -110,6 +110,7 @@ TEST(EpollEngineTest, ExternalDescriptorWakesAndReportsTag) const int fd = ::eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); ASSERT_GE(fd, 0); engine.add_external_fd(fd, 0x55AA); + engine.add_external_fd(fd, 0x55AA); uint64_t one = 1; ASSERT_EQ(::write(fd, &one, sizeof(one)), static_cast(sizeof(one))); diff --git a/tools/pixi-tasks/build-cpp.sh b/tools/pixi-tasks/build-cpp.sh index 89c1bbda1..00f8428c0 100644 --- a/tools/pixi-tasks/build-cpp.sh +++ b/tools/pixi-tasks/build-cpp.sh @@ -5,9 +5,9 @@ set -euo pipefail cd "${PIXI_PROJECT_ROOT:?}" cmake -S lib/crowdb-tree -B lib/crowdb-tree/build -DCMAKE_BUILD_TYPE=Release -DCROWDB_TREE_TEST_UTIL=ON -cmake --build lib/crowdb-tree/build -j -cmake --build lib/crowdb-tree/build -j --target crowdb_rpc_tests -cmake -S lib/crowdb-rpc -B lib/crowdb-rpc/build -DCMAKE_BUILD_TYPE=Release -cmake --build lib/crowdb-rpc/build -j +cmake --build lib/crowdb-tree/build -j32 +cmake --build lib/crowdb-tree/build -j32 --target crowdb_rpc_tests +cmake -S lib/crowdb-rpc -B lib/crowdb-rpc/build -DCMAKE_BUILD_TYPE=Release -DCROWDB_RPC_TEST_UTIL=ON +cmake --build lib/crowdb-rpc/build -j32 cmake -S app/crowdb-diskio -B app/crowdb-diskio/build -DCMAKE_BUILD_TYPE=Release -cmake --build app/crowdb-diskio/build -j +cmake --build app/crowdb-diskio/build -j32 diff --git a/tools/rdma/run-local-loop.sh b/tools/rdma/run-local-loop.sh index 37277b392..989363c09 100755 --- a/tools/rdma/run-local-loop.sh +++ b/tools/rdma/run-local-loop.sh @@ -140,21 +140,42 @@ echo "RDMA $mode loop: client=$client_device/$client_if@$client_addr server=$ser cleanup() { if [[ -n "$server_pid" ]]; then - kill "$server_pid" 2>/dev/null || true + kill -- "-$server_pid" 2>/dev/null || true wait "$server_pid" 2>/dev/null || true fi } trap cleanup EXIT INT TERM -common_env=(CROWDB_RDMA_E2E=1 CROWDB_RDMA_E2E_PORT="$port") -run_in_ns "$server_ns" env "${common_env[@]}" \ +wire_slot_capacity=${CROWDB_RDMA_WIRE_SLOT_CAPACITY:-1024} +sq_depth=${CROWDB_RDMA_SQ_DEPTH:-1024} +rq_depth=${CROWDB_RDMA_RQ_DEPTH:-1024} +posted_receive_target=${CROWDB_RDMA_POSTED_RECEIVE_TARGET:-$rq_depth} +connections=${CROWDB_RDMA_TEST_CONNECTIONS:-1} +cq_depth=${CROWDB_RDMA_CQ_DEPTH:-$((connections * (sq_depth + rq_depth)))} +send_arena_bytes=${CROWDB_RDMA_SEND_ARENA_BYTES:-$((sq_depth * wire_slot_capacity))} +receive_arena_bytes=${CROWDB_RDMA_RECEIVE_ARENA_BYTES:-$((posted_receive_target * wire_slot_capacity))} +echo "RDMA config: sq_depth=$sq_depth rq_depth=$rq_depth posted_receive_target=$posted_receive_target wire_slot_capacity=$wire_slot_capacity cq_depth=$cq_depth send_arena_bytes=$send_arena_bytes receive_arena_bytes=$receive_arena_bytes" +common_env=(CROWDB_RDMA_E2E=1 CROWDB_RDMA_E2E_PORT="$port" + CROWDB_RDMA_SQ_DEPTH="$sq_depth" CROWDB_RDMA_RQ_DEPTH="$rq_depth" + CROWDB_RDMA_POSTED_RECEIVE_TARGET="$posted_receive_target" + CROWDB_RDMA_WIRE_SLOT_CAPACITY="$wire_slot_capacity" CROWDB_RDMA_CQ_DEPTH="$cq_depth" + CROWDB_RDMA_SEND_ARENA_BYTES="$send_arena_bytes" CROWDB_RDMA_RECEIVE_ARENA_BYTES="$receive_arena_bytes") +server_command=(env "${common_env[@]}" CROWDB_RDMA_E2E_ROLE=server \ CROWDB_RDMA_E2E_LISTEN_ADDR="$server_addr" \ CROWDB_RDMA_DEVICE="$server_device" \ - "$pixi_bin" run "$test_bin" --gtest_filter="$filter" & + "$pixi_bin" run "$test_bin" --gtest_filter="$filter") +if [[ "$server_ns" != root ]]; then + server_command=(ip netns exec "$server_ns" "${server_command[@]}") +fi +setsid "${server_command[@]}" & server_pid=$! sleep 1 +if ! kill -0 "$server_pid" 2>/dev/null; then + wait "$server_pid" + die "server exited before the client started" +fi run_in_ns "$client_ns" env "${common_env[@]}" \ CROWDB_RDMA_E2E_ROLE=client \ CROWDB_RDMA_E2E_LISTEN_ADDR="$server_addr" \ diff --git a/tools/rdma/run-timed-load.py b/tools/rdma/run-timed-load.py new file mode 100644 index 000000000..86954e744 --- /dev/null +++ b/tools/rdma/run-timed-load.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +"""Run successive, fixed-duration RDMA RPC load profiles on the local HCA.""" + +import argparse +import csv +import datetime +import os +from pathlib import Path +import re +import signal +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--seconds", type=int, default=10) + parser.add_argument("--threads", type=int, nargs="+", default=[1, 2, 4, 8, 16, 32]) + parser.add_argument("--rq-depth", type=int, choices=[1024, 4096], default=1024) + parser.add_argument("--window", type=int, default=32) + parser.add_argument("--io-workers", type=int, help="hold I/O workers per peer fixed across profiles") + parser.add_argument("--connections", type=int, help="hold QP connections fixed across profiles") + parser.add_argument("--skip-serial", action="store_true", help="skip the one-in-flight baseline") + args = parser.parse_args() + if not 1 <= args.seconds <= 300 or not 1 <= args.window <= 1024: + parser.error("seconds must be 1..300 and window must be 1..1024") + if any(count < 1 or count > 32 for count in args.threads): + parser.error("thread counts must be 1..32") + if any(count is not None and not 1 <= count <= 32 for count in [args.io_workers, args.connections]): + parser.error("I/O workers and connections must be 1..32") + + root = Path(__file__).resolve().parents[2] + stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + output = root / ".crowdb-runtime/artifacts/rdma" / stamp + output.mkdir(parents=True) + pixi = os.environ.get("PIXI_BIN", "/home/cj/.pixi/bin/pixi") + runtime_env = os.environ.copy() + runtime_env["CROWDB_RUNTIME_ROOT"] = str(output / "runtime") + subprocess.run([pixi, "run", "clean-env"], cwd=root, env=runtime_env, check=True) + profiles = [] if args.skip_serial else [(1, 1)] + profiles.extend((count, args.window) for count in args.threads) + print(f"Timed RDMA load artifacts: {output}", flush=True) + + with (output / "results.csv").open("w", newline="") as results: + writer = None + for index, (count, window) in enumerate(profiles): + env = runtime_env.copy() + env.update({ + "CROWDB_RDMA_ENABLED": "1", + "CROWDB_RDMA_RPC_E2E": "1", + "CROWDB_RDMA_E2E_FILTER": "RdmaHardwareTest.RpcServerClientTimedLoad", + "CROWDB_RDMA_TEST_SECONDS": str(args.seconds), + "CROWDB_RDMA_TEST_WARMUP_SECONDS": "1", + "CROWDB_RDMA_TEST_PRODUCERS": str(count), + "CROWDB_RDMA_TEST_WORKERS": str(args.io_workers or count), + "CROWDB_RDMA_TEST_CONNECTIONS": str(args.connections or count), + "CROWDB_RDMA_TEST_WINDOW": str(window), + "CROWDB_RDMA_RQ_DEPTH": str(args.rq_depth), + "CROWDB_RDMA_POSTED_RECEIVE_TARGET": str(args.rq_depth), + "CROWDB_RDMA_E2E_PORT": str(19201 + index), + }) + command = [pixi, "run", "--", "bash", "tools/rdma/run-local-loop.sh", "local"] + log_path = output / f"{index:02d}-{count}threads-{window}inflight.log" + with log_path.open("w") as log: + process = subprocess.Popen(command, cwd=root, env=env, stdout=log, + stderr=subprocess.STDOUT, start_new_session=True) + try: + returncode = process.wait(timeout=args.seconds + 45) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGTERM) + process.wait(timeout=10) + returncode = 124 + text = log_path.read_text() + print(text, end="", flush=True) + if returncode != 0: + print(f"Profile failed: exit={returncode}, full log={log_path}", file=sys.stderr) + return returncode + matches = re.findall(r"^RDMA timed load: (.+)$", text, re.MULTILINE) + if len(matches) != 1: + raise RuntimeError(f"expected one timed load result in {log_path}") + row = dict(field.split("=", 1) for field in matches[0].split()) + row.update(rq_depth=args.rq_depth, sq_depth=env.get("CROWDB_RDMA_SQ_DEPTH", "1024"), + wire_slot_capacity=env.get("CROWDB_RDMA_WIRE_SLOT_CAPACITY", "1024"), log=log_path.name) + if writer is None: + writer = csv.DictWriter(results, fieldnames=list(row)) + writer.writeheader() + writer.writerow(row) + results.flush() + print(f"Completed timed RDMA load: {output / 'results.csv'}", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/rpc-test/src/workload.rs b/tools/rpc-test/src/workload.rs index e28bd7161..a15b502b3 100644 --- a/tools/rpc-test/src/workload.rs +++ b/tools/rpc-test/src/workload.rs @@ -40,7 +40,11 @@ use std::path::Path; const ECHO_MSG_TYPE: u16 = 100; pub async fn run(log_dir: &Path, args: RpcArgs) -> ExitCode { - let data_bytes = if args.value_size == 0 { Vec::new() } else { vec![0xAB; args.value_size] }; + let data_bytes = if args.value_size == 0 { + Vec::new() + } else { + vec![0xAB; args.value_size] + }; // Client-side server for connection management. let server = Arc::new(RpcServer::with_engines(None, args.io_engines, args.io_workers)); From beca4c038dea9bbbcaf0f2054fb5f946a0a7f025 Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 22 Oct 2026 18:48:27 +0800 Subject: [PATCH 020/258] Require explicit RDMA benchmark paths and reject invalid results --- doc/working/plan-rdma-rpc-transport.md | 92 ++++++++++++++++++ .../crowdb-rpc/transport/socket_transport.h | 1 + .../src/transport/socket_transport.cpp | 14 +++ .../tests/common/test_environment.h | 37 +++++++ lib/crowdb-rpc/tests/transport_path_test.cpp | 73 ++++++++++++++ tools/benchmark/bench-rdma-rpc-regression.sh | 46 ++++++--- tools/benchmark/tests/rdma_regression_test.py | 97 +++++++++++++++++++ tools/rdma/run-timed-load.py | 1 + 8 files changed, 349 insertions(+), 12 deletions(-) create mode 100644 lib/crowdb-rpc/tests/common/test_environment.h create mode 100644 lib/crowdb-rpc/tests/transport_path_test.cpp create mode 100644 tools/benchmark/tests/rdma_regression_test.py diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 6db0031bc..544343a66 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -150,6 +150,98 @@ and TPS/P50/P95/P99 output. profiles. Existing capability serialization and state predicates alone do not prove that every connection follows the negotiated setup contract. +### Benchmark audit and proposed unified measurement + +User decisions: use coroutine loaders for the primary throughput benchmark; +keep producer-thread stress as diagnostic coverage; include errors in every +performance result; prohibit TCP fallback in RDMA benchmarks; redesign the +latency histogram to resolve sub-65-us RTTs. + +- [x] **Audit historical baseline**: the retained 128-byte run at + `.crowdb-runtime/artifacts/bench/rdma-rpc-20261018-220617/server/metrics.log` + records TCP socket read/writev data for all cases. Its 32-connection count + is 3,560,660 payloads over approximately three seconds, matching the + script comment's 1,186,876 TPS. Treat that number as TCP-path evidence, + not a verified RDMA baseline. The old `results.tsv` files contain error + rows despite successful shell exit; the script also expects JSON while + the current CLI emits a human report. +- [x] **Add explicit fallback policy**: `CROWDB_RDMA_ALLOW_TCP_FALLBACK=0` + sets `SocketTransport::allow_tcp_fallback_` false. Unavailable, disabled, + or uncompiled RDMA then returns connection failure before opening a TCP + socket. The default remains enabled; invalid policy values disable TCP + fallback. Count only allowed fallback attempts. Five path-policy tests + and five existing capability tests pass. + The full 126-test native RPC suite passes (hardware-only cases skip there), + and the actual hardware worker round trip also passes with fallback disabled. +- [x] **Reject false benchmark success**: both RDMA load runners set the + strict policy. The regression script preserves complete per-case client + output, records `client_exit`, `report_errors`, and `valid`, and returns + nonzero for a failed client or invalid report. Missing RPC error counts + are `NA`, not zero. A rebuilt-client invalid-device reproduction returns + client exit 2 and script exit 1. Retained evidence: + `.crowdb-runtime/artifacts/rdma/strict-policy-script-fixed/`. + Changed C++ formatting/tree-lint and workspace clippy pass. The historical + table is now annotated as unverified RDMA data instead of a regression gate. + Four runner contract tests cover a successful report, a failed client with + preserved error count, a missing report despite exit zero, and reported RPC + errors despite exit zero. These use fixture processes without an HCA: + `pixi run -- python -m unittest discover -s tools/benchmark/tests -p rdma_regression_test.py -v`. +- [ ] **Unify the workload and report contract after the test-design + decision**: use one coroutine-driven RPC echo workload, one case manifest, + and one JSON/CSV schema for native processes and two host-network + containers. Require actual RDMA path and unique QP counts. Verify request + correlation and response length/content; include submission rejection and + native coroutine retry counters, which are not fully represented by the + current Rust `EchoHandler` response-error counter. + Audit the built-in echo allocator before using it as the common workload: + it allocates replies from `conn->pool()`, which is currently the RDMA + receive pool, while the diagnostic echo uses `server.pool()`. A completely + preposted receive pool leaves no allocation admission for those replies. +- [ ] **Unify histogram semantics**: use nanosecond input, fixed bounded + memory, mergeable per-polling-owner shards, explicit zero/overflow counts, + exact count/sum/min/max, and avg/P50/P90/P95/P99/P99.9 output. Remove the + common C++/Rust histograms' 65.5-us underflow floor; do not retain a separate + benchmark-only precision contract. Specify a shared bucket layout and + golden vectors. Record timing origin and exclude warmup/drain from TPS. +- [ ] **Confirm CQ ownership before expanding the performance matrix**: + current `RdmaTransport` has one `cq_` per peer, no dedicated polling + thread, and competing epoll workers protected by `cq_polling_`. + `RdmaTransport::start()` only changes lifecycle state. The recommended + target is one assigned polling/event thread per CQ, with QPs assigned + to one CQ, bounded busy polling under traffic, and arm/poll/epoll sleep + after the idle budget. Proposed `cq_count` and `poller_count` are not + existing configuration knobs. Compare native and OCI runs with the same + manifest; neither single-host topology establishes physical-node isolation. + +Proposed result schema (client/server resource values are separate): + +| Group | Required columns | +|-------------|------------------------------------------------------------------------------------------------------------------------------| +| Identity | case_id, run_id, git_sha, binary hashes, topology, CPU/HCA identity, CPU affinity, actual path, fallback policy/count | +| Workload | client_mode, loader_count, producer_threads, in_flight_per_loader, connection_count, logical_ops_per_rpc, payload_bytes, wire_bytes | +| Scheduling | io_engines, io_workers, cq_count, poller_count, poll_batch, verbs_poll_batch, poll_empty_budget, epoll_wakeups, CQ event/poll counts | +| Capacity | requested/effective sq_depth, rq_depth, posted_receive_target, cq_depth, wire_slot_capacity, arena bytes, queue/pool high-water marks | +| Measurement | warmup_s, measure_s, repeat_index, latency_scope, submitted, completed_in_window, drain_completed, rpc_tps, logical_ops_s, payload_Bps | +| Latency | latency_count, avg_ns, p50_ns, p90_ns, p95_ns, p99_ns, p999_ns, min_ns, max_ns, histogram_zero_count, histogram_overflow_count | +| Errors | errors_total, setup_errors, submit_rejects, timeouts, connection_errors, cq_errors, payload_mismatches, duplicate_completions, retries, report_errors, client_exit, server_exit, valid | +| Cost | client/server CPU time, context switches, CPU migrations, registered bytes, quarantined bytes, available RNR/provider counters | + +All fields belong to one machine-readable row; unavailable values are null. +Human tables can show projections joined by case_id but always include errors +and validity. Normal shutdown WR flushes are separate from workload failures. +Each measured operation is one validated RPC round trip; current code copies +one OutFrame into one SEND slot and does not combine multiple RPCs merely +because they share a QP. Application aggregation, if benchmarked, must report +logical_ops_per_rpc and logical_ops_s separately from rpc_tps. + +Recommended sweep: single-loader latency baseline; then sweep coroutine count +with one QP/CQ/poller; then QP count; then CQ/poller count; then queue depths +and payload/slot size. Use one-second warmup and ten-second measurement, repeat +candidate peaks, and retain all trials and errors. Choose the highest repeatable +zero-error TPS and show its latency rather than selecting unrelated best +throughput and latency fields from different trials. Wire header/control bytes +count toward slot capacity, including at the 1-MiB class boundary. + Provider references for the implementation choices: - [libibverbs QP capacity terminology](https://github.com/linux-rdma/rdma-core/blob/master/libibverbs/man/ibv_create_qp.3). diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h index 42635c7b3..a2874f491 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h @@ -520,6 +520,7 @@ class SocketTransport : public Transport private: BufferPool *pool_; + bool allow_tcp_fallback_{true}; #ifdef CROWDB_RPC_HAVE_RDMA std::unique_ptr rdma_transport_; RdmaConfig rdma_config_; diff --git a/lib/crowdb-rpc/src/transport/socket_transport.cpp b/lib/crowdb-rpc/src/transport/socket_transport.cpp index 7e343bdd4..651867768 100644 --- a/lib/crowdb-rpc/src/transport/socket_transport.cpp +++ b/lib/crowdb-rpc/src/transport/socket_transport.cpp @@ -549,6 +549,12 @@ SocketTransport::SocketTransport(uint32_t io_engines, uint32_t io_workers, Buffe { assert(io_engines >= 1 && io_workers >= 1); assert(io_workers % io_engines == 0); + if (const char *fallback = std::getenv("CROWDB_RDMA_ALLOW_TCP_FALLBACK"); fallback != nullptr) { + allow_tcp_fallback_ = std::strcmp(fallback, "1") == 0; + if (!allow_tcp_fallback_ && std::strcmp(fallback, "0") != 0) { + CRB_LOG_WARN("invalid RDMA TCP fallback policy value={}; TCP fallback disabled", fallback); + } + } if (pool_ == nullptr) { pool_ = new SystemBufferPool(); // own a default pool } @@ -916,6 +922,14 @@ std::shared_ptr SocketTransport::connect(const std::string &addr, in attach_rdma_connection(rdma_conn); return rdma_conn; } + } +#endif + if (!allow_tcp_fallback_) { + CRB_LOG_WARN("RDMA connection unavailable; TCP fallback disabled addr={} port={}", addr, port); + return nullptr; + } +#ifdef CROWDB_RPC_HAVE_RDMA + if (rdma_transport_ != nullptr) { rdma_transport_->stats().fallback_count.fetch_add(1, std::memory_order_relaxed); } #endif diff --git a/lib/crowdb-rpc/tests/common/test_environment.h b/lib/crowdb-rpc/tests/common/test_environment.h new file mode 100644 index 000000000..95e422db3 --- /dev/null +++ b/lib/crowdb-rpc/tests/common/test_environment.h @@ -0,0 +1,37 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include +#include +#include + +struct TestEnvironment +{ + std::string name; + std::optional previous; + + TestEnvironment(const char *key, const char *value) : name(key) + { + if (const char *old = std::getenv(key); old != nullptr) { + previous = old; + } + if (value == nullptr) { + ::unsetenv(key); + } + else { + ::setenv(key, value, 1); + } + } + + ~TestEnvironment() + { + if (previous) { + ::setenv(name.c_str(), previous->c_str(), 1); + } + else { + ::unsetenv(name.c_str()); + } + } +}; diff --git a/lib/crowdb-rpc/tests/transport_path_test.cpp b/lib/crowdb-rpc/tests/transport_path_test.cpp new file mode 100644 index 000000000..ce93ede0f --- /dev/null +++ b/lib/crowdb-rpc/tests/transport_path_test.cpp @@ -0,0 +1,73 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "common/test_environment.h" +#include "crowdb-rpc/server/server.h" + +#include + +using namespace crowdb::rpc; + +namespace +{ +class TransportPathTest : public ::testing::Test +{ + protected: + TestEnvironment disabled{"CROWDB_RDMA_ENABLED", "0"}; + TestEnvironment fallback{"CROWDB_RDMA_ALLOW_TCP_FALLBACK", nullptr}; + TestEnvironment device{"CROWDB_RDMA_DEVICE", "test-no-device"}; + RpcServer server; + + void SetUp() override + { + ASSERT_TRUE(server.listen("127.0.0.1", 0)); + server.start(); + } + + void check(const char *enabled, const char *policy, bool expect_tcp) + { + TestEnvironment requested{"CROWDB_RDMA_ENABLED", enabled}; + TestEnvironment selected{"CROWDB_RDMA_ALLOW_TCP_FALLBACK", policy}; + SocketTransport client(1, 1); + client.start(); + auto connection = client.connect("127.0.0.1", server.listen_port()); + if (expect_tcp) { + ASSERT_NE(connection, nullptr); + EXPECT_EQ(connection->path(), ConnectionPath::Tcp); + } + else { + EXPECT_EQ(connection, nullptr); + } +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) + if (const auto *stats = client.rdma_stats_for_tests(); stats != nullptr) { + EXPECT_EQ(stats->fallback_count.load(), expect_tcp ? 1U : 0U); + } +#endif + } +}; + +TEST_F(TransportPathTest, DefaultPolicyKeepsExplicitTcpUsable) +{ + check("0", nullptr, true); +} + +TEST_F(TransportPathTest, UnavailableRdmaFallsBackWhenAllowed) +{ + check("1", "1", true); +} + +TEST_F(TransportPathTest, StrictRdmaRejectsUnavailableDeviceWithoutTcpFallback) +{ + check("1", "0", false); +} + +TEST_F(TransportPathTest, StrictRdmaRejectsDisabledOrUncompiledTransport) +{ + check("0", "0", false); +} + +TEST_F(TransportPathTest, InvalidFallbackPolicyDoesNotSilentlyEnableTcp) +{ + check("0", "invalid", false); +} +} // namespace diff --git a/tools/benchmark/bench-rdma-rpc-regression.sh b/tools/benchmark/bench-rdma-rpc-regression.sh index 6ec1a1225..8f0b22da5 100755 --- a/tools/benchmark/bench-rdma-rpc-regression.sh +++ b/tools/benchmark/bench-rdma-rpc-regression.sh @@ -2,7 +2,9 @@ # RDMA RPC echo benchmark. Mirrors bench-rpc-regression.sh with an HCA-backed # connection so transport throughput and latency can be compared directly. # -# Baseline (2026-10-18, HCA rocep35s0f0:1, 3s per case, 0 errors): +# Historical run (2026-10-18, 3s per case, reported 0 errors): +# Audit on 2026-10-22 found TCP socket traffic matching the 128B counts. +# These values are not a verified RDMA baseline; retain them for provenance. # payload workers/connections ops/s avg-us p50-us p99-us # 128 B 1/1 43,161 22.1 65 65 # 128 B 4/4 441,779 143.6 124 215 @@ -16,8 +18,8 @@ # 4 KiB 4/4 376,107 168.8 149 266 # 4 KiB 8/8 582,940 872.2 811 1,810 # 4 KiB 16/32 834,076 1,184.5 606 6,881 -# These are reference values for the shared-CQ implementation; reruns should -# record the exact HCA, duration, payload, and worker/connection configuration. +# New RDMA results require strict path selection and a valid structured report. +# Record the exact HCA, binary, duration, payload, and concurrency configuration. set -euo pipefail cd "$(dirname "$0")/../.." @@ -27,6 +29,7 @@ RESULTS_FILE="${RDMA_BENCH_RESULTS:-$LOG_ROOT/results.tsv}" REGRESSION_LOG_ROOT="$LOG_ROOT" source tools/benchmark/bench-regression-common.sh export CROWDB_LOG_ROOT="$LOG_ROOT" +export CROWDB_RDMA_ALLOW_TCP_FALLBACK=0 regression_init DURATION="${RDMA_BENCH_DURATION:-10}" @@ -78,35 +81,54 @@ run_bench() { stop_server mkdir -p "$LOG_ROOT/server" start_server "$workers" - local output json + local output json client_exit=0 output=$(CROWDB_RDMA_ENABLED=1 CROWDB_RDMA_LISTEN=0 CROWDB_RDMA_E2E_PORT="$PORT" \ CROWDB_RDMA_DEVICE="$DEVICE" CROWDB_RDMA_PORT="$HCA_PORT" \ CROWDB_RDMA_SOURCE_ADDR="$ADDR" CROWDB_PORT_OFFSET="$PORT_OFFSET" \ pixi run -- "$CLI_BIN" bench rpc \ --duration-secs "$DURATION" --loader-num "$loaders" --connections "$connections" \ + --mode coroutine \ --value-size "$VALUE_SIZE" --io-engines 1 --io-workers "$workers" \ - --server-host "$ADDR" --server-port "$PORT" 2>&1) || true + --server-host "$ADDR" --server-port "$PORT" 2>&1) || client_exit=$? + printf '%s\n' "$output" >"$LOG_ROOT/$label.client.log" json=$(echo "$output" | sed -n '/^{/,/^}/p') - if [[ -z "$json" ]]; then - echo "$output" | tail -20 >&2 - printf '%s\t%s\t0\t0\t0\t0\t0\t1\n' "$label" "$workers" >>"$RESULTS_FILE" + if [[ -z "$json" ]] || ! jq -e '(.total_ops | type) == "number" and + (.total_errors | type) == "number" and (.duration_ms | type) == "number" and + (.by_op.write.latency_us.avg_us | type) == "number" and + (.by_op.write.latency_us.p50_us | type) == "number" and + (.by_op.write.latency_us.p99_us | type) == "number" and + .total_ops >= 0 and .total_errors >= 0 and .duration_ms > 0' <<<"$json" >/dev/null; then + printf '%s\n' "$output" >&2 + printf '%s\t%s\tNA\tNA\tNA\tNA\tNA\tNA\t%s\t1\t0\n' \ + "$label" "$workers" "$client_exit" >>"$RESULTS_FILE" + stop_server + echo " ERROR: missing or invalid structured benchmark report; client_exit=$client_exit" >&2 + return 1 else - local ops avg p50 p99 p999 errors + local ops avg p50 p99 p999 errors valid=0 ops=$(echo "$json" | jq -r '.total_ops * 1000 / .duration_ms' | awk '{printf "%.0f", $1}') avg=$(echo "$json" | jq -r '.by_op.write.latency_us.avg_us') p50=$(echo "$json" | jq -r '.by_op.write.latency_us.p50_us') p99=$(echo "$json" | jq -r '.by_op.write.latency_us.p99_us') p999=$(echo "$json" | jq -r '.by_op.write.latency_us.p999_us') errors=$(echo "$json" | jq -r '.total_errors') + if [[ "$client_exit" == 0 && "$errors" == 0 ]] && jq -e '.total_ops > 0' <<<"$json" >/dev/null; then + valid=1 + fi echo " ops/s=$ops avg=${avg}us p50=${p50}us p99=${p99}us p999=${p999}us err=$errors" - printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$label" "$workers" "$ops" "$avg" "$p50" "$p99" "$p999" "$errors" >>"$RESULTS_FILE" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t0\t%s\n' \ + "$label" "$workers" "$ops" "$avg" "$p50" "$p99" "$p999" "$errors" "$client_exit" "$valid" >>"$RESULTS_FILE" + stop_server + if [[ "$valid" != 1 ]]; then + echo " ERROR: failed benchmark profile; client_exit=$client_exit errors=$errors" >&2 + return 1 + fi fi - stop_server } trap stop_server EXIT mkdir -p "$LOG_ROOT" -printf 'label\twkr\tops_s\tavg_us\tp50_us\tp99_us\tp999_us\terrors\n' >"$RESULTS_FILE" +printf 'label\twkr\tops_s\tavg_us\tp50_us\tp99_us\tp999_us\terrors\tclient_exit\treport_errors\tvalid\n' >"$RESULTS_FILE" echo "=== RDMA RPC echo ($VALUE_SIZE B, ${DURATION}s, HCA $DEVICE:$HCA_PORT) ===" run_bench 1 1 rdma_1w_1l_1c 1 run_bench 64 4 rdma_4w_64l_4c 4 diff --git a/tools/benchmark/tests/rdma_regression_test.py b/tools/benchmark/tests/rdma_regression_test.py new file mode 100644 index 000000000..f4e9ad007 --- /dev/null +++ b/tools/benchmark/tests/rdma_regression_test.py @@ -0,0 +1,97 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +"""Result and exit-status contracts, using fixture processes without an HCA.""" + +import csv +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +class RdmaRegressionTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="rdma-runner-") + self.addCleanup(self.temporary.cleanup) + self.directory = Path(self.temporary.name) + self.root = Path(__file__).resolve().parents[3] + self.server = self.directory / "server.py" + self.server.write_text( + '#!/usr/bin/env python3\nimport signal\n' + 'print("listening port=1", flush=True)\nsignal.pause()\n' + ) + self.server.chmod(0o755) + self.client = self.directory / "client.py" + self.client.write_text( + '#!/usr/bin/env python3\nimport os, sys\n' + 'assert os.environ["CROWDB_RDMA_ALLOW_TCP_FALLBACK"] == "0"\n' + 'assert sys.argv[sys.argv.index("--mode") + 1] == "coroutine"\n' + 'print(os.environ["RDMA_FIXTURE_REPORT"])\n' + 'sys.exit(int(os.environ["RDMA_FIXTURE_EXIT"]))\n' + ) + self.client.chmod(0o755) + + def run_case(self, report, client_exit): + environment = os.environ.copy() + environment.update({ + "RDMA_BENCH_SERVER_BIN": str(self.server), + "RDMA_BENCH_CLI_BIN": str(self.client), + "RDMA_BENCH_CASES": "rdma_1w_1l_1c", + "RDMA_BENCH_LOG_ROOT": str(self.directory / "output"), + "RDMA_FIXTURE_REPORT": report, + "RDMA_FIXTURE_EXIT": str(client_exit), + }) + process = subprocess.run( + [shutil.which("pixi"), "run", "--", "bash", "tools/benchmark/bench-rdma-rpc-regression.sh"], + cwd=self.root, env=environment, capture_output=True, text=True, timeout=20, + ) + results = self.directory / "output/results.tsv" + self.assertTrue(results.exists(), process.stdout + process.stderr) + with results.open() as file: + rows = list(csv.DictReader(file, delimiter="\t")) + self.assertEqual(len(rows), 1, process.stdout + process.stderr) + return process, rows[0] + + @staticmethod + def report(errors=0): + return json.dumps({ + "total_ops": 100, + "duration_ms": 10000, + "total_errors": errors, + "by_op": {"write": {"latency_us": {"avg_us": 20.0, "p50_us": 18, "p99_us": 40}}}, + }, indent=2) + + def test_valid_report_succeeds_and_records_zero_errors(self): + process, row = self.run_case(self.report(), 0) + self.assertEqual(process.returncode, 0, process.stdout + process.stderr) + self.assertEqual(row["errors"], "0") + self.assertEqual(row["report_errors"], "0") + self.assertEqual(row["valid"], "1") + + def test_failed_client_preserves_reported_error_count(self): + process, row = self.run_case(self.report(errors=7), 2) + self.assertNotEqual(process.returncode, 0, process.stdout + process.stderr) + self.assertEqual(row["errors"], "7") + self.assertEqual(row["client_exit"], "2") + self.assertEqual(row["valid"], "0") + + def test_missing_report_is_not_success_even_when_client_exits_zero(self): + process, row = self.run_case("human report", 0) + self.assertNotEqual(process.returncode, 0, process.stdout + process.stderr) + self.assertEqual(row["errors"], "NA") + self.assertEqual(row["report_errors"], "1") + self.assertEqual(row["valid"], "0") + + def test_error_report_is_failure_even_when_client_exits_zero(self): + process, row = self.run_case(self.report(errors=3), 0) + self.assertNotEqual(process.returncode, 0, process.stdout + process.stderr) + self.assertEqual(row["errors"], "3") + self.assertEqual(row["valid"], "0") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/rdma/run-timed-load.py b/tools/rdma/run-timed-load.py index 86954e744..bb69e3345 100644 --- a/tools/rdma/run-timed-load.py +++ b/tools/rdma/run-timed-load.py @@ -50,6 +50,7 @@ def main(): env = runtime_env.copy() env.update({ "CROWDB_RDMA_ENABLED": "1", + "CROWDB_RDMA_ALLOW_TCP_FALLBACK": "0", "CROWDB_RDMA_RPC_E2E": "1", "CROWDB_RDMA_E2E_FILTER": "RdmaHardwareTest.RpcServerClientTimedLoad", "CROWDB_RDMA_TEST_SECONDS": str(args.seconds), From 27870426301a9a50488d63dd53b5fa53d96b7765 Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 22 Oct 2026 19:09:33 +0800 Subject: [PATCH 021/258] Plan unified RDMA benchmarks and completion workers --- doc/backlog/R239-rdma-rpc-transport.md | 42 ++++-- doc/working/plan-rdma-rpc-transport.md | 177 ++++++++++++++++++++++--- doc/working/plan-rdma.md | 19 ++- 3 files changed, 203 insertions(+), 35 deletions(-) diff --git a/doc/backlog/R239-rdma-rpc-transport.md b/doc/backlog/R239-rdma-rpc-transport.md index 4260e762f..923f778b3 100644 --- a/doc/backlog/R239-rdma-rpc-transport.md +++ b/doc/backlog/R239-rdma-rpc-transport.md @@ -14,17 +14,24 @@ without changing request semantics. ## Solution 1. Add an RC SEND/RECV transport behind the common RPC `Transport` interface. - Each message is copied into one fixed-size registered wire slot and sent - with one SGE. Receive slots are pre-posted from a registered arena and are + One or more complete RPC frames are copied into one fixed-size registered + wire slot and sent with one SGE. Optional bounded transport aggregation + preserves each frame's identity and ordering, drains available frames + without a batching timer, and never exceeds the local send slot or the + peer's advertised receive slot, including all headers and metadata. + Receive slots are pre-posted from a registered arena and are reused only after Frame ownership is released. The complete wire message must fit the slot capacity. 2. Use service discovery for coarse RDMA capability and TCP handshake for the current CM endpoint, device/GID identity, capability generation, protocol version, and supported data classes. Local configuration and probing can disable RDMA before CM setup. -3. Register the RDMA completion and CM event descriptors with the existing - epoll engine. Epoll wakes the worker; the RPC library acknowledges events, - re-arms notifications, polls bounded CQ batches, and dispatches completions. +3. Assign each QP to one shared CQ and each CQ to one fixed polling/event + worker. Busy workers poll and dispatch bounded batches continuously; idle + workers arm notifications, poll again to close the race, then sleep in + epoll. Acknowledge completion events before resuming polling. Generic + socket workers do not compete for CQ ownership. CM remains a separately + serialized lifecycle event source. 4. Select RDMA as the preferred path for one logical connection and TCP as its one-way fallback. A TCP connection is never upgraded to RDMA. The shared request ID is used for idempotent or duplicate-suppressed retry. @@ -48,6 +55,15 @@ without changing request semantics. its documented default port. The RDMA HCA device and physical HCA port are shared and are not offset. The default RDMA CM service port is `19101`, so the effective port is `19101 + CROWDB_PORT_OFFSET`. +9. Provide `crowdb-cli bench rdma` sharing the coroutine echo workload with + `bench rpc`, driven by the existing RPC regression scripts. Explicitly + configure loaders, QPs, CQs, queue depths, slot capacity and aggregation; + prohibit TCP fallback in RDMA tests and verify the actual path. Measure + validated completions for ten seconds after warmup, with separate drain + accounting, nanosecond latency histograms, categorized errors and canonical + machine-readable results. Report all workload/resource settings, RPC TPS, + SEND rate, avg/P90/tail latency and validity together. Application logical + operations remain a separate metric from correlated RPC completions. ## Dependencies @@ -71,9 +87,16 @@ without changing request semantics. - Given an RDMA payload partially completes, when the connection fails, no TCP bytes are appended and uncertain buffers remain quarantined. Invariant: partial transfer isolation. Integration test. -- Given CQ and CM events arrive, when epoll wakes the worker, all currently - available completions are drained without a private polling thread. +- Given QPs assigned to multiple CQs and busy/idle traffic transitions, when + completions and CM events arrive, each CQ's fixed owner drains completions, + arms/polls before idle sleep, acknowledges wakeups and stops cleanly without + competing socket workers or lost wakeups. Invariant: event/poll composition. Integration test. +- Given aggregation is enabled and frames approach the peer slot limit, when + frames are submitted, only complete fitting frames share a SEND; every RPC + retains identity, ordering and completion ownership. A malformed suffix + publishes no frames, and a held frame view prevents receive-slot repost. + Invariant: bounded aggregate ownership. Integration test. - Given an OCI image with no RDMA devices, when the TCP-only profile starts, service discovery selects TCP without attempting RDMA. Invariant: optional capability. E2E test. @@ -88,7 +111,10 @@ without changing request semantics. is separated from unprivileged application execution. E2E test. - Given the RDMA benchmark runner uses the same echo workload as the TCP regression benchmark, when it runs on a prepared HCA, it reports throughput, - latency percentiles, and errors for matching concurrency cases. Invariant: + latency including avg/P90/tails, categorized errors, all queue/QP/CQ/slot + settings and actual RDMA path for matching ten-second coroutine cases. + Missing reports, nonzero errors, fallback and failed processes fail the run. + Invariant: transport regressions and large flow bugs are visible before downstream use. Benchmark test. - Given an application linked against `crowdb-rpc`, when deployment provides diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 544343a66..dd4bafbda 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -7,8 +7,9 @@ Upstream: [R239](../backlog/R239-rdma-rpc-transport.md), [root RDMA plan](plan-rdma.md), and [RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md). -Goal: add RC SEND/RECV to the common RPC transport with epoll/CQ event -composition and one-way RDMA-to-TCP fallback. +Goal: complete RC SEND/RECV ownership and fallback acceptance, together with +CLI coroutine benchmarks, bounded RPC aggregation, and one polling/event +worker per CQ, using the existing RPC regression runners. ## Progress @@ -96,7 +97,7 @@ and TPS/P50/P95/P99 output. rerunning under `CROWDB_RUNTIME_ROOT=.crowdb-runtime/rdma-check` passed. Full gate artifacts: `.crowdb-runtime/artifacts/unit/cpp-3ab318e0c5a04f27a8328be78ca1cab6/`. -- [~] **Complete remaining transport and OCI acceptance**: negotiated TCP +- [ ] **Complete remaining transport and OCI acceptance**: negotiated TCP capability preflight, real registered-lease PD/MR lifetime verification, and multi-container profiles remain open. Do not treat the historical gate record below as verification of these unfinished contracts. @@ -150,7 +151,7 @@ and TPS/P50/P95/P99 output. profiles. Existing capability serialization and state predicates alone do not prove that every connection follows the negotiated setup contract. -### Benchmark audit and proposed unified measurement +### Benchmark audit and unified measurement User decisions: use coroutine loaders for the primary throughput benchmark; keep producer-thread stress as diagnostic coverage; include errors in every @@ -186,8 +187,8 @@ latency histogram to resolve sub-65-us RTTs. preserved error count, a missing report despite exit zero, and reported RPC errors despite exit zero. These use fixture processes without an HCA: `pixi run -- python -m unittest discover -s tools/benchmark/tests -p rdma_regression_test.py -v`. -- [ ] **Unify the workload and report contract after the test-design - decision**: use one coroutine-driven RPC echo workload, one case manifest, +- [ ] **Unify the workload and report contract**: use one + coroutine-driven RPC echo workload, one case manifest, and one JSON/CSV schema for native processes and two host-network containers. Require actual RDMA path and unique QP counts. Verify request correlation and response length/content; include submission rejection and @@ -203,22 +204,22 @@ latency histogram to resolve sub-65-us RTTs. common C++/Rust histograms' 65.5-us underflow floor; do not retain a separate benchmark-only precision contract. Specify a shared bucket layout and golden vectors. Record timing origin and exclude warmup/drain from TPS. -- [ ] **Confirm CQ ownership before expanding the performance matrix**: +- [ ] **Replace competing CQ consumers before expanding the performance matrix**: current `RdmaTransport` has one `cq_` per peer, no dedicated polling thread, and competing epoll workers protected by `cq_polling_`. `RdmaTransport::start()` only changes lifecycle state. The recommended - target is one assigned polling/event thread per CQ, with QPs assigned + agreed target is one assigned polling/event thread per CQ, with QPs assigned to one CQ, bounded busy polling under traffic, and arm/poll/epoll sleep - after the idle budget. Proposed `cq_count` and `poller_count` are not - existing configuration knobs. Compare native and OCI runs with the same + after the idle budget. Add `cq_count`; derive `poller_count = cq_count` + rather than exposing two inconsistent knobs. Compare native and OCI runs with the same manifest; neither single-host topology establishes physical-node isolation. -Proposed result schema (client/server resource values are separate): +Required result schema (client/server resource values are separate): | Group | Required columns | |-------------|------------------------------------------------------------------------------------------------------------------------------| | Identity | case_id, run_id, git_sha, binary hashes, topology, CPU/HCA identity, CPU affinity, actual path, fallback policy/count | -| Workload | client_mode, loader_count, producer_threads, in_flight_per_loader, connection_count, logical_ops_per_rpc, payload_bytes, wire_bytes | +| Workload | client_mode, loader_count, producer_threads, in_flight_per_loader, connection_count, qpair_count, logical_ops_per_rpc, payload_bytes, wire_bytes, aggregation_enabled, aggregation_max_bytes, aggregation_max_frames, frames_per_send | | Scheduling | io_engines, io_workers, cq_count, poller_count, poll_batch, verbs_poll_batch, poll_empty_budget, epoll_wakeups, CQ event/poll counts | | Capacity | requested/effective sq_depth, rq_depth, posted_receive_target, cq_depth, wire_slot_capacity, arena bytes, queue/pool high-water marks | | Measurement | warmup_s, measure_s, repeat_index, latency_scope, submitted, completed_in_window, drain_completed, rpc_tps, logical_ops_s, payload_Bps | @@ -229,10 +230,11 @@ Proposed result schema (client/server resource values are separate): All fields belong to one machine-readable row; unavailable values are null. Human tables can show projections joined by case_id but always include errors and validity. Normal shutdown WR flushes are separate from workload failures. -Each measured operation is one validated RPC round trip; current code copies -one OutFrame into one SEND slot and does not combine multiple RPCs merely -because they share a QP. Application aggregation, if benchmarked, must report -logical_ops_per_rpc and logical_ops_s separately from rpc_tps. +Each measured operation is one validated RPC round trip. Transport aggregation +may put multiple complete RPC frames in one SEND slot; count every correlated +RPC once and report SEND count and frames per SEND separately. Application +aggregation, if benchmarked, must report logical_ops_per_rpc and logical_ops_s +separately from rpc_tps. Recommended sweep: single-loader latency baseline; then sweep coroutine count with one QP/CQ/poller; then QP count; then CQ/poller count; then queue depths @@ -242,6 +244,140 @@ zero-error TPS and show its latency rather than selecting unrelated best throughput and latency fields from different trials. Wire header/control bytes count toward slot capacity, including at the 1-MiB class boundary. +## Agreed implementation sequence + +The user selected joint implementation of the benchmark and CQ architecture +on 2026-10-22, then requested this plan before code changes. The primary runner +is `tools/benchmark/bench-rdma-rpc-regression.sh`: start the RPC server and use +`crowdb-cli bench rdma` as its client, sharing the workload implementation with +`bench rpc`. Thread-driven gtests remain transport diagnostics. Transport +aggregation is permitted for max TPS; `qpair_count` and `wire_slot_capacity` +are independent, reported parameters. These decisions replace earlier notes +deferring multiple CQs or forbidding transport coalescing. + +### A. Contract and reliable timing + +- [x] **Record the agreed scope**: amend this plan, the root decisions, and + R239's high-level contract; preserve historical measurements as historical. + Files: this plan, `plan-rdma.md`, `../backlog/R239-rdma-rpc-transport.md`. +- [~] **Replace the histogram floor**: use a shared C++/Rust logarithmic + nanosecond bucket layout with bounded memory, explicit zero/overflow, exact + count/sum/min/max, and P50/P90/P95/P99/P99.9. Keep existing metric callers + compatible, expose mergeable snapshots, and verify shared golden vectors, + sub-microsecond values, bucket boundaries, empty/reset and overflow cases. + Files: `lib/crowdb-common/{cpp,rust}/` histogram implementations and tests; + `app/crowdb-cli/src/commands/bench/{loader,result}.rs`. + +### B. CQ owners and resource ownership + +- [ ] **Create the bounded CQ pool**: add `RdmaConfig::cq_count`, + `verbs_poll_batch`, and strict environment validation. Each CQ owns its + completion channel, wake fd, fixed poller and bounded WR table. Assign each + QP to one CQ at creation; SEND/RECV and repost retain that assignment. + Distinguish requested from provider-effective depths and reject insufficient + CQ capacity for the configured QPs and posted receives before traffic. + Files: `rdma_transport.h`, `rdma_config.cpp`, `rdma_connection.cpp`, + `rdma_work_request.cpp`, new private CQ implementation under `transport/rdma/`. +- [ ] **Run one polling/event worker per CQ**: continuously poll while busy, + dispatch callbacks on the owner, arm notification after the empty budget, + poll again before sleeping, acknowledge events, and wake promptly on submit + and stop. Keep CM management separate from CQ completion ownership; generic + epoll workers cannot compete for that CQ. Stop producers, wake/join pollers, + drain/quarantine WRs, then destroy QPs/CQs/channels. Test idle/busy transitions, + arm races, contention, multi-CQ mapping, partial setup and repeated teardown. + Files: RDMA transport/completion sources, `socket_transport.cpp`, RPC tests. +- [ ] **Separate application and receive allocators**: built-in echo and + default handlers allocate from the common application pool, not a fully + preposted receive arena. Receive posting and repost explicitly use the + connection's retained RDMA receive pool. Test `posted_receive_target == + rq_depth` with request and response payload validation. Files: `connection.h`, + RDMA setup/completion, `c_api.cpp`, `server/handler.cpp`, transport tests. +- [ ] **Finish real registered-lease shutdown**: retain PD/context/MR owners + until the last held receive lease is released, including after transport + shutdown; no deregister-before-release, double repost, premature recycle or + silent PD leak. Files: RDMA buffer pool and lifecycle sources, lease tests. + +### C. Bounded aggregation + +- [ ] **Coalesce complete frames on the CQ owner**: use bounded lock-free + admission and per-QP queues; drain available frames without adding a waiting + timer. Keep per-QP ordering and common frame/request IDs. Limit a SEND to + `min(aggregation_max_bytes, local send slot capacity, peer receive slot + capacity)` including every header/control byte and to + `aggregation_max_frames`. Disabled mode sends one frame. Reject oversized + single frames before admission; do not fragment or silently fall back. + Files: RDMA submit/owner loop, `out_frame.h`, config/capability validation. +- [ ] **Retain aggregate owners through completion**: one SEND WR retains all + constituent frames until CQE; failure closes the uncertain stream and + quarantines its slot once. Validate the entire received aggregate before + publishing any frame. Give each frame an independent immutable buffer view + sharing the receive-slot lease; repost only after all C++/FFI views release. + Test exact slot fit, one byte over, multi-frame correlation/order, malformed + suffix, held middle frame, send errors, bounded exhaustion and shutdown. + Files: RDMA completion/WR code, buffer views and focused RPC tests. +- [ ] **Expose aggregation observations**: export actual frames/SEND, wire + bytes, queue high-water and rejects. Keep RPC TPS distinct from transport + SEND rate; compare aggregation disabled/enabled for both TCP and RDMA. + Files: transport statistics, C API/Rust FFI and benchmark reports. + +### D. CLI coroutine workload and canonical results + +- [ ] **Add `bench rdma` with shared echo workload**: reuse `bench rpc` + coroutine machinery; expose `qpair_count`, loader count, warmup/duration, + slot/aggregation and CQ/queue knobs using the same code variable names as + the report. RDMA command requires RDMA and disables TCP fallback. Preserve + existing TCP command behavior; no OS thread per loader. Reject inconsistent + settings before starting. Files: CLI `bench.rs`, `bench/verb.rs`, + `bench/rpc.rs`, shared workload/config modules. +- [ ] **Validate and measure each completion**: expose response data to + `CoBenchHandler`, verify request ID, length and payload, count submission + rejects/retries/timeouts/connection/payload/duplicate errors, and reconcile + C++ coroutine statistics with Rust results. Warm up for one second, measure + completions for ten seconds using a monotonic deadline, then drain separately; + exclude setup/warmup/drain from TPS. Define latency from submission to + validated response in nanoseconds. Never destroy a coroutine while a late + completion can resume it. Files: `co_client.cpp`, FFI `co_bench.rs`, CLI echo + handler and lifecycle tests. +- [ ] **Write the canonical report**: emit a versioned JSON result directly + to a requested file, plus human output; serialize all schema columns above. + Use explicit null for unavailable counters. Record measured errors and mark + invalid on setup/client/server/report failure or RDMA path mismatch. Include + client/server resource settings and actual QP/CQ/poller/path/fallback stats. + Files: CLI benchmark result/transport FFI modules and report tests. + +### E. One runner, staged evidence and requirement completion + +- [ ] **Unify regression orchestration**: keep the named RDMA regression + script as the entry point and share common orchestration with TCP regression. + Launch the RPC server then the CLI client from one case manifest; wait for + actual RDMA readiness, not just the TCP listen message. Save complete logs, + canonical JSON and a wide CSV/TSV with errors/validity; propagate failures. + Record git/binary hashes, topology, CPU/HCA, affinity, CPU time and supported + provider counters. Tests cover invalid/missing reports, reported errors, + failed/early-exit server, disabled fallback and actual argument propagation. + Files: `tools/benchmark/bench-{rdma-,}rpc-regression.sh`, common runner and + `tools/benchmark/tests/rdma_regression_test.py`. +- [ ] **Run gradual native hardware profiles**: first one coroutine/QP/CQ, + then loader sweep 1/2/4/8/16/32 and higher as needed, then QP and CQ sweeps; + compare aggregation off/on, SQ/RQ 1024 and 4096, small slot classes and larger + payload/slot pairs. Increase depth only when counters show it limits load. + Each case uses 1 s warmup + 10 s measured traffic. Repeat candidate peaks + three times and retain all rows. Require zero workload errors/fallback and + report avg/P90/tails with the highest repeatable TPS. Do not claim a hardware + maximum when CPU/affinity/topology still limits the result. +- [ ] **Run matching OCI profiles and finish protocol tests**: use the same + CLI workload and manifest in two host-network containers on this physical + HCA; record separate process/container topology. Complete negotiated TCP + preflight, fallback-before-payload, no mixed stream, no-device TCP and + non-root device/memlock acceptance. Compare matching native/container rows + and explain any gap. Files: RDMA OCI runner, capability/path tests. +- [ ] **Gate and document each coherent change**: build with `-j32`; run all + executables/tests/lints through `pixi run`, use `clean-env` before server + tests, and keep complete failure output. Run affected tests, fmt, clippy, + changed C++ format/tree-lint, then full C++/FFI gates. Update permanent design + to implemented behavior; remove requirements/plans only after all acceptance + passes. Keep an explicit verified/pending checkpoint after each commit. + Provider references for the implementation choices: - [libibverbs QP capacity terminology](https://github.com/linux-rdma/rdma-core/blob/master/libibverbs/man/ibv_create_qp.3). @@ -287,13 +423,12 @@ Remaining implementation and acceptance work: built image; - remove temporary hardware diagnostics, normalize failure logging, and run the full C++ format/tree-lint gates; -- keep multi-CQ and a dedicated polling worker as follow-up performance work; - they are not R239 blockers unless the acceptance profile shows contention; +- implement multi-CQ and a fixed polling/event worker per CQ together with + the CLI coroutine benchmark, as agreed in the implementation sequence above; - rerun the RDMA benchmark concurrency and payload cases after the acceptance changes. - Completed on the local HCA at 128B, 1KiB, and 4KiB with 0 errors for 1, 4, - 8, and 32 connections. The 4-connection cases reached about 442k ops/s - (128B), 414k ops/s (1KiB), and 376k ops/s (4KiB). + Earlier script results claiming 442k/414k/376k ops/s did not establish the + RDMA path; replace them with validated canonical results from the CLI runner. ## Phase 1: common transport seam diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index 87a61b316..b7bbc83d3 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -43,8 +43,9 @@ and the fixed-I/O-worker control; protocol and OCI acceptance remain open. This is the root plan. The five child plans contain implementation details; only this plan controls phase order, cross-backlog decisions, and final -acceptance. No child plan may broaden the transport contract without updating -the formal design first. +acceptance. The agreed joint CQ/benchmark/aggregation work is detailed in +[the transport plan](plan-rdma-rpc-transport.md#agreed-implementation-sequence). +Update the formal design with each implemented architecture change. ## Fixed design decisions @@ -53,8 +54,13 @@ the formal design first. - Use service discovery for coarse RDMA capability and TCP handshake for the current RDMA endpoint, capability generation, and data classes. - Select RDMA before the first payload; fallback is RDMA to TCP only. -- Reuse existing ChunkIO and Chunk-KV aggregation. RDMA does not aggregate or - flush independently. +- Preserve existing ChunkIO and Chunk-KV aggregation. Additionally permit + bounded transport coalescing of complete RPC frames without a batching timer; + every SEND fits both the local send slot and peer receive slot, including + headers, and retains each RPC's identity and completion ownership. +- Assign each QP to one CQ and each CQ to one fixed polling/event worker. + Implement this together with `crowdb-cli bench rdma` and the existing RPC + regression runner; report QP count, slot size, aggregation and errors. - Use 64 KiB metadata, 256 KiB aggregated Chunk-KV, and 1 MiB datapath capacities, configured as transport defaults. - Use posted receives as implicit data credit; do not add an extra credit @@ -210,8 +216,9 @@ device, selected path, fallback count, buffer pressure, and final result. - [ ] **No mixed stream**: never append TCP bytes to a partially completed RDMA stream. -- [ ] **No hidden aggregation**: only ChunkIO/Chunk-KV decides message boundary - and flush; RDMA transports the result. +- [ ] **Bounded aggregation**: application message boundaries remain intact; + transport coalescing preserves complete frames, order and per-RPC identity + and stays within both peers' slot limits. - [ ] **No premature reuse**: completion, disk completion, and caller release all precede buffer reuse for their respective ownership path. - [ ] **Bounded memory**: registered bytes, posted receives, send depth, disk From 03b745e628549b426e4b7ea251080eae456efe2a Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 22 Oct 2026 19:20:51 +0800 Subject: [PATCH 022/258] Unify nanosecond latency histograms and benchmark percentiles --- app/crowdb-cli/src/commands/bench/loader.rs | 7 +- app/crowdb-cli/src/commands/bench/result.rs | 66 ++- doc/working/plan-rdma-rpc-transport.md | 45 +- .../crowdb-common/metrics/latency_histogram.h | 85 ++-- .../cpp/src/metrics/latency_histogram.cpp | 228 +++++----- .../cpp/tests/latency_histogram_test.cpp | 113 +++++ .../rust/src/metrics/histogram.rs | 400 ++++-------------- .../rust/src/metrics/histogram/data.rs | 136 ++++++ .../rust/src/metrics/registry.rs | 5 +- .../rust/tests/latency_histogram_test.rs | 80 ++++ .../testdata/latency_histogram.csv | 21 + .../tests/common/test_rdma_load.cpp | 28 -- lib/crowdb-rpc/tests/common/test_rdma_load.h | 31 +- lib/crowdb-rpc/tests/rdma_timed_load_test.cpp | 25 +- lib/crowdb-tree/tests/unit/metrics_test.cpp | 29 +- tools/rpc-test/src/workload.rs | 2 +- 16 files changed, 746 insertions(+), 555 deletions(-) create mode 100644 lib/crowdb-common/cpp/tests/latency_histogram_test.cpp create mode 100644 lib/crowdb-common/rust/src/metrics/histogram/data.rs create mode 100644 lib/crowdb-common/rust/tests/latency_histogram_test.rs create mode 100644 lib/crowdb-common/testdata/latency_histogram.csv diff --git a/app/crowdb-cli/src/commands/bench/loader.rs b/app/crowdb-cli/src/commands/bench/loader.rs index 9df08ab0d..1ad30f7db 100644 --- a/app/crowdb-cli/src/commands/bench/loader.rs +++ b/app/crowdb-cli/src/commands/bench/loader.rs @@ -57,7 +57,12 @@ impl BenchRecorder { /// Record one successful op with its latency (µs) and payload bytes. pub fn record_ok(&self, us: u64, bytes: u64) { - self.latency.observe(us.saturating_mul(1000)); + self.record_ok_ns(us.saturating_mul(1000), bytes); + } + + /// Preserve nanosecond precision for RPC completion callbacks. + pub fn record_ok_ns(&self, ns: u64, bytes: u64) { + self.latency.observe(ns); self.bytes.observe(bytes); } diff --git a/app/crowdb-cli/src/commands/bench/result.rs b/app/crowdb-cli/src/commands/bench/result.rs index 97cb5fd73..ecea040fd 100644 --- a/app/crowdb-cli/src/commands/bench/result.rs +++ b/app/crowdb-cli/src/commands/bench/result.rs @@ -83,6 +83,7 @@ impl BenchOps { #[derive(Debug, Serialize)] pub struct OpStats { pub latency_us: LatencyUs, + pub latency_ns: LatencyNs, } impl From for OpStats { @@ -94,10 +95,57 @@ impl From for OpStats { p50: s.p50 / 1000, p99: s.p99 / 1000, }, + latency_ns: LatencyNs { + count: s.count, + avg: s.avg, + min: s.min, + max: s.max, + p50: s.p50, + p90: s.p90, + p95: s.p95, + p99: s.p99, + p999: s.p999, + zero_count: s.zero_count, + overflow_count: s.overflow_count, + }, } } } +/// Nanosecond values preserve sub-microsecond precision in machine reports. +#[derive(Debug, Serialize)] +pub struct LatencyNs { + pub count: u64, + pub avg: f64, + pub min: u64, + pub max: u64, + pub p50: u64, + pub p90: u64, + pub p95: u64, + pub p99: u64, + pub p999: u64, + pub zero_count: u64, + pub overflow_count: u64, +} + +impl fmt::Display for LatencyNs { + #[allow(clippy::cast_precision_loss)] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "avg={:.3}us p50={:.3}us p90={:.3}us p95={:.3}us p99={:.3}us p99.9={:.3}us max={:.3}us overflow={}", + self.avg / 1000.0, + self.p50 as f64 / 1000.0, + self.p90 as f64 / 1000.0, + self.p95 as f64 / 1000.0, + self.p99 as f64 / 1000.0, + self.p999 as f64 / 1000.0, + self.max as f64 / 1000.0, + self.overflow_count, + ) + } +} + /// Per-op latency percentiles. Field names are serialized as /// `*_us` (the contract the regression scripts parse); the Rust names /// drop the shared `_us` postfix to satisfy `clippy::struct_field_names`. @@ -208,25 +256,13 @@ impl fmt::Display for BenchResult { writeln!(f, " correctness_errors: {ce}")?; } if let Some(s) = &self.by_op.read { - writeln!( - f, - " read: avg={}us p50={}us p99={}us", - s.latency_us.avg, s.latency_us.p50, s.latency_us.p99, - )?; + writeln!(f, " read: {}", s.latency_ns)?; } if let Some(s) = &self.by_op.write { - writeln!( - f, - " write: avg={}us p50={}us p99={}us", - s.latency_us.avg, s.latency_us.p50, s.latency_us.p99, - )?; + writeln!(f, " write: {}", s.latency_ns)?; } if let Some(s) = &self.by_op.list { - writeln!( - f, - " list: avg={}us p50={}us p99={}us", - s.latency_us.avg, s.latency_us.p50, s.latency_us.p99, - )?; + writeln!(f, " list: {}", s.latency_ns)?; } if let Some(cts) = &self.client_transport_stats { writeln!( diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index dd4bafbda..a2080f12f 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -260,13 +260,45 @@ deferring multiple CQs or forbidding transport coalescing. - [x] **Record the agreed scope**: amend this plan, the root decisions, and R239's high-level contract; preserve historical measurements as historical. Files: this plan, `plan-rdma.md`, `../backlog/R239-rdma-rpc-transport.md`. -- [~] **Replace the histogram floor**: use a shared C++/Rust logarithmic +- [x] **Replace the histogram floor**: use a shared C++/Rust logarithmic nanosecond bucket layout with bounded memory, explicit zero/overflow, exact count/sum/min/max, and P50/P90/P95/P99/P99.9. Keep existing metric callers compatible, expose mergeable snapshots, and verify shared golden vectors, sub-microsecond values, bucket boundaries, empty/reset and overflow cases. Files: `lib/crowdb-common/{cpp,rust}/` histogram implementations and tests; `app/crowdb-cli/src/commands/bench/{loader,result}.rs`. +- [~] **Make coroutine completion safe before parallel CQ benchmarks**: + `co_on_complete` currently fails a CAS against `CO_RUNNING`, then stores + `CO_RESPONSE_READY`; `await_suspend` can change to `CO_SUSPENDED` between + those operations and lose the resume. Replace that handoff with one atomic + publication. Replace concurrent `coroutine_handle::done()` inspection and + forced timeout resume/destruction with explicit final-suspend publication + and callback quiescence. Keep timeout handling in the client's request + completion/reaper path. Add deterministic completion-before/after-suspend + and concurrent handoff tests, plus real coroutine echo/error coverage. + Files: `client/co_client.cpp`, a private coroutine state header, RPC tests + and FFI coroutine API. This is prerequisite to attributing multi-CQ errors + or performance to the transport. + +Verified histogram checkpoint: + +- C++/Rust share 18 golden vectors, inclusive bucket bounds, nearest-rank + quantiles, exact 0..255 ns resolution, 128 sub-buckets per magnitude below + `2^40 ns`, explicit zero/overflow counts and a 128-bit exact sum. Owned + shards merge without atomics; final cumulative captures are exact after + producers join. Concurrent live/window captures remain approximate. +- Both language suites cover sub-microsecond samples, empty/reset, nearest + rank, exact extrema, overflow/carry, shard merge and concurrent final counts. + All 79 Rust common tests, workspace clippy, formatting and scoped tree-lint + pass. Latest full C++/FFI gate: + `.crowdb-runtime/artifacts/unit/cpp-0837d1ed9b1e4171a8c7afdd54f955c8/`. +- The diagnostic thread workload now uses the common histogram. One 10 s + hardware run with 1 s warmup, one producer/QP/worker, 32 in flight and 1 KiB + slots completed 1,767,704 measured RPCs with zero errors or histogram overflow + (176,770 TPS; avg 174.61 us; P90 195.58 us; P99 215.04 us). Submitted count + equals measured + drained. This verifies the migrated diagnostic histogram, + not the future coroutine max-TPS claim. Artifact: + `.crowdb-runtime/artifacts/rdma/20261022T111938Z/results.csv`. ### B. CQ owners and resource ownership @@ -336,7 +368,8 @@ deferring multiple CQs or forbidding transport coalescing. completions for ten seconds using a monotonic deadline, then drain separately; exclude setup/warmup/drain from TPS. Define latency from submission to validated response in nanoseconds. Never destroy a coroutine while a late - completion can resume it. Files: `co_client.cpp`, FFI `co_bench.rs`, CLI echo + completion can resume it (the lifecycle prerequisite is in phase A). + Files: `co_client.cpp`, FFI `co_bench.rs`, CLI echo handler and lifecycle tests. - [ ] **Write the canonical report**: emit a versioned JSON result directly to a requested file, plus human output; serialize all schema columns above. @@ -531,6 +564,14 @@ E2E tests: ## Consolidated files +- `lib/crowdb-common/cpp/include/crowdb-common/metrics/latency_histogram.h` +- `lib/crowdb-common/cpp/src/metrics/latency_histogram.cpp` +- `lib/crowdb-common/rust/src/metrics/histogram.rs` and `histogram/data.rs` +- `lib/crowdb-common/testdata/latency_histogram.csv` and both language tests +- `lib/crowdb-rpc/src/client/co_client.cpp` and private coroutine state header +- `lib/crowdb-rpc/ffi/src/co_bench.rs` and C/Rust statistics bindings +- `app/crowdb-cli/src/commands/bench/{verb,rpc,loader,result}.rs` +- `tools/benchmark/bench-rdma-rpc-regression.sh`, shared RPC runner and tests - `lib/crowdb-rpc/include/crowdb-rpc/transport.h` - `lib/crowdb-rpc/include/crowdb-rpc/connection.h` - `lib/crowdb-rpc/include/crowdb-rpc/transport/epoll/epoll_engine.h` diff --git a/lib/crowdb-common/cpp/include/crowdb-common/metrics/latency_histogram.h b/lib/crowdb-common/cpp/include/crowdb-common/metrics/latency_histogram.h index cb09cfce1..bbe1b4f99 100644 --- a/lib/crowdb-common/cpp/include/crowdb-common/metrics/latency_histogram.h +++ b/lib/crowdb-common/cpp/include/crowdb-common/metrics/latency_histogram.h @@ -1,45 +1,64 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -// HDR-style latency histogram with logarithmic buckets. -// -// Each power-of-2 magnitude is divided into 128 sub-buckets, giving a -// bounded relative error of <= 0.78% everywhere. Values below the -// lowest discernible value (2^16 ≈ 65.5us) go into a single underflow -// bucket; values above the max trackable value (2^34 ≈ 17.2s) go into -// a single overflow bucket. Observation is lock-free and -// allocation-free on the hot path (O(1) bit manipulation + relaxed -// atomic fetch_add). #pragma once +#include #include #include -#include #include -#include -#include namespace crowdb::common::metrics { +// Shared with Rust: exact 0..255 ns, then 128 buckets per power of two +// below 2^40 ns. Overflow quantiles are UINT64_MAX, with an explicit count. class LatencyHistogram { public: - explicit LatencyHistogram(std::string name); + struct Snapshot + { + uint64_t count = 0; + double avg = 0; + uint64_t p50 = 0; + uint64_t p99 = 0; + uint64_t max = 0; + uint64_t total_count = 0; + uint64_t min = 0; + uint64_t p90 = 0; + uint64_t p95 = 0; + uint64_t p999 = 0; + uint64_t zero_count = 0; + uint64_t overflow_count = 0; + uint64_t sum_low = 0; + uint64_t sum_high = 0; + }; - void observe(uint64_t ns); + static constexpr size_t NUM_BUCKETS = 128 + (33 * 128) + 1; - struct Snapshot + // One polling owner's shard. Observe does not allocate or use atomics. + struct Data { - uint64_t count; - double avg; // f64 average (ns), exact sum/count - uint64_t p50; // ns - uint64_t p99; // ns - uint64_t max; // ns - uint64_t total_count; + std::array buckets{}; + uint64_t sum_low = 0; + uint64_t sum_high = 0; + uint64_t min = UINT64_MAX; + uint64_t max = 0; + + void observe(uint64_t ns); + void merge(const Data &other); + [[nodiscard]] Snapshot snapshot(uint64_t total_count = 0) const; }; - Snapshot flush(); + explicit LatencyHistogram(std::string name); + void observe(uint64_t ns); + + // Concurrent captures are approximate; join writers before the exact + // final cumulative snapshot. Window flush never resets cumulative data. + Snapshot flush(); + [[nodiscard]] Snapshot snapshot() const; + [[nodiscard]] Snapshot snapshot_total() const; + [[nodiscard]] Data capture_total() const; [[nodiscard]] const std::string &name() const { @@ -47,11 +66,23 @@ class LatencyHistogram } private: - std::string name_; - std::vector>> buckets_; - std::atomic count_; - std::atomic sum_; - std::atomic total_count_; + struct AtomicData + { + std::array, NUM_BUCKETS> buckets{}; + std::atomic sum_low{0}; + std::atomic sum_high{0}; + std::atomic min{UINT64_MAX}; + std::atomic max{0}; + + void observe(uint64_t ns); + Data capture(bool reset); + [[nodiscard]] Data capture() const; + }; + + static size_t bucket_index(uint64_t ns); + std::string name_; + AtomicData window_; + AtomicData total_; }; } // namespace crowdb::common::metrics diff --git a/lib/crowdb-common/cpp/src/metrics/latency_histogram.cpp b/lib/crowdb-common/cpp/src/metrics/latency_histogram.cpp index 4a65ba86c..eaf247e4b 100644 --- a/lib/crowdb-common/cpp/src/metrics/latency_histogram.cpp +++ b/lib/crowdb-common/cpp/src/metrics/latency_histogram.cpp @@ -4,144 +4,172 @@ #include "crowdb-common/metrics/latency_histogram.h" #include -#include +#include +#include #include namespace crowdb::common::metrics { -// ── HDR bucket parameters ──────────────────────────────────────── -// -// HDR-style logarithmic buckets: each power-of-2 magnitude is divided -// into SUB_BUCKET_COUNT equal sub-buckets, giving a bounded relative -// error of <= 1 / SUB_BUCKET_COUNT ≈ 0.78% everywhere. -// -// Configuration tuned for the crowdb latency range: -// - LVD (lowest discernible value) = 2^UNIT_MAGNITUDE ≈ 65.5us. -// Values below this go into a single underflow bucket (index 0). -// - Max magnitude covers 2^(UNIT_MAGNITUDE + NUM_MAGNITUDES) = 2^34 -// ≈ 17.2s. Values above go into the overflow bucket (last index). -// - 2 significant figures → SUB_BUCKET_COUNT = 128 (2^7). - -static constexpr uint32_t UNIT_MAGNITUDE = 16; // LVD ≈ 65.5us -static constexpr size_t SUB_BUCKET_COUNT = 128; // 2^7 → ≤0.78% error -static constexpr uint32_t SUB_BUCKET_BITS = 7; // log2(SUB_BUCKET_COUNT) -static constexpr uint64_t SUB_BUCKET_MASK = 127; // SUB_BUCKET_COUNT - 1 -static constexpr size_t NUM_MAGNITUDES = 18; // magnitudes 0..17 → up to 2^34 -// 1 underflow + NUM_MAGNITUDES * SUB_BUCKET_COUNT regular + 1 overflow. -static constexpr size_t NUM_BUCKETS = 1 + (NUM_MAGNITUDES * SUB_BUCKET_COUNT) + 1; -static constexpr size_t kUnderflow = 0; -static constexpr size_t kOverflow = NUM_BUCKETS - 1; - -// ── HDR index / boundary math ───────────────────────────────────── - -// Map a value to its HDR bucket index via O(1) bit manipulation. -// Values < LVD go to the underflow bucket (0); values > max go to -// the overflow bucket (NUM_BUCKETS - 1). -static size_t bucket_index(uint64_t v) -{ - if (v < (1ULL << UNIT_MAGNITUDE)) { - return kUnderflow; +size_t LatencyHistogram::bucket_index(uint64_t ns) +{ + if (ns < 128) { + return static_cast(ns); } - uint32_t highest_bit = 63 - static_cast(__builtin_clzll(v)); - uint32_t magnitude = highest_bit - UNIT_MAGNITUDE; - if (magnitude >= NUM_MAGNITUDES) { - return kOverflow; + const auto magnitude = static_cast(std::bit_width(ns) - 1); + if (magnitude >= 40) { + return NUM_BUCKETS - 1; } - uint64_t sub_bucket = (v >> (highest_bit - SUB_BUCKET_BITS)) & SUB_BUCKET_MASK; - return 1 + (static_cast(magnitude) * SUB_BUCKET_COUNT) + static_cast(sub_bucket); + return 128 + (static_cast(magnitude - 7) * 128) + static_cast((ns >> (magnitude - 7)) & 127); } -// Upper bound (exclusive) of the bucket at `index`, in nanoseconds. -// The underflow bucket returns the LVD; the overflow bucket returns -// the max trackable value (2^(UNIT_MAGNITUDE + NUM_MAGNITUDES)). -static uint64_t bucket_upper_bound(size_t index) +static uint64_t upper_bound(size_t index) { - if (index == kUnderflow) { - return 1ULL << UNIT_MAGNITUDE; + if (index < 128) { + return index; } - if (index == kOverflow) { - return 1ULL << (UNIT_MAGNITUDE + static_cast(NUM_MAGNITUDES)); + if (index == LatencyHistogram::NUM_BUCKETS - 1) { + return UINT64_MAX; } - size_t linear = index - 1; - size_t magnitude = linear / SUB_BUCKET_COUNT; - size_t sub_bucket = linear % SUB_BUCKET_COUNT; - uint64_t magnitude_base = 1ULL << (UNIT_MAGNITUDE + static_cast(magnitude)); - uint64_t sub_bucket_width = 1ULL << (UNIT_MAGNITUDE + static_cast(magnitude) - SUB_BUCKET_BITS); - return magnitude_base + ((static_cast(sub_bucket) + 1) * sub_bucket_width); + const size_t offset = index - 128; + const auto magnitude = 7 + static_cast(offset / 128); + return (uint64_t{1} << magnitude) + (((offset % 128) + 1) * (uint64_t{1} << (magnitude - 7))) - 1; } -// Compute the p-th percentile from bucket counts. Returns the upper -// bound of the bucket that contains the p-th percentile value. The -// overflow bucket is capped at the last regular bucket's upper bound -// to avoid reporting the max-trackable sentinel. -static uint64_t percentile(const std::vector &bucket_counts, uint64_t count, uint64_t p) +static uint64_t percentile(const LatencyHistogram::Data &data, uint64_t count, uint64_t permille) { if (count == 0) { return 0; } - uint64_t target = count * p / 100; - uint64_t cumulative = 0; - for (size_t i = 0; i < bucket_counts.size(); ++i) { - cumulative += bucket_counts[i]; + const uint64_t target = (count / 1000 * permille) + (((count % 1000 * permille) + 999) / 1000); + uint64_t cumulative = 0; + for (size_t i = 0; i < data.buckets.size(); ++i) { + cumulative += data.buckets[i]; if (cumulative >= target) { - size_t capped = std::min(i, kOverflow - 1); - return bucket_upper_bound(capped); + return i == LatencyHistogram::NUM_BUCKETS - 1 ? UINT64_MAX : std::min(upper_bound(i), data.max); } } - return bucket_upper_bound(kOverflow - 1); + return 0; } -// Find the highest non-empty bucket's upper bound. The overflow -// bucket is capped at the last regular bucket's upper bound. -static uint64_t max_latency(const std::vector &bucket_counts) +void LatencyHistogram::Data::observe(uint64_t ns) { - for (size_t i = bucket_counts.size(); i-- > 0;) { - if (bucket_counts[i] > 0) { - size_t capped = std::min(i, kOverflow - 1); - return bucket_upper_bound(capped); - } + ++buckets[bucket_index(ns)]; + const uint64_t previous = sum_low; + sum_low += ns; + sum_high += sum_low < previous ? 1 : 0; + min = std::min(min, ns); + max = std::max(max, ns); +} + +void LatencyHistogram::Data::merge(const Data &other) +{ + for (size_t i = 0; i < buckets.size(); ++i) { + buckets[i] += other.buckets[i]; } - return 0; + const uint64_t previous = sum_low; + sum_low += other.sum_low; + sum_high += other.sum_high + (sum_low < previous ? 1 : 0); + min = std::min(min, other.min); + max = std::max(max, other.max); } -// ── LatencyHistogram ───────────────────────────────────────────── +LatencyHistogram::Snapshot LatencyHistogram::Data::snapshot(uint64_t total_count) const +{ + Snapshot result; + result.count = std::accumulate(buckets.begin(), buckets.end(), uint64_t{0}); + result.sum_low = sum_low; + result.sum_high = sum_high; + result.avg = result.count == 0 + ? 0.0 + : ((static_cast(sum_high) * 18446744073709551616.0) + static_cast(sum_low)) / + static_cast(result.count); + result.min = result.count == 0 ? 0 : min; + result.max = max; + result.p50 = percentile(*this, result.count, 500); + result.p90 = percentile(*this, result.count, 900); + result.p95 = percentile(*this, result.count, 950); + result.p99 = percentile(*this, result.count, 990); + result.p999 = percentile(*this, result.count, 999); + result.zero_count = buckets[0]; + result.overflow_count = buckets.back(); + result.total_count = std::max(result.count, total_count); + return result; +} -LatencyHistogram::LatencyHistogram(std::string name) : name_(std::move(name)), count_(0), sum_(0), total_count_(0) +LatencyHistogram::LatencyHistogram(std::string name) : name_(std::move(name)) { - buckets_.reserve(NUM_BUCKETS); - for (size_t i = 0; i < NUM_BUCKETS; ++i) { - buckets_.push_back(std::make_unique>(0)); +} + +void LatencyHistogram::AtomicData::observe(uint64_t ns) +{ + buckets[bucket_index(ns)].fetch_add(1, std::memory_order_relaxed); + const uint64_t previous = sum_low.fetch_add(ns, std::memory_order_relaxed); + if (previous > UINT64_MAX - ns) { + sum_high.fetch_add(1, std::memory_order_relaxed); + } + uint64_t bound = min.load(std::memory_order_relaxed); + while (ns < bound && !min.compare_exchange_weak(bound, ns, std::memory_order_relaxed)) { + } + bound = max.load(std::memory_order_relaxed); + while (ns > bound && !max.compare_exchange_weak(bound, ns, std::memory_order_relaxed)) { } } +LatencyHistogram::Data LatencyHistogram::AtomicData::capture(bool reset) +{ + if (!reset) { + return std::as_const(*this).capture(); + } + Data result; + for (size_t i = 0; i < buckets.size(); ++i) { + result.buckets[i] = buckets[i].exchange(0, std::memory_order_relaxed); + } + result.sum_low = sum_low.exchange(0, std::memory_order_relaxed); + result.sum_high = sum_high.exchange(0, std::memory_order_relaxed); + result.min = min.exchange(UINT64_MAX, std::memory_order_relaxed); + result.max = max.exchange(0, std::memory_order_relaxed); + return result; +} + +LatencyHistogram::Data LatencyHistogram::AtomicData::capture() const +{ + Data result; + for (size_t i = 0; i < buckets.size(); ++i) { + result.buckets[i] = buckets[i].load(std::memory_order_relaxed); + } + result.sum_low = sum_low.load(std::memory_order_relaxed); + result.sum_high = sum_high.load(std::memory_order_relaxed); + result.min = min.load(std::memory_order_relaxed); + result.max = max.load(std::memory_order_relaxed); + return result; +} + void LatencyHistogram::observe(uint64_t ns) { - size_t idx = bucket_index(ns); - buckets_[idx]->fetch_add(1, std::memory_order_relaxed); - count_.fetch_add(1, std::memory_order_relaxed); - sum_.fetch_add(ns, std::memory_order_relaxed); - total_count_.fetch_add(1, std::memory_order_relaxed); + window_.observe(ns); + total_.observe(ns); } LatencyHistogram::Snapshot LatencyHistogram::flush() { - std::vector bucket_counts(NUM_BUCKETS); - for (size_t i = 0; i < NUM_BUCKETS; ++i) { - bucket_counts[i] = buckets_[i]->exchange(0, std::memory_order_relaxed); - } - uint64_t count = count_.exchange(0, std::memory_order_relaxed); - uint64_t sum = sum_.exchange(0, std::memory_order_relaxed); - uint64_t total = total_count_.load(std::memory_order_relaxed); - - Snapshot snap; - snap.count = count; - snap.avg = count > 0 ? static_cast(sum) / static_cast(count) : 0.0; - snap.p50 = percentile(bucket_counts, count, 50); - snap.p99 = percentile(bucket_counts, count, 99); - snap.max = max_latency(bucket_counts); - snap.total_count = total; - return snap; + const auto cumulative = total_.capture().snapshot().count; + return window_.capture(true).snapshot(cumulative); +} + +LatencyHistogram::Snapshot LatencyHistogram::snapshot() const +{ + return window_.capture().snapshot(total_.capture().snapshot().count); +} + +LatencyHistogram::Snapshot LatencyHistogram::snapshot_total() const +{ + return total_.capture().snapshot(); +} + +LatencyHistogram::Data LatencyHistogram::capture_total() const +{ + return total_.capture(); } } // namespace crowdb::common::metrics diff --git a/lib/crowdb-common/cpp/tests/latency_histogram_test.cpp b/lib/crowdb-common/cpp/tests/latency_histogram_test.cpp new file mode 100644 index 000000000..c63bc01b0 --- /dev/null +++ b/lib/crowdb-common/cpp/tests/latency_histogram_test.cpp @@ -0,0 +1,113 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-common/metrics/latency_histogram.h" + +#include + +#include +#include +#include +#include +#include + +using crowdb::common::metrics::LatencyHistogram; + +TEST(LatencyHistogram, SharedNanosecondGoldenVectors) +{ + const auto path = + std::filesystem::path(__FILE__).parent_path().parent_path().parent_path() / "testdata/latency_histogram.csv"; + std::ifstream input(path); + ASSERT_TRUE(input.is_open()) << path; + std::string line; + size_t cases = 0; + while (std::getline(input, line)) { + if (line.empty() || line[0] == '#') { + continue; + } + const auto comma = line.find(','); + ASSERT_NE(comma, std::string::npos); + const uint64_t sample = std::stoull(line.substr(0, comma)); + const uint64_t upper = std::stoull(line.substr(comma + 1)); + LatencyHistogram h("test.latency"); + h.observe(sample); + h.observe(UINT64_MAX); + const auto s = h.snapshot_total(); + EXPECT_EQ(s.p50, upper) << sample; + EXPECT_EQ(s.p90, UINT64_MAX); + EXPECT_EQ(s.min, sample); + EXPECT_EQ(s.max, UINT64_MAX); + EXPECT_EQ(s.sum_low, sample - 1); + EXPECT_EQ(s.sum_high, sample == 0 ? 0 : 1); + EXPECT_EQ(s.zero_count, sample == 0 ? 1 : 0); + EXPECT_EQ(s.overflow_count, 1 + (sample >= (uint64_t{1} << 40) ? 1 : 0)); + ++cases; + } + EXPECT_EQ(cases, 18); +} + +TEST(LatencyHistogram, NearestRankAndExactExtrema) +{ + LatencyHistogram::Data h; + for (uint64_t ns = 1; ns <= 1000; ++ns) { + h.observe(ns); + } + const auto s = h.snapshot(); + EXPECT_EQ(s.count, 1000); + EXPECT_EQ(s.sum_low, 500500); + EXPECT_EQ(s.min, 1); + EXPECT_EQ(s.max, 1000); + EXPECT_EQ(s.p50, 501); + EXPECT_EQ(s.p90, 903); + EXPECT_EQ(s.p95, 951); + EXPECT_EQ(s.p99, 991); + EXPECT_EQ(s.p999, 999); + LatencyHistogram::Data one; + one.observe(123); + EXPECT_EQ(one.snapshot().p50, 123); +} + +TEST(LatencyHistogram, WindowResetAndShardMerge) +{ + LatencyHistogram h("test.latency"); + EXPECT_EQ(h.snapshot_total().min, 0); + h.observe(100); + EXPECT_EQ(h.flush().sum_low, 100); + EXPECT_EQ(h.flush().count, 0); + h.observe(900); + const auto s = h.snapshot_total(); + EXPECT_EQ(s.count, 2); + EXPECT_EQ(s.sum_low, 1000); + EXPECT_EQ(s.min, 100); + EXPECT_EQ(s.max, 900); + LatencyHistogram::Data a; + LatencyHistogram::Data b; + a.observe(100); + b.observe(900); + a.merge(b); + EXPECT_EQ(a.snapshot().p50, s.p50); + EXPECT_EQ(a.snapshot().sum_low, s.sum_low); +} + +TEST(LatencyHistogram, ConcurrentFinalSnapshot) +{ + LatencyHistogram h("test.latency"); + std::vector threads; + threads.reserve(4); + for (int i = 0; i < 4; ++i) { + threads.emplace_back([&h] { + for (uint64_t ns = 0; ns < 10000; ++ns) { + h.observe(ns); + } + }); + } + for (auto &thread : threads) { + thread.join(); + } + const auto s = h.snapshot_total(); + EXPECT_EQ(s.count, 40000); + EXPECT_EQ(s.sum_low, 199980000); + EXPECT_EQ(s.min, 0); + EXPECT_EQ(s.max, 9999); + EXPECT_EQ(s.zero_count, 4); +} diff --git a/lib/crowdb-common/rust/src/metrics/histogram.rs b/lib/crowdb-common/rust/src/metrics/histogram.rs index f0a17e645..8081ec8e6 100644 --- a/lib/crowdb-common/rust/src/metrics/histogram.rs +++ b/lib/crowdb-common/rust/src/metrics/histogram.rs @@ -1,369 +1,113 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +//! Lock-free window and cumulative nanosecond histograms. +//! +//! Live snapshots/flushes are approximate while writers are running. Stop or +//! join producers before taking the exact final snapshot. Owned `HistogramData` +//! shards avoid shared atomics and merge after their polling workers stop. + use std::sync::atomic::{AtomicU64, Ordering}; use super::MetricName; -// ── HDR bucket parameters ──────────────────────────────────────── -// -// HDR-style logarithmic buckets: each power-of-2 magnitude is divided -// into `SUB_BUCKET_COUNT` equal sub-buckets, giving a bounded relative -// error of ≤ 1 / SUB_BUCKET_COUNT ≈ 0.78% everywhere. -// -// Configuration tuned for the crowdb latency range: -// - LVD (lowest discernible value) = 2^UNIT_MAGNITUDE ≈ 65.5µs. -// Values below this go into a single underflow bucket (index 0). -// The user's workload starts at ~100µs, so sub-65µs granularity is -// irrelevant. -// - Max magnitude covers 2^(UNIT_MAGNITUDE + NUM_MAGNITUDES) = 2^34 -// ≈ 17.2s. Values above go into the overflow bucket (last index). -// - 2 significant figures → SUB_BUCKET_COUNT = 128 (2^7). - -const UNIT_MAGNITUDE: u32 = 16; // floor(log2(100_000)) → LVD ≈ 65.5µs -const SUB_BUCKET_COUNT: usize = 128; // 2^7 → ≤0.78% relative error -const SUB_BUCKET_BITS: u32 = 7; // log2(SUB_BUCKET_COUNT) -const SUB_BUCKET_MASK: u64 = (SUB_BUCKET_COUNT as u64) - 1; // 127 -const NUM_MAGNITUDES: usize = 18; // magnitudes 0..17 → up to 2^34 ≈ 17.2s -/// 1 underflow + `NUM_MAGNITUDES` * `SUB_BUCKET_COUNT` regular + 1 overflow. -const NUM_BUCKETS: usize = 1 + NUM_MAGNITUDES * SUB_BUCKET_COUNT + 1; -const UNDERFLOW: usize = 0; -const OVERFLOW: usize = NUM_BUCKETS - 1; +mod data; +use data::{bucket_index, NUM_BUCKETS}; +pub use data::{HistogramData, HistogramSnapshot}; -/// Fixed-bucket HDR latency histogram with window + cumulative tracking. -/// -/// Each `observe(ns)` does O(1) bit manipulation to find the bucket, -/// then `fetch_add(1)` on both the window and cumulative bucket arrays, -/// and on `count`/`total_count` + `sum`/`total_sum`. No allocation, no -/// locks. Relative error ≤ ~0.78% for values in [65.5µs, 17.2s]. -/// -/// `flush()` resets window state (buckets, count, sum) but keeps -/// cumulative state (`total_buckets`, `total_count`, `total_sum`) — so -/// `snapshot_total()` returns full-run percentiles for a final report -/// even after periodic flushes have reset the window. -#[allow(dead_code)] #[derive(Debug)] -pub struct LatencyHistogram { - name: MetricName, - // Window state (reset on flush). +struct AtomicHistogram { buckets: [AtomicU64; NUM_BUCKETS], - count: AtomicU64, - sum: AtomicU64, - // Cumulative state (never reset). - total_buckets: [AtomicU64; NUM_BUCKETS], - total_count: AtomicU64, - total_sum: AtomicU64, + sum_low: AtomicU64, + sum_high: AtomicU64, + min: AtomicU64, + max: AtomicU64, } -#[allow(dead_code)] -impl LatencyHistogram { - #[must_use] - pub fn new(name: MetricName) -> Self { +impl AtomicHistogram { + fn new() -> Self { Self { - name, buckets: std::array::from_fn(|_| AtomicU64::new(0)), - count: AtomicU64::new(0), - sum: AtomicU64::new(0), - total_buckets: std::array::from_fn(|_| AtomicU64::new(0)), - total_count: AtomicU64::new(0), - total_sum: AtomicU64::new(0), + sum_low: AtomicU64::new(0), + sum_high: AtomicU64::new(0), + min: AtomicU64::new(u64::MAX), + max: AtomicU64::new(0), } } - /// Record a latency observation in nanoseconds. Updates both - /// window and cumulative state in a single call. - pub fn observe(&self, ns: u64) { - let idx = bucket_index(ns); - self.buckets[idx].fetch_add(1, Ordering::Relaxed); - self.total_buckets[idx].fetch_add(1, Ordering::Relaxed); - self.count.fetch_add(1, Ordering::Relaxed); - self.sum.fetch_add(ns, Ordering::Relaxed); - self.total_count.fetch_add(1, Ordering::Relaxed); - self.total_sum.fetch_add(ns, Ordering::Relaxed); - } - - /// Snapshot and reset window state. Returns count, p50, p99, max - /// (all in nanoseconds), and `total_count`. Cumulative state is - /// preserved for `snapshot_total()`. - pub fn flush(&self) -> HistogramSnapshot { - let mut bucket_counts = vec![0u64; NUM_BUCKETS]; - for (i, b) in self.buckets.iter().enumerate() { - bucket_counts[i] = b.swap(0, Ordering::Relaxed); + fn observe(&self, ns: u64) { + self.buckets[bucket_index(ns)].fetch_add(1, Ordering::Relaxed); + let previous = self.sum_low.fetch_add(ns, Ordering::Relaxed); + if previous > u64::MAX - ns { + self.sum_high.fetch_add(1, Ordering::Relaxed); } - let count = self.count.swap(0, Ordering::Relaxed); - let sum = self.sum.swap(0, Ordering::Relaxed); - let total_count = self.total_count.load(Ordering::Relaxed); - - let p50 = percentile(&bucket_counts, count, 50); - let p99 = percentile(&bucket_counts, count, 99); - let max = max_latency(&bucket_counts); - let avg = compute_avg(sum, count); - - HistogramSnapshot { - count, - avg, - p50, - p99, - max, - total_count, + if ns < self.min.load(Ordering::Relaxed) { + self.min.fetch_min(ns, Ordering::Relaxed); } - } - - /// Current window values without resetting. - pub fn snapshot(&self) -> HistogramSnapshot { - let mut bucket_counts = vec![0u64; NUM_BUCKETS]; - for (i, b) in self.buckets.iter().enumerate() { - bucket_counts[i] = b.load(Ordering::Relaxed); - } - let count = self.count.load(Ordering::Relaxed); - let sum = self.sum.load(Ordering::Relaxed); - let total_count = self.total_count.load(Ordering::Relaxed); - - let p50 = percentile(&bucket_counts, count, 50); - let p99 = percentile(&bucket_counts, count, 99); - let max = max_latency(&bucket_counts); - let avg = compute_avg(sum, count); - - HistogramSnapshot { - count, - avg, - p50, - p99, - max, - total_count, + if ns > self.max.load(Ordering::Relaxed) { + self.max.fetch_max(ns, Ordering::Relaxed); } } - /// Cumulative values across all observations (never reset by - /// `flush()`). Use for a final report after periodic flushes. - #[must_use] - pub fn snapshot_total(&self) -> HistogramSnapshot { - let mut bucket_counts = vec![0u64; NUM_BUCKETS]; - for (i, b) in self.total_buckets.iter().enumerate() { - bucket_counts[i] = b.load(Ordering::Relaxed); + fn capture(&self, reset: bool) -> HistogramData { + let get = |atomic: &AtomicU64, empty| { + if reset { + atomic.swap(empty, Ordering::Relaxed) + } else { + atomic.load(Ordering::Relaxed) + } + }; + HistogramData { + buckets: self.buckets.iter().map(|b| get(b, 0)).collect(), + sum: (u128::from(get(&self.sum_high, 0)) << 64) | u128::from(get(&self.sum_low, 0)), + min: get(&self.min, u64::MAX), + max: get(&self.max, 0), } - let count = self.total_count.load(Ordering::Relaxed); - let sum = self.total_sum.load(Ordering::Relaxed); - - let p50 = percentile(&bucket_counts, count, 50); - let p99 = percentile(&bucket_counts, count, 99); - let max = max_latency(&bucket_counts); - let avg = compute_avg(sum, count); - - HistogramSnapshot { - count, - avg, - p50, - p99, - max, - total_count: count, - } - } - - pub fn name(&self) -> &str { - &self.name } } -// ── HDR index / boundary math ───────────────────────────────────── - -/// Map a value to its HDR bucket index via O(1) bit manipulation. -/// Values < LVD go to the underflow bucket (0); values > max go to -/// the overflow bucket (`NUM_BUCKETS` - 1). -#[allow(dead_code)] -fn bucket_index(v: u64) -> usize { - if v < (1u64 << UNIT_MAGNITUDE) { - return UNDERFLOW; - } - let highest_bit = v.ilog2(); - let magnitude = highest_bit - UNIT_MAGNITUDE; - if magnitude as usize >= NUM_MAGNITUDES { - return OVERFLOW; - } - let sub_bucket = ((v >> (highest_bit - SUB_BUCKET_BITS)) & SUB_BUCKET_MASK) as usize; - 1 + magnitude as usize * SUB_BUCKET_COUNT + sub_bucket -} - -/// Upper bound (exclusive) of the bucket at `index`, in nanoseconds. -/// The underflow bucket returns the LVD; the overflow bucket returns -/// the max trackable value (`2^(UNIT_MAGNITUDE + NUM_MAGNITUDES)`). -#[allow(dead_code, clippy::cast_possible_truncation)] -fn bucket_upper_bound(index: usize) -> u64 { - if index == UNDERFLOW { - return 1u64 << UNIT_MAGNITUDE; - } - if index == OVERFLOW { - return 1u64 << (UNIT_MAGNITUDE + NUM_MAGNITUDES as u32); - } - let linear = index - 1; - let magnitude = linear / SUB_BUCKET_COUNT; - let sub_bucket = linear % SUB_BUCKET_COUNT; - let magnitude_base = 1u64 << (UNIT_MAGNITUDE + magnitude as u32); - let sub_bucket_width = 1u64 << (UNIT_MAGNITUDE + magnitude as u32 - SUB_BUCKET_BITS); - magnitude_base + (sub_bucket as u64 + 1) * sub_bucket_width -} - -/// Compute the p-th percentile from bucket counts. Returns the upper -/// bound of the bucket that contains the p-th percentile value. The -/// overflow bucket is capped at the last regular bucket's upper bound -/// to avoid reporting the max-trackable sentinel. -#[allow(dead_code)] -fn percentile(bucket_counts: &[u64], count: u64, p: u64) -> u64 { - if count == 0 { - return 0; - } - let target = count * p / 100; - let mut cumulative = 0u64; - for (i, &bc) in bucket_counts.iter().enumerate() { - cumulative += bc; - if cumulative >= target { - let capped = i.min(OVERFLOW - 1); - return bucket_upper_bound(capped); - } - } - bucket_upper_bound(OVERFLOW - 1) -} - -/// Find the highest non-empty bucket's upper bound. The overflow -/// bucket is capped at the last regular bucket's upper bound. -#[allow(dead_code)] -fn max_latency(bucket_counts: &[u64]) -> u64 { - for (i, &bc) in bucket_counts.iter().enumerate().rev() { - if bc > 0 { - let capped = i.min(OVERFLOW - 1); - return bucket_upper_bound(capped); - } - } - 0 -} - -/// Arithmetic mean of `sum` / `count` as `f64`. Returns `0.0` if `count` -/// is zero (avoids division by zero). -#[allow(clippy::cast_precision_loss)] -fn compute_avg(sum: u64, count: u64) -> f64 { - if count > 0 { - sum as f64 / count as f64 - } else { - 0.0 - } -} - -#[derive(Debug, Clone, Copy, Default)] -pub struct HistogramSnapshot { - pub count: u64, - pub avg: f64, - pub p50: u64, - pub p99: u64, - pub max: u64, - pub total_count: u64, +#[derive(Debug)] +pub struct LatencyHistogram { + name: MetricName, + window: AtomicHistogram, + total: AtomicHistogram, } -#[cfg(test)] -mod tests { - use super::*; - - /// `500_000` ns → magnitude 2, sub-bucket 116, upper bound `501_760`. - const BOUND_500K: u64 = 501_760; - /// `200_000` ns → magnitude 1, sub-bucket 67, upper bound `200_704`. - const BOUND_200K: u64 = 200_704; - /// `10_000_000` ns → magnitude 7, sub-bucket 24, upper bound `10_027_008`. - const BOUND_10MS: u64 = 10_027_008; - - #[test] - #[allow(clippy::float_cmp)] - fn histogram_p50_p99_with_known_distribution() { - let h = LatencyHistogram::new(MetricName::Static("test.lh")); - for _ in 0..100 { - h.observe(500_000); +impl LatencyHistogram { + #[must_use] + pub fn new(name: MetricName) -> Self { + Self { + name, + window: AtomicHistogram::new(), + total: AtomicHistogram::new(), } - let s = h.flush(); - assert_eq!(s.count, 100); - assert_eq!(s.total_count, 100); - // p50 and p99 fall in the HDR bucket containing 500µs. - assert_eq!(s.p50, BOUND_500K); - assert_eq!(s.p99, BOUND_500K); - assert_eq!(s.max, BOUND_500K); - // avg is exact (f64). - assert_eq!(s.avg, 500_000.0); } - #[test] - fn histogram_mixed_distribution() { - let h = LatencyHistogram::new(MetricName::Static("test.lh")); - // 80 fast (200µs), 20 slow (10ms) — both above LVD. - for _ in 0..80 { - h.observe(200_000); - } - for _ in 0..20 { - h.observe(10_000_000); - } - let s = h.flush(); - assert_eq!(s.count, 100); - // p50 should fall in the 200µs bucket (first 80 are at 200µs). - assert_eq!(s.p50, BOUND_200K); - // p99 should fall in the 10ms bucket (cumulative at 200µs = 80, target = 99). - assert_eq!(s.p99, BOUND_10MS); - assert_eq!(s.max, BOUND_10MS); + pub fn observe(&self, ns: u64) { + self.window.observe(ns); + self.total.observe(ns); } - #[test] - fn histogram_window_resets_after_flush() { - let h = LatencyHistogram::new(MetricName::Static("test.lh")); - h.observe(100_000); - h.observe(200_000); - let s1 = h.flush(); - assert_eq!(s1.count, 2); - assert_eq!(s1.total_count, 2); - - let s2 = h.flush(); - assert_eq!(s2.count, 0); - assert_eq!(s2.p50, 0); - assert_eq!(s2.total_count, 2); // total accumulates + pub fn flush(&self) -> HistogramSnapshot { + self.window + .capture(true) + .snapshot(self.total.buckets.iter().map(|b| b.load(Ordering::Relaxed)).sum()) } - #[test] - fn histogram_snapshot_does_not_reset() { - let h = LatencyHistogram::new(MetricName::Static("test.lh")); - h.observe(100_000); - let s = h.snapshot(); - assert_eq!(s.count, 1); - let s2 = h.snapshot(); - assert_eq!(s2.count, 1); + #[must_use] + pub fn snapshot(&self) -> HistogramSnapshot { + self.window + .capture(false) + .snapshot(self.total.buckets.iter().map(|b| b.load(Ordering::Relaxed)).sum()) } - #[test] - fn bucket_index_correctness() { - // Underflow: below LVD (2^16 = 65536). - assert_eq!(bucket_index(0), UNDERFLOW); - assert_eq!(bucket_index(65_535), UNDERFLOW); - // First regular bucket: exactly LVD. - assert_eq!(bucket_index(65_536), 1); - // 100µs → magnitude 0, sub-bucket 67. - assert_eq!(bucket_index(100_000), 1 + 67); - // 500µs → magnitude 2, sub-bucket 116. - assert_eq!(bucket_index(500_000), 1 + 2 * 128 + 116); - // Overflow: beyond max trackable (2^34). - assert_eq!(bucket_index(1u64 << 34), OVERFLOW); - assert_eq!(bucket_index(u64::MAX), OVERFLOW); + #[must_use] + pub fn snapshot_total(&self) -> HistogramSnapshot { + self.total.capture(false).snapshot(0) } - #[test] - #[allow(clippy::cast_precision_loss)] - fn hdr_relative_error_within_one_percent() { - // Verify the HDR guarantee: for any value in range, the bucket - // upper bound is within 1% of the value. - for v in [ - 100_000u64, - 500_000, - 1_000_000, - 10_000_000, - 100_000_000, - 1_000_000_000, - ] { - let idx = bucket_index(v); - let bound = bucket_upper_bound(idx); - let error = ((bound as f64 - v as f64) / v as f64 * 100.0).abs(); - assert!(error < 1.0, "v={v} bound={bound} error={error:.3}% exceeds 1%"); - } + #[must_use] + pub fn name(&self) -> &str { + &self.name } } diff --git a/lib/crowdb-common/rust/src/metrics/histogram/data.rs b/lib/crowdb-common/rust/src/metrics/histogram/data.rs new file mode 100644 index 000000000..d671ca717 --- /dev/null +++ b/lib/crowdb-common/rust/src/metrics/histogram/data.rs @@ -0,0 +1,136 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Shared layout with C++: exact 0..255 ns, then 128 buckets per power of +//! two up to 2^40 ns. Quantiles use inclusive bucket bounds and nearest rank; +//! relative error is below 1/128. Overflow quantiles return `u64::MAX`. + +const SUB_BUCKETS: usize = 128; +const FIRST_MAGNITUDE: u32 = 7; +const LIMIT_MAGNITUDE: u32 = 40; +pub(super) const NUM_BUCKETS: usize = 128 + 33 * SUB_BUCKETS + 1; +const OVERFLOW: usize = NUM_BUCKETS - 1; + +pub(super) fn bucket_index(ns: u64) -> usize { + if ns < 128 { + return usize::try_from(ns).unwrap_or(0); + } + let magnitude = ns.ilog2(); + if magnitude >= LIMIT_MAGNITUDE { + return OVERFLOW; + } + 128 + (magnitude - FIRST_MAGNITUDE) as usize * SUB_BUCKETS + + ((ns >> (magnitude - FIRST_MAGNITUDE)) & 127) as usize +} + +fn upper_bound(index: usize) -> u64 { + if index < 128 { + return index as u64; + } + if index == OVERFLOW { + return u64::MAX; + } + let offset = index - 128; + let magnitude = FIRST_MAGNITUDE + u32::try_from(offset / SUB_BUCKETS).unwrap_or(0); + (1_u64 << magnitude) + ((offset % SUB_BUCKETS) as u64 + 1) * (1_u64 << (magnitude - 7)) - 1 +} + +/// One polling owner's allocation-free observations, mergeable after joining. +#[derive(Debug, Clone)] +pub struct HistogramData { + pub(super) buckets: Vec, + pub(super) sum: u128, + pub(super) min: u64, + pub(super) max: u64, +} + +impl Default for HistogramData { + fn default() -> Self { + Self { + buckets: vec![0; NUM_BUCKETS], + sum: 0, + min: u64::MAX, + max: 0, + } + } +} + +impl HistogramData { + pub fn observe(&mut self, ns: u64) { + self.buckets[bucket_index(ns)] += 1; + self.sum += u128::from(ns); + self.min = self.min.min(ns); + self.max = self.max.max(ns); + } + + pub fn merge(&mut self, other: &Self) { + for (dest, source) in self.buckets.iter_mut().zip(&other.buckets) { + *dest += source; + } + self.sum += other.sum; + self.min = self.min.min(other.min); + self.max = self.max.max(other.max); + } + + /// Pass the cumulative count for a window, or zero for an owned shard. + #[must_use] + #[allow(clippy::cast_precision_loss)] + pub fn snapshot(&self, total_count: u64) -> HistogramSnapshot { + let count = self.buckets.iter().sum(); + HistogramSnapshot { + count, + sum: self.sum, + avg: if count == 0 { + 0.0 + } else { + self.sum as f64 / count as f64 + }, + min: if count == 0 { 0 } else { self.min }, + max: self.max, + p50: self.percentile(count, 500), + p90: self.percentile(count, 900), + p95: self.percentile(count, 950), + p99: self.percentile(count, 990), + p999: self.percentile(count, 999), + zero_count: self.buckets[0], + overflow_count: self.buckets[OVERFLOW], + total_count: total_count.max(count), + } + } + + fn percentile(&self, count: u64, permille: u64) -> u64 { + if count == 0 { + return 0; + } + let target = count / 1000 * permille + (count % 1000 * permille).div_ceil(1000); + let mut cumulative = 0; + for (i, bucket) in self.buckets.iter().enumerate() { + cumulative += bucket; + if cumulative >= target { + return if i == OVERFLOW { + u64::MAX + } else { + upper_bound(i).min(self.max) + }; + } + } + 0 + } +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct HistogramSnapshot { + pub count: u64, + pub sum: u128, + pub avg: f64, + pub min: u64, + pub max: u64, + pub p50: u64, + pub p90: u64, + pub p95: u64, + pub p99: u64, + pub p999: u64, + pub zero_count: u64, + pub overflow_count: u64, + pub total_count: u64, +} diff --git a/lib/crowdb-common/rust/src/metrics/registry.rs b/lib/crowdb-common/rust/src/metrics/registry.rs index b1d6a16e1..3310202f9 100644 --- a/lib/crowdb-common/rust/src/metrics/registry.rs +++ b/lib/crowdb-common/rust/src/metrics/registry.rs @@ -477,8 +477,7 @@ mod tests { bw.observe(100); bw.observe(300); let h = reg.register_histogram("s.1.kv.get.lh"); - // Values must be ≥ LVD (65_536 ns) to land in regular HDR buckets. - // 100_000 ns → bucket upper bound 100_352 ns. + // Identical samples have quantiles capped at the exact maximum. h.observe(100_000); h.observe(100_000); let s = reg.register_summary("s.1.kv.scan.l"); @@ -532,7 +531,7 @@ mod tests { }); assert_eq!( histogram, - Some((2, 100_352, 100_352, 2)), + Some((2, 100_000, 100_000, 2)), "histogram count/p50/p99/total" ); diff --git a/lib/crowdb-common/rust/tests/latency_histogram_test.rs b/lib/crowdb-common/rust/tests/latency_histogram_test.rs new file mode 100644 index 000000000..baac8fe57 --- /dev/null +++ b/lib/crowdb-common/rust/tests/latency_histogram_test.rs @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_common::metrics::histogram::HistogramData; +use crowdb_common::metrics::{LatencyHistogram, MetricName}; + +#[test] +fn shared_nanosecond_golden_vectors() { + for line in include_str!("../../testdata/latency_histogram.csv").lines() { + if line.starts_with('#') || line.is_empty() { + continue; + } + let (sample, upper) = line.split_once(',').unwrap(); + let sample: u64 = sample.parse().unwrap(); + let upper: u64 = upper.parse().unwrap(); + let h = LatencyHistogram::new(MetricName::Static("test.latency")); + h.observe(sample); + h.observe(u64::MAX); + let s = h.snapshot_total(); + assert_eq!(s.p50, upper, "sample={sample}"); + assert_eq!(s.p90, u64::MAX); + assert_eq!(s.min, sample); + assert_eq!(s.max, u64::MAX); + assert_eq!(s.sum, u128::from(sample) + u128::from(u64::MAX)); + assert_eq!(s.zero_count, u64::from(sample == 0)); + assert_eq!(s.overflow_count, 1 + u64::from(sample >= 1 << 40)); + } +} + +#[test] +fn quantiles_use_nearest_rank_and_preserve_exact_extrema() { + let mut h = HistogramData::default(); + for ns in 1..=1000 { + h.observe(ns); + } + let s = h.snapshot(0); + assert_eq!((s.count, s.sum, s.min, s.max), (1000, 500_500, 1, 1000)); + assert_eq!((s.p50, s.p90, s.p95, s.p99, s.p999), (501, 903, 951, 991, 999)); + let mut one = HistogramData::default(); + one.observe(123); + assert_eq!(one.snapshot(0).p50, 123); +} + +#[test] +fn flush_resets_only_the_window_and_shards_merge_without_losing_samples() { + let h = LatencyHistogram::new(MetricName::Static("test.latency")); + let empty = h.snapshot_total(); + assert_eq!((empty.count, empty.min, empty.max, empty.p999), (0, 0, 0, 0)); + h.observe(100); + assert_eq!(h.flush().sum, 100); + assert_eq!(h.flush().count, 0); + h.observe(900); + let s = h.snapshot_total(); + assert_eq!((s.count, s.sum, s.min, s.max), (2, 1000, 100, 900)); + let mut a = HistogramData::default(); + let mut b = HistogramData::default(); + a.observe(100); + b.observe(900); + a.merge(&b); + assert_eq!(a.snapshot(0).p50, s.p50); + assert_eq!(a.snapshot(0).sum, s.sum); +} + +#[test] +fn concurrent_writers_have_exact_final_counts_and_sum() { + let h = std::sync::Arc::new(LatencyHistogram::new(MetricName::Static("test.latency"))); + std::thread::scope(|scope| { + for _ in 0..4 { + let h = &h; + scope.spawn(move || { + for ns in 0..10_000 { + h.observe(ns); + } + }); + } + }); + let s = h.snapshot_total(); + assert_eq!((s.count, s.sum, s.min, s.max), (40_000, 199_980_000, 0, 9999)); + assert_eq!(s.zero_count, 4); +} diff --git a/lib/crowdb-common/testdata/latency_histogram.csv b/lib/crowdb-common/testdata/latency_histogram.csv new file mode 100644 index 000000000..71bf7f6ec --- /dev/null +++ b/lib/crowdb-common/testdata/latency_histogram.csv @@ -0,0 +1,21 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +# sample_ns,inclusive_bucket_upper_ns (paired with UINT64_MAX for P50) +0,0 +1,1 +127,127 +128,128 +255,255 +256,257 +511,511 +512,515 +1000,1003 +10000,10047 +65535,65535 +65536,66047 +200000,200703 +500000,501759 +10000000,10027007 +1099511627775,1099511627775 +1099511627776,18446744073709551615 +18446744073709551615,18446744073709551615 diff --git a/lib/crowdb-rpc/tests/common/test_rdma_load.cpp b/lib/crowdb-rpc/tests/common/test_rdma_load.cpp index 98536cd9f..04679d042 100644 --- a/lib/crowdb-rpc/tests/common/test_rdma_load.cpp +++ b/lib/crowdb-rpc/tests/common/test_rdma_load.cpp @@ -13,34 +13,6 @@ namespace crowdb::rpc { -void TestRdmaLatency::observe(uint64_t ns) -{ - ns = std::max(uint64_t{1}, ns); - const unsigned exponent = std::bit_width(ns) - 1; - const unsigned shift = exponent > 7 ? exponent - 7 : 0; - const auto sub = static_cast((ns - (uint64_t{1} << exponent)) >> shift); - buckets[exponent * SUB_BUCKETS + sub].fetch_add(1, std::memory_order_relaxed); - sum_ns.fetch_add(ns, std::memory_order_relaxed); - uint64_t maximum = max_ns.load(std::memory_order_relaxed); - while (maximum < ns && !max_ns.compare_exchange_weak(maximum, ns, std::memory_order_relaxed)) { - } -} - -uint64_t TestRdmaLatency::percentile(const std::array &counts, uint64_t count, uint32_t percent) -{ - const uint64_t rank = std::max(uint64_t{1}, (count * percent + 99) / 100); - uint64_t seen = 0; - for (size_t index = 0; index < counts.size(); ++index) { - seen += counts[index]; - if (seen >= rank) { - const size_t exponent = index / SUB_BUCKETS; - const size_t shift = exponent > 7 ? exponent - 7 : 0; - return (uint64_t{1} << exponent) + ((index % SUB_BUCKETS + 1) << shift); - } - } - return 0; -} - TestRdmaLoad::TestRdmaLoad(uint32_t count, uint32_t in_flight, uint32_t seconds) : producer_count(count), in_flight_per_producer(in_flight), diff --git a/lib/crowdb-rpc/tests/common/test_rdma_load.h b/lib/crowdb-rpc/tests/common/test_rdma_load.h index 9cd38fc99..1d4cbec0d 100644 --- a/lib/crowdb-rpc/tests/common/test_rdma_load.h +++ b/lib/crowdb-rpc/tests/common/test_rdma_load.h @@ -3,6 +3,7 @@ #pragma once +#include "crowdb-common/metrics/latency_histogram.h" #include "crowdb-rpc/client/client.h" #include "crowdb-rpc/transport/socket_transport.h" @@ -25,18 +26,6 @@ inline uint64_t test_rdma_clock_ns() .count()); } -struct TestRdmaLatency -{ - static constexpr size_t SUB_BUCKETS = 128; - static constexpr size_t BUCKETS = 64 * SUB_BUCKETS; - std::array, BUCKETS> buckets{}; - std::atomic sum_ns{0}; - std::atomic max_ns{0}; - - void observe(uint64_t ns); - static uint64_t percentile(const std::array &buckets, uint64_t count, uint32_t percent); -}; - struct TestRdmaProducer; struct TestRdmaLoadSlot @@ -72,15 +61,15 @@ struct TestRdmaLoad struct TestRdmaProducer { - TestRdmaLoad &load; - uint32_t index; - std::unique_ptr slots; - TestRdmaLatency latency; - uint64_t total_submitted = 0; - uint64_t measured_submitted = 0; - std::atomic total_completed{0}; - std::atomic window_completed{0}; - std::atomic drain_completed{0}; + TestRdmaLoad &load; + uint32_t index; + std::unique_ptr slots; + crowdb::common::metrics::LatencyHistogram latency{"rdma.test.latency"}; + uint64_t total_submitted = 0; + uint64_t measured_submitted = 0; + std::atomic total_completed{0}; + std::atomic window_completed{0}; + std::atomic drain_completed{0}; TestRdmaProducer(TestRdmaLoad &owner, uint32_t producer_index); void run(RpcClient &client, SocketTransport &transport, Connection *connection); diff --git a/lib/crowdb-rpc/tests/rdma_timed_load_test.cpp b/lib/crowdb-rpc/tests/rdma_timed_load_test.cpp index 4f45f775b..50059614e 100644 --- a/lib/crowdb-rpc/tests/rdma_timed_load_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_timed_load_test.cpp @@ -99,34 +99,33 @@ void test_timed_server(const char *address, int port, const TestLoadSettings &se void test_report_load(const TestLoadSettings &settings, TestRdmaLoad &load) { - std::array buckets{}; - uint64_t completed = 0, submitted = 0, drained = 0, sum_ns = 0, maximum_ns = 0; + crowdb::common::metrics::LatencyHistogram::Data histogram; + uint64_t completed = 0, submitted = 0, drained = 0; for (const auto &producer : load.producers) { completed += producer->window_completed.load(); submitted += producer->measured_submitted; drained += producer->drain_completed.load(); - sum_ns += producer->latency.sum_ns.load(); - maximum_ns = std::max(maximum_ns, producer->latency.max_ns.load()); - for (size_t index = 0; index < buckets.size(); ++index) - buckets[index] += producer->latency.buckets[index].load(); + histogram.merge(producer->latency.capture_total()); } - auto percentile_us = [&](uint32_t percent) { - return static_cast(TestRdmaLatency::percentile(buckets, completed, percent)) / 1000.0; - }; + const auto latency = histogram.snapshot(); + EXPECT_EQ(latency.count, completed); EXPECT_EQ(load.errors.load(), 0U); EXPECT_GT(completed, 0U); EXPECT_EQ(completed + drained, submitted); std::fprintf(stderr, "RDMA timed load: duration_s=%u warmup_s=%u producer_count=%u io_workers=%u connection_count=%u " "in_flight_per_producer=%u payload_bytes=%u submitted=%llu completed=%llu drain_completed=%llu " - "TPS=%.0f p50_us=%.2f p95_us=%.2f p99_us=%.2f mean_us=%.2f max_us=%.2f errors=%llu\n", + "TPS=%.0f p50_us=%.2f p95_us=%.2f p99_us=%.2f mean_us=%.2f max_us=%.2f errors=%llu " + "p90_us=%.2f p999_us=%.2f histogram_overflow=%llu\n", settings.duration_seconds, settings.warmup_seconds, settings.producer_count, settings.io_workers, settings.connection_count, settings.in_flight_per_producer, TestRdmaLoad::PAYLOAD_BYTES, static_cast(submitted), static_cast(completed), static_cast(drained), static_cast(completed) / settings.duration_seconds, - percentile_us(50), percentile_us(95), percentile_us(99), - completed == 0 ? 0.0 : static_cast(sum_ns) / completed / 1000.0, - static_cast(maximum_ns) / 1000.0, static_cast(load.errors.load())); + static_cast(latency.p50) / 1000.0, static_cast(latency.p95) / 1000.0, + static_cast(latency.p99) / 1000.0, latency.avg / 1000.0, + static_cast(latency.max) / 1000.0, static_cast(load.errors.load()), + static_cast(latency.p90) / 1000.0, static_cast(latency.p999) / 1000.0, + static_cast(latency.overflow_count)); } void test_timed_client(const char *address, int port, const TestLoadSettings &settings) diff --git a/lib/crowdb-tree/tests/unit/metrics_test.cpp b/lib/crowdb-tree/tests/unit/metrics_test.cpp index 21f2eb081..2aad878e5 100644 --- a/lib/crowdb-tree/tests/unit/metrics_test.cpp +++ b/lib/crowdb-tree/tests/unit/metrics_test.cpp @@ -92,10 +92,10 @@ TEST(MetricsHistogram, P50P99WithKnownDistribution) } auto snap = h.flush(); EXPECT_EQ(snap.count, 100U); - // HDR bucket upper bound for 500us is 501'760ns (≤0.78% error). - EXPECT_EQ(snap.p50, 501'760U); - EXPECT_EQ(snap.p99, 501'760U); - EXPECT_EQ(snap.max, 501'760U); + // Quantile bucket bounds cannot exceed the exact observed maximum. + EXPECT_EQ(snap.p50, 500'000U); + EXPECT_EQ(snap.p99, 500'000U); + EXPECT_EQ(snap.max, 500'000U); // avg is exact (f64). EXPECT_DOUBLE_EQ(snap.avg, 500'000.0); } @@ -103,7 +103,7 @@ TEST(MetricsHistogram, P50P99WithKnownDistribution) TEST(MetricsHistogram, MixedDistribution) { LatencyHistogram h("test.lh"); - // 80 fast (200us), 20 slow (10ms) — both above LVD (65.5us). + // 80 fast (200us), 20 slow (10ms). for (int i = 0; i < 80; ++i) { h.observe(200'000); } @@ -112,17 +112,15 @@ TEST(MetricsHistogram, MixedDistribution) } auto snap = h.flush(); EXPECT_EQ(snap.count, 100U); - // p50 falls in the 200us bucket (upper bound 200'704ns). - EXPECT_EQ(snap.p50, 200'704U); - // p99 falls in the 10ms bucket (upper bound 10'027'008ns). - EXPECT_EQ(snap.p99, 10'027'008U); - EXPECT_EQ(snap.max, 10'027'008U); + EXPECT_EQ(snap.p50, 200'703U); + EXPECT_EQ(snap.p99, 10'000'000U); + EXPECT_EQ(snap.max, 10'000'000U); } -TEST(MetricsHistogram, UnderflowAndOverflow) +TEST(MetricsHistogram, LowLatencyResolution) { LatencyHistogram h("test.lh"); - // Below LVD (65.5us) → underflow bucket. + // Low latencies now occupy separate nanosecond buckets. h.observe(1'000); h.observe(65'535); // In-range values (majority so p50 falls in-range). @@ -132,10 +130,9 @@ TEST(MetricsHistogram, UnderflowAndOverflow) h.observe(500'000); auto snap = h.flush(); EXPECT_EQ(snap.count, 6U); - // p50 (target=3) falls in the 100us bucket (upper bound 100'352ns). - EXPECT_EQ(snap.p50, 100'352U); - // max falls in the 500us bucket (upper bound 501'760ns). - EXPECT_EQ(snap.max, 501'760U); + EXPECT_EQ(snap.p50, 100'351U); + EXPECT_EQ(snap.min, 1'000U); + EXPECT_EQ(snap.max, 500'000U); } TEST(MetricsHistogram, WindowResetsAfterFlush) diff --git a/tools/rpc-test/src/workload.rs b/tools/rpc-test/src/workload.rs index a15b502b3..24cfb72aa 100644 --- a/tools/rpc-test/src/workload.rs +++ b/tools/rpc-test/src/workload.rs @@ -282,7 +282,7 @@ impl CoBenchHandler for EchoHandler { fn on_response(&self, _request_id: u64, status: Result<(), RpcError>, latency_ns: u64) -> bool { if status.is_ok() { self.recorder - .record_ok(latency_ns / 1000, self.data_bytes.len() as u64); + .record_ok_ns(latency_ns, self.data_bytes.len() as u64); } else { self.recorder.record_err(); } From 34c97e0d4c6750ee4273a855343c0fd40a989dde Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 22 Oct 2026 19:30:41 +0800 Subject: [PATCH 023/258] Make coroutine completion handoff and joining safe --- doc/working/plan-rdma-rpc-transport.md | 26 +- lib/crowdb-rpc/ffi/build.rs | 2 + lib/crowdb-rpc/ffi/src/co_bench.rs | 52 +++- lib/crowdb-rpc/ffi/src/sys.rs | 11 + lib/crowdb-rpc/ffi/tests/co_bench_test.rs | 76 ++++++ lib/crowdb-rpc/ffi/tests/common/co_bench.rs | 77 ++++++ lib/crowdb-rpc/ffi/tests/common/mod.rs | 4 + .../include/crowdb-rpc/client/co_client.h | 3 + lib/crowdb-rpc/src/client/co_client.cpp | 223 ++---------------- lib/crowdb-rpc/src/client/co_state.h | 180 ++++++++++++++ lib/crowdb-rpc/tests/co_state_test.cpp | 48 ++++ lib/crowdb-rpc/tests/common/test_co_state.h | 42 ++++ 12 files changed, 533 insertions(+), 211 deletions(-) create mode 100644 lib/crowdb-rpc/ffi/tests/co_bench_test.rs create mode 100644 lib/crowdb-rpc/ffi/tests/common/co_bench.rs create mode 100644 lib/crowdb-rpc/ffi/tests/common/mod.rs create mode 100644 lib/crowdb-rpc/src/client/co_state.h create mode 100644 lib/crowdb-rpc/tests/co_state_test.cpp create mode 100644 lib/crowdb-rpc/tests/common/test_co_state.h diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index a2080f12f..e4365d064 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -267,7 +267,7 @@ deferring multiple CQs or forbidding transport coalescing. sub-microsecond values, bucket boundaries, empty/reset and overflow cases. Files: `lib/crowdb-common/{cpp,rust}/` histogram implementations and tests; `app/crowdb-cli/src/commands/bench/{loader,result}.rs`. -- [~] **Make coroutine completion safe before parallel CQ benchmarks**: +- [x] **Make coroutine completion safe before parallel CQ benchmarks**: `co_on_complete` currently fails a CAS against `CO_RUNNING`, then stores `CO_RESPONSE_READY`; `await_suspend` can change to `CO_SUSPENDED` between those operations and lose the resume. Replace that handoff with one atomic @@ -280,6 +280,28 @@ deferring multiple CQs or forbidding transport coalescing. and FFI coroutine API. This is prerequisite to attributing multi-CQ errors or performance to the transport. +Verified coroutine checkpoint: + +- `co_on_complete` publishes readiness with one atomic exchange, and final + suspension publishes completion once. The joining thread sleeps while + waiting and checks callback quiescence before reclaiming state or Rust + context; it never inspects a concurrently running coroutine handle or + force-resumes a pending await. Closed-connection submission rejection is + counted and stops that coroutine. Configure the existing request reaper for + stalled requests; its ordinary completion path handles timeouts. +- Rust `co_spawn` returns native `CoStats`, including submission rejection + errors; `CoBenchHandler::on_response_data` borrows response bytes for payload + validation. CLI report integration remains pending in phase D. +- Three C++ handoff cases pass, including 2,000 concurrent publication trials. + Three Rust integration cases pass: 2,048 validated echoes across 64 + coroutines/four connections/four I/O workers, timeout completion, and a + rejected closed connection. The full 29-test RPC FFI suite, 133-case native + RPC CTest suite (four hardware cases skipped), fmt, workspace clippy and + scoped tree-lint pass. No sanitizer or multi-CQ claim is made by these tests. +- The FFI build now tracks private source headers as well as public headers; + changes to the coroutine state or private transport headers cannot silently + leave the native client stale. + Verified histogram checkpoint: - C++/Rust share 18 golden vectors, inclusive bucket bounds, nearest-rank @@ -302,7 +324,7 @@ Verified histogram checkpoint: ### B. CQ owners and resource ownership -- [ ] **Create the bounded CQ pool**: add `RdmaConfig::cq_count`, +- [~] **Create the bounded CQ pool**: add `RdmaConfig::cq_count`, `verbs_poll_batch`, and strict environment validation. Each CQ owns its completion channel, wake fd, fixed poller and bounded WR table. Assign each QP to one CQ at creation; SEND/RECV and repost retain that assignment. diff --git a/lib/crowdb-rpc/ffi/build.rs b/lib/crowdb-rpc/ffi/build.rs index 685ba2b5b..538debc67 100644 --- a/lib/crowdb-rpc/ffi/build.rs +++ b/lib/crowdb-rpc/ffi/build.rs @@ -254,6 +254,8 @@ fn main() -> Result<(), Box> { for src in &sources { println!("cargo:rerun-if-changed={}", src.display()); } + // Private implementation headers participate in the native build too. + println!("cargo:rerun-if-changed={}", src.display()); println!("cargo:rerun-if-changed={}", include.display()); println!("cargo:rerun-if-changed={}", common_include.display()); diff --git a/lib/crowdb-rpc/ffi/src/co_bench.rs b/lib/crowdb-rpc/ffi/src/co_bench.rs index 854767249..5c8bd9375 100644 --- a/lib/crowdb-rpc/ffi/src/co_bench.rs +++ b/lib/crowdb-rpc/ffi/src/co_bench.rs @@ -13,6 +13,8 @@ //! //! `co_spawn` blocks until all coroutines finish; callers should run it //! on a blocking thread (e.g. `tokio::task::spawn_blocking`). +//! Configure the client's request timeout reaper before spawning so stalled +//! requests complete through the ordinary callback path. use std::ffi::c_void; use std::sync::Arc; @@ -31,8 +33,24 @@ pub trait CoBenchHandler: Send + Sync { /// to stop this coroutine. The C++ layer releases the response /// buffers after this returns — do not release them here. fn on_response(&self, request_id: u64, status: Result<(), RpcError>, latency_ns: u64) -> bool; + + /// Borrow response bytes for validation without cloning buffer owners. + /// The slices are valid only during this call. + fn on_response_data( + &self, + request_id: u64, + status: Result<(), RpcError>, + latency_ns: u64, + _control: &[u8], + _data: &[u8], + ) -> bool { + self.on_response(request_id, status, latency_ns) + } } +/// Native coroutine totals include submission rejections and response errors. +pub type CoStats = sys::CrowdbRpcCoStats; + /// Shared state passed through the C ABI as `ctx`. struct CoCtx { handler: Arc, @@ -54,9 +72,9 @@ pub fn co_spawn( num_coroutines: u32, msg_type: u16, handler: Arc, -) { +) -> CoStats { if conns.is_empty() || num_coroutines == 0 { - return; + return CoStats::default(); } let ctx = Box::new(CoCtx { handler }); let ctx_ptr = Box::into_raw(ctx) as *mut c_void; @@ -81,6 +99,10 @@ pub fn co_spawn( } // SAFETY: co_spawn has returned; no callback will fire again. unsafe { drop(Box::from_raw(ctx_ptr.cast::())) }; + let mut stats = CoStats::default(); + // SAFETY: all coroutines and callbacks joined; client remains borrowed. + unsafe { sys::crowdb_rpc_co_get_stats(client.handle(), &mut stats) }; + stats } // SAFETY: ctx is a valid CoCtx* set up by co_spawn. out_control / @@ -107,8 +129,8 @@ unsafe extern "C" fn build_cb( unsafe extern "C" fn resp_cb( ctx: *mut c_void, request_id: u64, - _control: sys::crowdb_rpc_buffer_t, - _data: sys::crowdb_rpc_buffer_t, + control: sys::crowdb_rpc_buffer_t, + data: sys::crowdb_rpc_buffer_t, status: sys::crowdb_rpc_status, latency_ns: u64, ) -> bool { @@ -118,5 +140,25 @@ unsafe extern "C" fn resp_cb( } else { Err(RpcError::from_status(status)) }; - c.handler.on_response(request_id, r, latency_ns) + c.handler.on_response_data( + request_id, + r, + latency_ns, + response_bytes(control), + response_bytes(data), + ) +} + +// SAFETY: response buffer owners remain in the C++ coroutine until resp_cb +// returns. Empty/null buffers produce an empty slice; no owner is transferred. +unsafe fn response_bytes<'a>(handle: sys::crowdb_rpc_buffer_t) -> &'a [u8] { + if handle.is_null() { + return &[]; + } + let length = sys::crowdb_rpc_buffer_len(handle) as usize; + let data = sys::crowdb_rpc_buffer_data(handle); + if data.is_null() || length == 0 { + return &[]; + } + std::slice::from_raw_parts(data, length) } diff --git a/lib/crowdb-rpc/ffi/src/sys.rs b/lib/crowdb-rpc/ffi/src/sys.rs index bb32f4ec8..1bfff75e1 100644 --- a/lib/crowdb-rpc/ffi/src/sys.rs +++ b/lib/crowdb-rpc/ffi/src/sys.rs @@ -350,6 +350,7 @@ extern "C" { pub fn crowdb_common_metrics_global_free(s: *mut c_char); // ── Coroutine client (Option 3: C++ coroutine + Rust FFI) ──── + pub fn crowdb_rpc_co_get_stats(client: crowdb_rpc_client_t, out: *mut CrowdbRpcCoStats); pub fn crowdb_rpc_co_spawn( client: crowdb_rpc_client_t, server: crowdb_rpc_server_t, @@ -362,3 +363,13 @@ extern "C" { ctx: *mut c_void, ); } + +#[repr(C)] +#[derive(Debug, Default, Clone, Copy)] +pub struct CrowdbRpcCoStats { + pub total_ops: u64, + pub total_errors: u64, + pub total_latency_ns: u64, + pub min_latency_ns: u64, + pub max_latency_ns: u64, +} diff --git a/lib/crowdb-rpc/ffi/tests/co_bench_test.rs b/lib/crowdb-rpc/ffi/tests/co_bench_test.rs new file mode 100644 index 000000000..4c9b6f4e7 --- /dev/null +++ b/lib/crowdb-rpc/ffi/tests/co_bench_test.rs @@ -0,0 +1,76 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +mod common; + +use std::sync::atomic::Ordering; +use std::sync::Arc; + +use common::co_bench::TestCoHandler; +use crowdb_rpc_ffi::co_bench::co_spawn; +use crowdb_rpc_ffi::{RpcClient, RpcServer}; + +#[test] +fn coroutine_echo_validates_borrowed_bytes_and_native_totals() { + let server = RpcServer::with_engines(None, 1, 4); + server.listen("127.0.0.1", 0).unwrap(); + server.register_echo_handler(100); + server.start(); + let connections: Vec<_> = (0..4) + .map(|_| server.connect("127.0.0.1", server.port()).unwrap()) + .collect(); + let client = RpcClient::new(); + client.set_completion_pool_size(64); + for connection in &connections { + client.attach(connection); + } + client.start_reaper(5_000_000_000, 1_000_000); + let handler = Arc::new(TestCoHandler::new(64, 32, false)); + let stats = co_spawn(&client, &server, &connections, 64, 100, handler.clone()); + client.stop_reaper(); + server.stop(); + assert_eq!(handler.errors.load(Ordering::Relaxed), 0); + assert_eq!(handler.responses.load(Ordering::Relaxed), 2048); + assert_eq!(stats.total_ops, 2048); + assert_eq!(stats.total_errors, 0); + assert!(stats.max_latency_ns >= stats.min_latency_ns); +} + +#[test] +fn request_timeout_completes_coroutines_before_context_is_released() { + let server = RpcServer::new(None); + server.listen("127.0.0.1", 0).unwrap(); + server.register_handler(101, |_request| {}); + server.start(); + let connections = [server.connect("127.0.0.1", server.port()).unwrap()]; + let client = RpcClient::new(); + client.set_completion_pool_size(8); + client.attach(&connections[0]); + client.start_reaper(2_000_000, 1_000_000); + let handler = Arc::new(TestCoHandler::new(8, 1, true)); + let stats = co_spawn(&client, &server, &connections, 8, 101, handler.clone()); + client.stop_reaper(); + server.stop(); + assert_eq!(handler.errors.load(Ordering::Relaxed), 0); + assert_eq!(handler.responses.load(Ordering::Relaxed), 8); + assert_eq!(stats.total_ops, 8); + assert_eq!(stats.total_errors, 8); +} + +#[test] +fn closed_connection_rejection_is_counted_without_retrying_forever() { + let server = RpcServer::new(None); + server.listen("127.0.0.1", 0).unwrap(); + server.start(); + let connections = [server.connect("127.0.0.1", server.port()).unwrap()]; + let client = RpcClient::new(); + client.set_completion_pool_size(1); + client.attach(&connections[0]); + connections[0].close(); + let handler = Arc::new(TestCoHandler::new(1, u64::MAX, false)); + let stats = co_spawn(&client, &server, &connections, 1, 100, handler.clone()); + server.stop(); + assert_eq!(handler.responses.load(Ordering::Relaxed), 0); + assert_eq!(stats.total_ops, 0); + assert_eq!(stats.total_errors, 1); +} diff --git a/lib/crowdb-rpc/ffi/tests/common/co_bench.rs b/lib/crowdb-rpc/ffi/tests/common/co_bench.rs new file mode 100644 index 000000000..983100556 --- /dev/null +++ b/lib/crowdb-rpc/ffi/tests/common/co_bench.rs @@ -0,0 +1,77 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::atomic::{AtomicU64, Ordering}; + +use crowdb_protocol::fb::{ConnectionPingRequest, ConnectionPingRequestArgs}; +use crowdb_rpc_ffi::co_bench::CoBenchHandler; +use crowdb_rpc_ffi::{Buffer, RpcError}; +use flatbuffers::FlatBufferBuilder; + +pub struct TestCoHandler { + pub stride: u64, + pub iterations: u64, + pub expect_timeout: bool, + pub responses: AtomicU64, + pub errors: AtomicU64, +} + +impl TestCoHandler { + pub fn new(stride: u64, iterations: u64, expect_timeout: bool) -> Self { + Self { + stride, + iterations, + expect_timeout, + responses: AtomicU64::new(0), + errors: AtomicU64::new(0), + } + } +} + +impl CoBenchHandler for TestCoHandler { + fn build_request(&self, id: u64) -> Option<(Buffer, Buffer)> { + if id / self.stride >= self.iterations { + return None; + } + let mut fbb = FlatBufferBuilder::new(); + let request = ConnectionPingRequest::create( + &mut fbb, + &ConnectionPingRequestArgs { + id, + rpc_create_nano: 0, + }, + ); + fbb.finish(request, None); + let mut data = vec![0xA5; 128]; + data[..8].copy_from_slice(&id.to_le_bytes()); + Some((Buffer::from_bytes(fbb.finished_data()), Buffer::from_vec(data))) + } + + fn on_response(&self, _id: u64, _status: Result<(), RpcError>, _latency_ns: u64) -> bool { + self.errors.fetch_add(1, Ordering::Relaxed); + false + } + + fn on_response_data( + &self, + id: u64, + status: Result<(), RpcError>, + _latency_ns: u64, + _control: &[u8], + data: &[u8], + ) -> bool { + let valid = if self.expect_timeout { + matches!(status, Err(RpcError::Timeout)) + } else { + status.is_ok() + && data.len() == 128 + && data[..8] == id.to_le_bytes() + && data[8..].iter().all(|&b| b == 0xA5) + }; + self.responses.fetch_add(1, Ordering::Relaxed); + if !valid { + self.errors.fetch_add(1, Ordering::Relaxed); + } + valid && !self.expect_timeout + } +} diff --git a/lib/crowdb-rpc/ffi/tests/common/mod.rs b/lib/crowdb-rpc/ffi/tests/common/mod.rs new file mode 100644 index 000000000..ed5bbbea3 --- /dev/null +++ b/lib/crowdb-rpc/ffi/tests/common/mod.rs @@ -0,0 +1,4 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +pub mod co_bench; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/client/co_client.h b/lib/crowdb-rpc/include/crowdb-rpc/client/co_client.h index 9517c7119..ade85eec1 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/client/co_client.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/client/co_client.h @@ -46,6 +46,9 @@ typedef bool (*crowdb_rpc_co_on_response)(void *ctx, uint64_t request_id, crowdb // the given connection (round-robin if multiple). The coroutines run // until either rust_build_request or rust_on_response returns false. // Blocks until all coroutines complete. +// Configure the client's timeout reaper before calling this for requests +// that may stall. Timeouts complete the pending request normally; a pending +// coroutine is never force-resumed or destroyed while a callback owns it. // // The client must have its completion pool sized // (crowdb_rpc_client_set_completion_pool_size) to >= num_coroutines diff --git a/lib/crowdb-rpc/src/client/co_client.cpp b/lib/crowdb-rpc/src/client/co_client.cpp index 55079866c..5880f5442 100644 --- a/lib/crowdb-rpc/src/client/co_client.cpp +++ b/lib/crowdb-rpc/src/client/co_client.cpp @@ -3,6 +3,7 @@ #include "crowdb-rpc/client/co_client.h" +#include "co_state.h" #include "crowdb-rpc/buffer.h" #include "crowdb-rpc/c_api.h" #include "crowdb-rpc/c_api_internal.h" @@ -24,175 +25,6 @@ namespace crowdb::rpc { -// Coroutine state machine: synchronizes co_on_complete (I/O worker -// thread) with await_suspend (coroutine thread). The race is: -// 1. Coroutine submits request, then tries to suspend. -// 2. I/O worker receives response, calls co_on_complete. -// 3. If co_on_complete runs before await_suspend sets the handle, -// resuming a null handle crashes. -// The state machine ensures co_on_complete only resumes when the -// coroutine is actually suspended (handle is valid). If the response -// arrives while RUNNING, it sets RESPONSE_READY — the coroutine sees -// this in await_suspend and doesn't suspend (returns false → resumes). -enum CoStateFlag : int { - CO_IDLE = 0, // coroutine not started or already exited - CO_RUNNING = 1, // coroutine is running (handle may be stale) - CO_SUSPENDED = 2, // coroutine is suspended, handle is valid - CO_RESPONSE_READY = 3, // response arrived while RUNNING — don't suspend -}; - -// Per-coroutine state. Allocated on the heap (one per coroutine), -// lives until the coroutine exits + is joined. -struct CoState -{ - // Rust callbacks. - crowdb_rpc_co_build_request build_fn; - crowdb_rpc_co_on_response on_response_fn; - void *rust_ctx; - - // C++ handles. - RpcClient *client; - Transport *transport; - Connection *conn; - uint16_t msg_type; - - // Per-coroutine stats. - uint64_t total_ops = 0; - uint64_t total_errors = 0; - uint64_t total_latency_ns = 0; - uint64_t min_latency_ns = UINT64_MAX; - uint64_t max_latency_ns = 0; - - // The awaitable for the current in-flight request. Filled by - // co_on_complete; read by the coroutine after resume. - crowdb_rpc_buffer_t resp_control = nullptr; - crowdb_rpc_buffer_t resp_data = nullptr; - crowdb_rpc_status resp_status = CROWDB_RPC_OK; - std::coroutine_handle<> handle; // set by await_suspend - - std::atomic *running; - - // State machine: IDLE → RUNNING → SUSPENDED ↔ RESPONSE_READY → IDLE. - // Replaces the old `active` boolean — see CoStateFlag above. - std::atomic co_state{CO_IDLE}; - - // Slot index for this coroutine (fixed, no collision with others). - // request_id = slot_index + iteration * (pool_mask + 1). - // This guarantees each coroutine always uses the same slab slot, - // so no two coroutines ever collide on the same slot. - size_t slot_index = 0; - size_t pool_mask = 0; - uint64_t next_req_id = 0; // per-coroutine request_id counter -}; - -// The C++ → Rust callback that resumes the coroutine. Set as the slab -// slot's callback. Called inline on the I/O worker thread when the -// response arrives. -[[maybe_unused]] static void co_on_complete(uint64_t /*request_id*/, crowdb_rpc_buffer_t control, - crowdb_rpc_buffer_t data, crowdb_rpc_status status, void *user_data) -{ - auto *s = static_cast(user_data); - // Store the response — the coroutine reads it after resume (or - // immediately if it hasn't suspended yet). - s->resp_control = control; - s->resp_data = data; - s->resp_status = status; - // Try to transition SUSPENDED → RESPONSE_READY. If successful, - // the coroutine was suspended (handle is valid) — resume it. - int expected = CO_SUSPENDED; - if (s->co_state.compare_exchange_strong(expected, CO_RESPONSE_READY, std::memory_order_acq_rel)) { - s->handle.resume(); - return; - } - // If RUNNING, the coroutine hasn't suspended yet (handle may be - // null). Set RESPONSE_READY so await_suspend sees it and doesn't - // suspend — the coroutine picks up the response immediately. - if (expected == CO_RUNNING) { - s->co_state.store(CO_RESPONSE_READY, std::memory_order_release); - return; - } - // IDLE or RESPONSE_READY — late/duplicate response, drop it. -} - -// Awaitable: suspends the coroutine, resumed by co_on_complete. -// The CoState pointer is passed to the awaitable so co_on_complete -// can store the response + resume. -struct CoAwait -{ - CoState *state; - - [[nodiscard]] constexpr bool await_ready() const noexcept - { - return false; - } - - bool await_suspend(std::coroutine_handle<> h) const noexcept - { - state->handle = h; - // Try to transition RUNNING → SUSPENDED. If successful, the - // response hasn't arrived yet — suspend and wait for - // co_on_complete to resume us. - int expected = CO_RUNNING; - if (state->co_state.compare_exchange_strong(expected, CO_SUSPENDED, std::memory_order_acq_rel)) { - return true; // suspend - } - // State is not RUNNING — co_on_complete already set - // RESPONSE_READY (response arrived before we suspended). - // Don't suspend — resume immediately. The response is already - // stored in state->resp_*. - return false; // don't suspend — resume inline - } - - void await_resume() const noexcept - { - // Coroutine is about to continue — mark it as running. - state->co_state.store(CO_RUNNING, std::memory_order_release); - } -}; - -// ── Coroutine task type ─────────────────────────────────────────── -// -// We use a minimal coroutine type. The promise stores nothing — all -// state is in the CoState struct, accessed via the captured pointer -// in the coroutine body. The coroutine body is a lambda that captures -// CoState* by value. - -struct CoTask -{ - struct promise_type - { - CoTask get_return_object() - { - return CoTask{std::coroutine_handle::from_promise(*this)}; - } - - std::suspend_always initial_suspend() noexcept - { - return {}; - } - - std::suspend_always final_suspend() noexcept - { - return {}; - } - - void return_void() noexcept - { - } - - void unhandled_exception() noexcept - { - std::abort(); - } - }; - - std::coroutine_handle handle; - - explicit CoTask(std::coroutine_handle h) : handle(h) - { - } -}; - // The coroutine body. Captures CoState* by value (it's a pointer, // stable on the heap). Loops: build → submit → co_await → on_response. // @@ -202,7 +34,7 @@ struct CoTask static CoTask co_run(CoState *s) { s->co_state.store(CO_RUNNING, std::memory_order_release); - while (s->running->load(std::memory_order_relaxed)) { + while (true) { // 1. Build request via Rust callback. crowdb_rpc_buffer_t control = nullptr; crowdb_rpc_buffer_t data = nullptr; @@ -215,10 +47,8 @@ static CoTask co_run(CoState *s) } // 2. Record start time (after build, before submit). - // e2e = now - start measures submit + network + parse + callback. - // This excludes the build time (negligible for echo). The design - // suggests using rpc_create_nano from the echoed response, but - // that requires passing it through the C callback ABI. + // Measures submission, transport and response parsing on the client + // clock. Request construction and the Rust response handler are outside. auto start = std::chrono::steady_clock::now(); // 3. Submit via send(). user_data = s (CoState*). @@ -245,10 +75,9 @@ static CoTask co_run(CoState *s) // the inner Buffer* via release_frame. Just yield and retry. s->total_errors++; rpc_submit_retry().inc(); - // Yield this thread — suspend until resumed. We use a - // self-resume: set handle, then suspend. A helper thread - // (or the spin-wait loop) will resume us shortly. - // For now, just yield the OS thread. + if (!s->conn->is_open()) { + break; + } std::this_thread::yield(); continue; } @@ -309,7 +138,7 @@ extern "C" void crowdb_rpc_co_spawn(crowdb_rpc_client_t client, crowdb_rpc_serve auto *client_ptr = client->client.get(); auto *transport = server->server->transport(); - std::atomic running{true}; + std::atomic finished_count{0}; // Create + start N coroutines. std::vector tasks; @@ -334,7 +163,7 @@ extern "C" void crowdb_rpc_co_spawn(crowdb_rpc_client_t client, crowdb_rpc_serve state->transport = transport; state->conn = conns[i % num_conns]->conn.get(); state->msg_type = msg_type; - state->running = &running; + state->finished_count = &finished_count; state->slot_index = i; state->pool_mask = pool_mask; state->next_req_id = 0; @@ -363,31 +192,17 @@ extern "C" void crowdb_rpc_co_spawn(crowdb_rpc_client_t client, crowdb_rpc_serve // Wait for all coroutines to complete. The coroutines exit when // build_fn or on_response_fn returns false (Rust sets this when // the deadline is reached). - auto wait_start = std::chrono::steady_clock::now(); - while (true) { - uint32_t not_done = 0; - for (auto &task : tasks) { - if (!task.handle.done()) { - not_done++; - } - } - if (not_done == 0) { - break; - } - // After 5s past the expected deadline, force-stop: set running=false - // and resume any still-suspended coroutines. - auto elapsed = std::chrono::steady_clock::now() - wait_start; - if (elapsed > std::chrono::seconds(30)) { - std::fprintf(stderr, "co_spawn: timeout, %u coroutines still running, forcing exit\n", not_done); - running.store(false, std::memory_order_release); - for (auto &task : tasks) { - if (!task.handle.done()) { - task.handle.resume(); - } - } - break; + while (finished_count.load(std::memory_order_acquire) != num_coroutines) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + // Final suspension publishes statistics, but a completing callback can + // still be unwinding after resume(). Keep both state and Rust context + // alive until every callback has returned. Request timeouts use RpcClient's + // reaper and ordinary completion path; never resume a pending await here. + for (const auto &state : states) { + while (state->active_callbacks.load(std::memory_order_acquire) != 0) { + std::this_thread::yield(); } - std::this_thread::yield(); } // Aggregate stats. diff --git a/lib/crowdb-rpc/src/client/co_state.h b/lib/crowdb-rpc/src/client/co_state.h new file mode 100644 index 000000000..a1d43dcd9 --- /dev/null +++ b/lib/crowdb-rpc/src/client/co_state.h @@ -0,0 +1,180 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/client/co_client.h" + +#include +#include +#include + +namespace crowdb::rpc +{ + +class RpcClient; +class Transport; +class Connection; + +enum CoStateFlag : int { CO_IDLE, CO_RUNNING, CO_SUSPENDED, CO_RESPONSE_READY }; + +struct CoState +{ + crowdb_rpc_co_build_request build_fn = nullptr; + crowdb_rpc_co_on_response on_response_fn = nullptr; + void *rust_ctx = nullptr; + RpcClient *client = nullptr; + Transport *transport = nullptr; + Connection *conn = nullptr; + uint16_t msg_type = 0; + + uint64_t total_ops = 0; + uint64_t total_errors = 0; + uint64_t total_latency_ns = 0; + uint64_t min_latency_ns = UINT64_MAX; + uint64_t max_latency_ns = 0; + + crowdb_rpc_buffer_t resp_control = nullptr; + crowdb_rpc_buffer_t resp_data = nullptr; + crowdb_rpc_status resp_status = CROWDB_RPC_OK; + std::coroutine_handle<> handle; + std::atomic *finished_count = nullptr; + std::atomic active_callbacks{0}; + std::atomic co_state{CO_IDLE}; + size_t slot_index = 0; + size_t pool_mask = 0; + uint64_t next_req_id = 0; +}; + +inline void co_drop_response(crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data) +{ + if (control != nullptr) { + crowdb_rpc_buffer_release(control); + } + if (data != nullptr) { + crowdb_rpc_buffer_release(data); + } +} + +struct CoCallbackOperation +{ + CoState *state; + + explicit CoCallbackOperation(CoState *owner) : state(owner) + { + state->active_callbacks.fetch_add(1, std::memory_order_acquire); + } + + ~CoCallbackOperation() + { + state->active_callbacks.fetch_sub(1, std::memory_order_release); + } + + CoCallbackOperation(const CoCallbackOperation &) = delete; + CoCallbackOperation &operator=(const CoCallbackOperation &) = delete; +}; + +// RpcClient claims each request completion once. Publication must be one +// atomic operation: a failed CAS followed by store could miss a concurrent +// RUNNING -> SUSPENDED transition and leave the coroutine asleep forever. +inline void co_on_complete(uint64_t, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, crowdb_rpc_status status, + void *user_data) +{ + auto *state = static_cast(user_data); + const CoCallbackOperation operation(state); + if (state->co_state.load(std::memory_order_acquire) == CO_IDLE) { + co_drop_response(control, data); + return; + } + state->resp_control = control; + state->resp_data = data; + state->resp_status = status; + const int previous = state->co_state.exchange(CO_RESPONSE_READY, std::memory_order_acq_rel); + if (previous == CO_SUSPENDED) { + state->handle.resume(); + } +} + +struct CoAwait +{ + CoState *state; + + [[nodiscard]] constexpr bool await_ready() const noexcept + { + return false; + } + + bool await_suspend(std::coroutine_handle<> handle) const noexcept + { + state->handle = handle; + int expected = CO_RUNNING; + return state->co_state.compare_exchange_strong(expected, CO_SUSPENDED, std::memory_order_acq_rel); + } + + void await_resume() const noexcept + { + state->co_state.store(CO_RUNNING, std::memory_order_release); + } +}; + +struct CoFinalAwait +{ + CoState *state; + + [[nodiscard]] constexpr bool await_ready() const noexcept + { + return false; + } + + void await_suspend(std::coroutine_handle<>) const noexcept + { + // The coroutine is suspended before publication. Do not access the + // frame after publishing; the joining thread may destroy it. + state->co_state.store(CO_IDLE, std::memory_order_release); + state->finished_count->fetch_add(1, std::memory_order_release); + } + + void await_resume() const noexcept + { + } +}; + +struct CoTask +{ + struct promise_type + { + CoState *state; + + explicit promise_type(CoState *owner) : state(owner) + { + } + + CoTask get_return_object() + { + return CoTask{std::coroutine_handle::from_promise(*this)}; + } + + std::suspend_always initial_suspend() noexcept + { + return {}; + } + + CoFinalAwait final_suspend() noexcept + { + return {state}; + } + + void return_void() noexcept + { + } + + void unhandled_exception() noexcept + { + std::abort(); + } + }; + + std::coroutine_handle handle; +}; + +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/tests/co_state_test.cpp b/lib/crowdb-rpc/tests/co_state_test.cpp new file mode 100644 index 000000000..3ca533bd5 --- /dev/null +++ b/lib/crowdb-rpc/tests/co_state_test.cpp @@ -0,0 +1,48 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "common/test_co_state.h" + +#include + +#include + +using namespace crowdb::rpc; + +TEST(CoroutineCompletion, ResponseBeforeSuspendPublishesOnce) +{ + TestCoroutine coroutine; + coroutine.complete(); + EXPECT_EQ(coroutine.state.co_state.load(), CO_RESPONSE_READY); + coroutine.task.handle.resume(); + EXPECT_EQ(coroutine.finished.load(), 1U); + EXPECT_EQ(coroutine.state.total_ops, 1U); + EXPECT_EQ(coroutine.state.active_callbacks.load(), 0U); +} + +TEST(CoroutineCompletion, ResponseAfterSuspendResumesOnce) +{ + TestCoroutine coroutine; + coroutine.task.handle.resume(); + EXPECT_EQ(coroutine.state.co_state.load(), CO_SUSPENDED); + coroutine.complete(); + EXPECT_EQ(coroutine.finished.load(), 1U); + EXPECT_EQ(coroutine.state.total_ops, 1U); + EXPECT_EQ(coroutine.state.active_callbacks.load(), 0U); + coroutine.complete(); + EXPECT_EQ(coroutine.finished.load(), 1U); + EXPECT_EQ(coroutine.state.co_state.load(), CO_IDLE); +} + +TEST(CoroutineCompletion, ConcurrentPublicationCannotLoseResume) +{ + for (int iteration = 0; iteration < 2000; ++iteration) { + TestCoroutine coroutine; + std::thread completing([&coroutine] { coroutine.complete(); }); + coroutine.task.handle.resume(); + completing.join(); + ASSERT_EQ(coroutine.finished.load(std::memory_order_acquire), 1U) << iteration; + EXPECT_EQ(coroutine.state.total_ops, 1U); + EXPECT_EQ(coroutine.state.active_callbacks.load(), 0U); + } +} diff --git a/lib/crowdb-rpc/tests/common/test_co_state.h b/lib/crowdb-rpc/tests/common/test_co_state.h new file mode 100644 index 000000000..0e947a992 --- /dev/null +++ b/lib/crowdb-rpc/tests/common/test_co_state.h @@ -0,0 +1,42 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "../../src/client/co_state.h" + +namespace crowdb::rpc +{ + +inline CoTask test_co_wait(CoState *state) +{ + co_await CoAwait{state}; + ++state->total_ops; + co_drop_response(state->resp_control, state->resp_data); +} + +struct TestCoroutine +{ + std::atomic finished{0}; + CoState state; + CoTask task; + + TestCoroutine() + { + state.finished_count = &finished; + state.co_state.store(CO_RUNNING); + task = test_co_wait(&state); + } + + ~TestCoroutine() + { + task.handle.destroy(); + } + + void complete() + { + co_on_complete(1, nullptr, nullptr, CROWDB_RPC_OK, &state); + } +}; + +} // namespace crowdb::rpc From ddb79155f24e866e3a74f40531c5216af3e48f48 Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 22 Oct 2026 20:52:21 +0800 Subject: [PATCH 024/258] Assign bounded completion queues to dedicated RDMA pollers --- doc/design/rpc/design-crowdb-rpc-rdma.md | 91 ++++++---- doc/working/plan-rdma-rpc-transport.md | 145 +++++++++------- .../include/crowdb-rpc/connection.h | 27 ++- .../transport/rdma/rdma_transport.h | 115 +++++++++---- lib/crowdb-rpc/src/connection.cpp | 10 ++ .../src/transport/rdma/rdma_completion.cpp | 125 +++++++++----- .../src/transport/rdma/rdma_config.cpp | 20 ++- .../src/transport/rdma/rdma_connection.cpp | 136 +++++++-------- lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp | 94 +++++++++++ lib/crowdb-rpc/src/transport/rdma/rdma_cq.h | 42 +++++ .../src/transport/rdma/rdma_poller.cpp | 124 ++++++++++++++ lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp | 57 +++++++ .../src/transport/rdma/rdma_transport.cpp | 135 ++++++++------- .../src/transport/rdma/rdma_work_request.cpp | 79 ++++++--- .../tests/common/test_rdma_transport.h | 45 ++++- lib/crowdb-rpc/tests/rdma_capability_test.cpp | 19 +++ lib/crowdb-rpc/tests/rdma_completion_test.cpp | 157 +++++++++++++++++- lib/crowdb-rpc/tests/rdma_config_test.cpp | 43 +++++ lib/crowdb-rpc/tests/rdma_timed_load_test.cpp | 13 +- 19 files changed, 1113 insertions(+), 364 deletions(-) create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_cq.h create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index f916586af..ca89084cc 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -109,36 +109,45 @@ message. A slot is never posted twice concurrently. ## 5. Shared CQ and completion model -Connections share a bounded CQ pool. Each CQ owns one completion channel and -is assigned to a polling worker. Every QP uses the same shared CQ for SEND and -RECV completions. The CQ maps `qp_num` to the owning RPC connection; a -completion never requires scanning every connection CQ. - -The polling worker is the primary completion path: - -1. polls the assigned CQ in a bounded busy-poll loop; -2. dispatches all currently available completions; -3. after the configured empty-poll budget, arms CQ notification with - `ibv_req_notify_cq`; -4. polls once more to close the notification race; -5. registers the CQ completion fd with epoll only while idle. - -Epoll is a wakeup path, not the completion processor. When the CQ completion -fd becomes readable, the worker acknowledges the event, removes the idle -state, and returns to polling. The RDMA-CM event fd follows the same worker -ownership rule but remains a connection-management event source. - -The worker processes a CQ batch as follows: - -1. resolves the completion's `qp_num` to a connection; -2. resolves the work request through its stable `wr_id` record; -3. dispatches SEND or RECV completion; -4. returns or quarantines the slot according to completion status. - -The RPC library controls polling. FFI receives only a completion callback or a -buffer lease after C++ has validated the frame and resolved ownership. A -connection does not own a CQ or completion channel; it owns only its QP and -slot rings. +Connections share a bounded CQ pool. Each QP uses one assigned CQ for both +SEND and RECV. Each CQ has its own completion channel, generation-tagged WR +table, wake descriptor and fixed polling/event thread. QP admission reserves +`sq_depth + posted_receive_target` entries on its assigned CQ; insufficient +capacity rejects setup. Failed setup rolls the reservation back, and QP +retirement releases it. Each WR retains its connection, arena owners, buffers +and request metadata. Its token resolves the CQ partition, local entry and +generation without scanning connections or trusting an error CQE's opcode. + +Normal completion processing belongs to the assigned CQ thread: + +1. poll bounded verbs batches and dispatch completions; +2. while traffic continues, return directly to polling; +3. after the empty-poll budget, arm notification with `ibv_req_notify_cq`; +4. poll again to close the notification race; +5. enter the CQ's private epoll wait only when still idle; +6. acknowledge completion events after waking and resume polling. + +The completion and wake descriptors stay registered with the private epoll +engine. Generic socket workers handle the separately serialized CM channel and +do not drain managed CQs. CM retirement can acquire exclusive CQ ownership to +drain acknowledged responses before closing a connection, then destroy its QP +and cancel unresolved work. Per-connection posting admission prevents QP +destruction from racing SEND, receive posting or repost. Stop wakes and joins +CQ threads before destroying their QPs, CQs and channels. + +Completed receives that precede application handler installation stay in a +bounded deferred list with their WR and arena owners. Publishing the handler +wakes the CQ thread; only then are those frames dispatched. Connection +capability frames are consumed by the common connection seam before application +routing. Handlerless data is never discarded merely because CM setup and CQ +processing ran on different threads. + +A completed WR entry is released before invoking application callbacks or +releasing original send-buffer owners. Immediate replies and receive reposts +can therefore reuse bounded WR admission. The registered slot itself remains +leased until all frame/C++/FFI owners release it. Work cancellation and deferred +receive retirement take the same exclusive CQ ownership gate. FFI receives a +callback or buffer lease only after frame validation and ownership resolution. ## 6. Failure and fallback @@ -174,14 +183,24 @@ changing the lease contract. The transport configuration supplies defaults when the transport is created: ```toml -rdma_wire_slot_size = "1MiB" -rdma_send_slot_count = 64 -rdma_receive_slot_count = 64 -rdma_cq_count = 4 -rdma_poll_empty_budget = 50000 -rdma_registered_bytes = "128MiB" +wire_slot_capacity = 1048576 +send_arena_bytes = 67108864 +receive_arena_bytes = 1073741824 +sq_depth = 1024 +rq_depth = 1024 +posted_receive_target = 1024 +cq_count = 1 +cq_depth = 8192 +poll_batch = 256 +verbs_poll_batch = 32 +poll_empty_budget = 256 ``` +CQ depth is per CQ; arena budgets and SQ/RQ depths are per QP. For example, +one 8192-entry CQ admits four QPs with the default send and posted-receive +limits. `poller_count` follows the number of initialized CQs. Small-message +benchmarks select 1024-byte slots and matching arena budgets explicitly. + The effective slot counts are bounded by registered bytes, QP send/receive depth, DiskIO queue bytes, and SDK-held slots. A configured slot size is a capacity limit, not a required wire length; SEND uses the actual message diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index e4365d064..77696ca0d 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -13,53 +13,30 @@ worker per CQ, using the existing RPC regression runners. ## Progress -The transport implementation now includes validated endpoint capability data -(protocol, generation, device, GID, port, and supported capacity classes), -explicit RDMA path states, and a pre-payload-only TCP fallback predicate. -Implemented since this checkpoint: active/passive RDMA-CM setup, RC QP -creation, one-MR arena-backed wire slots, one-SGE SEND/RECV posting, bounded receive posting, and the initial completion-channel -polling with CQ acknowledgement, error quarantine, validated receive-frame -dispatch into `Connection::on_frame`, and epoll external-fd registration. -The CQ path now uses one transport-wide shared CQ for all QPs. Its completion -channel is registered with one epoll engine; the worker drains completions with -a bounded busy-poll budget and rearms notification after the queue is idle. -Epoll is therefore only the wakeup path, while completion dispatch remains in -the CQ drain. Stop now wakes the owning engine, CQ polling and completion-error -counters are exposed, and CM disconnect events close and retire their connection -pools before QP/ID teardown. The OCI runner now also has an unprivileged -shared-HCA profile with explicit verbs/CM devices, IPC_LOCK, and bounded -memlock. Capability handshake bytes -have a fixed wire representation with round-trip tests. Negotiated capability -exchange through the existing TCP handshake, common transport path -selection/fallback, receive repost after owner release, and two-peer OCI E2E -execution are now wired at the RPC seam: TCP connections exchange the fixed -capability frame, connections retain the peer capability, RDMA-enabled -connections are selected before TCP fallback, and completed RDMA frames retain -their buffers until the owning `Frame` is destroyed, then repost the receive -pair. The live host HCA local-loop CM/QP/send/receive flow now passes, including -receive repost and connection-level CQ handling. The actual `RpcServer`/ -`RpcClient` worker path now passes a live request/response round trip with -multiple sequential frames on one QP, including CQ wakeup, frame dispatch, -request correlation, response completion, and clean teardown. Remaining -acceptance includes failure-injection coverage, pre-payload TCP fallback proof, -stable completion ownership mapping, and executing the single-node -multi-container OCI profiles against a built `crowdb:test` image. Physical -multi-node validation is deferred to [the separate follow-up plan](plan-rdma-multi-node.md). - -The completion ownership mapping now uses a bounded, generation-tagged -`WorkRequest` table for SEND and RECV completions. It retains the connection, -request, stream, sequence, and logical offset. Error dispatch uses the retained -work kind rather than an undefined completion opcode. A receive lease is -reposted by `Buffer::last_release_cb` only after the final C++/FFI owner drops -it; held caller leases survive frame deletion and transport shutdown. - -The current code also preposts receives before `rdma_connect`/`rdma_accept`, -uses a short configurable RNR timer, drains acknowledged CQ work before CM -close, and fixes CQ registration and buffer-pool teardown ordering. The earlier -64-request burst exposed RNR stalls and premature test-server teardown. The -worker test now has a client-finished barrier, configurable workload size, -worker count, QP count, bounded in-flight window, per-request payload validation, -and TPS/P50/P95/P99 output. +The registered-buffer foundation and initial RC SEND/RECV flow are committed. +SEND/RECV tokens retain stable owners, receive leases wait for final C++/FFI +release, and strict RDMA tests reject TCP fallback. Nanosecond histogram and +coroutine handoff/joining fixes are also committed and verified below. + +The current CQ implementation assigns each QP to one bounded CQ, with one +fixed polling/event thread per CQ and separate CM event handling. Early +receives wait for application handler publication; QP teardown gates concurrent +posting. Per-connection arena ownership no longer occupies application +`Connection::user_data`. CQ and provider-returned QP capacity values are +retained, and poll-call counters use per-CQ shards. The CQ diff passed focused +tests, live HCA diagnostics, the full C++/FFI gate, formatting and workspace +clippy and scoped tree-lint. Tree-lint exits zero with style warnings. + +Still pending: application reply allocation independent of receive admission, +real PD/MR lifetime after held leases and transport teardown, bounded transport +aggregation, `bench rdma`/shared coroutine reporting, unified regression +orchestration, negotiated TCP capability preflight, failure injection, and the +two-container acceptance profiles. Capability serialization and automatic +consumption at the connection seam do not establish negotiated preflight: +`connect` still attempts CM before the required TCP preflight. Do not treat the +historical benchmark comments or older subtask ledger as completed acceptance. +Physical multi-node validation remains outside this scope in +[the follow-up plan](plan-rdma-multi-node.md). ## Current continuation checkpoint @@ -198,21 +175,19 @@ latency histogram to resolve sub-65-us RTTs. it allocates replies from `conn->pool()`, which is currently the RDMA receive pool, while the diagnostic echo uses `server.pool()`. A completely preposted receive pool leaves no allocation admission for those replies. -- [ ] **Unify histogram semantics**: use nanosecond input, fixed bounded +- [x] **Unify histogram semantics**: use nanosecond input, fixed bounded memory, mergeable per-polling-owner shards, explicit zero/overflow counts, exact count/sum/min/max, and avg/P50/P90/P95/P99/P99.9 output. Remove the common C++/Rust histograms' 65.5-us underflow floor; do not retain a separate benchmark-only precision contract. Specify a shared bucket layout and golden vectors. Record timing origin and exclude warmup/drain from TPS. -- [ ] **Replace competing CQ consumers before expanding the performance matrix**: - current `RdmaTransport` has one `cq_` per peer, no dedicated polling - thread, and competing epoll workers protected by `cq_polling_`. - `RdmaTransport::start()` only changes lifecycle state. The recommended - agreed target is one assigned polling/event thread per CQ, with QPs assigned - to one CQ, bounded busy polling under traffic, and arm/poll/epoll sleep - after the idle budget. Add `cq_count`; derive `poller_count = cq_count` - rather than exposing two inconsistent knobs. Compare native and OCI runs with the same - manifest; neither single-host topology establishes physical-node isolation. +- [x] **Replace competing CQ consumers before expanding the performance matrix**: + `RdmaTransport` now assigns QPs to a bounded CQ pool and starts one fixed + polling/event thread per CQ. Generic epoll workers handle CM events only. + Busy polling continues under traffic; idle workers arm, poll again, then + sleep on their private epoll engine. `poller_count` derives from `cq_count`. + Native and OCI comparison with the same manifest remains in phase E; + neither single-host topology establishes physical-node isolation. Required result schema (client/server resource values are separate): @@ -221,7 +196,7 @@ Required result schema (client/server resource values are separate): | Identity | case_id, run_id, git_sha, binary hashes, topology, CPU/HCA identity, CPU affinity, actual path, fallback policy/count | | Workload | client_mode, loader_count, producer_threads, in_flight_per_loader, connection_count, qpair_count, logical_ops_per_rpc, payload_bytes, wire_bytes, aggregation_enabled, aggregation_max_bytes, aggregation_max_frames, frames_per_send | | Scheduling | io_engines, io_workers, cq_count, poller_count, poll_batch, verbs_poll_batch, poll_empty_budget, epoll_wakeups, CQ event/poll counts | -| Capacity | requested/effective sq_depth, rq_depth, posted_receive_target, cq_depth, wire_slot_capacity, arena bytes, queue/pool high-water marks | +| Capacity | requested/effective sq_depth, rq_depth, posted_receive_target, cq_depth, wire_slot_capacity, arena bytes, application_buffer_capacity, send_queue_capacity, queue/pool high-water marks | | Measurement | warmup_s, measure_s, repeat_index, latency_scope, submitted, completed_in_window, drain_completed, rpc_tps, logical_ops_s, payload_Bps | | Latency | latency_count, avg_ns, p50_ns, p90_ns, p95_ns, p99_ns, p999_ns, min_ns, max_ns, histogram_zero_count, histogram_overflow_count | | Errors | errors_total, setup_errors, submit_rejects, timeouts, connection_errors, cq_errors, payload_mismatches, duplicate_completions, retries, report_errors, client_exit, server_exit, valid | @@ -324,7 +299,7 @@ Verified histogram checkpoint: ### B. CQ owners and resource ownership -- [~] **Create the bounded CQ pool**: add `RdmaConfig::cq_count`, +- [x] **Create the bounded CQ pool**: add `RdmaConfig::cq_count`, `verbs_poll_batch`, and strict environment validation. Each CQ owns its completion channel, wake fd, fixed poller and bounded WR table. Assign each QP to one CQ at creation; SEND/RECV and repost retain that assignment. @@ -332,7 +307,7 @@ Verified histogram checkpoint: CQ capacity for the configured QPs and posted receives before traffic. Files: `rdma_transport.h`, `rdma_config.cpp`, `rdma_connection.cpp`, `rdma_work_request.cpp`, new private CQ implementation under `transport/rdma/`. -- [ ] **Run one polling/event worker per CQ**: continuously poll while busy, +- [x] **Run one polling/event worker per CQ**: continuously poll while busy, dispatch callbacks on the owner, arm notification after the empty budget, poll again before sleeping, acknowledge events, and wake promptly on submit and stop. Keep CM management separate from CQ completion ownership; generic @@ -340,7 +315,22 @@ Verified histogram checkpoint: drain/quarantine WRs, then destroy QPs/CQs/channels. Test idle/busy transitions, arm races, contention, multi-CQ mapping, partial setup and repeated teardown. Files: RDMA transport/completion sources, `socket_transport.cpp`, RPC tests. -- [ ] **Separate application and receive allocators**: built-in echo and + Retain arenas in the connection's private transport context; leave + `Connection::user_data` to its caller. Gate per-QP posting before destruction, + and publish migrated CM IDs together with their owner map under the existing + lifecycle mutex. Reclaim a completed WR entry before invoking an application + callback or releasing source-buffer owners, so exact-depth handoffs can + admit a reply/repost. Poll-call counters count verbs invocations and use + per-CQ atomics with aggregation on status reads. + A CQ worker may receive data before CM returns and the caller installs its + frame handler. Retain those completed receives in a bounded per-CQ deferred + list, with their WR/connection/pool owners, until handler publication wakes + the CQ owner. Connection-level capability frames are consumed at the common + connection seam. Cancellation takes exclusive CQ ownership to retire deferred + work; shutdown joins pollers before destroying QPs/CQs. No hot-path lock is + introduced. Test early receive/late handler, deferred cancellation, fixed-owner + busy/idle wakeups and late buffer release. +- [~] **Separate application and receive allocators**: built-in echo and default handlers allocate from the common application pool, not a fully preposted receive arena. Receive posting and repost explicitly use the connection's retained RDMA receive pool. Test `posted_receive_target == @@ -351,6 +341,37 @@ Verified histogram checkpoint: shutdown; no deregister-before-release, double repost, premature recycle or silent PD leak. Files: RDMA buffer pool and lifecycle sources, lease tests. +CQ verification: + +- 27 focused capability/configuration/completion tests pass, including CQ + reservation rollback, partitioned tokens, independent polling gates, deferred + receive ownership, wake-on-handler-publication, cancellation and exact-depth + callback admission. A fixture allocation failure was diagnosed in GDB and + fixed by separating application-buffer admission from the WR exhaustion + being tested; it was not a transport CQ crash. +- The final diagnostic uses two native processes on one HCA, two QPs/two CQs + per peer, one polling thread per CQ, one generic I/O worker per peer, two + producer threads, 32 in flight each, 128-byte payload, 1 KiB slots, + SQ/RQ/post target 1024, CQ depth 4096 per CQ, and 1 MiB send/receive arenas + per QP. After 1 s warmup, 10 s measured traffic completed 8,278,071 RPCs: + 827,807 TPS, avg 74.92 us, P90 86.02 us, P99 97.28 us, zero RPC errors and + zero histogram overflow; 62 requests drained afterward. Actual QP/CQ/poller + counts are asserted. This is a transport diagnostic, not the primary CLI + coroutine max-TPS result. Artifact and binary/source identity: + `.crowdb-runtime/artifacts/rdma/20261022T124101Z/`. +- A prior successful diagnostic at + `.crowdb-runtime/artifacts/rdma/20261022T122640Z/` measured 829,227 TPS with + its retained binary hash. The subsequent source revision corrected/sharded + poll-call statistics; do not combine measurements from these two binaries. +- `.crowdb-runtime/artifacts/rdma/20261022T122423Z/` is invalid: the runner + started with the previous binary after a failed incremental build. Its + `INVALID.md` explicitly excludes it from acceptance/comparisons. Later runs + require a successful build before launch and retain binary/source hashes. +- The full C++ gate and all 29 Rust RPC FFI tests pass on the final CQ source. + Complete logs and results: + `.crowdb-runtime/artifacts/unit/cpp-c333ccb1d15145bfadc725c037c7be19/`. + Workspace clippy and changed C++ formatting also pass. + ### C. Bounded aggregation - [ ] **Coalesce complete frames on the CQ owner**: use bounded lock-free @@ -458,7 +479,7 @@ Completed: - connection-level PD and registered send/receive slot pools; - initial receive posting, CQ completion dispatch, owner-release repost, and RNR retry configuration; -- RDMA-to-TCP capability negotiation and pre-payload fallback wiring; +- capability serialization and explicit pre-payload fallback policy; TCP preflight remains pending; - epoll registration for CM and CQ wakeup events; - live HCA local-loop CM/QP/send/receive E2E with all frames received and clean CQ status, and clean client/server exit; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/connection.h b/lib/crowdb-rpc/include/crowdb-rpc/connection.h index 4f1d2fe16..8d3109085 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/connection.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/connection.h @@ -186,7 +186,25 @@ class Connection : public std::enable_shared_from_this // Callbacks (set by RpcClient / RpcServer). void set_on_frame(OnFrameCallback cb) { + const bool ready = static_cast(cb); on_frame_callback_.set(std::move(cb)); + const auto *notify = on_frame_ready_.get(); + if (ready && notify != nullptr && *notify) { + (*notify)(); + } + } + + [[nodiscard]] bool has_frame_handler() const + { + const auto *callback = on_frame_callback_.get(); + return callback != nullptr && static_cast(*callback); + } + + // The transport installs this before publishing the connection. Handler + // publication wakes an owner retaining an early completed receive. + void set_frame_ready_notifier(std::function notify) + { + on_frame_ready_.set(std::move(notify)); } void set_on_close(OnCloseCallback cb) @@ -250,6 +268,8 @@ class Connection : public std::enable_shared_from_this } private: + friend class RdmaTransport; + std::shared_ptr transport_context_; int64_t id_; std::string name_; BufferPool *pool_; @@ -285,9 +305,10 @@ class Connection : public std::enable_shared_from_this OutFrame *pending_frames_[BATCH_MAX]; int pending_count_{0}; - ConnectionCallback on_frame_callback_; - ConnectionCallback on_close_callback_; - ConnectionCallback on_rpc_close_callback_; + ConnectionCallback on_frame_callback_; + ConnectionCallback> on_frame_ready_; + ConnectionCallback on_close_callback_; + ConnectionCallback on_rpc_close_callback_; }; } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index deea3cb06..61c68f5fe 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -34,6 +34,7 @@ struct RdmaConfig std::string device; uint8_t port = 1; uint32_t cq_depth = 8192; + uint32_t cq_count = 1; uint32_t wire_slot_capacity = RDMA_WIRE_SLOT_CAPACITY; uint64_t send_arena_bytes = 64ULL << 20; uint64_t receive_arena_bytes = 1ULL << 30; @@ -42,6 +43,7 @@ struct RdmaConfig uint32_t qp_max_send_sge = 2; uint32_t qp_max_recv_sge = 2; uint32_t poll_batch = 256; + uint32_t verbs_poll_batch = 32; uint32_t poll_empty_budget = 256; uint32_t shutdown_drain_ms = 3000; uint32_t posted_receive_target = 1024; @@ -124,6 +126,10 @@ class RdmaTransport : public Transport std::atomic local_protection_errors{0}; std::atomic retry_exceeded_errors{0}; std::atomic remote_access_errors{0}; + std::atomic poller_count{0}; + std::atomic cq_events{0}; + std::atomic cq_wakeups{0}; + std::atomic pending_deferred_receives{0}; }; explicit RdmaTransport(RdmaConfig config = {}); @@ -146,10 +152,10 @@ class RdmaTransport : public Transport // with the common RPC worker before submitting frames. std::shared_ptr accept(); - // Enable completion handling. CQ draining is driven by the epoll worker. + // Start one dedicated polling/event worker per initialized CQ. void start(); void stop(); - [[nodiscard]] int completion_fd() const; + [[nodiscard]] int completion_fd(uint32_t index = 0) const; [[nodiscard]] int cm_event_fd() const; int drain_cq(uint32_t max_completions = 0); void register_events(SocketEngine *engine); @@ -162,17 +168,19 @@ class RdmaTransport : public Transport [[nodiscard]] const Stats &stats() const { + refresh_poll_stats(); return stats_; } Stats &stats() { + refresh_poll_stats(); return stats_; } [[nodiscard]] bool ready() const { - return capability_available_; + return capability_available_.load(std::memory_order_acquire); } private: @@ -180,14 +188,12 @@ class RdmaTransport : public Transport friend struct TestRdmaTransport; # endif RdmaConfig config_; - bool capability_available_ = false; - struct ibv_context *context_ = nullptr; - bool owns_context_ = true; - struct ibv_pd *pd_ = nullptr; - struct ibv_comp_channel *comp_channel_ = nullptr; - struct ibv_cq *cq_ = nullptr; - struct rdma_event_channel *cm_channel_ = nullptr; - struct rdma_cm_id *listen_id_ = nullptr; + std::atomic capability_available_{false}; + struct ibv_context *context_ = nullptr; + bool owns_context_ = true; + struct ibv_pd *pd_ = nullptr; + struct rdma_event_channel *cm_channel_ = nullptr; + struct rdma_cm_id *listen_id_ = nullptr; std::string requested_device_; uint8_t port_ = 1; std::vector cm_ids_; @@ -200,39 +206,74 @@ class RdmaTransport : public Transport ConnectionHandler connection_handler_; std::atomic running_{false}; - std::atomic_flag cq_polling_ = ATOMIC_FLAG_INIT; - std::atomic_flag cm_polling_ = ATOMIC_FLAG_INIT; - int continuation_fd_ = -1; + std::atomic_flag cm_polling_ = ATOMIC_FLAG_INIT; - bool prime_receive(const std::shared_ptr &connection); - Stats stats_; + bool prime_receive(const std::shared_ptr &connection); + mutable Stats stats_; + void refresh_poll_stats() const; enum class WorkKind : uint8_t { Send, Receive }; + struct ConnectionPools; struct WorkRequest { - std::atomic token{0}; - WorkKind kind; - std::shared_ptr conn; - Buffer *slot = nullptr; - OutFrame *frame = nullptr; - uint64_t request_id = 0; - uint64_t stream_id = 0; - uint64_t sequence = 0; - uint64_t logical_offset = 0; + std::atomic token{0}; + WorkKind kind; + std::shared_ptr conn; + std::shared_ptr pools; + Buffer *slot = nullptr; + OutFrame *frame = nullptr; + uint64_t request_id = 0; + uint64_t stream_id = 0; + uint64_t sequence = 0; + uint64_t logical_offset = 0; }; - static constexpr uint64_t WORK_RESERVED = UINT64_MAX; - std::unique_ptr work_requests_; - std::atomic next_work_{0}; - std::atomic work_generation_{1}; + static constexpr uint64_t WORK_RESERVED = UINT64_MAX; + + struct CompletionQueue; + + struct CqReservation + { + CompletionQueue *queue; + uint32_t capacity; + bool committed = false; + + CqReservation(CompletionQueue *owner, uint32_t entries) : queue(owner), capacity(entries) + { + } + + ~CqReservation(); + CqReservation(const CqReservation &) = delete; + CqReservation &operator=(const CqReservation &) = delete; + }; - WorkRequest *acquire_work(WorkKind kind, Connection *conn, Buffer *slot, OutFrame *frame = nullptr); - WorkRequest *claim_work(uint64_t token); - void release_work(WorkRequest *work); - void dispatch_completion(const struct ibv_wc &wc); - void cancel_work(Connection *connection = nullptr); - void close_after_drain(Connection *connection); + std::vector> completion_queues_; + std::atomic next_cq_{0}; + CompletionQueue *reserve_cq(uint32_t capacity); + bool create_completion_queues(ibv_context *context); + void destroy_completion_queues(); + int drain_queue(uint32_t index, uint32_t max_completions, bool wait_for_owner = false); + uint32_t process_deferred(CompletionQueue &queue, uint32_t max_completions); + void cancel_deferred(CompletionQueue &queue, Connection *connection); + void start_pollers(); + void run_poller(CompletionQueue *queue); + void notify_queue(uint32_t index); + void install_frame_notifier(const std::shared_ptr &connection, uint32_t index); + static constexpr uint64_t CM_EVENT_TAG = 1; + static constexpr uint64_t CQ_EVENT_BASE = 2; + + WorkRequest *acquire_work(WorkKind kind, Connection *conn, Buffer *slot, OutFrame *frame = nullptr); + WorkRequest *claim_work(uint64_t token); + void release_work(WorkRequest *work); + void dispatch_completion(const struct ibv_wc &wc); + void cancel_work(Connection *connection = nullptr); + void close_after_drain(Connection *connection); + void destroy_connection_qp(Connection *connection, rdma_cm_id *id); + static ConnectionPools *pools_for(Connection *connection); + static void set_connection_pools(Connection *connection, std::shared_ptr pools); + bool publish_cm_connection(rdma_cm_id *id, const std::shared_ptr &connection, + std::shared_ptr pools); struct RepostState { @@ -276,8 +317,12 @@ class RdmaTransport : public Transport struct ConnectionPools : std::enable_shared_from_this { + ibv_qp_cap effective_capacity{}; + std::shared_ptr posting_state = std::make_shared(); std::unique_ptr send; std::unique_ptr recv; + uint32_t cq_index = 0; + uint32_t reserved_capacity = 0; }; struct RepostContext diff --git a/lib/crowdb-rpc/src/connection.cpp b/lib/crowdb-rpc/src/connection.cpp index a98c2e1d6..c4c1be418 100644 --- a/lib/crowdb-rpc/src/connection.cpp +++ b/lib/crowdb-rpc/src/connection.cpp @@ -422,9 +422,19 @@ void Connection::close() void Connection::on_frame(Frame *frame) { + if (frame->header.msg_type == RDMA_HANDSHAKE_MSG_TYPE) { + RdmaEndpointCapability capability; + if (parse_rdma_handshake(frame->control, &capability)) { + set_peer_capability(capability); + } + delete frame; + return; + } if (const auto *callback = on_frame_callback_.get(); callback != nullptr && *callback) { (*callback)(frame, this); + return; } + delete frame; } } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp index 31920a6f3..87b48b101 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp @@ -4,6 +4,7 @@ #include "crowdb-common/log.h" #include "crowdb-rpc/transport/rdma/rdma_transport.h" #include "crowdb-rpc/transport/socket_transport.h" +#include "rdma_cq.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -16,29 +17,46 @@ namespace crowdb::rpc int RdmaTransport::drain_cq(uint32_t max_completions) { - if (cq_ == nullptr || cq_polling_.test_and_set(std::memory_order_acquire)) + int drained = 0; + for (uint32_t index = 0; index < completion_queues_.size(); ++index) + drained += drain_queue(index, max_completions); + return drained; +} + +int RdmaTransport::drain_queue(uint32_t index, uint32_t max_completions, bool wait_for_owner) +{ + auto &queue = *completion_queues_[index]; + if (queue.cq == nullptr) + return 0; + if (wait_for_owner) { + while (queue.polling.test_and_set(std::memory_order_acquire)) + std::this_thread::yield(); + } + else if (queue.polling.test_and_set(std::memory_order_acquire)) { return 0; + } // Only the CQ consumer may clear its continuation. A competing worker // must leave the wakeup readable when it cannot acquire the CQ. - if (continuation_fd_ >= 0) { + if (queue.continuation_fd >= 0) { uint64_t value = 0; - (void)::read(continuation_fd_, &value, sizeof(value)); + (void)::read(queue.continuation_fd, &value, sizeof(value)); } if (max_completions == 0) max_completions = std::max(1U, config_.poll_batch); - stats_.cq_polls.fetch_add(1, std::memory_order_relaxed); - uint32_t drained = 0; - uint32_t empty_polls = 0; - const uint32_t kEmptyPollBudget = config_.poll_empty_budget; - std::array completions{}; - auto poll_one = [&](ibv_cq *cq) { + uint32_t drained = process_deferred(queue, max_completions); + uint32_t empty_polls = 0; + const uint32_t kEmptyPollBudget = config_.poll_empty_budget; + std::array completions{}; + auto poll_one = [&](ibv_cq *cq) { while (drained < max_completions && cq != nullptr) { - const int batch = ibv_poll_cq(cq, static_cast(std::min(32, max_completions - drained)), - completions.data()); + queue.poll_calls.fetch_add(1, std::memory_order_relaxed); + const int batch = + ibv_poll_cq(cq, static_cast(std::min(config_.verbs_poll_batch, max_completions - drained)), + completions.data()); if (batch < 0) break; if (batch == 0) { - stats_.cq_empty_polls.fetch_add(1, std::memory_order_relaxed); + queue.empty_polls.fetch_add(1, std::memory_order_relaxed); if (++empty_polls >= kEmptyPollBudget) break; continue; @@ -50,24 +68,30 @@ int RdmaTransport::drain_cq(uint32_t max_completions) drained += static_cast(batch); } }; - if (comp_channel_ != nullptr) { + const bool managed = queue.managed.load(std::memory_order_acquire); + if (queue.channel != nullptr && (!managed || queue.event_wakeup.exchange(false))) { ibv_cq *event_cq = nullptr; void *context = nullptr; - while (ibv_get_cq_event(comp_channel_, &event_cq, &context) == 0) { + while (ibv_get_cq_event(queue.channel, &event_cq, &context) == 0) { ibv_ack_cq_events(event_cq, 1); + stats_.cq_events.fetch_add(1, std::memory_order_relaxed); } } - poll_one(cq_); - (void)ibv_req_notify_cq(cq_, 0); + poll_one(queue.cq); + if (managed && drained > 0) { + queue.polling.clear(std::memory_order_release); + return static_cast(drained); + } + (void)ibv_req_notify_cq(queue.cq, 0); // Completions arriving between the empty poll and notification arming // need one final poll. A full batch gets an explicit epoll continuation. empty_polls = 0; - poll_one(cq_); - if (drained == max_completions && continuation_fd_ >= 0) { + poll_one(queue.cq); + if (!managed && drained == max_completions && queue.continuation_fd >= 0) { const uint64_t value = 1; - (void)::write(continuation_fd_, &value, sizeof(value)); + (void)::write(queue.continuation_fd, &value, sizeof(value)); } - cq_polling_.clear(std::memory_order_release); + queue.polling.clear(std::memory_order_release); return static_cast(drained); } @@ -75,26 +99,33 @@ void RdmaTransport::register_events(SocketEngine *engine) { if (engine == nullptr) return; - // A shared CQ has one completion channel. Register that channel with one - // epoll engine only; registering the same fd in every engine causes - // duplicate wakeups and lets different workers race the CQ event. if (event_engine_ != nullptr && event_engine_ != engine) return; event_engine_ = engine; - if (completion_fd() >= 0) - engine->add_external_fd(completion_fd(), 1); if (cm_event_fd() >= 0) - engine->add_external_fd(cm_event_fd(), 2); - if (continuation_fd_ >= 0) - engine->add_external_fd(continuation_fd_, 3); + engine->add_external_fd(cm_event_fd(), CM_EVENT_TAG); + for (const auto &queue : completion_queues_) { + if (queue->managed.load(std::memory_order_acquire)) + continue; + const uint64_t tag = CQ_EVENT_BASE + 2 * queue->index; + if (queue->channel != nullptr) + engine->add_external_fd(queue->channel->fd, tag); + if (queue->continuation_fd >= 0) + engine->add_external_fd(queue->continuation_fd, tag + 1); + } + if (running_.load(std::memory_order_acquire)) + start_pollers(); } int RdmaTransport::handle_external_event(uint64_t tag, uint32_t max_completions) { - if (tag == 1 || tag == 3) { - return drain_cq(max_completions); + if (tag >= CQ_EVENT_BASE && tag < CQ_EVENT_BASE + 2 * config_.cq_count) { + const auto index = static_cast((tag - CQ_EVENT_BASE) / 2); + if (completion_queues_[index]->managed.load(std::memory_order_acquire)) + return 0; + return drain_queue(index, max_completions); } - if (tag == 2) { + if (tag == CM_EVENT_TAG) { // The shared CM channel has one consumer while passive setup waits // for ESTABLISHED. Another worker must not consume that event. if (cm_polling_.test_and_set(std::memory_order_acquire)) @@ -223,16 +254,17 @@ void RdmaTransport::release_completed_receive(WorkRequest *work, const ibv_wc &w delete frame; return; } - auto pools = work->conn->user_data == nullptr - ? nullptr - : static_cast(work->conn->user_data)->shared_from_this(); - auto *repost = new RepostContext{repost_state_, work->conn, std::move(pools)}; + auto connection = std::move(work->conn); + auto pools = std::move(work->pools); + auto *repost = new RepostContext{repost_state_, connection, std::move(pools)}; frame->data_buf = work->slot; work->slot->last_release_cb = &RdmaTransport::repost_frame_release; work->slot->last_release_ctx = repost; - work->conn->on_frame(frame); - stats_.receives.fetch_add(1, std::memory_order_relaxed); release_work(work); + // A callback may immediately release/repost the slot or submit a reply. + // Return this completed WR entry before either can need bounded admission. + stats_.receives.fetch_add(1, std::memory_order_relaxed); + connection->on_frame(frame); } void RdmaTransport::repost_frame_release(void *context, Buffer *slot) @@ -251,7 +283,10 @@ void RdmaTransport::repost_frame_release(void *context, Buffer *slot) void RdmaTransport::release_completed_frame(WorkRequest *work, const ibv_wc &wc) { - OutFrame *frame = work->frame; + OutFrame *frame = work->frame; + auto connection = std::move(work->conn); + const auto pools = std::move(work->pools); + Buffer *slot = work->slot; if (wc.status != IBV_WC_SUCCESS) { record_completion_error(wc); @@ -261,22 +296,22 @@ void RdmaTransport::release_completed_frame(WorkRequest *work, const ibv_wc &wc) (void)work->slot->quarantine(); for (uint8_t i = 0; i < frame->data_views(); ++i) (void)frame->data_view(i)->quarantine(); - if (work->conn != nullptr) - work->conn->close(); + if (connection != nullptr) + connection->close(); } else if (work->slot != nullptr && !work->slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available)) { (void)work->slot->quarantine(); - if (work->conn != nullptr) - work->conn->close(); + if (connection != nullptr) + connection->close(); } - if (work->slot != nullptr) - work->slot->release(); + release_work(work); + if (slot != nullptr) + slot->release(); if (frame->control != nullptr) frame->control->release(); frame->release_data(); delete frame; - release_work(work); } } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp index a80c97492..4d2815f60 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp @@ -32,21 +32,27 @@ bool RdmaConfig::apply_environment() destination = static_cast(parsed); return true; }; - return read("CROWDB_RDMA_SQ_DEPTH", sq_depth) && read("CROWDB_RDMA_RQ_DEPTH", rq_depth) && - read("CROWDB_RDMA_POSTED_RECEIVE_TARGET", posted_receive_target) && + return (port != 1 || read("CROWDB_RDMA_PORT", port)) && read("CROWDB_RDMA_SQ_DEPTH", sq_depth) && + read("CROWDB_RDMA_RQ_DEPTH", rq_depth) && read("CROWDB_RDMA_POSTED_RECEIVE_TARGET", posted_receive_target) && read("CROWDB_RDMA_WIRE_SLOT_CAPACITY", wire_slot_capacity) && read("CROWDB_RDMA_SEND_ARENA_BYTES", send_arena_bytes) && - read("CROWDB_RDMA_RECEIVE_ARENA_BYTES", receive_arena_bytes) && read("CROWDB_RDMA_CQ_DEPTH", cq_depth); + read("CROWDB_RDMA_RECEIVE_ARENA_BYTES", receive_arena_bytes) && read("CROWDB_RDMA_CQ_DEPTH", cq_depth) && + read("CROWDB_RDMA_CQ_COUNT", cq_count) && read("CROWDB_RDMA_POLL_BATCH", poll_batch) && + read("CROWDB_RDMA_VERBS_POLL_BATCH", verbs_poll_batch) && + read("CROWDB_RDMA_POLL_EMPTY_BUDGET", poll_empty_budget); } bool RdmaConfig::valid() const { BufferCapacityClass capacity_class; uint32_t capacity = 0; - return classify_rdma_capacity(wire_slot_capacity, &capacity_class, &capacity) && capacity == wire_slot_capacity && - sq_depth > 0 && rq_depth > 0 && posted_receive_target > 0 && posted_receive_target <= rq_depth && - cq_depth > posted_receive_target && cq_depth <= static_cast(INT32_MAX) && - send_arena_bytes >= wire_slot_capacity && + return port > 0 && classify_rdma_capacity(wire_slot_capacity, &capacity_class, &capacity) && + capacity == wire_slot_capacity && sq_depth > 0 && rq_depth > 0 && posted_receive_target > 0 && + posted_receive_target <= rq_depth && cq_depth > posted_receive_target && + cq_depth <= static_cast(INT32_MAX) && cq_count > 0 && cq_count <= 256 && + static_cast(cq_count) * cq_depth < UINT32_MAX && poll_batch > 0 && + poll_batch <= static_cast(INT32_MAX) && verbs_poll_batch > 0 && verbs_poll_batch <= 256 && + poll_empty_budget > 0 && send_arena_bytes >= wire_slot_capacity && receive_arena_bytes >= static_cast(posted_receive_target) * wire_slot_capacity && min_rnr_timer <= 31 && retry_count <= 7 && rnr_retry_count <= 7; } diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp index cee33286b..a0a4662af 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp @@ -4,6 +4,7 @@ #include "crowdb-common/log.h" #include "crowdb-rpc/transport/rdma/rdma_transport.h" #include "rdma_channel.h" +#include "rdma_cq.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -20,8 +21,7 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int { if (!ready() || addr.empty() || port <= 0 || port > 65535) { - CRB_LOG_WARN("rdma connect: transport unavailable pd={} cq={} addr={} port={}", static_cast(pd_), - static_cast(cq_), addr, port); + CRB_LOG_WARN("rdma connect: transport unavailable addr={} port={}", addr, port); return nullptr; } { @@ -116,9 +116,17 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int rdma_destroy_id(id); return nullptr; } + const uint64_t required_capacity = static_cast(config_.sq_depth) + config_.posted_receive_target; + CqReservation reservation(required_capacity > UINT32_MAX ? nullptr + : reserve_cq(static_cast(required_capacity)), + static_cast(required_capacity)); + if (reservation.queue == nullptr) { + rdma_destroy_id(id); + return nullptr; + } ibv_qp_init_attr qp_attr{}; - qp_attr.send_cq = cq_; - qp_attr.recv_cq = cq_; + qp_attr.send_cq = reservation.queue->cq; + qp_attr.recv_cq = reservation.queue->cq; qp_attr.qp_type = IBV_QPT_RC; qp_attr.cap.max_send_wr = std::min(config_.sq_depth, static_cast(device_attributes.max_qp_wr)); qp_attr.cap.max_recv_wr = std::min(config_.rq_depth, static_cast(device_attributes.max_qp_wr)); @@ -133,22 +141,25 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int rdma_destroy_id(id); return nullptr; } - auto pools = std::make_shared(); - pools->send = std::make_unique(id->pd, config_.send_arena_bytes, config_.sq_depth); - pools->recv = std::make_unique(id->pd, config_.receive_arena_bytes, config_.rq_depth); + auto pools = std::make_shared(); + pools->effective_capacity = qp_attr.cap; + pools->cq_index = reservation.queue->index; + pools->reserved_capacity = reservation.capacity; + pools->send = std::make_unique(id->pd, config_.send_arena_bytes, config_.sq_depth); + pools->recv = std::make_unique(id->pd, config_.receive_arena_bytes, config_.rq_depth); if (pools->send == nullptr || pools->recv == nullptr) { rdma_destroy_qp(id); rdma_destroy_id(id); return nullptr; } - auto connection = std::make_shared(reinterpret_cast(id), addr, pools->recv.get()); - connection->user_data = pools.get(); + auto connection = std::make_shared(reinterpret_cast(id), addr, pools->recv.get()); + set_connection_pools(connection.get(), pools); + reservation.committed = true; connection->set_path(ConnectionPath::Rdma); connection->transport_handle = reinterpret_cast(id->qp); + install_frame_notifier(connection, pools->cq_index); if (!prime_receive(connection)) { - connection->close(); - rdma_destroy_qp(id); - cancel_work(connection.get()); + destroy_connection_qp(connection.get(), id); rdma_destroy_id(id); return nullptr; } @@ -160,9 +171,7 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int if (rdma_connect(id, ¶ms) != 0) { CRB_LOG_WARN("rdma connect: rdma_connect failed addr={} errno={}", addr, errno); - connection->close(); - rdma_destroy_qp(id); - cancel_work(connection.get()); + destroy_connection_qp(connection.get(), id); rdma_destroy_id(id); return nullptr; } @@ -173,34 +182,22 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int event == nullptr ? -1 : event->event, event == nullptr ? 0 : event->status, errno); if (event != nullptr) rdma_ack_cm_event(event); - connection->close(); - rdma_destroy_qp(id); - cancel_work(connection.get()); + destroy_connection_qp(connection.get(), id); rdma_destroy_id(id); return nullptr; } rdma_ack_cm_event(event); - if (rdma_migrate_id(id, cm_channel_) != 0) { - connection->close(); - rdma_destroy_qp(id); - cancel_work(connection.get()); - rdma_destroy_id(id); - return nullptr; - } ibv_qp_attr receive_attributes{}; receive_attributes.min_rnr_timer = config_.min_rnr_timer; if (ibv_modify_qp(id->qp, &receive_attributes, IBV_QP_MIN_RNR_TIMER) != 0) { - connection->close(); - rdma_destroy_qp(id); - cancel_work(connection.get()); + destroy_connection_qp(connection.get(), id); rdma_destroy_id(id); return nullptr; } - { - std::scoped_lock lock(cm_mu_); - cm_ids_.push_back(id); - cm_connections_.emplace(id, connection); - cm_pools_.emplace(id, std::move(pools)); + if (!publish_cm_connection(id, connection, std::move(pools))) { + destroy_connection_qp(connection.get(), id); + rdma_destroy_id(id); + return nullptr; } return connection; } @@ -266,16 +263,15 @@ std::shared_ptr RdmaTransport::accept() } cm_ids_.erase(std::remove(cm_ids_.begin(), cm_ids_.end(), event_id), cm_ids_.end()); } - if (closed_connection != nullptr) - close_after_drain(closed_connection.get()); + if (closed_connection == nullptr) + continue; + close_after_drain(closed_connection.get()); if (event_type == RDMA_CM_EVENT_DISCONNECTED || event_type == RDMA_CM_EVENT_CONNECT_ERROR) stats_.cm_disconnects.fetch_add(1, std::memory_order_relaxed); - if (event_id->qp != nullptr) + if (closed_connection != nullptr) + destroy_connection_qp(closed_connection.get(), event_id); + else if (event_id->qp != nullptr) rdma_destroy_qp(event_id); - if (closed_connection != nullptr) { - closed_connection->transport_handle = 0; - cancel_work(closed_connection.get()); - } rdma_destroy_id(event_id); } } @@ -301,9 +297,17 @@ std::shared_ptr RdmaTransport::accept() return nullptr; } + const uint64_t required_capacity = static_cast(config_.sq_depth) + config_.posted_receive_target; + CqReservation reservation(required_capacity > UINT32_MAX ? nullptr + : reserve_cq(static_cast(required_capacity)), + static_cast(required_capacity)); + if (reservation.queue == nullptr) { + rdma_destroy_id(id); + return nullptr; + } ibv_qp_init_attr qp_attr{}; - qp_attr.send_cq = cq_; - qp_attr.recv_cq = cq_; + qp_attr.send_cq = reservation.queue->cq; + qp_attr.recv_cq = reservation.queue->cq; qp_attr.qp_type = IBV_QPT_RC; qp_attr.cap.max_send_wr = config_.sq_depth; qp_attr.cap.max_recv_wr = config_.rq_depth; @@ -317,22 +321,25 @@ std::shared_ptr RdmaTransport::accept() rdma_destroy_id(id); return nullptr; } - auto pools = std::make_shared(); - pools->send = std::make_unique(id->pd, config_.send_arena_bytes, config_.sq_depth); - pools->recv = std::make_unique(id->pd, config_.receive_arena_bytes, config_.rq_depth); + auto pools = std::make_shared(); + pools->effective_capacity = qp_attr.cap; + pools->cq_index = reservation.queue->index; + pools->reserved_capacity = reservation.capacity; + pools->send = std::make_unique(id->pd, config_.send_arena_bytes, config_.sq_depth); + pools->recv = std::make_unique(id->pd, config_.receive_arena_bytes, config_.rq_depth); if (pools->send == nullptr || pools->recv == nullptr) { rdma_destroy_qp(id); rdma_destroy_id(id); return nullptr; } - auto connection = std::make_shared(reinterpret_cast(id), "rdma-peer", pools->recv.get()); - connection->user_data = pools.get(); + auto connection = std::make_shared(reinterpret_cast(id), "rdma-peer", pools->recv.get()); + set_connection_pools(connection.get(), pools); + reservation.committed = true; connection->set_path(ConnectionPath::Rdma); connection->transport_handle = reinterpret_cast(id->qp); + install_frame_notifier(connection, pools->cq_index); if (!prime_receive(connection)) { - connection->close(); - rdma_destroy_qp(id); - cancel_work(connection.get()); + destroy_connection_qp(connection.get(), id); rdma_destroy_id(id); return nullptr; } @@ -346,9 +353,7 @@ std::shared_ptr RdmaTransport::accept() if (accept_rc != 0) { CRB_LOG_WARN("rdma accept: rdma_accept failed errno={}", errno); - connection->close(); - rdma_destroy_qp(id); - cancel_work(connection.get()); + destroy_connection_qp(connection.get(), id); rdma_destroy_id(id); return nullptr; } @@ -360,36 +365,23 @@ std::shared_ptr RdmaTransport::accept() event == nullptr ? -1 : event->event, event == nullptr ? 0 : event->status, errno); if (event != nullptr) rdma_ack_cm_event(event); - connection->close(); - rdma_destroy_qp(id); - cancel_work(connection.get()); + destroy_connection_qp(connection.get(), id); rdma_destroy_id(id); return nullptr; } rdma_ack_cm_event(event); - if (rdma_migrate_id(id, cm_channel_) != 0) { - connection->close(); - rdma_destroy_qp(id); - cancel_work(connection.get()); - rdma_destroy_id(id); - return nullptr; - } - ibv_qp_attr receive_attributes{}; receive_attributes.min_rnr_timer = config_.min_rnr_timer; if (ibv_modify_qp(id->qp, &receive_attributes, IBV_QP_MIN_RNR_TIMER) != 0) { - connection->close(); - rdma_destroy_qp(id); - cancel_work(connection.get()); + destroy_connection_qp(connection.get(), id); rdma_destroy_id(id); return nullptr; } - { - std::scoped_lock lock(cm_mu_); - cm_ids_.push_back(id); - cm_connections_.emplace(id, connection); - cm_pools_.emplace(id, std::move(pools)); + if (!publish_cm_connection(id, connection, std::move(pools))) { + destroy_connection_qp(connection.get(), id); + rdma_destroy_id(id); + return nullptr; } return connection; } diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp new file mode 100644 index 000000000..ee3e97844 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp @@ -0,0 +1,94 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_cq.h" + +#include "crowdb-rpc/transport/rdma/rdma_transport.h" + +#ifdef CROWDB_RPC_HAVE_RDMA + +# include "crowdb-common/log.h" +# include "rdma_channel.h" + +# include + +# include + +namespace crowdb::rpc +{ + +RdmaTransport::CqReservation::~CqReservation() +{ + if (queue != nullptr && !committed) { + queue->reserved_capacity.fetch_sub(capacity, std::memory_order_release); + } +} + +RdmaTransport::CompletionQueue *RdmaTransport::reserve_cq(uint32_t capacity) +{ + const uint32_t first = next_cq_.fetch_add(1, std::memory_order_relaxed) % config_.cq_count; + for (uint32_t offset = 0; offset < config_.cq_count; ++offset) { + auto &queue = *completion_queues_[(first + offset) % config_.cq_count]; + uint32_t used = queue.reserved_capacity.load(std::memory_order_relaxed); + while (used <= config_.cq_depth && capacity <= config_.cq_depth - used) { + if (queue.reserved_capacity.compare_exchange_weak(used, used + capacity, std::memory_order_acq_rel)) { + return &queue; + } + } + } + CRB_LOG_WARN("rdma CQ capacity exhausted: cq_count={} cq_depth={} required_entries={}", config_.cq_count, + config_.cq_depth, capacity); + return nullptr; +} + +bool RdmaTransport::create_completion_queues(ibv_context *context) +{ + for (const auto &owner : completion_queues_) { + auto &queue = *owner; + queue.channel = ibv_create_comp_channel(context); + if (queue.channel == nullptr) { + destroy_completion_queues(); + return false; + } + rdma_detail::make_nonblocking(queue.channel->fd); + queue.cq = ibv_create_cq(context, static_cast(config_.cq_depth), &queue, queue.channel, 0); + if (queue.cq == nullptr || std::cmp_less(queue.cq->cqe, config_.cq_depth) || + ibv_req_notify_cq(queue.cq, 0) != 0) { + destroy_completion_queues(); + return false; + } + queue.effective_depth = static_cast(queue.cq->cqe); + } + return true; +} + +void RdmaTransport::refresh_poll_stats() const +{ + uint64_t polls = 0; + uint64_t empty = 0; + for (const auto &queue : completion_queues_) { + polls += queue->poll_calls.load(std::memory_order_relaxed); + empty += queue->empty_polls.load(std::memory_order_relaxed); + } + stats_.cq_polls.store(polls, std::memory_order_relaxed); + stats_.cq_empty_polls.store(empty, std::memory_order_relaxed); +} + +void RdmaTransport::destroy_completion_queues() +{ + for (const auto &owner : completion_queues_) { + auto &queue = *owner; + if (queue.cq != nullptr) { + ibv_destroy_cq(queue.cq); + queue.cq = nullptr; + } + if (queue.channel != nullptr) { + ibv_destroy_comp_channel(queue.channel); + queue.channel = nullptr; + } + } +} + +} // namespace crowdb::rpc + +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h new file mode 100644 index 000000000..f2a3b0732 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h @@ -0,0 +1,42 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/transport/rdma/rdma_transport.h" + +#ifdef CROWDB_RPC_HAVE_RDMA +# include "crowdb-rpc/transport/socket_transport.h" + +namespace crowdb::rpc +{ +struct RdmaTransport::CompletionQueue +{ + uint32_t index = 0; + uint32_t effective_depth = 0; + ibv_cq *cq = nullptr; + ibv_comp_channel *channel = nullptr; + int continuation_fd = -1; + std::unique_ptr work_requests; + + struct DeferredReceive + { + WorkRequest *work; + ibv_wc completion; + }; + + std::vector deferred; + std::unique_ptr engine; + std::thread worker; + std::atomic managed{false}; + std::atomic event_wakeup{true}; + std::atomic_flag polling = ATOMIC_FLAG_INIT; + std::atomic next_work{0}; + std::atomic generation{1}; + std::atomic reserved_capacity{0}; + std::atomic poll_calls{0}; + std::atomic empty_polls{0}; +}; + +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp new file mode 100644 index 000000000..cfe9714b3 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp @@ -0,0 +1,124 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_cq.h" + +#ifdef CROWDB_RPC_HAVE_RDMA + +# include "crowdb-common/log.h" +# include "crowdb-rpc/transport/epoll/epoll_engine.h" + +# include + +namespace crowdb::rpc +{ + +void RdmaTransport::start_pollers() +{ + for (const auto &owner : completion_queues_) { + auto &queue = *owner; + if (queue.cq == nullptr || queue.worker.joinable()) + continue; + auto engine = std::make_unique(); + if (engine->init() != 0) { + capability_available_.store(false, std::memory_order_release); + CRB_LOG_ERROR("rdma CQ poller epoll initialization failed; RDMA setup unavailable: cq_index={}", + queue.index); + return; + } + if (event_engine_ != nullptr) { + if (queue.channel != nullptr) + event_engine_->remove_external_fd(queue.channel->fd); + event_engine_->remove_external_fd(queue.continuation_fd); + } + if (queue.channel != nullptr) + engine->add_external_fd(queue.channel->fd, 1); + engine->add_external_fd(queue.continuation_fd, 2); + queue.engine = std::move(engine); + queue.managed.store(true, std::memory_order_release); + queue.event_wakeup.store(true, std::memory_order_release); + try { + queue.worker = std::thread([this, pointer = &queue] { run_poller(pointer); }); + } + catch (const std::system_error &error) { + queue.managed.store(false, std::memory_order_release); + capability_available_.store(false, std::memory_order_release); + CRB_LOG_ERROR("rdma CQ poller thread creation failed; RDMA setup unavailable: cq_index={} error={}", + queue.index, error.what()); + return; + } + stats_.poller_count.fetch_add(1, std::memory_order_relaxed); + } +} + +void RdmaTransport::run_poller(CompletionQueue *queue) +{ + const std::string name = "rdma-cq-" + std::to_string(queue->index); + crowdb::common::set_current_thread_name(name.c_str()); + std::array events{}; + while (running_.load(std::memory_order_acquire)) { + if (drain_queue(queue->index, config_.poll_batch) > 0) + continue; + const int count = queue->engine->wait(events.data(), events.size(), 1000); + if (count > 0) { + queue->event_wakeup.store(true, std::memory_order_release); + stats_.cq_wakeups.fetch_add(1, std::memory_order_relaxed); + } + } +} + +void RdmaTransport::notify_queue(uint32_t index) +{ + const uint64_t value = 1; + (void)::write(completion_queues_[index]->continuation_fd, &value, sizeof(value)); +} + +void RdmaTransport::install_frame_notifier(const std::shared_ptr &connection, uint32_t index) +{ + connection->set_frame_ready_notifier([state = repost_state_, index] { + Operation operation(state); + if (operation) + state->transport->notify_queue(index); + }); +} + +uint32_t RdmaTransport::process_deferred(CompletionQueue &queue, uint32_t max_completions) +{ + uint32_t completed = 0; + size_t retained = 0; + for (auto pending : queue.deferred) { + if (completed < max_completions && + (pending.work->conn->has_frame_handler() || !pending.work->conn->is_open())) { + if (!pending.work->conn->is_open()) + pending.completion.status = IBV_WC_WR_FLUSH_ERR; + release_completed_receive(pending.work, pending.completion); + stats_.pending_deferred_receives.fetch_sub(1, std::memory_order_relaxed); + ++completed; + } + else { + queue.deferred[retained++] = pending; + } + } + queue.deferred.resize(retained); + return completed; +} + +void RdmaTransport::cancel_deferred(CompletionQueue &queue, Connection *connection) +{ + size_t retained = 0; + for (auto pending : queue.deferred) { + if (connection == nullptr || pending.work->conn.get() == connection) { + pending.completion.status = IBV_WC_WR_FLUSH_ERR; + release_completed_receive(pending.work, pending.completion); + stats_.pending_deferred_receives.fetch_sub(1, std::memory_order_relaxed); + } + else { + queue.deferred[retained++] = pending; + } + } + queue.deferred.resize(retained); +} + +} // namespace crowdb::rpc + +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp new file mode 100644 index 000000000..b78973dac --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_cq.h" + +#ifdef CROWDB_RPC_HAVE_RDMA + +namespace crowdb::rpc +{ + +RdmaTransport::ConnectionPools *RdmaTransport::pools_for(Connection *connection) +{ + return static_cast(connection->transport_context_.get()); +} + +void RdmaTransport::set_connection_pools(Connection *connection, std::shared_ptr pools) +{ + connection->transport_context_ = std::move(pools); +} + +bool RdmaTransport::publish_cm_connection(rdma_cm_id *id, const std::shared_ptr &connection, + std::shared_ptr pools) +{ + // The shared CM consumer must see the owner before a migrated terminal + // event can retire this ID. Setup events stay private until this point. + std::scoped_lock lock(cm_mu_); + if (rdma_migrate_id(id, cm_channel_) != 0) + return false; + cm_ids_.push_back(id); + cm_connections_.emplace(id, connection); + cm_pools_.emplace(id, std::move(pools)); + return true; +} + +void RdmaTransport::destroy_connection_qp(Connection *connection, rdma_cm_id *id) +{ + connection->close(); + auto *pools = pools_for(connection); + if (pools != nullptr) { + pools->posting_state->users.fetch_or(RepostState::STOPPED, std::memory_order_acq_rel); + while ((pools->posting_state->users.load(std::memory_order_acquire) & ~RepostState::STOPPED) != 0) + std::this_thread::yield(); + } + if (id->qp != nullptr) + rdma_destroy_qp(id); + connection->transport_handle = 0; + cancel_work(connection); + if (pools != nullptr && pools->reserved_capacity != 0) { + completion_queues_[pools->cq_index]->reserved_capacity.fetch_sub(pools->reserved_capacity, + std::memory_order_release); + pools->reserved_capacity = 0; + } +} + +} // namespace crowdb::rpc + +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index ec10539d5..11ee15ce4 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -6,6 +6,7 @@ #include "crowdb-common/log.h" #include "crowdb-rpc/transport/socket_transport.h" #include "rdma_channel.h" +#include "rdma_cq.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -31,10 +32,18 @@ namespace crowdb::rpc RdmaTransport::RdmaTransport(RdmaConfig config) : config_(std::move(config)) { const bool valid_config = config_.apply_environment() && config_.valid(); - if (!valid_config) + if (!valid_config) { config_.cq_depth = 1; - work_requests_ = std::make_unique(config_.cq_depth); - continuation_fd_ = ::eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + config_.cq_count = 1; + } + for (uint32_t index = 0; index < config_.cq_count; ++index) { + auto queue = std::make_unique(); + queue->index = index; + queue->work_requests = std::make_unique(config_.cq_depth); + queue->deferred.reserve(config_.cq_depth); + queue->continuation_fd = ::eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + completion_queues_.push_back(std::move(queue)); + } repost_state_ = std::make_shared(); repost_state_->transport = this; if (!valid_config) { @@ -46,12 +55,7 @@ RdmaTransport::RdmaTransport(RdmaConfig config) : config_(std::move(config)) config_.device = requested_device; } requested_device_ = config_.device; - if (const char *requested_port = std::getenv("CROWDB_RDMA_PORT"); requested_port != nullptr) { - const int value = std::atoi(requested_port); - if (value > 0 && value <= 255 && config.port == 1) - config_.port = static_cast(value); - } - port_ = config_.port; + port_ = config_.port; int device_count = 0; ibv_device **devices = ibv_get_device_list(&device_count); @@ -81,17 +85,12 @@ RdmaTransport::RdmaTransport(RdmaConfig config) : config_(std::move(config)) RdmaTransport::~RdmaTransport() { RdmaTransport::shutdown(); - if (continuation_fd_ >= 0) - ::close(continuation_fd_); send_pool_.reset(); recv_pool_.reset(); - if (cq_ != nullptr) { - ibv_destroy_cq(cq_); - cq_ = nullptr; - } - if (comp_channel_ != nullptr) { - ibv_destroy_comp_channel(comp_channel_); - comp_channel_ = nullptr; + destroy_completion_queues(); + for (const auto &queue : completion_queues_) { + if (queue->continuation_fd >= 0) + ::close(queue->continuation_fd); } if (pd_ != nullptr) { ibv_dealloc_pd(pd_); @@ -107,55 +106,28 @@ bool RdmaTransport::adopt_cm_context(struct ibv_context *context) { if (context == nullptr) return false; + std::scoped_lock lock(cm_mu_); if (context_ == context) - return true; + return ready(); + // A transport uses one device context for its CQ pool and registered + // arenas. Do not replace live QPs/registrations when another HCA appears. + if (context_ != nullptr) + return false; auto *new_pd = ibv_alloc_pd(context); if (new_pd == nullptr) return false; - auto *new_channel = ibv_create_comp_channel(context); - if (new_channel == nullptr) { + if (!create_completion_queues(context)) { ibv_dealloc_pd(new_pd); return false; } - rdma_detail::make_nonblocking(new_channel->fd); - auto *new_cq = ibv_create_cq(context, static_cast(config_.cq_depth), this, new_channel, 0); - if (new_cq == nullptr || ibv_req_notify_cq(new_cq, 0) != 0) { - if (new_cq != nullptr) - ibv_destroy_cq(new_cq); - ibv_destroy_comp_channel(new_channel); - ibv_dealloc_pd(new_pd); - return false; - } - auto new_send = std::make_unique(new_pd, config_.send_arena_bytes, config_.sq_depth); - auto new_recv = std::make_unique(new_pd, config_.send_arena_bytes, config_.sq_depth); - if (new_send == nullptr || new_recv == nullptr) { - ibv_destroy_cq(new_cq); - ibv_destroy_comp_channel(new_channel); - ibv_dealloc_pd(new_pd); - return false; - } - - send_pool_.reset(); - recv_pool_.reset(); - if (cq_ != nullptr) - ibv_destroy_cq(cq_); - if (comp_channel_ != nullptr) - ibv_destroy_comp_channel(comp_channel_); - if (pd_ != nullptr) - ibv_dealloc_pd(pd_); - if (context_ != nullptr && owns_context_) - ibv_close_device(context_); - context_ = context; owns_context_ = false; pd_ = new_pd; - comp_channel_ = new_channel; - cq_ = new_cq; - send_pool_ = std::move(new_send); - recv_pool_ = std::move(new_recv); + send_pool_ = std::make_unique(pd_, config_.send_arena_bytes, config_.sq_depth); + recv_pool_ = std::make_unique(pd_, config_.send_arena_bytes, config_.sq_depth); register_events(event_engine_); - return true; + return ready(); } bool RdmaTransport::submit(Connection *conn, OutFrame *frame) @@ -163,18 +135,22 @@ bool RdmaTransport::submit(Connection *conn, OutFrame *frame) Operation operation(repost_state_); if (!operation) return false; - if (conn == nullptr || !conn->is_open() || frame == nullptr || cq_ == nullptr || frame->control == nullptr || - conn->transport_handle == 0) { + if (conn == nullptr || !conn->is_open() || frame == nullptr || frame->control == nullptr) { return false; } if (frame->data_views() > 1) { return false; } - auto *pools = static_cast(conn->user_data); - if (pools == nullptr || pools->send == nullptr) { + auto *pools = pools_for(conn); + if (pools == nullptr || pools->send == nullptr || pools->cq_index >= completion_queues_.size() || + completion_queues_[pools->cq_index]->cq == nullptr) { return false; } + Operation posting(pools->posting_state); + if (!posting || !conn->is_open() || conn->transport_handle == 0) + return false; + // TCP serializes the header and buffers independently. RDMA materializes // the complete wire message in one registered slot so the receiver can // post and recycle one contiguous buffer. @@ -259,10 +235,16 @@ bool RdmaTransport::post_receive(Connection *conn, Buffer *slot) Operation operation(repost_state_); if (!operation) return false; - if (conn == nullptr || !conn->is_open() || slot == nullptr || slot->data == nullptr || cq_ == nullptr || - conn->transport_handle == 0 || slot->capacity < config_.wire_slot_capacity) { + if (conn == nullptr || !conn->is_open() || slot == nullptr || slot->data == nullptr || + slot->capacity < config_.wire_slot_capacity) { return false; } + auto *pools = pools_for(conn); + if (pools == nullptr) + return false; + Operation posting(pools->posting_state); + if (!posting || !conn->is_open() || conn->transport_handle == 0) + return false; ibv_sge sge{}; sge.addr = reinterpret_cast(slot->data); sge.length = slot->capacity; @@ -335,10 +317,12 @@ void RdmaTransport::shutdown() const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(config_.shutdown_drain_ms); while (true) { bool pending_sends = false; - for (uint32_t index = 0; index < config_.cq_depth; ++index) { - const auto &work = work_requests_[index]; - const uint64_t token = work.token.load(std::memory_order_acquire); - pending_sends |= token != 0 && token != WORK_RESERVED && work.kind == WorkKind::Send; + for (const auto &queue : completion_queues_) { + for (uint32_t index = 0; index < config_.cq_depth; ++index) { + const auto &work = queue->work_requests[index]; + const uint64_t token = work.token.load(std::memory_order_acquire); + pending_sends |= token != 0 && token != WORK_RESERVED && work.kind == WorkKind::Send; + } } if (!pending_sends || std::chrono::steady_clock::now() >= deadline) break; @@ -370,6 +354,8 @@ void RdmaTransport::shutdown() while (drain_cq(config_.poll_batch) > 0) { } cancel_work(); + for (const auto &queue : completion_queues_) + queue->reserved_capacity.store(0, std::memory_order_release); if (listen_id_ != nullptr) { if (listen_id_->qp != nullptr) rdma_destroy_qp(listen_id_); @@ -384,23 +370,36 @@ void RdmaTransport::shutdown() void RdmaTransport::start() { + if (running_.exchange(true, std::memory_order_acq_rel)) + return; if ((repost_state_->users.load(std::memory_order_acquire) & RepostState::STOPPED) != 0) { repost_state_ = std::make_shared(); repost_state_->transport = this; } - running_.store(true, std::memory_order_relaxed); + stats_.poller_count.store(0, std::memory_order_relaxed); + start_pollers(); } void RdmaTransport::stop() { running_.store(false, std::memory_order_release); + for (const auto &queue : completion_queues_) + if (queue->engine != nullptr) + queue->engine->notify_worker(); + for (const auto &queue : completion_queues_) { + if (queue->worker.joinable()) + queue->worker.join(); + queue->managed.store(false, std::memory_order_release); + } if (event_engine_ != nullptr) event_engine_->notify_worker(); } -int RdmaTransport::completion_fd() const +int RdmaTransport::completion_fd(uint32_t index) const { - return comp_channel_ == nullptr ? -1 : comp_channel_->fd; + if (index >= completion_queues_.size() || completion_queues_[index]->channel == nullptr) + return -1; + return completion_queues_[index]->channel->fd; } int RdmaTransport::cm_event_fd() const diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp index a82e489d3..49fdc8180 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. #include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "rdma_cq.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -13,22 +14,31 @@ RdmaTransport::WorkRequest *RdmaTransport::acquire_work(WorkKind kind, Connectio auto owner = conn->weak_from_this().lock(); if (owner == nullptr) return nullptr; + auto *pools = pools_for(conn); + const uint32_t cq_index = pools == nullptr ? 0 : pools->cq_index; + if (cq_index >= completion_queues_.size()) + return nullptr; + auto &queue = *completion_queues_[cq_index]; for (uint32_t attempt = 0; attempt < config_.cq_depth; ++attempt) { - const uint32_t index = next_work_.fetch_add(1, std::memory_order_relaxed) % config_.cq_depth; - auto &work = work_requests_[index]; + const uint32_t index = queue.next_work.fetch_add(1, std::memory_order_relaxed) % config_.cq_depth; + auto &work = queue.work_requests[index]; uint64_t empty = 0; if (!work.token.compare_exchange_strong(empty, WORK_RESERVED, std::memory_order_acq_rel)) continue; - work.kind = kind; - work.conn = std::move(owner); - work.slot = slot; - work.frame = frame; - work.request_id = frame == nullptr ? slot->request_id : frame->request_id; - work.stream_id = slot->stream_id; - work.sequence = slot->sequence; - work.logical_offset = slot->logical_offset; - const uint64_t generation = work_generation_.fetch_add(1, std::memory_order_relaxed); - work.token.store((generation << 32) | index, std::memory_order_release); + work.kind = kind; + work.conn = std::move(owner); + work.pools = pools == nullptr ? nullptr : pools->shared_from_this(); + work.slot = slot; + work.frame = frame; + work.request_id = frame == nullptr ? slot->request_id : frame->request_id; + work.stream_id = slot->stream_id; + work.sequence = slot->sequence; + work.logical_offset = slot->logical_offset; + uint64_t generation = queue.generation.fetch_add(1, std::memory_order_relaxed); + if (generation == 0) + generation = queue.generation.fetch_add(1, std::memory_order_relaxed); + const uint64_t global_index = static_cast(cq_index) * config_.cq_depth + index; + work.token.store((generation << 32) | global_index, std::memory_order_release); return &work; } return nullptr; @@ -37,15 +47,17 @@ RdmaTransport::WorkRequest *RdmaTransport::acquire_work(WorkKind kind, Connectio RdmaTransport::WorkRequest *RdmaTransport::claim_work(uint64_t token) { const auto index = static_cast(token); - if (token == 0 || token == WORK_RESERVED || index >= config_.cq_depth) + if (token == 0 || token == WORK_RESERVED || + static_cast(index) >= static_cast(config_.cq_depth) * config_.cq_count) return nullptr; - auto &work = work_requests_[index]; + auto &work = completion_queues_[index / config_.cq_depth]->work_requests[index % config_.cq_depth]; return work.token.compare_exchange_strong(token, WORK_RESERVED, std::memory_order_acq_rel) ? &work : nullptr; } void RdmaTransport::release_work(WorkRequest *work) { work->conn.reset(); + work->pools.reset(); work->slot = nullptr; work->frame = nullptr; work->token.store(0, std::memory_order_release); @@ -58,6 +70,16 @@ void RdmaTransport::dispatch_completion(const ibv_wc &wc) return; // Error completions do not provide a trustworthy opcode. The retained // work record owns dispatch, even after disconnect or QP destruction. + if (work->kind == WorkKind::Receive && wc.status == IBV_WC_SUCCESS && !work->conn->has_frame_handler()) { + const bool handshake = wc.byte_len >= HEADER_SIZE && work->slot->capacity >= HEADER_SIZE && + parse_header(work->slot->data).msg_type == RDMA_HANDSHAKE_MSG_TYPE; + if (!handshake) { + const uint32_t index = static_cast(wc.wr_id) / config_.cq_depth; + completion_queues_[index]->deferred.push_back({work, wc}); + stats_.pending_deferred_receives.fetch_add(1, std::memory_order_relaxed); + return; + } + } if (work->kind == WorkKind::Receive) release_completed_receive(work, wc); else @@ -68,15 +90,21 @@ void RdmaTransport::cancel_work(Connection *connection) { // Called only after the affected QPs are destroyed and device access has // stopped. Missing/uncertain completions must still release their owners. - for (uint32_t index = 0; index < config_.cq_depth; ++index) { - auto &work = work_requests_[index]; - const uint64_t token = work.token.load(std::memory_order_acquire); - if (token == 0 || token == WORK_RESERVED || (connection != nullptr && work.conn.get() != connection)) - continue; - ibv_wc completion{}; - completion.wr_id = token; - completion.status = IBV_WC_WR_FLUSH_ERR; - dispatch_completion(completion); + for (const auto &queue : completion_queues_) { + while (queue->polling.test_and_set(std::memory_order_acquire)) + std::this_thread::yield(); + cancel_deferred(*queue, connection); + for (uint32_t index = 0; index < config_.cq_depth; ++index) { + auto &work = queue->work_requests[index]; + const uint64_t token = work.token.load(std::memory_order_acquire); + if (token == 0 || token == WORK_RESERVED || (connection != nullptr && work.conn.get() != connection)) + continue; + ibv_wc completion{}; + completion.wr_id = token; + completion.status = IBV_WC_WR_FLUSH_ERR; + dispatch_completion(completion); + } + queue->polling.clear(std::memory_order_release); } } @@ -84,8 +112,9 @@ void RdmaTransport::close_after_drain(Connection *connection) { // A disconnect notification can race CQ readiness. Publish completions // already acknowledged by the NIC before failing the remaining requests. - while (drain_cq(config_.poll_batch) > 0) { - } + const auto *pools = pools_for(connection); + const uint32_t index = pools == nullptr ? 0 : pools->cq_index; + (void)drain_queue(index, config_.cq_depth, true); connection->close(); } diff --git a/lib/crowdb-rpc/tests/common/test_rdma_transport.h b/lib/crowdb-rpc/tests/common/test_rdma_transport.h index fb8947f74..1b56ddc7d 100644 --- a/lib/crowdb-rpc/tests/common/test_rdma_transport.h +++ b/lib/crowdb-rpc/tests/common/test_rdma_transport.h @@ -3,6 +3,7 @@ #pragma once +#include "../../src/transport/rdma/rdma_cq.h" #include "crowdb-rpc/transport/rdma/rdma_transport.h" #if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) @@ -50,19 +51,47 @@ struct TestRdmaTransport transport.close_after_drain(connection); } - static void set_cq(RdmaTransport &transport, ibv_cq *cq) + static void set_cq(RdmaTransport &transport, ibv_cq *cq, uint32_t index = 0) { - transport.cq_ = cq; + transport.completion_queues_[index]->cq = cq; if (cq == nullptr) transport.event_engine_ = nullptr; } - static void set_cq_polling(RdmaTransport &transport, bool polling) + static void set_cq_polling(RdmaTransport &transport, bool polling, uint32_t index = 0) { if (polling) - transport.cq_polling_.test_and_set(std::memory_order_acquire); + transport.completion_queues_[index]->polling.test_and_set(std::memory_order_acquire); else - transport.cq_polling_.clear(std::memory_order_release); + transport.completion_queues_[index]->polling.clear(std::memory_order_release); + } + + static std::shared_ptr bind_cq(const std::shared_ptr &connection, uint32_t index) + { + auto pools = std::make_shared(); + pools->cq_index = index; + RdmaTransport::set_connection_pools(connection.get(), pools); + return pools; + } + + static bool reserve(RdmaTransport &transport, uint32_t capacity, bool commit) + { + RdmaTransport::CqReservation reservation(transport.reserve_cq(capacity), capacity); + reservation.committed = commit; + return reservation.queue != nullptr; + } + + static void watch_frames(RdmaTransport &transport, const std::shared_ptr &connection, uint32_t index) + { + transport.install_frame_notifier(connection, index); + } + + static bool pending(RdmaTransport &transport, uint64_t token) + { + const auto index = static_cast(token); + return transport.completion_queues_[index / transport.config_.cq_depth] + ->work_requests[index % transport.config_.cq_depth] + .token.load(std::memory_order_acquire) == token; } }; @@ -72,14 +101,16 @@ struct TestCompletionQueue ibv_cq cq{}; std::deque ready; std::deque on_arm; - uint32_t arms = 0; + uint32_t arms = 0; + uint64_t poll_calls = 0; TestCompletionQueue() { cq.context = &context; cq.cq_context = this; context.ops.poll_cq = [](ibv_cq *queue, int capacity, ibv_wc *output) { - auto &self = *static_cast(queue->cq_context); + auto &self = *static_cast(queue->cq_context); + ++self.poll_calls; const auto count = std::min(static_cast(capacity), self.ready.size()); for (size_t index = 0; index < count; ++index) { output[index] = self.ready.front(); diff --git a/lib/crowdb-rpc/tests/rdma_capability_test.cpp b/lib/crowdb-rpc/tests/rdma_capability_test.cpp index 02466b5b3..89581ff11 100644 --- a/lib/crowdb-rpc/tests/rdma_capability_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_capability_test.cpp @@ -1,6 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#include "crowdb-rpc/connection.h" #include "crowdb-rpc/transport/rdma/rdma_capability.h" #include @@ -56,6 +57,24 @@ TEST(RdmaCapabilityTest, FallbackStopsAfterPayloadSubmission) EXPECT_FALSE(crowdb::rpc::can_fallback_to_tcp(RdmaPathState::Failed, true)); } +TEST(RdmaCapabilityTest, ConnectionConsumesCapabilityBeforeApplicationHandlerPublication) +{ + SystemBufferPool pool; + crowdb::rpc::Connection connection(1, "handshake", &pool); + RdmaEndpointCapability source; + source.rdma_available = true; + source.protocol_version = 1; + source.generation = 17; + const auto wire = crowdb::rpc::serialize_rdma_handshake(source); + auto *frame = new crowdb::rpc::Frame; + frame->header.msg_type = crowdb::rpc::RDMA_HANDSHAKE_MSG_TYPE; + frame->control.assign(wire.begin(), wire.end()); + connection.on_frame(frame); + EXPECT_TRUE(connection.peer_handshake_received()); + EXPECT_EQ(connection.peer_capability().generation, source.generation); + EXPECT_FALSE(connection.has_frame_handler()); +} + TEST(RdmaCapabilityTest, ConnectionStateRejectsIllegalUpgradeAndFallback) { RdmaConnectionState state; diff --git a/lib/crowdb-rpc/tests/rdma_completion_test.cpp b/lib/crowdb-rpc/tests/rdma_completion_test.cpp index afe735595..8a6378aab 100644 --- a/lib/crowdb-rpc/tests/rdma_completion_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_completion_test.cpp @@ -8,6 +8,9 @@ # include +# include +# include + using namespace crowdb::rpc; namespace @@ -17,6 +20,7 @@ RdmaConfig test_config() RdmaConfig config; config.device = "test-no-device"; config.cq_depth = 8; + config.cq_count = 2; config.rq_depth = 4; config.posted_receive_target = 4; config.poll_empty_budget = 1; @@ -27,14 +31,18 @@ class RdmaCompletionTest : public ::testing::Test { protected: RegisteredBufferPool slots{16U << 20, 32}; - SystemBufferPool buffers{32}; + SystemBufferPool buffers{64}; TestCompletionQueue queue; RdmaTransport transport{test_config()}; std::shared_ptr connection = std::make_shared(1, "test", &slots); void TearDown() override { + transport.stop(); + TestRdmaTransport::set_cq_polling(transport, false); + TestRdmaTransport::set_cq_polling(transport, false, 1); TestRdmaTransport::set_cq(transport, nullptr); + TestRdmaTransport::set_cq(transport, nullptr, 1); TestRdmaTransport::cancel(transport); } @@ -48,7 +56,16 @@ class RdmaCompletionTest : public ::testing::Test frame->request_id = 239; frame->control = buffers.alloc(1); frame->data = buffers.alloc(16); - const auto token = TestRdmaTransport::send(transport, owner == nullptr ? connection : owner, slot, frame); + if (frame->control == nullptr || frame->data == nullptr) { + (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); + slot->release(); + if (frame->control != nullptr) + frame->control->release(); + frame->release_data(); + delete frame; + return 0; + } + const auto token = TestRdmaTransport::send(transport, owner == nullptr ? connection : owner, slot, frame); if (token == 0) { (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); slot->release(); @@ -103,6 +120,54 @@ TEST_F(RdmaCompletionTest, BoundedWorkTableRejectsExhaustionAndStaleCompletions) TestRdmaTransport::complete(transport, tokens[index]); } +TEST_F(RdmaCompletionTest, QueueCapacityReservationRollsBackFailedSetup) +{ + EXPECT_TRUE(TestRdmaTransport::reserve(transport, 8, false)); + EXPECT_TRUE(TestRdmaTransport::reserve(transport, 8, true)); + EXPECT_TRUE(TestRdmaTransport::reserve(transport, 8, true)); + EXPECT_FALSE(TestRdmaTransport::reserve(transport, 1, true)); +} + +TEST_F(RdmaCompletionTest, WorkTablesAndPollingOwnershipAreIndependentPerCq) +{ + auto other = std::make_shared(6, "other-cq", &slots); + const auto binding = TestRdmaTransport::bind_cq(other, 1); + int user_marker = 42; + other->user_data = &user_marker; + std::array first{}; + std::array second{}; + for (size_t index = 0; index < first.size(); ++index) { + first[index] = send(); + second[index] = send(other); + ASSERT_NE(first[index], 0U); + ASSERT_NE(second[index], 0U); + EXPECT_LT(static_cast(first[index]), 8U); + EXPECT_GE(static_cast(second[index]), 8U); + EXPECT_LT(static_cast(second[index]), 16U); + } + EXPECT_EQ(send(), 0U); + EXPECT_EQ(send(other), 0U); + TestCompletionQueue other_queue; + queue.ready.push_back(TestCompletionQueue::completion(first.front())); + other_queue.ready.push_back(TestCompletionQueue::completion(second.front())); + TestRdmaTransport::set_cq(transport, &queue.cq); + TestRdmaTransport::set_cq(transport, &other_queue.cq, 1); + TestRdmaTransport::set_cq_polling(transport, true); + EXPECT_EQ(transport.drain_cq(8), 1); + EXPECT_EQ(queue.ready.size(), 1U); + EXPECT_TRUE(other_queue.ready.empty()); + TestRdmaTransport::set_cq_polling(transport, false); + EXPECT_EQ(transport.drain_cq(8), 1); + for (size_t index = 1; index < first.size(); ++index) { + TestRdmaTransport::complete(transport, first[index]); + TestRdmaTransport::complete(transport, second[index]); + } + TestRdmaTransport::set_cq(transport, nullptr, 1); + EXPECT_TRUE(connection->is_open()); + EXPECT_TRUE(other->is_open()); + EXPECT_EQ(other->user_data, &user_marker); +} + TEST_F(RdmaCompletionTest, WorkRetainsConnectionUntilCompletion) { auto owner = std::make_shared(3, "retained", &slots); @@ -139,6 +204,92 @@ TEST_F(RdmaCompletionTest, ReceiveWaitsForFrameReleaseBeforeRecycling) EXPECT_EQ(slot->ownership.load(), BufferOwnership::Available); } +TEST_F(RdmaCompletionTest, CompletedReceiveReturnsWorkAdmissionBeforeApplicationCallback) +{ + std::array sends{}; + for (auto &token : sends) { + token = send(); + ASSERT_NE(token, 0U); + } + uint64_t reply = 0; + connection->set_on_frame([&](Frame *frame, Connection *) { + reply = send(); + delete frame; + }); + Buffer *slot = slots.alloc(RDMA_WIRE_SLOT_CAPACITY); + ASSERT_NE(slot, nullptr); + serialize_header(slot->data, Header{}); + ASSERT_TRUE(slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive)); + const auto received = TestRdmaTransport::receive(transport, connection, slot); + ASSERT_NE(received, 0U); + TestRdmaTransport::complete(transport, received, IBV_WC_SUCCESS, HEADER_SIZE); + ASSERT_NE(reply, 0U); + EXPECT_TRUE(TestRdmaTransport::pending(transport, reply)); + EXPECT_FALSE(TestRdmaTransport::pending(transport, received)); + TestRdmaTransport::complete(transport, reply); + for (const auto token : sends) + TestRdmaTransport::complete(transport, token); +} + +TEST_F(RdmaCompletionTest, FixedPollerRetainsEarlyReceiveAndWakesOnHandlerPublication) +{ + Buffer *slot = slots.alloc(RDMA_WIRE_SLOT_CAPACITY); + ASSERT_NE(slot, nullptr); + Header header; + header.data_size = 4; + serialize_header(slot->data, header); + std::memcpy(slot->data + HEADER_SIZE, "late", 4); + ASSERT_TRUE(slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive)); + const auto token = TestRdmaTransport::receive(transport, connection, slot); + auto completion = TestCompletionQueue::completion(token); + completion.byte_len = HEADER_SIZE + 4; + queue.ready.push_back(completion); + TestRdmaTransport::set_cq(transport, &queue.cq); + TestRdmaTransport::watch_frames(transport, connection, 0); + EpollEngine generic; + ASSERT_EQ(generic.init(), 0); + transport.register_events(&generic); + transport.start(); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(1); + while (transport.stats().pending_deferred_receives.load() == 0 && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + ASSERT_EQ(transport.stats().pending_deferred_receives.load(), 1U); + EXPECT_EQ(transport.handle_external_event(3), 0); + std::array events{}; + EXPECT_EQ(generic.wait(events.data(), events.size(), 0), 0); + std::atomic received{false}; + std::thread::id callback_thread; + connection->set_on_frame([&](Frame *frame, Connection *) { + callback_thread = std::this_thread::get_id(); + EXPECT_EQ(std::memcmp(frame->data_buf->payload_data(), "late", 4), 0); + delete frame; + received.store(true, std::memory_order_release); + }); + while (!received.load(std::memory_order_acquire) && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + transport.stop(); + EXPECT_TRUE(received.load()); + EXPECT_NE(callback_thread, std::this_thread::get_id()); + EXPECT_EQ(transport.stats().poller_count.load(), 1U); + EXPECT_EQ(transport.stats().pending_deferred_receives.load(), 0U); + EXPECT_EQ(slot->ref->load(), 0); +} + +TEST_F(RdmaCompletionTest, CancellationRetiresReceiveDeferredBeforeHandlerInstallation) +{ + Buffer *slot = slots.alloc(RDMA_WIRE_SLOT_CAPACITY); + ASSERT_NE(slot, nullptr); + serialize_header(slot->data, Header{}); + ASSERT_TRUE(slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive)); + const auto token = TestRdmaTransport::receive(transport, connection, slot); + TestRdmaTransport::complete(transport, token, IBV_WC_SUCCESS, HEADER_SIZE); + ASSERT_EQ(transport.stats().pending_deferred_receives.load(), 1U); + TestRdmaTransport::cancel(transport); + EXPECT_EQ(transport.stats().pending_deferred_receives.load(), 0U); + EXPECT_EQ(slot->ref->load(), 0); + EXPECT_EQ(slots.quarantined_bytes(), RDMA_WIRE_SLOT_CAPACITY); +} + TEST_F(RdmaCompletionTest, ReceiveErrorAndShutdownQuarantineUncertainBuffers) { Buffer *slot = slots.alloc(RDMA_WIRE_SLOT_CAPACITY); @@ -216,6 +367,8 @@ TEST_F(RdmaCompletionTest, PollAfterArmingClosesNotificationRace) EXPECT_EQ(transport.drain_cq(8), 1); EXPECT_TRUE(queue.ready.empty()); EXPECT_EQ(queue.arms, 1U); + EXPECT_EQ(transport.stats().cq_polls.load(), queue.poll_calls); + EXPECT_LE(transport.stats().cq_empty_polls.load(), queue.poll_calls); } TEST_F(RdmaCompletionTest, CmDisconnectPublishesAlreadyCompletedResponseBeforeFailingRequests) diff --git a/lib/crowdb-rpc/tests/rdma_config_test.cpp b/lib/crowdb-rpc/tests/rdma_config_test.cpp index 8909a295f..6bba7f83c 100644 --- a/lib/crowdb-rpc/tests/rdma_config_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_config_test.cpp @@ -1,6 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#include "common/test_environment.h" #include "crowdb-rpc/transport/rdma/rdma_transport.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -44,4 +45,46 @@ TEST(RdmaConfigTest, LargeWireSlotsAnd4096ReceivesUse64BitArenaBudget) EXPECT_EQ(config.receive_arena_bytes, uint64_t{4096} * config.wire_slot_capacity); } +TEST(RdmaConfigTest, CqCountAndWorkTokensAreBounded) +{ + RdmaConfig config; + config.cq_count = 0; + EXPECT_FALSE(config.valid()); + config.cq_count = 257; + EXPECT_FALSE(config.valid()); + config.cq_count = 256; + EXPECT_TRUE(config.valid()); + config.cq_depth = INT32_MAX; + EXPECT_FALSE(config.valid()); +} + +TEST(RdmaConfigTest, PollingBudgetsMustBeNonzeroAndFitTheVerbsBatch) +{ + RdmaConfig config; + config.poll_batch = 0; + EXPECT_FALSE(config.valid()); + config.poll_batch = 256; + config.verbs_poll_batch = 0; + EXPECT_FALSE(config.valid()); + config.verbs_poll_batch = 257; + EXPECT_FALSE(config.valid()); + config.verbs_poll_batch = 256; + EXPECT_TRUE(config.valid()); + config.poll_empty_budget = 0; + EXPECT_FALSE(config.valid()); +} + +TEST(RdmaConfigTest, EnvironmentParsesCqAndPollingKnobsStrictly) +{ + const TestEnvironment count("CROWDB_RDMA_CQ_COUNT", "2"); + const TestEnvironment batch("CROWDB_RDMA_VERBS_POLL_BATCH", "64"); + RdmaConfig config; + ASSERT_TRUE(config.apply_environment()); + EXPECT_EQ(config.cq_count, 2U); + EXPECT_EQ(config.verbs_poll_batch, 64U); + EXPECT_TRUE(config.valid()); + const TestEnvironment invalid("CROWDB_RDMA_CQ_COUNT", "2x"); + EXPECT_FALSE(config.apply_environment()); +} + #endif diff --git a/lib/crowdb-rpc/tests/rdma_timed_load_test.cpp b/lib/crowdb-rpc/tests/rdma_timed_load_test.cpp index 50059614e..f3abb7114 100644 --- a/lib/crowdb-rpc/tests/rdma_timed_load_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_timed_load_test.cpp @@ -138,19 +138,28 @@ void test_timed_client(const char *address, int port, const TestLoadSettings &se transport.start(); std::vector> connections; std::unordered_set qp_numbers; + std::unordered_set cq_pointers; for (uint32_t index = 0; index < settings.connection_count; ++index) { auto connection = transport.connect(address, port); ASSERT_NE(connection, nullptr); ASSERT_EQ(connection->path(), ConnectionPath::Rdma); auto *qp = reinterpret_cast(connection->transport_handle); ASSERT_NE(qp, nullptr); + ASSERT_EQ(qp->send_cq, qp->recv_cq); + cq_pointers.insert(qp->send_cq); qp_numbers.insert(qp->qp_num); client.attach(connection.get()); connections.push_back(std::move(connection)); } ASSERT_EQ(qp_numbers.size(), settings.connection_count); - std::fprintf(stderr, "RDMA topology: physical_hosts=1 shared_hca=%s independent_client_qps=%zu\n", - std::getenv("CROWDB_RDMA_DEVICE"), qp_numbers.size()); + ASSERT_EQ(cq_pointers.size(), std::min(settings.connection_count, test_setting("CROWDB_RDMA_CQ_COUNT", 1))); +# ifdef CROWDB_RPC_TEST_UTIL + const auto *stats = transport.rdma_stats_for_tests(); + ASSERT_NE(stats, nullptr); + ASSERT_EQ(stats->poller_count.load(), test_setting("CROWDB_RDMA_CQ_COUNT", 1)); +# endif + std::fprintf(stderr, "RDMA topology: physical_hosts=1 shared_hca=%s independent_client_qps=%zu active_cqs=%zu\n", + std::getenv("CROWDB_RDMA_DEVICE"), qp_numbers.size(), cq_pointers.size()); std::vector threads; for (uint32_t index = 0; index < settings.producer_count; ++index) threads.emplace_back([&, index] { From 0108f9aaf76e9fc40cdd52d89f0b9ecbc7a6a774 Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 22 Oct 2026 21:19:22 +0800 Subject: [PATCH 025/258] Retain registered arenas through final lease release --- doc/design/rpc/design-crowdb-rpc-rdma.md | 22 ++ doc/working/plan-rdma-rpc-transport.md | 114 ++++++- doc/working/plan-rdma.md | 12 +- lib/crowdb-rpc/include/crowdb-rpc/buffer.h | 16 +- lib/crowdb-rpc/include/crowdb-rpc/c_api.h | 13 +- .../include/crowdb-rpc/server/handler.h | 1 + .../transport/rdma/rdma_transport.h | 59 ++-- lib/crowdb-rpc/src/c_api.cpp | 37 +-- lib/crowdb-rpc/src/server/handler.cpp | 25 ++ .../src/transport/rdma/rdma_arena.h | 44 +++ .../src/transport/rdma/rdma_buffer_pool.cpp | 290 ++++++++++-------- .../src/transport/rdma/rdma_completion.cpp | 25 +- .../src/transport/rdma/rdma_connection.cpp | 13 +- .../src/transport/rdma/rdma_device.cpp | 36 +++ .../src/transport/rdma/rdma_device.h | 23 ++ .../src/transport/rdma/rdma_transport.cpp | 35 +-- .../src/transport/socket_transport.cpp | 2 +- lib/crowdb-rpc/tests/c_api_buffer_test.cpp | 39 +++ .../tests/common/test_rdma_device.h | 42 +++ .../tests/common/test_rdma_transport.h | 5 + lib/crowdb-rpc/tests/rdma_completion_test.cpp | 11 + lib/crowdb-rpc/tests/rdma_lifetime_test.cpp | 162 ++++++++++ lib/crowdb-rpc/tests/rdma_worker_test.cpp | 17 +- 23 files changed, 771 insertions(+), 272 deletions(-) create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_arena.h create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_device.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_device.h create mode 100644 lib/crowdb-rpc/tests/c_api_buffer_test.cpp create mode 100644 lib/crowdb-rpc/tests/common/test_rdma_device.h create mode 100644 lib/crowdb-rpc/tests/rdma_lifetime_test.cpp diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index ca89084cc..98536f83b 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -107,6 +107,23 @@ SDK lease budget, or a full DiskIO write queue applies backpressure to the existing producer or aggregation layer. RDMA adds no application-level credit message. A slot is never posted twice concurrently. +`Connection::pool()` is the application allocator used by echo and default +handlers. Initial receives and reposts use the separate retained receive arena; +fully posting the RQ therefore does not consume reply allocation admission. +Send and receive arenas enforce both byte and total descriptor limits. Tagged +entry indices with atomic links protect their class freelists from concurrent +allocation/recycle ABA; quarantined entries remain outside those freelists. + +Each outstanding registered buffer retains its shared arena even after the +`RdmaBufferPool` facade or transport is destroyed. The arena retains the PD and +the RDMA-CM device context list. The last lease release destroys arena +descriptors, deregisters its MR, then drops the device owner; the last device +owner deallocates its PD before releasing the context list. Completed receive +leases additionally retain their connection and repost admission state. After +shutdown, final release retires the slot without using the destroyed transport +or posting another receive. QP retirement does not accumulate arenas in a +transport-wide retired list. + ## 5. Shared CQ and completion model Connections share a bounded CQ pool. Each QP uses one assigned CQ for both @@ -151,6 +168,11 @@ callback or buffer lease only after frame validation and ownership resolution. ## 6. Failure and fallback +WR flushes on a closed connection are counted as `shutdown_flushes` separately +from `cq_errors`. A flush on an open connection still fails that stream and +counts as a workload error. Slots with uncertain completion are quarantined in +both cases. + RDMA setup failure, missing capability, or a transport error before the first payload may select TCP. The request ID is shared by both transports. Retry is allowed only when existing request semantics provide idempotency or duplicate diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 77696ca0d..edf81aee3 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -27,9 +27,12 @@ retained, and poll-call counters use per-CQ shards. The CQ diff passed focused tests, live HCA diagnostics, the full C++/FFI gate, formatting and workspace clippy and scoped tree-lint. Tree-lint exits zero with style warnings. -Still pending: application reply allocation independent of receive admission, -real PD/MR lifetime after held leases and transport teardown, bounded transport -aggregation, `bench rdma`/shared coroutine reporting, unified regression +Application reply allocation now uses a separate pool, and real HCA tests +prove held registered/receive leases survive pool/transport/CM-ID destruction. +The C ABI now returns independent handles for retained buffer references. + +Still pending: portable registered-pool concurrency/lifetime parity, bounded +transport aggregation, `bench rdma`/shared coroutine reporting, unified regression orchestration, negotiated TCP capability preflight, failure injection, and the two-container acceptance profiles. Capability serialization and automatic consumption at the connection seam do not establish negotiated preflight: @@ -330,16 +333,81 @@ Verified histogram checkpoint: work; shutdown joins pollers before destroying QPs/CQs. No hot-path lock is introduced. Test early receive/late handler, deferred cancellation, fixed-owner busy/idle wakeups and late buffer release. -- [~] **Separate application and receive allocators**: built-in echo and +- [x] **Separate application and receive allocators**: built-in echo and default handlers allocate from the common application pool, not a fully preposted receive arena. Receive posting and repost explicitly use the connection's retained RDMA receive pool. Test `posted_receive_target == rq_depth` with request and response payload validation. Files: `connection.h`, RDMA setup/completion, `c_api.cpp`, `server/handler.cpp`, transport tests. -- [ ] **Finish real registered-lease shutdown**: retain PD/context/MR owners +- [x] **Finish real registered-lease shutdown**: retain PD/context/MR owners until the last held receive lease is released, including after transport shutdown; no deregister-before-release, double repost, premature recycle or silent PD leak. Files: RDMA buffer pool and lifecycle sources, lease tests. + `RdmaBufferPool` becomes a facade over a shared arena; each outstanding + buffer retains that arena, including buffers returned by `register_buffer`. + The arena retains a shared `RdmaDevice` PD owner and the `rdma_get_devices` + context list. Deregister MR before releasing PD, then release the list. + Replace raw-pointer freelist heads with tagged entry indices and atomic + links to prevent concurrent allocation/recycle ABA and link data races. + Enforce a hard total buffer-entry bound as well as the arena byte bound. + Verify the C ABI handle layer too: `crowdb_rpc_buffer_ref` currently + increments the inner buffer reference but returns the same wrapper, while + `crowdb_rpc_buffer_release` deletes that wrapper. Return an independent + opaque handle per retained reference and test both release orders, an + external owner released exactly once, and a cloned real registered lease + surviving pool/PD facade destruction. This is part of the lease contract. +- [~] **Finish portable pool parity**: the no-verbs `RegisteredBufferPool` + still uses raw-pointer freelist heads and a facade pointer in outstanding + buffers. Reuse the bounded shared-arena/index freelist machinery for that + pool too, retaining detach semantics and optional registration. Add no-device + concurrent recycle, held lease after facade destruction, byte/descriptor + bounds, quarantine and detach tests. Make `SystemBufferPool` outstanding + admission one atomic reservation so parallel reply allocation cannot exceed + its declared count. Files: `buffer.h`, `buffer.cpp`, private common arena + sources, RDMA pool wrapper and buffer tests. This closes the foundation + contract without leaving concurrency safety dependent on HCA availability. + +Application allocator verification: + +- `SocketTransport` passes its application pool into RDMA connections; + `prime_receive` allocates from private `ConnectionPools::recv` explicitly. + The C API and native test use the same built-in `handle_echo` implementation. +- A real HCA worker round trip with RQ/post target 1024 and exactly 1 MiB of + 1 KiB receive slots completed 1000 payload-validated requests with zero + request errors. The server asserts the connection application pool equals + `RpcServer::pool()`. This fixed-count case is functional evidence only. + +Registered lifetime verification: + +- A real registered lease, cloned through the C ABI, remains readable after + its pool facade is destroyed. Final release destroys the arena/MR before + the retained PD owner; `ibv_dealloc_pd` returns zero. Two separate C ABI cases + reproduce the original shared-wrapper bug before the fix and pass afterward, + covering both release orders and exactly one external-owner callback. +- Eight producer threads complete 160,000 allocate/clone/recycle iterations + without duplicate live slot ownership or payload corruption. Oversized + allocation failures preserve smaller-class descriptor admission. +- The live local-HCA CM case holds a receive lease and a `register_buffer` + copy across destruction of both transports, both CM IDs, and caller-held + connection handles. Both payloads remain readable; releasing their final + references expires the respective device owners. Ten repeated setup/teardown + cycles pass. This covers real MRs rather than only the emulated pool. +- Closed-connection flushes count in `shutdown_flushes`; open-connection flush + and other CQ errors still fail the stream. The focused regression verifies + these counters independently. +- Full C++/FFI gate before the final C ABI delta passes at + `.crowdb-runtime/artifacts/unit/cpp-df0a8b16e65c44dfb32c065e6a50cd1e/`. + After that delta, all 149 native RPC CTest cases pass (five opt-in cases skip + there), all 29 Rust RPC FFI tests pass, changed formatting and scoped + tree-lint and workspace clippy pass. +- The same 2-QP/2-CQ/2-poller, 2-producer/window-32 diagnostic profile now + completes 7,828,933 measured RPCs in 10 s: 782,893 TPS, avg 78.96 us, + P90 88.58 us, P99 100.35 us, zero request errors/overflow, 57 drained. + The earlier CQ-only revision measured 827,807 TPS; this single run is about + 5.4% lower and is not sufficient to attribute a stable regression or identify + max TPS. Keep both source/binary identities; do not sacrifice lease ownership + to recover throughput. Latest artifact: + `.crowdb-runtime/artifacts/rdma/20261022T131359Z/`. CQ verification: @@ -374,6 +442,19 @@ CQ verification: ### C. Bounded aggregation +- [ ] **Negotiate receive admission before coalescing**: finish TCP capability + preflight before the CM attempt; reuse its TCP connection only for allowed + pre-payload fallback. Publish actual CM address/port, per-instance generation, + device/GID identity, receive slot capacity and supported aggregate frame count. + Bind the CM private-data exchange to the preflight generation and capacities; + reject stale/mismatched peers before publishing an RDMA connection. Direct + low-level CM diagnostics also negotiate receive slot capacities. Strict RDMA + permits the TCP control preflight but never TCP workload bytes. Keep no-device + strict rejection before opening a socket. Retire a successful preflight TCP + connection through the common worker cleanup path. This is a prerequisite + for the peer-slot bound below; the remaining OCI/failure matrix stays in E. + Files: RDMA capability/CM sources, `socket_transport.cpp`, server readiness, + capability/path and hardware tests. - [ ] **Coalesce complete frames on the CQ owner**: use bounded lock-free admission and per-QP queues; drain available frames without adding a waiting timer. Keep per-QP ordering and common frame/request IDs. Limit a SEND to @@ -381,7 +462,15 @@ CQ verification: capacity)` including every header/control byte and to `aggregation_max_frames`. Disabled mode sends one frame. Reject oversized single frames before admission; do not fragment or silently fall back. - Files: RDMA submit/owner loop, `out_frame.h`, config/capability validation. + Files: RDMA submit/owner loop, `transport.h`, config/capability validation. + Reuse `Connection::send_queue` for MPSC frame admission. Keep a bounded + per-CQ connection directory, changed only under its existing CQ ownership + gate during cold setup/retirement. The CQ owner drains per-QP queues with a + fixed SEND budget and retains one overflow frame for the next slot; do not + read a lifecycle-mutated owner map in the polling loop or introduce an + atomic-shared-pointer load with hidden locking on every poll. Wake a sleeping + owner with a pending-publication/idle handshake rather than an eventfd write + for every RPC. Retire admitted/pending frames on every failure path. - [ ] **Retain aggregate owners through completion**: one SEND WR retains all constituent frames until CQE; failure closes the uncertain stream and quarantines its slot once. Validate the entire received aggregate before @@ -390,6 +479,13 @@ CQ verification: Test exact slot fit, one byte over, multi-frame correlation/order, malformed suffix, held middle frame, send errors, bounded exhaustion and shutdown. Files: RDMA completion/WR code, buffer views and focused RPC tests. + Link constituent `OutFrame` owners in the SEND WR without a per-frame vector + allocation. Parse the full receive slot into bounded header/offset records + before the first callback. Each published payload view has its own descriptor + and view offset, sharing only the parent slot lease; `ref_clone` on the parent + alone cannot represent different immutable offsets safely. Keep a parent + reference until every frame has been dispatched. Defer a mixed capability/data + aggregate if the application handler is not yet published. - [ ] **Expose aggregation observations**: export actual frames/SEND, wire bytes, queue high-water and rejects. Keep RPC TPS distinct from transport SEND rate; compare aggregation disabled/enabled for both TCP and RDMA. @@ -459,6 +555,10 @@ Provider references for the implementation choices: - [libibverbs QP capacity terminology](https://github.com/linux-rdma/rdma-core/blob/master/libibverbs/man/ibv_create_qp.3). - [RDMA-CM receive posting before connection readiness](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/man/rdma_create_qp.3). - [RDMA-CM RNR behavior and default delay](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/man/rdma_accept.3). +- [RDMA-CM device context retention](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/cma.c) + (`rdma_get_devices` increments device references; `rdma_free_devices` + releases them). Retaining the matching CM context list avoids substituting + an independently opened context for a CM-owned QP. ## Historical verification record @@ -567,7 +667,7 @@ Remaining implementation and acceptance work: ## Phase 5: fallback and observability -- [~] **Implement pre-payload fallback**: if CM/QP setup or first submission +- [ ] **Implement pre-payload fallback**: if CM/QP setup or first submission fails before payload, select TCP with the same request identity. - [x] **Reject mixed completion**: after any payload submission, fail the stream on uncertainty and never enqueue a TCP suffix. diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index b7bbc83d3..26609d5d8 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -25,8 +25,12 @@ and C/Rust lease metadata accessors. R239's transport path is implemented and passes the prepared HCA local-loop plus the real `RpcServer`/`RpcClient` worker round trip. Its requirement is still open because negotiated capability preflight, -pre-payload fallback, real PD/MR lifetime with held caller leases, and -multi-container acceptance have not all passed. +pre-payload fallback, portable pool parity, aggregation/CLI benchmarking and +multi-container acceptance have not all passed. Live HCA tests now verify +held registered/receive leases after pool, transport and CM-ID destruction; +the final release deregisters MR before PD release. Application reply allocation +is separate from receive admission, and every CQ has a fixed polling/event +owner. The transport plan records the corresponding gates and diagnostics. The latest transport handoff closes the completion ownership gap: SEND and RECV work requests now carry stable connection/request/stream metadata, receive @@ -149,6 +153,10 @@ TCP-only profile and build/mocked tests. uncertain completion cases. - [x] **Review handoff**: verify that R239 can obtain control/data buffers and that every asynchronous owner has a release transition. +- [ ] **Close portable pool parity**: share the verified arena lifetime and + tagged-index freelist machinery with the no-verbs registered pool, then + verify concurrent bounds, held leases, quarantine and detach without an HCA. + The detailed transport plan owns this continuation before aggregation. Exit condition: no buffer can be recycled while transport, disk, or caller ownership remains unresolved. diff --git a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h index ec2945235..39547ded6 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace crowdb::rpc { @@ -50,12 +51,15 @@ constexpr uint32_t RDMA_DATAPATH_CAPACITY = 1U << 20; // recycle to pool when ref == 0. struct Buffer { - uint8_t *data = nullptr; - uint32_t len = 0; // bytes written (payload length) - uint32_t capacity = 0; // allocated capacity - BufferType type = BufferType::System; - class BufferPool *pool = nullptr; - std::atomic *ref = nullptr; // shared refcount (pool-allocated or standalone) + uint8_t *data = nullptr; + uint32_t len = 0; // bytes written (payload length) + uint32_t capacity = 0; // allocated capacity + BufferType type = BufferType::System; + class BufferPool *pool = nullptr; + // Registered allocations retain their arena independently of its facade. + std::shared_ptr pool_owner; + uint32_t pool_index = 0; + std::atomic *ref = nullptr; // shared refcount (pool-allocated or standalone) std::atomic ownership{BufferOwnership::Available}; BufferCapacityClass capacity_class = BufferCapacityClass::Control; uint64_t request_id = 0; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h index 091a9e418..85a2cc0e2 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h @@ -44,12 +44,13 @@ uint64_t crowdb_rpc_buffer_sequence(crowdb_rpc_buffer_t buf); uint64_t crowdb_rpc_buffer_logical_offset(crowdb_rpc_buffer_t buf); uint32_t crowdb_rpc_buffer_registration_id(crowdb_rpc_buffer_t buf); int crowdb_rpc_buffer_ownership(crowdb_rpc_buffer_t buf); -void crowdb_rpc_buffer_set_metadata(crowdb_rpc_buffer_t buf, uint64_t request_id, - uint64_t stream_id, uint64_t sequence, - uint64_t logical_offset, uint32_t registration_id); -int crowdb_rpc_buffer_transition(crowdb_rpc_buffer_t buf, int from, int to); -int crowdb_rpc_buffer_quarantine(crowdb_rpc_buffer_t buf); -bool crowdb_rpc_buffer_detach(crowdb_rpc_buffer_t buf); +void crowdb_rpc_buffer_set_metadata(crowdb_rpc_buffer_t buf, uint64_t request_id, uint64_t stream_id, uint64_t sequence, + uint64_t logical_offset, uint32_t registration_id); +int crowdb_rpc_buffer_transition(crowdb_rpc_buffer_t buf, int from, int to); +int crowdb_rpc_buffer_quarantine(crowdb_rpc_buffer_t buf); +bool crowdb_rpc_buffer_detach(crowdb_rpc_buffer_t buf); +// Return an independent handle retaining the same immutable allocation. +// Each handle must be released once; releasing one leaves other handles valid. crowdb_rpc_buffer_t crowdb_rpc_buffer_ref(crowdb_rpc_buffer_t buf); void crowdb_rpc_buffer_release(crowdb_rpc_buffer_t buf); // Create a standalone buffer (not pool-allocated) from raw bytes. The diff --git a/lib/crowdb-rpc/include/crowdb-rpc/server/handler.h b/lib/crowdb-rpc/include/crowdb-rpc/server/handler.h index 32b16332c..e7a7a00b2 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/server/handler.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/server/handler.h @@ -34,6 +34,7 @@ using HandlerFn = std::function; // Built-in handlers. OutFrame *handle_ping(Frame *request, Connection *conn); +OutFrame *handle_echo(Frame *request, Connection *conn); OutFrame *handle_unknown(Frame *request, Connection *conn); // HandlerRegistry: maps msg_type → HandlerFn. Thread-safe. diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index 61c68f5fe..3e45b6408 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -26,6 +26,7 @@ namespace crowdb::rpc { class SocketEngine; +struct RdmaDevice; constexpr uint32_t RDMA_WIRE_SLOT_CAPACITY = RDMA_DATAPATH_CAPACITY; @@ -59,48 +60,26 @@ struct RdmaConfig // The pool pre-registers a large memory region and carves out buffers // from it, avoiding per-buffer ibv_reg_mr calls (which are expensive). // Every buffer is a slice of the single MR, so registration is bounded by the -// pool region and deregistration happens exactly once at pool shutdown. Class -// freelists and arena admission are atomic; shutdown is the only lifecycle -// operation that walks the full allocation list. +// pool region. Leases retain the arena and its registration owner; the final +// release deregisters the MR before dropping the PD/context owner. class RdmaBufferPool : public BufferPool { public: // mr_size: total registered memory size. max_buffers: pool capacity. - RdmaBufferPool(struct ibv_pd *pd, uint64_t mr_size = 1 << 20, uint32_t max_buffers = 1024); + RdmaBufferPool(struct ibv_pd *pd, uint64_t mr_size = 1 << 20, uint32_t max_buffers = 1024, + std::shared_ptr registration_owner = {}); ~RdmaBufferPool() override; Buffer *alloc(uint32_t capacity) override; void recycle(Buffer *buf) override; - [[nodiscard]] uint64_t quarantined_bytes() const - { - return quarantined_bytes_.load(std::memory_order_acquire); - } - - [[nodiscard]] bool quiescent() const - { - return outstanding_.load(std::memory_order_acquire) == 0; - } - - void quarantine_inflight(); + [[nodiscard]] uint64_t quarantined_bytes() const; + [[nodiscard]] bool quiescent() const; + void quarantine_inflight(); private: - struct ibv_pd *pd_; - struct ibv_mr *mr_ = nullptr; - void *region_ = nullptr; - uint64_t mr_size_; - uint32_t max_buffers_; - std::atomic outstanding_{0}; - std::atomic next_offset_{0}; - std::array, 4> free_heads_{}; - std::atomic quarantine_head_{nullptr}; - std::atomic all_head_{nullptr}; - std::atomic quarantined_bytes_{0}; - - Buffer *alloc_fresh(uint32_t capacity); - Buffer *pop(std::atomic &head); - void push(std::atomic &head, Buffer *buf); - bool reserve_outstanding(); + struct Arena; + std::shared_ptr arena_; }; // RdmaTransport: RDMA transport using libibverbs + librdmacm. @@ -123,6 +102,7 @@ class RdmaTransport : public Transport std::atomic cq_empty_polls{0}; std::atomic cm_disconnects{0}; std::atomic flush_errors{0}; + std::atomic shutdown_flushes{0}; std::atomic local_protection_errors{0}; std::atomic retry_exceeded_errors{0}; std::atomic remote_access_errors{0}; @@ -132,7 +112,7 @@ class RdmaTransport : public Transport std::atomic pending_deferred_receives{0}; }; - explicit RdmaTransport(RdmaConfig config = {}); + explicit RdmaTransport(RdmaConfig config = {}, BufferPool *application_pool = nullptr); ~RdmaTransport() override; bool submit(Connection *conn, OutFrame *frame) override; @@ -189,11 +169,11 @@ class RdmaTransport : public Transport # endif RdmaConfig config_; std::atomic capability_available_{false}; - struct ibv_context *context_ = nullptr; - bool owns_context_ = true; - struct ibv_pd *pd_ = nullptr; - struct rdma_event_channel *cm_channel_ = nullptr; - struct rdma_cm_id *listen_id_ = nullptr; + struct ibv_context *context_ = nullptr; + std::shared_ptr device_; + struct ibv_pd *pd_ = nullptr; + struct rdma_event_channel *cm_channel_ = nullptr; + struct rdma_cm_id *listen_id_ = nullptr; std::string requested_device_; uint8_t port_ = 1; std::vector cm_ids_; @@ -201,6 +181,8 @@ class RdmaTransport : public Transport SocketEngine *event_engine_ = nullptr; std::mutex cm_mu_; + SystemBufferPool default_application_pool_; + BufferPool *application_pool_; std::unique_ptr send_pool_; std::unique_ptr recv_pool_; ConnectionHandler connection_handler_; @@ -333,11 +315,10 @@ class RdmaTransport : public Transport }; std::unordered_map> cm_pools_; - std::vector> retired_pools_; void release_completed_frame(WorkRequest *work, const struct ibv_wc &wc); void release_completed_receive(WorkRequest *work, const struct ibv_wc &wc); - void record_completion_error(const struct ibv_wc &wc); + void record_completion_error(const struct ibv_wc &wc, bool expected_flush = false); static void repost_frame_release(void *context, Buffer *data); bool adopt_cm_context(struct ibv_context *context); }; diff --git a/lib/crowdb-rpc/src/c_api.cpp b/lib/crowdb-rpc/src/c_api.cpp index ac72e0725..9cc1f9085 100644 --- a/lib/crowdb-rpc/src/c_api.cpp +++ b/lib/crowdb-rpc/src/c_api.cpp @@ -143,8 +143,8 @@ uint32_t crowdb_rpc_buffer_registration_id(crowdb_rpc_buffer_t buf) int crowdb_rpc_buffer_ownership(crowdb_rpc_buffer_t buf) { return (buf != nullptr && buf->buf != nullptr) - ? static_cast(buf->buf->ownership.load(std::memory_order_acquire)) - : -1; + ? static_cast(buf->buf->ownership.load(std::memory_order_acquire)) + : -1; } void crowdb_rpc_buffer_set_metadata(crowdb_rpc_buffer_t buf, uint64_t request_id, uint64_t stream_id, uint64_t sequence, @@ -191,7 +191,9 @@ crowdb_rpc_buffer_t crowdb_rpc_buffer_ref(crowdb_rpc_buffer_t buf) if (buf == nullptr || buf->buf == nullptr) { return nullptr; } - return buf->buf->ref_clone() == nullptr ? nullptr : buf; + auto clone = std::make_unique(); + clone->buf = buf->buf->ref_clone(); + return clone->buf == nullptr ? nullptr : clone.release(); } catch (...) { return nullptr; @@ -894,40 +896,13 @@ void crowdb_rpc_conn_close(crowdb_rpc_conn_t conn) // ── Built-in echo handler ───────────────────────────────────────── -// Echo handler: returns the request data as the response data, with a -// ConnectionPingResponse control buffer echoing the request_id. Same -// logic as the load_test.cpp echo handler, compiled into the library. -static crowdb::rpc::OutFrame *echo_handler(crowdb::rpc::Frame *request, crowdb::rpc::Connection *conn) -{ - uint64_t req_id = request->request_id; - uint64_t create_nano = request->rpc_create_nano; - crowdb::rpc::BufferPool *pool = conn->pool(); - uint64_t resp_nano = static_cast(std::chrono::steady_clock::now().time_since_epoch().count()); - crowdb::rpc::Buffer *resp_ctrl = crowdb::rpc::build_ping_response(pool, req_id, create_nano, resp_nano); - - crowdb::rpc::Buffer *resp_data = nullptr; - if (request->data_buf != nullptr && request->data_buf->len > 0) { - resp_data = pool->alloc(request->data_buf->len); - if (resp_data != nullptr) { - std::memcpy(resp_data->data, request->data_buf->payload_data(), request->data_buf->len); - resp_data->write(resp_data->data, request->data_buf->len); - } - } - - // Capture msg_type from the request header so the response matches. - uint16_t msg_type = request->header.msg_type; - - delete request; - return crowdb::rpc::build_out_frame(req_id, msg_type, resp_ctrl, resp_data); -} - void crowdb_rpc_server_register_echo_handler(crowdb_rpc_server_t server, uint16_t msg_type) { try { if (server == nullptr) { return; } - server->server->register_handler(msg_type, echo_handler); + server->server->register_handler(msg_type, crowdb::rpc::handle_echo); } catch (...) { } diff --git a/lib/crowdb-rpc/src/server/handler.cpp b/lib/crowdb-rpc/src/server/handler.cpp index 263622d5d..0be65902d 100644 --- a/lib/crowdb-rpc/src/server/handler.cpp +++ b/lib/crowdb-rpc/src/server/handler.cpp @@ -44,4 +44,29 @@ OutFrame *handle_unknown(Frame *request, Connection *conn) return build_out_frame(req_id, msg_type, resp_ctrl, nullptr); } +OutFrame *handle_echo(Frame *request, Connection *conn) +{ + const auto request_id = request->request_id; + const auto msg_type = request->header.msg_type; + const auto now = static_cast(std::chrono::steady_clock::now().time_since_epoch().count()); + auto *control = build_ping_response(conn->pool(), request_id, request->rpc_create_nano, now); + Buffer *data = nullptr; + const auto length = request->data_buf == nullptr ? 0 : request->data_buf->len; + if (length != 0) { + data = conn->pool()->alloc(length); + if (data != nullptr) + data->write(request->data_buf->payload_data(), length); + } + delete request; + if (control == nullptr || (length != 0 && data == nullptr)) { + if (control != nullptr) + control->release(); + if (data != nullptr) + data->release(); + conn->close(); + return nullptr; + } + return build_out_frame(request_id, msg_type, control, data); +} + } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_arena.h b/lib/crowdb-rpc/src/transport/rdma/rdma_arena.h new file mode 100644 index 000000000..f8532fb6e --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_arena.h @@ -0,0 +1,44 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/transport/rdma/rdma_transport.h" + +#ifdef CROWDB_RPC_HAVE_RDMA +namespace crowdb::rpc +{ +struct RdmaBufferPool::Arena : BufferPool, std::enable_shared_from_this +{ + struct Entry + { + std::atomic buffer{nullptr}; + std::atomic next{0}; + }; + + std::shared_ptr registration_owner; + ibv_mr *mr = nullptr; + void *region = nullptr; + uint64_t bytes; + uint32_t capacity; + std::unique_ptr entries; + std::array, 4> free_heads{}; + std::atomic unused_entries{0}; + std::atomic next_entry{0}; + std::atomic outstanding{0}; + std::atomic next_offset{0}; + std::atomic quarantined{0}; + + Arena(ibv_pd *pd, uint64_t size, uint32_t count, std::shared_ptr owner); + ~Arena() override; + Buffer *alloc(uint32_t requested) override; + void recycle(Buffer *buffer) override; + Buffer *fresh(uint32_t size); + Buffer *pop(size_t bucket); + void push(Buffer *buffer); + uint32_t pop_entry(std::atomic &head); + void push_entry(std::atomic &head, uint32_t index); + void quarantine_inflight(); +}; +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp index 8690c6662..d05989907 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp @@ -1,188 +1,214 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -#include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "rdma_arena.h" #ifdef CROWDB_RPC_HAVE_RDMA +# include "crowdb-common/log.h" +# include # include namespace crowdb::rpc { +RdmaBufferPool::RdmaBufferPool(ibv_pd *pd, uint64_t mr_size, uint32_t max_buffers, + std::shared_ptr registration_owner) + : arena_(std::make_shared(pd, mr_size, max_buffers, std::move(registration_owner))) +{ +} + +RdmaBufferPool::~RdmaBufferPool() = default; + +Buffer *RdmaBufferPool::alloc(uint32_t capacity) +{ + return arena_->alloc(capacity); +} + +void RdmaBufferPool::recycle(Buffer *buffer) +{ + arena_->recycle(buffer); +} + +bool RdmaBufferPool::quiescent() const +{ + return arena_->outstanding.load(std::memory_order_acquire) == 0; +} -// ── RdmaBufferPool ──────────────────────────────────────────────── +uint64_t RdmaBufferPool::quarantined_bytes() const +{ + return arena_->quarantined.load(std::memory_order_relaxed); +} -RdmaBufferPool::RdmaBufferPool(struct ibv_pd *pd, uint64_t mr_size, uint32_t max_buffers) - : pd_(pd), - mr_size_(mr_size), - max_buffers_(max_buffers) +void RdmaBufferPool::quarantine_inflight() { - for (auto &head : free_heads_) - head.store(nullptr, std::memory_order_relaxed); - if (pd_ == nullptr || mr_size_ == 0 || max_buffers_ == 0) { + arena_->quarantine_inflight(); +} + +RdmaBufferPool::Arena::Arena(ibv_pd *pd, uint64_t size, uint32_t count, std::shared_ptr owner) + : registration_owner(std::move(owner)), + bytes(size), + capacity(count) +{ + if (pd == nullptr || size == 0 || count == 0 || count == UINT32_MAX) return; - } - if (posix_memalign(®ion_, 64, mr_size_) != 0) { - region_ = nullptr; + entries = std::make_unique(count); + if (posix_memalign(®ion, 64, size) != 0) { + region = nullptr; return; } - mr_ = ibv_reg_mr(pd_, region_, mr_size_, - IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | - IBV_ACCESS_REMOTE_ATOMIC); - if (mr_ == nullptr) { - std::free(region_); - region_ = nullptr; - return; + mr = ibv_reg_mr(pd, region, size, IBV_ACCESS_LOCAL_WRITE); + if (mr == nullptr) { + std::free(region); + region = nullptr; } } -RdmaBufferPool::~RdmaBufferPool() +RdmaBufferPool::Arena::~Arena() { - if (outstanding_.load(std::memory_order_acquire) != 0) { - // Keep the MR registered while a caller, disk operation, or device - // completion still owns a slice. Process teardown reclaims it. - return; - } - Buffer *buf = all_head_.exchange(nullptr, std::memory_order_acq_rel); - while (buf != nullptr) { - Buffer *next = buf->next_all; - delete buf->ref; - delete buf; - buf = next; - } - if (mr_ != nullptr) { - ibv_dereg_mr(mr_); - mr_ = nullptr; + assert(outstanding.load(std::memory_order_acquire) == 0); + if (entries != nullptr) { + for (uint32_t index = 0; index < capacity; ++index) { + if (auto *buffer = entries[index].buffer.load(std::memory_order_relaxed); buffer != nullptr) { + delete buffer->ref; + delete buffer; + } + } } - if (region_ != nullptr) { - std::free(region_); - region_ = nullptr; + if (mr != nullptr) { + const int error = ibv_dereg_mr(mr); + if (error != 0) + CRB_LOG_ERROR("rdma MR deregistration failed error={}", error); } + std::free(region); } -Buffer *RdmaBufferPool::alloc_fresh(uint32_t capacity) +Buffer *RdmaBufferPool::Arena::fresh(uint32_t size) { - if (mr_ == nullptr || region_ == nullptr) { + if (mr == nullptr) return nullptr; - } - uint64_t offset = next_offset_.load(std::memory_order_relaxed); - uint64_t aligned_offset; - while (true) { - if (offset > mr_size_ || offset > UINT64_MAX - 63U) + uint32_t index = pop_entry(unused_entries); + if (index == UINT32_MAX) { + index = next_entry.load(std::memory_order_relaxed); + do { + if (index >= capacity) + return nullptr; + } while (!next_entry.compare_exchange_weak(index, index + 1, std::memory_order_relaxed)); + } + uint64_t offset = next_offset.load(std::memory_order_relaxed); + uint64_t aligned; + do { + if (offset > bytes || offset > UINT64_MAX - 63U) { + push_entry(unused_entries, index); return nullptr; - aligned_offset = (offset + 63U) & ~uint64_t{63}; - if (aligned_offset > mr_size_ || capacity > mr_size_ - aligned_offset) + } + aligned = (offset + 63U) & ~uint64_t{63}; + if (aligned > bytes || size > bytes - aligned) { + push_entry(unused_entries, index); return nullptr; - if (next_offset_.compare_exchange_weak(offset, aligned_offset + capacity, std::memory_order_acq_rel, - std::memory_order_relaxed)) - break; - } - - auto *buf = new Buffer; - buf->capacity = capacity; - buf->type = BufferType::Registered; - buf->pool = this; - buf->ref = new std::atomic(1); - - buf->data = static_cast(region_) + aligned_offset; - buf->len = 0; - buf->allocation_data = buf->data; - buf->allocation_capacity = capacity; - buf->registration_handle = reinterpret_cast(mr_); - buf->registration_key = mr_->lkey; - Buffer *all = all_head_.load(std::memory_order_acquire); - do { - buf->next_all = all; - } while (!all_head_.compare_exchange_weak(all, buf, std::memory_order_acq_rel, std::memory_order_acquire)); - return buf; + } + } while (!next_offset.compare_exchange_weak(offset, aligned + size, std::memory_order_relaxed)); + auto *buffer = new Buffer; + buffer->capacity = size; + buffer->type = BufferType::Registered; + buffer->pool = this; + buffer->pool_index = index; + buffer->ref = new std::atomic(1); + buffer->data = static_cast(region) + aligned; + buffer->allocation_data = buffer->data; + buffer->allocation_capacity = size; + buffer->registration_handle = reinterpret_cast(mr); + buffer->registration_key = mr->lkey; + buffer->registration_id = mr->lkey; + entries[index].buffer.store(buffer, std::memory_order_release); + return buffer; } -Buffer *RdmaBufferPool::pop(std::atomic &head) +Buffer *RdmaBufferPool::Arena::pop(size_t bucket) { - Buffer *current = head.load(std::memory_order_acquire); - while (current != nullptr && !head.compare_exchange_weak(current, current->next_pool, std::memory_order_acq_rel, - std::memory_order_acquire)) { - } - if (current != nullptr) - current->next_pool = nullptr; - return current; + const auto index = pop_entry(free_heads[bucket]); + return index == UINT32_MAX ? nullptr : entries[index].buffer.load(std::memory_order_acquire); } -void RdmaBufferPool::push(std::atomic &head, Buffer *buf) +uint32_t RdmaBufferPool::Arena::pop_entry(std::atomic &head) { - Buffer *current = head.load(std::memory_order_acquire); - do { - buf->next_pool = current; - } while (!head.compare_exchange_weak(current, buf, std::memory_order_acq_rel, std::memory_order_acquire)); + uint64_t current = head.load(std::memory_order_acquire); + while (static_cast(current) != 0) { + const auto index = static_cast(current) - 1; + const auto next = entries[index].next.load(std::memory_order_relaxed); + const auto replacement = ((current + (uint64_t{1} << 32)) & 0xffffffff00000000ULL) | next; + if (head.compare_exchange_weak(current, replacement, std::memory_order_acq_rel)) + return index; + } + return UINT32_MAX; } -bool RdmaBufferPool::reserve_outstanding() +void RdmaBufferPool::Arena::push(Buffer *buffer) { - uint32_t count = outstanding_.load(std::memory_order_relaxed); - while (count < max_buffers_ && !outstanding_.compare_exchange_weak(count, count + 1, std::memory_order_acq_rel, - std::memory_order_relaxed)) { - } - return count < max_buffers_; + push_entry(free_heads[static_cast(buffer->capacity_class)], buffer->pool_index); } -void RdmaBufferPool::quarantine_inflight() +void RdmaBufferPool::Arena::push_entry(std::atomic &head, uint32_t index) { - for (Buffer *buf = all_head_.load(std::memory_order_acquire); buf != nullptr; buf = buf->next_all) { - const BufferOwnership state = buf->ownership.load(std::memory_order_acquire); - if (state != BufferOwnership::Available && state != BufferOwnership::Quarantined) - (void)buf->quarantine(); - } + auto &entry = entries[index]; + uint64_t current = head.load(std::memory_order_relaxed); + uint64_t replacement; + do { + entry.next.store(static_cast(current), std::memory_order_relaxed); + replacement = ((current + (uint64_t{1} << 32)) & 0xffffffff00000000ULL) | (index + 1); + } while (!head.compare_exchange_weak(current, replacement, std::memory_order_release, std::memory_order_relaxed)); } -Buffer *RdmaBufferPool::alloc(uint32_t capacity) +Buffer *RdmaBufferPool::Arena::alloc(uint32_t requested) { BufferCapacityClass cls; - uint32_t bucket; - if (!classify_rdma_capacity(capacity, &cls, &bucket)) { + uint32_t size; + if (!classify_rdma_capacity(requested, &cls, &size)) return nullptr; - } - Buffer *buf = pop(free_heads_[static_cast(cls)]); - if (buf != nullptr) { - buf->reset_view(); - buf->len = 0; - buf->ref->store(1, std::memory_order_relaxed); - buf->capacity_class = cls; - buf->ownership.store(BufferOwnership::Available, std::memory_order_release); - if (!reserve_outstanding()) { - push(free_heads_[static_cast(cls)], buf); - return nullptr; - } - return buf; - } - if (!reserve_outstanding()) + auto *buffer = pop(static_cast(cls)); + if (buffer == nullptr) + buffer = fresh(size); + if (buffer == nullptr) return nullptr; - buf = alloc_fresh(bucket); - if (buf != nullptr) { - buf->capacity_class = cls; - buf->registration_id = mr_ == nullptr ? 0 : mr_->lkey; - } - else - outstanding_.fetch_sub(1, std::memory_order_acq_rel); - return buf; + buffer->reset_view(); + buffer->len = 0; + buffer->ref->store(1, std::memory_order_relaxed); + buffer->ownership.store(BufferOwnership::Available, std::memory_order_release); + buffer->capacity_class = cls; + buffer->pool_owner = shared_from_this(); + outstanding.fetch_add(1, std::memory_order_relaxed); + return buffer; } -void RdmaBufferPool::recycle(Buffer *buf) +void RdmaBufferPool::Arena::recycle(Buffer *buffer) { - outstanding_.fetch_sub(1, std::memory_order_acq_rel); - buf->reset_view(); - const BufferOwnership state = buf->ownership.load(std::memory_order_acquire); + // Retain this receiver until after publication; another allocator may + // immediately replace buffer->pool_owner after push. + const auto owner = std::move(buffer->pool_owner); + buffer->reset_view(); + const auto state = buffer->ownership.load(std::memory_order_acquire); if (state != BufferOwnership::Available && state != BufferOwnership::Quarantined) - (void)buf->quarantine(); - if (buf->ownership.load(std::memory_order_acquire) == BufferOwnership::Quarantined) { - quarantined_bytes_.fetch_add(buf->capacity, std::memory_order_acq_rel); - push(quarantine_head_, buf); + (void)buffer->quarantine(); + outstanding.fetch_sub(1, std::memory_order_release); + if (buffer->ownership.load(std::memory_order_acquire) == BufferOwnership::Quarantined) { + quarantined.fetch_add(buffer->capacity, std::memory_order_relaxed); return; } - buf->len = 0; - buf->ownership.store(BufferOwnership::Available, std::memory_order_release); - push(free_heads_[static_cast(buf->capacity_class)], buf); + buffer->len = 0; + push(buffer); } +void RdmaBufferPool::Arena::quarantine_inflight() +{ + if (entries == nullptr) + return; + for (uint32_t index = 0; index < capacity; ++index) { + auto *buffer = entries[index].buffer.load(std::memory_order_acquire); + if (buffer == nullptr) + continue; + const auto state = buffer->ownership.load(std::memory_order_acquire); + if (state != BufferOwnership::Available && state != BufferOwnership::Quarantined) + (void)buffer->quarantine(); + } +} } // namespace crowdb::rpc - #endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp index 87b48b101..79812851c 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp @@ -148,15 +148,17 @@ int RdmaTransport::handle_external_event(uint64_t tag, uint32_t max_completions) bool RdmaTransport::prime_receive(const std::shared_ptr &connection) { - if (connection == nullptr || connection->pool() == nullptr) { - + if (connection == nullptr) { return false; } + auto *pools = pools_for(connection.get()); + if (pools == nullptr || pools->recv == nullptr) + return false; const uint32_t depth = config_.posted_receive_target; if (depth == 0 || depth > config_.rq_depth || static_cast(depth) * config_.wire_slot_capacity > config_.receive_arena_bytes) return false; - BufferPool *pool = connection->pool(); + BufferPool *pool = pools->recv.get(); for (uint32_t i = 0; i < depth; ++i) { Buffer *slot = pool->alloc(config_.wire_slot_capacity); if (slot == nullptr) { @@ -173,8 +175,12 @@ bool RdmaTransport::prime_receive(const std::shared_ptr &connection) return true; } -void RdmaTransport::record_completion_error(const ibv_wc &wc) +void RdmaTransport::record_completion_error(const ibv_wc &wc, bool expected_flush) { + if (wc.status == IBV_WC_WR_FLUSH_ERR && expected_flush) { + stats_.shutdown_flushes.fetch_add(1, std::memory_order_relaxed); + return; + } stats_.cq_errors.fetch_add(1, std::memory_order_relaxed); switch (wc.status) { case IBV_WC_WR_FLUSH_ERR: @@ -198,16 +204,17 @@ void RdmaTransport::release_completed_receive(WorkRequest *work, const ibv_wc &w { auto quarantine_and_release = [&] { - record_completion_error(wc); + record_completion_error(wc, !work->conn->is_open()); work->conn->close(); (void)work->slot->quarantine(); work->slot->release(); release_work(work); }; if (wc.status != IBV_WC_SUCCESS) { - CRB_LOG_WARN("rdma receive: completion failed status={} vendor_err={} opcode={}", - static_cast(wc.status), static_cast(wc.vendor_err), - static_cast(wc.opcode)); + if (wc.status != IBV_WC_WR_FLUSH_ERR || work->conn->is_open()) + CRB_LOG_WARN("rdma receive: completion failed status={} vendor_err={} opcode={}", + static_cast(wc.status), static_cast(wc.vendor_err), + static_cast(wc.opcode)); quarantine_and_release(); return; } @@ -289,7 +296,7 @@ void RdmaTransport::release_completed_frame(WorkRequest *work, const ibv_wc &wc) Buffer *slot = work->slot; if (wc.status != IBV_WC_SUCCESS) { - record_completion_error(wc); + record_completion_error(wc, connection != nullptr && !connection->is_open()); // A failed SEND makes the stream uncertain; do not allow a TCP // suffix to be appended to this RDMA connection. if (work->slot != nullptr) diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp index a0a4662af..6d476ca62 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp @@ -145,14 +145,14 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int pools->effective_capacity = qp_attr.cap; pools->cq_index = reservation.queue->index; pools->reserved_capacity = reservation.capacity; - pools->send = std::make_unique(id->pd, config_.send_arena_bytes, config_.sq_depth); - pools->recv = std::make_unique(id->pd, config_.receive_arena_bytes, config_.rq_depth); + pools->send = std::make_unique(id->pd, config_.send_arena_bytes, config_.sq_depth, device_); + pools->recv = std::make_unique(id->pd, config_.receive_arena_bytes, config_.rq_depth, device_); if (pools->send == nullptr || pools->recv == nullptr) { rdma_destroy_qp(id); rdma_destroy_id(id); return nullptr; } - auto connection = std::make_shared(reinterpret_cast(id), addr, pools->recv.get()); + auto connection = std::make_shared(reinterpret_cast(id), addr, application_pool_); set_connection_pools(connection.get(), pools); reservation.committed = true; connection->set_path(ConnectionPath::Rdma); @@ -258,7 +258,6 @@ std::shared_ptr RdmaTransport::accept() cm_connections_.erase(it); } if (auto it = cm_pools_.find(event_id); it != cm_pools_.end()) { - retired_pools_.push_back(std::move(it->second)); cm_pools_.erase(it); } cm_ids_.erase(std::remove(cm_ids_.begin(), cm_ids_.end(), event_id), cm_ids_.end()); @@ -325,14 +324,14 @@ std::shared_ptr RdmaTransport::accept() pools->effective_capacity = qp_attr.cap; pools->cq_index = reservation.queue->index; pools->reserved_capacity = reservation.capacity; - pools->send = std::make_unique(id->pd, config_.send_arena_bytes, config_.sq_depth); - pools->recv = std::make_unique(id->pd, config_.receive_arena_bytes, config_.rq_depth); + pools->send = std::make_unique(id->pd, config_.send_arena_bytes, config_.sq_depth, device_); + pools->recv = std::make_unique(id->pd, config_.receive_arena_bytes, config_.rq_depth, device_); if (pools->send == nullptr || pools->recv == nullptr) { rdma_destroy_qp(id); rdma_destroy_id(id); return nullptr; } - auto connection = std::make_shared(reinterpret_cast(id), "rdma-peer", pools->recv.get()); + auto connection = std::make_shared(reinterpret_cast(id), "rdma-peer", application_pool_); set_connection_pools(connection.get(), pools); reservation.committed = true; connection->set_path(ConnectionPath::Rdma); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_device.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_device.cpp new file mode 100644 index 000000000..9bfc92f97 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_device.cpp @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_device.h" + +#ifdef CROWDB_RPC_HAVE_RDMA +# include "crowdb-common/log.h" + +# include + +namespace crowdb::rpc +{ +std::shared_ptr RdmaDevice::retain(ibv_context *context) +{ + auto owner = std::make_shared(); + int count = 0; + owner->devices = rdma_get_devices(&count); + if (owner->devices == nullptr || + std::find(owner->devices, owner->devices + count, context) == owner->devices + count) + return nullptr; + owner->pd = ibv_alloc_pd(context); + return owner->pd == nullptr ? nullptr : owner; +} + +RdmaDevice::~RdmaDevice() +{ + if (pd != nullptr) { + const int error = ibv_dealloc_pd(pd); + if (error != 0) + CRB_LOG_ERROR("rdma PD release failed error={}", error); + } + if (devices != nullptr) + rdma_free_devices(devices); +} +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_device.h b/lib/crowdb-rpc/src/transport/rdma/rdma_device.h new file mode 100644 index 000000000..378added9 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_device.h @@ -0,0 +1,23 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#ifdef CROWDB_RPC_HAVE_RDMA +# include +# include + +# include + +namespace crowdb::rpc +{ +struct RdmaDevice +{ + ibv_context **devices = nullptr; + ibv_pd *pd = nullptr; + + static std::shared_ptr retain(ibv_context *context); + ~RdmaDevice(); +}; +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index 11ee15ce4..f9e91761b 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -7,6 +7,7 @@ #include "crowdb-rpc/transport/socket_transport.h" #include "rdma_channel.h" #include "rdma_cq.h" +#include "rdma_device.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -29,7 +30,9 @@ namespace crowdb::rpc // ── RdmaTransport ───────────────────────────────────────────────── -RdmaTransport::RdmaTransport(RdmaConfig config) : config_(std::move(config)) +RdmaTransport::RdmaTransport(RdmaConfig config, BufferPool *application_pool) + : config_(std::move(config)), + application_pool_(application_pool == nullptr ? &default_application_pool_ : application_pool) { const bool valid_config = config_.apply_environment() && config_.valid(); if (!valid_config) { @@ -92,14 +95,9 @@ RdmaTransport::~RdmaTransport() if (queue->continuation_fd >= 0) ::close(queue->continuation_fd); } - if (pd_ != nullptr) { - ibv_dealloc_pd(pd_); - pd_ = nullptr; - } - if (context_ != nullptr && owns_context_) { - ibv_close_device(context_); - context_ = nullptr; - } + device_.reset(); + pd_ = nullptr; + context_ = nullptr; } bool RdmaTransport::adopt_cm_context(struct ibv_context *context) @@ -114,18 +112,17 @@ bool RdmaTransport::adopt_cm_context(struct ibv_context *context) // arenas. Do not replace live QPs/registrations when another HCA appears. if (context_ != nullptr) return false; - auto *new_pd = ibv_alloc_pd(context); - if (new_pd == nullptr) + auto device = RdmaDevice::retain(context); + if (device == nullptr) return false; if (!create_completion_queues(context)) { - ibv_dealloc_pd(new_pd); return false; } - context_ = context; - owns_context_ = false; - pd_ = new_pd; - send_pool_ = std::make_unique(pd_, config_.send_arena_bytes, config_.sq_depth); - recv_pool_ = std::make_unique(pd_, config_.send_arena_bytes, config_.sq_depth); + context_ = context; + device_ = std::move(device); + pd_ = device_->pd; + send_pool_ = std::make_unique(pd_, config_.send_arena_bytes, config_.sq_depth, device_); + recv_pool_ = std::make_unique(pd_, config_.send_arena_bytes, config_.sq_depth, device_); register_events(event_engine_); return ready(); } @@ -289,6 +286,9 @@ bool RdmaTransport::post_receive(Connection *conn, Buffer *slot) Buffer *RdmaTransport::register_buffer(Buffer *buf) { + Operation operation(repost_state_); + if (!operation) + return nullptr; if (buf == nullptr || buf->type == BufferType::Registered) { return buf; // already registered } @@ -339,7 +339,6 @@ void RdmaTransport::shutdown() if (auto it = cm_pools_.find(id); it != cm_pools_.end()) { it->second->send->quarantine_inflight(); it->second->recv->quarantine_inflight(); - retired_pools_.push_back(std::move(it->second)); cm_pools_.erase(it); } if (id->qp != nullptr) diff --git a/lib/crowdb-rpc/src/transport/socket_transport.cpp b/lib/crowdb-rpc/src/transport/socket_transport.cpp index 651867768..cba359c80 100644 --- a/lib/crowdb-rpc/src/transport/socket_transport.cpp +++ b/lib/crowdb-rpc/src/transport/socket_transport.cpp @@ -562,7 +562,7 @@ SocketTransport::SocketTransport(uint32_t io_engines, uint32_t io_workers, Buffe rdma_config_ = std::move(rdma_config); if (const char *enabled = std::getenv("CROWDB_RDMA_ENABLED"); enabled != nullptr && std::strcmp(enabled, "1") == 0) { - rdma_transport_ = std::make_unique(rdma_config_); + rdma_transport_ = std::make_unique(rdma_config_, pool_); } #endif // Create N independent engines, each with M workers. When M>1, the diff --git a/lib/crowdb-rpc/tests/c_api_buffer_test.cpp b/lib/crowdb-rpc/tests/c_api_buffer_test.cpp new file mode 100644 index 000000000..26a1c876c --- /dev/null +++ b/lib/crowdb-rpc/tests/c_api_buffer_test.cpp @@ -0,0 +1,39 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/c_api.h" + +#include + +#include + +TEST(CApiBufferTest, ClonedHandlesReleaseIndependently) +{ + const uint8_t bytes[] = {4, 8, 16}; + auto original = crowdb_rpc_buffer_create(bytes, sizeof(bytes)); + ASSERT_NE(original, nullptr); + auto clone = crowdb_rpc_buffer_ref(original); + ASSERT_NE(clone, nullptr); + ASSERT_NE(original, clone); + EXPECT_EQ(crowdb_rpc_buffer_data(original), crowdb_rpc_buffer_data(clone)); + crowdb_rpc_buffer_release(original); + EXPECT_EQ(crowdb_rpc_buffer_len(clone), sizeof(bytes)); + EXPECT_EQ(std::memcmp(crowdb_rpc_buffer_data(clone), bytes, sizeof(bytes)), 0); + crowdb_rpc_buffer_release(clone); +} + +TEST(CApiBufferTest, ExternalOwnerReleasesOnlyAfterLastIndependentHandle) +{ + uint32_t releases = 0; + const uint8_t bytes[] = {3, 5, 7}; + auto original = crowdb_rpc_buffer_create_external( + bytes, sizeof(bytes), [](void *context) { ++*static_cast(context); }, &releases); + ASSERT_NE(original, nullptr); + auto clone = crowdb_rpc_buffer_ref(original); + ASSERT_NE(original, clone); + crowdb_rpc_buffer_release(clone); + EXPECT_EQ(releases, 0U); + EXPECT_EQ(std::memcmp(crowdb_rpc_buffer_data(original), bytes, sizeof(bytes)), 0); + crowdb_rpc_buffer_release(original); + EXPECT_EQ(releases, 1U); +} diff --git a/lib/crowdb-rpc/tests/common/test_rdma_device.h b/lib/crowdb-rpc/tests/common/test_rdma_device.h new file mode 100644 index 000000000..65175ceb5 --- /dev/null +++ b/lib/crowdb-rpc/tests/common/test_rdma_device.h @@ -0,0 +1,42 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/transport/rdma/rdma_transport.h" + +#ifdef CROWDB_RPC_HAVE_RDMA +namespace crowdb::rpc +{ +struct TestRdmaDevice +{ + ibv_context *context = nullptr; + ibv_pd *pd = nullptr; + std::shared_ptr> release_result = std::make_shared>(-1); + + static std::shared_ptr open() + { + auto device = std::make_shared(); + int count = 0; + auto **devices = ibv_get_device_list(&count); + if (devices == nullptr) + return nullptr; + for (int index = 0; index < count && device->context == nullptr; ++index) + device->context = ibv_open_device(devices[index]); + ibv_free_device_list(devices); + if (device->context == nullptr) + return nullptr; + device->pd = ibv_alloc_pd(device->context); + return device->pd == nullptr ? nullptr : device; + } + + ~TestRdmaDevice() + { + if (pd != nullptr) + release_result->store(ibv_dealloc_pd(pd), std::memory_order_release); + if (context != nullptr) + ibv_close_device(context); + } +}; +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/tests/common/test_rdma_transport.h b/lib/crowdb-rpc/tests/common/test_rdma_transport.h index 1b56ddc7d..2a263dbd1 100644 --- a/lib/crowdb-rpc/tests/common/test_rdma_transport.h +++ b/lib/crowdb-rpc/tests/common/test_rdma_transport.h @@ -16,6 +16,11 @@ namespace crowdb::rpc struct TestRdmaTransport { + static std::weak_ptr device_owner(RdmaTransport &transport) + { + return std::shared_ptr(transport.device_); + } + static uint64_t send(RdmaTransport &transport, const std::shared_ptr &connection, Buffer *slot, OutFrame *frame) { diff --git a/lib/crowdb-rpc/tests/rdma_completion_test.cpp b/lib/crowdb-rpc/tests/rdma_completion_test.cpp index 8a6378aab..2d39722f4 100644 --- a/lib/crowdb-rpc/tests/rdma_completion_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_completion_test.cpp @@ -128,6 +128,17 @@ TEST_F(RdmaCompletionTest, QueueCapacityReservationRollsBackFailedSetup) EXPECT_FALSE(TestRdmaTransport::reserve(transport, 1, true)); } +TEST_F(RdmaCompletionTest, ClosedConnectionFlushIsSeparateFromWorkloadErrors) +{ + const auto token = send(); + ASSERT_NE(token, 0U); + connection->close(); + TestRdmaTransport::complete(transport, token, IBV_WC_WR_FLUSH_ERR); + EXPECT_EQ(transport.stats().cq_errors.load(), 0U); + EXPECT_EQ(transport.stats().flush_errors.load(), 0U); + EXPECT_EQ(transport.stats().shutdown_flushes.load(), 1U); +} + TEST_F(RdmaCompletionTest, WorkTablesAndPollingOwnershipAreIndependentPerCq) { auto other = std::make_shared(6, "other-cq", &slots); diff --git a/lib/crowdb-rpc/tests/rdma_lifetime_test.cpp b/lib/crowdb-rpc/tests/rdma_lifetime_test.cpp new file mode 100644 index 000000000..d5c2825c3 --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_lifetime_test.cpp @@ -0,0 +1,162 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "common/test_rdma_device.h" +#include "common/test_rdma_transport.h" +#include "crowdb-rpc/c_api_internal.h" +#include "crowdb-rpc/server/message.h" + +#ifdef CROWDB_RPC_HAVE_RDMA +# include + +# include +# include +# include +# include +# include + +using namespace crowdb::rpc; + +TEST(RdmaBufferPoolTest, HeldLeaseRetainsMrAndPdAfterPoolDestruction) +{ + auto device = TestRdmaDevice::open(); + if (device == nullptr) + GTEST_SKIP() << "no accessible RDMA device"; + auto result = device->release_result; + std::weak_ptr weak_device = device; + auto pool = std::make_unique(device->pd, 4096, 4, device); + auto *buffer = pool->alloc(32); + ASSERT_NE(buffer, nullptr); + buffer->write("held registered payload", 23); + auto original = new crowdb_rpc_buffer_s{buffer}; + auto clone = crowdb_rpc_buffer_ref(original); + ASSERT_NE(original, clone); + pool.reset(); + device.reset(); + EXPECT_FALSE(weak_device.expired()); + EXPECT_EQ(result->load(), -1); + EXPECT_EQ(std::memcmp(crowdb_rpc_buffer_data(clone), "held registered payload", 23), 0); + crowdb_rpc_buffer_release(original); + EXPECT_FALSE(weak_device.expired()); + crowdb_rpc_buffer_release(clone); + EXPECT_TRUE(weak_device.expired()); + EXPECT_EQ(result->load(), 0) << "MR must deregister before PD release"; +} + +TEST(RdmaBufferPoolTest, FailedLargeAllocationPreservesSmallBufferAdmission) +{ + auto device = TestRdmaDevice::open(); + if (device == nullptr) + GTEST_SKIP() << "no accessible RDMA device"; + RdmaBufferPool pool(device->pd, 2048, 2, device); + for (int attempt = 0; attempt < 8; ++attempt) + EXPECT_EQ(pool.alloc(65536), nullptr); + auto *first = pool.alloc(1); + auto *second = pool.alloc(1); + ASSERT_NE(first, nullptr); + ASSERT_NE(second, nullptr); + EXPECT_EQ(pool.alloc(1), nullptr); + first->release(); + second->release(); + EXPECT_TRUE(pool.quiescent()); +} + +TEST(RdmaBufferPoolTest, ConcurrentRecyclingNeverPublishesOneSlotTwice) +{ + auto device = TestRdmaDevice::open(); + if (device == nullptr) + GTEST_SKIP() << "no accessible RDMA device"; + RdmaBufferPool pool(device->pd, 32 * 1024, 32, device); + std::array, 32> occupied{}; + std::atomic errors{0}; + std::vector threads; + for (uint8_t producer = 1; producer <= 8; ++producer) { + threads.emplace_back([&, producer] { + for (uint32_t iteration = 0; iteration < 20000; ++iteration) { + Buffer *buffer; + while ((buffer = pool.alloc(32)) == nullptr) + std::this_thread::yield(); + if (occupied[buffer->pool_index].exchange(true)) + errors.fetch_add(1); + std::memset(buffer->data, producer, 32); + auto *clone = buffer->ref_clone(); + if (!std::all_of(buffer->data, buffer->data + 32, + [producer](uint8_t value) { return value == producer; })) + errors.fetch_add(1); + buffer->release(); + occupied[clone->pool_index].store(false); + clone->release(); + } + }); + } + for (auto &thread : threads) + thread.join(); + EXPECT_EQ(errors.load(), 0U); + EXPECT_TRUE(pool.quiescent()); +} + +# ifdef CROWDB_RPC_TEST_UTIL +TEST(RdmaHardwareTest, HeldReceiveOutlivesTransportAndCmIds) +{ + const char *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) + GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; + RdmaConfig config; + config.wire_slot_capacity = 1024; + config.sq_depth = config.rq_depth = config.posted_receive_target = 8; + config.send_arena_bytes = config.receive_arena_bytes = 8192; + config.cq_depth = 16; + SystemBufferPool application_pool; + auto server = std::make_unique(config, &application_pool); + auto client = std::make_unique(config, &application_pool); + ASSERT_TRUE(server->listen(address, 19388)); + std::shared_ptr incoming; + std::thread acceptor([&] { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (incoming == nullptr && std::chrono::steady_clock::now() < deadline) { + incoming = server->accept(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + }); + auto outgoing = client->connect(address, 19388); + acceptor.join(); + ASSERT_NE(incoming, nullptr); + ASSERT_NE(outgoing, nullptr); + Buffer *held = nullptr; + incoming->set_on_frame([&](Frame *frame, Connection *) { + held = frame->data_buf->ref_clone(); + delete frame; + }); + auto device = TestRdmaTransport::device_owner(*server); + auto client_device = TestRdmaTransport::device_owner(*client); + auto *control = build_ping_request(&application_pool, 71, 0); + auto *data = application_pool.alloc(32); + ASSERT_NE(control, nullptr); + ASSERT_NE(data, nullptr); + data->write("receive lease", 13); + auto *registered = client->register_buffer(data); + ASSERT_NE(registered, nullptr); + auto *frame = build_out_frame(71, 239, control, data); + ASSERT_TRUE(client->submit(outgoing.get(), frame)); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (held == nullptr && std::chrono::steady_clock::now() < deadline) { + client->drain_cq(); + server->drain_cq(); + } + ASSERT_NE(held, nullptr); + server.reset(); + client.reset(); + incoming.reset(); + outgoing.reset(); + EXPECT_FALSE(device.expired()); + EXPECT_NE(held->registration_handle, 0U); + EXPECT_EQ(std::memcmp(held->payload_data(), "receive lease", 13), 0); + EXPECT_FALSE(client_device.expired()); + EXPECT_EQ(std::memcmp(registered->payload_data(), "receive lease", 13), 0); + registered->release(); + EXPECT_TRUE(client_device.expired()); + held->release(); + EXPECT_TRUE(device.expired()); +} +# endif +#endif diff --git a/lib/crowdb-rpc/tests/rdma_worker_test.cpp b/lib/crowdb-rpc/tests/rdma_worker_test.cpp index d570cd827..983bef89a 100644 --- a/lib/crowdb-rpc/tests/rdma_worker_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_worker_test.cpp @@ -127,21 +127,10 @@ TEST(RdmaHardwareTest, RpcServerClientWorkerRoundTrip) delete frame; return static_cast(nullptr); }); - server.register_handler(kMessageType, [&received, &server](Frame *frame, Connection *) { + server.register_handler(kMessageType, [&received, &server](Frame *frame, Connection *connection) { + EXPECT_EQ(connection->pool(), server.pool()); received.fetch_add(1, std::memory_order_relaxed); - OutFrame *response = new OutFrame; - response->request_id = frame->request_id; - response->header.msg_type = static_cast(kMessageType); - response->control = - crowdb::rpc::build_ping_response(server.pool(), frame->request_id, frame->rpc_create_nano, 0); - response->header.msg_size = response->control == nullptr ? 0 : response->control->len; - if (frame->data_buf != nullptr && frame->data_buf->len != 0) { - response->data = server.pool()->alloc(frame->data_buf->len); - if (response->data != nullptr) - response->data->write(frame->data_buf->payload_data(), frame->data_buf->len); - } - delete frame; - return response; + return handle_echo(frame, connection); }); server.start(); for (int i = 0; i < 100 && !client_finished.load(std::memory_order_acquire); ++i) From da5d9f57a44fc1050c752e82a90320001d1deaf7 Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 22 Oct 2026 21:37:34 +0800 Subject: [PATCH 026/258] Share bounded registered arena ownership across transport builds --- doc/design/rpc/design-crowdb-rpc-rdma.md | 10 + doc/working/plan-rdma-rpc-transport.md | 45 +++- doc/working/plan-rdma.md | 10 +- lib/crowdb-rpc/CMakeLists.txt | 15 +- lib/crowdb-rpc/include/crowdb-rpc/buffer.h | 32 +-- .../transport/rdma/rdma_transport.h | 3 +- lib/crowdb-rpc/src/buffer.cpp | 127 +---------- lib/crowdb-rpc/src/buffer_arena.cpp | 198 ++++++++++++++++ lib/crowdb-rpc/src/buffer_arena.h | 65 ++++++ lib/crowdb-rpc/src/registered_buffer_pool.cpp | 40 ++++ .../src/transport/rdma/rdma_arena.h | 44 ---- .../src/transport/rdma/rdma_buffer_pool.cpp | 211 ++++-------------- .../tests/registered_buffer_test.cpp | 142 ++++++++++++ 13 files changed, 569 insertions(+), 373 deletions(-) create mode 100644 lib/crowdb-rpc/src/buffer_arena.cpp create mode 100644 lib/crowdb-rpc/src/buffer_arena.h create mode 100644 lib/crowdb-rpc/src/registered_buffer_pool.cpp delete mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_arena.h create mode 100644 lib/crowdb-rpc/tests/registered_buffer_test.cpp diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index 98536f83b..8434f8663 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -124,6 +124,16 @@ shutdown, final release retires the slot without using the destroyed transport or posting another receive. QP retirement does not accumulate arenas in a transport-wide retired list. +`RegisteredBufferPool` uses the same `BufferArena` descriptor admission, +ownership transitions and tagged class freelists without requiring verbs or an +HCA. `BufferRegion` owns the bounded allocation; a verbs-backed region adds +registration metadata and retains its PD owner. `committed_bytes` reports carved +arena bytes, while the region capacity bounds the reserved allocation. Detached +portable buffers keep the region alive and reserve their descriptor until final +release. Their detached ranges remain committed and are not reused by another +stream. Cached and quarantined descriptors both count toward the hard total. +Application-pool count admission also uses one atomic reservation. + ## 5. Shared CQ and completion model Connections share a bounded CQ pool. Each QP uses one assigned CQ for both diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index edf81aee3..69d644ebc 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -31,7 +31,8 @@ Application reply allocation now uses a separate pool, and real HCA tests prove held registered/receive leases survive pool/transport/CM-ID destruction. The C ABI now returns independent handles for retained buffer references. -Still pending: portable registered-pool concurrency/lifetime parity, bounded +Portable and verbs-backed pools now share the same arena/lifetime machinery. +Still pending: registration admission across retired connections, bounded transport aggregation, `bench rdma`/shared coroutine reporting, unified regression orchestration, negotiated TCP capability preflight, failure injection, and the two-container acceptance profiles. Capability serialization and automatic @@ -356,7 +357,7 @@ Verified histogram checkpoint: opaque handle per retained reference and test both release orders, an external owner released exactly once, and a cloned real registered lease surviving pool/PD facade destruction. This is part of the lease contract. -- [~] **Finish portable pool parity**: the no-verbs `RegisteredBufferPool` +- [x] **Finish portable pool parity**: the no-verbs `RegisteredBufferPool` still uses raw-pointer freelist heads and a facade pointer in outstanding buffers. Reuse the bounded shared-arena/index freelist machinery for that pool too, retaining detach semantics and optional registration. Add no-device @@ -366,6 +367,44 @@ Verified histogram checkpoint: its declared count. Files: `buffer.h`, `buffer.cpp`, private common arena sources, RDMA pool wrapper and buffer tests. This closes the foundation contract without leaving concurrency safety dependent on HCA availability. + The shared private `BufferArena` owns descriptors; `BufferRegion` owns its + allocation and optional MR/PD owner. Detached portable descriptors keep arena + ownership until their final callback, returning index admission only then; + the detached byte range stays committed. Add `CROWDB_RPC_ENABLE_RDMA=OFF` + as a supported CMake build switch so the no-verbs gate is reproducible even + on this host with rdma-core installed. +- [~] **Bound registration across connection retirement**: add a byte + reservation to the existing `RdmaDevice` resource owner. A held lease must + retain that reservation after its QP releases CQ admission; a new QP cannot + bypass the registered-memory bound by replacing an arena that is still held. + Add `registered_memory_budget_bytes` (zero derives a checked bound from the + CQ-admitted QP count, per-QP arenas and compatibility copy pools), current and + high-water reserved bytes, and registration rejection/failure counters. Keep + reservations atomic and retained by the MR owner until deregistration. + Reject setup when either send or receive MR creation fails: a non-null pool + facade alone does not prove registration succeeded. Test retained-lease + exhaustion/release and concurrent reservations, plus invalid/overflowed + budgets. Files: `rdma_device.{h,cpp}`, RDMA config/setup/pool sources and + resource lifetime tests. Expose these values in phase D's status/report API. + Include final detached-buffer quarantine accounting in the shared arena and + verify it with a no-device case. + +Portable pool verification: + +- `BufferArena`/`BufferRegion` back both registered pool facades; detach keeps + its arena owner, and final detach release returns descriptor admission. Six + portable tests cover held/detached leases, the hard descriptor and byte + bounds, quarantine, 160,000 concurrent recycling iterations and concurrent + `SystemBufferPool` count admission. +- `CROWDB_RPC_ENABLE_RDMA=OFF` builds successfully; all 123 CTest cases pass + in that build. The enabled build passes all 42 focused buffer/completion + cases (one compatibility-copy case skips before CM adoption). The real HCA + held receive/registered-copy test passes ten repeated CM setup/teardown cycles + after the shared-arena extraction. +- Full enabled C++/FFI gate and all 29 Rust RPC FFI tests pass: + `.crowdb-runtime/artifacts/unit/cpp-26a123727c20408cbc7ef7f73be952f1/`. + Formatting, scoped tree-lint and workspace clippy pass. Tree-lint retains its + non-blocking style warning set. Application allocator verification: @@ -712,6 +751,8 @@ E2E tests: - `lib/crowdb-common/rust/src/metrics/histogram.rs` and `histogram/data.rs` - `lib/crowdb-common/testdata/latency_histogram.csv` and both language tests - `lib/crowdb-rpc/src/client/co_client.cpp` and private coroutine state header +- `lib/crowdb-rpc/src/buffer_arena.{h,cpp}`, `registered_buffer_pool.cpp`, + shared buffer facade and registered/real-MR lifetime tests - `lib/crowdb-rpc/ffi/src/co_bench.rs` and C/Rust statistics bindings - `app/crowdb-cli/src/commands/bench/{verb,rpc,loader,result}.rs` - `tools/benchmark/bench-rdma-rpc-regression.sh`, shared RPC runner and tests diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index 26609d5d8..b61586018 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -25,7 +25,8 @@ and C/Rust lease metadata accessors. R239's transport path is implemented and passes the prepared HCA local-loop plus the real `RpcServer`/`RpcClient` worker round trip. Its requirement is still open because negotiated capability preflight, -pre-payload fallback, portable pool parity, aggregation/CLI benchmarking and +pre-payload fallback, registration admission across retired connections, +aggregation/CLI benchmarking and multi-container acceptance have not all passed. Live HCA tests now verify held registered/receive leases after pool, transport and CM-ID destruction; the final release deregisters MR before PD release. Application reply allocation @@ -153,10 +154,15 @@ TCP-only profile and build/mocked tests. uncertain completion cases. - [x] **Review handoff**: verify that R239 can obtain control/data buffers and that every asynchronous owner has a release transition. -- [ ] **Close portable pool parity**: share the verified arena lifetime and +- [x] **Close portable pool parity**: share the verified arena lifetime and tagged-index freelist machinery with the no-verbs registered pool, then verify concurrent bounds, held leases, quarantine and detach without an HCA. The detailed transport plan owns this continuation before aggregation. +- [ ] **Bound retired registration admission**: keep registered bytes charged + to the existing device owner while old caller leases retain their MRs, reject + failed MR setup before publishing a connection, and verify the current/high + water budget and rejection counters. The transport plan owns this final + resource admission continuation. Exit condition: no buffer can be recycled while transport, disk, or caller ownership remains unresolved. diff --git a/lib/crowdb-rpc/CMakeLists.txt b/lib/crowdb-rpc/CMakeLists.txt index 110604303..a88fd4678 100644 --- a/lib/crowdb-rpc/CMakeLists.txt +++ b/lib/crowdb-rpc/CMakeLists.txt @@ -50,16 +50,19 @@ endif() # Gated behind CROWDB_RPC_HAVE_RDMA. No macOS RNICs, so RDMA sources are # excluded from the build entirely when not found (mirrors crowdb-tree's # liburing gate). rdma-core (conda-forge) provides both libraries on Linux. -find_path(IBVERBS_INCLUDE_DIR infiniband/verbs.h) -find_library(IBVERBS_LIB NAMES ibverbs) -find_path(RDMACM_INCLUDE_DIR rdma/rdma_cma.h) -find_library(RDMACM_LIB NAMES rdmacm) -if(IBVERBS_INCLUDE_DIR AND IBVERBS_LIB AND RDMACM_INCLUDE_DIR AND RDMACM_LIB) +option(CROWDB_RPC_ENABLE_RDMA "Enable RDMA when libibverbs and librdmacm are available" ON) +if(CROWDB_RPC_ENABLE_RDMA) + find_path(IBVERBS_INCLUDE_DIR infiniband/verbs.h) + find_library(IBVERBS_LIB NAMES ibverbs) + find_path(RDMACM_INCLUDE_DIR rdma/rdma_cma.h) + find_library(RDMACM_LIB NAMES rdmacm) +endif() +if(CROWDB_RPC_ENABLE_RDMA AND IBVERBS_INCLUDE_DIR AND IBVERBS_LIB AND RDMACM_INCLUDE_DIR AND RDMACM_LIB) set(CROWDB_RPC_HAVE_RDMA TRUE) message(STATUS "crowdb-rpc RDMA: enabled (${IBVERBS_LIB}, ${RDMACM_LIB})") else() set(CROWDB_RPC_HAVE_RDMA FALSE) - message(STATUS "crowdb-rpc RDMA: not found (TCP-only build)") + message(STATUS "crowdb-rpc RDMA: disabled or unavailable (TCP-only build)") endif() # ── Library ─────────────────────────────────────────────────────── diff --git a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h index 39547ded6..c867261e3 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h @@ -12,6 +12,8 @@ namespace crowdb::rpc { +class BufferArena; + // Buffer type determines the allocation strategy and memory registration. // System: aligned system malloc (posix_memalign), recycled to a // SystemBufferPool. Registered: ibv_reg_mr'd for RDMA, recycled to an @@ -178,15 +180,8 @@ class RegisteredBufferPool : public BufferPool void recycle(Buffer *buf) override; bool detach(Buffer *buf) override; - [[nodiscard]] uint64_t committed_bytes() const - { - return committed_bytes_.load(); - } - - [[nodiscard]] uint64_t quarantined_bytes() const - { - return quarantined_bytes_.load(); - } + [[nodiscard]] uint64_t committed_bytes() const; + [[nodiscard]] uint64_t quarantined_bytes() const; [[nodiscard]] uint32_t registration_id() const { @@ -194,23 +189,8 @@ class RegisteredBufferPool : public BufferPool } private: - static constexpr size_t CLASS_COUNT = 4; - uint64_t max_bytes_; - uint32_t max_buffers_; - uint32_t registration_id_; - std::atomic committed_bytes_{0}; - std::atomic outstanding_{0}; - std::array, CLASS_COUNT> free_heads_{}; - std::atomic quarantine_head_{nullptr}; - std::atomic quarantined_bytes_{0}; - - static size_t index(BufferCapacityClass cls) - { - return static_cast(cls); - } - - Buffer *pop(std::atomic &head); - void push(std::atomic &head, Buffer *buf); + std::shared_ptr arena_; + uint32_t registration_id_; }; } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index 3e45b6408..7249d79c8 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -78,8 +78,7 @@ class RdmaBufferPool : public BufferPool void quarantine_inflight(); private: - struct Arena; - std::shared_ptr arena_; + std::shared_ptr arena_; }; // RdmaTransport: RDMA transport using libibverbs + librdmacm. diff --git a/lib/crowdb-rpc/src/buffer.cpp b/lib/crowdb-rpc/src/buffer.cpp index 2dd3560d9..acc4680e5 100644 --- a/lib/crowdb-rpc/src/buffer.cpp +++ b/lib/crowdb-rpc/src/buffer.cpp @@ -143,10 +143,11 @@ SystemBufferPool::SystemBufferPool(uint32_t max_buffers) : max_buffers_(max_buff Buffer *SystemBufferPool::alloc(uint32_t capacity) { - if (outstanding_.load(std::memory_order_relaxed) >= max_buffers_) { - return nullptr; // bound exceeded - } - outstanding_.fetch_add(1, std::memory_order_relaxed); + uint32_t count = outstanding_.load(std::memory_order_relaxed); + do { + if (count >= max_buffers_) + return nullptr; + } while (!outstanding_.compare_exchange_weak(count, count + 1, std::memory_order_relaxed)); auto *buf = new Buffer; buf->capacity = capacity; @@ -188,122 +189,4 @@ bool SystemBufferPool::detach(Buffer *buf) return true; } -RegisteredBufferPool::RegisteredBufferPool(uint64_t max_bytes, uint32_t max_buffers, uint32_t registration_id) - : max_bytes_(max_bytes), - max_buffers_(max_buffers), - registration_id_(registration_id) -{ - for (auto &head : free_heads_) - head.store(nullptr, std::memory_order_relaxed); -} - -RegisteredBufferPool::~RegisteredBufferPool() -{ - auto destroy = [](Buffer *head) { - while (head != nullptr) { - Buffer *next = head->next_pool; - delete head->ref; - std::free(head->data); - delete head; - head = next; - } - }; - for (auto &head : free_heads_) - destroy(head.exchange(nullptr, std::memory_order_acq_rel)); - destroy(quarantine_head_.exchange(nullptr, std::memory_order_acq_rel)); -} - -Buffer *RegisteredBufferPool::pop(std::atomic &head) -{ - Buffer *current = head.load(std::memory_order_acquire); - while (current != nullptr && !head.compare_exchange_weak(current, current->next_pool, std::memory_order_acq_rel, - std::memory_order_acquire)) { - } - if (current != nullptr) - current->next_pool = nullptr; - return current; -} - -void RegisteredBufferPool::push(std::atomic &head, Buffer *buf) -{ - Buffer *current = head.load(std::memory_order_acquire); - do { - buf->next_pool = current; - } while (!head.compare_exchange_weak(current, buf, std::memory_order_acq_rel, std::memory_order_acquire)); -} - -Buffer *RegisteredBufferPool::alloc(uint32_t requested) -{ - BufferCapacityClass cls; - uint32_t capacity; - if (!classify_rdma_capacity(requested, &cls, &capacity)) - return nullptr; - Buffer *buf = pop(free_heads_[index(cls)]); - if (buf == nullptr) { - uint64_t bytes = committed_bytes_.load(std::memory_order_relaxed); - while (bytes + capacity <= max_bytes_ && - !committed_bytes_.compare_exchange_weak(bytes, bytes + capacity, std::memory_order_acq_rel, - std::memory_order_relaxed)) { - } - if (bytes + capacity > max_bytes_) - return nullptr; - uint32_t count = outstanding_.fetch_add(1, std::memory_order_acq_rel); - if (count >= max_buffers_) { - outstanding_.fetch_sub(1, std::memory_order_acq_rel); - committed_bytes_.fetch_sub(capacity, std::memory_order_acq_rel); - return nullptr; - } - void *data = nullptr; - if (posix_memalign(&data, 64, capacity) != 0) { - outstanding_.fetch_sub(1, std::memory_order_acq_rel); - committed_bytes_.fetch_sub(capacity, std::memory_order_acq_rel); - return nullptr; - } - buf = new Buffer; - buf->data = static_cast(data); - buf->capacity = capacity; - buf->type = BufferType::Registered; - buf->pool = this; - buf->ref = new std::atomic(1); - } - else { - outstanding_.fetch_add(1, std::memory_order_acq_rel); - buf->ref->store(1, std::memory_order_release); - } - buf->len = 0; - buf->capacity_class = cls; - buf->registration_id = registration_id_; - buf->ownership.store(BufferOwnership::Available, std::memory_order_release); - return buf; -} - -void RegisteredBufferPool::recycle(Buffer *buf) -{ - outstanding_.fetch_sub(1, std::memory_order_acq_rel); - const BufferOwnership state = buf->ownership.load(std::memory_order_acquire); - if (state != BufferOwnership::Available && state != BufferOwnership::Quarantined) { - (void)buf->quarantine(); - } - if (buf->ownership.load(std::memory_order_acquire) == BufferOwnership::Quarantined) { - quarantined_bytes_.fetch_add(buf->capacity, std::memory_order_acq_rel); - push(quarantine_head_, buf); - return; - } - buf->len = 0; - buf->ownership.store(BufferOwnership::Available, std::memory_order_release); - push(free_heads_[index(buf->capacity_class)], buf); -} - -bool RegisteredBufferPool::detach(Buffer *buf) -{ - const BufferOwnership state = - buf == nullptr ? BufferOwnership::Quarantined : buf->ownership.load(std::memory_order_acquire); - if (buf == nullptr || buf->pool != this || buf->ref == nullptr || buf->ref->load(std::memory_order_acquire) != 1 || - (state != BufferOwnership::Available && state != BufferOwnership::CallerOwned)) - return false; - buf->pool = nullptr; - outstanding_.fetch_sub(1, std::memory_order_acq_rel); - return true; -} - } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/buffer_arena.cpp b/lib/crowdb-rpc/src/buffer_arena.cpp new file mode 100644 index 000000000..d9a75bead --- /dev/null +++ b/lib/crowdb-rpc/src/buffer_arena.cpp @@ -0,0 +1,198 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "buffer_arena.h" + +#include +#include + +namespace crowdb::rpc +{ +std::shared_ptr BufferRegion::allocate(uint64_t bytes, uint32_t registration_id) +{ + auto region = std::make_shared(); + region->bytes = bytes; + region->registration_id = registration_id; + void *memory = nullptr; + if (bytes != 0 && posix_memalign(&memory, 64, bytes) == 0) { + region->data = static_cast(memory); + region->owner = std::shared_ptr(memory, std::free); + } + return region; +} + +BufferArena::BufferArena(std::shared_ptr allocation, uint32_t count) + : region(std::move(allocation)), + bytes(region == nullptr ? 0 : region->bytes), + capacity(count) +{ + if (region != nullptr && region->data != nullptr && count != 0 && count != UINT32_MAX) + entries_ = std::make_unique(count); +} + +BufferArena::~BufferArena() +{ + assert(outstanding.load(std::memory_order_acquire) == 0); + if (entries_ != nullptr) { + for (uint32_t index = 0; index < capacity; ++index) { + if (auto *buffer = entries_[index].buffer.load(std::memory_order_relaxed); buffer != nullptr) { + delete buffer->ref; + delete buffer; + } + } + } +} + +Buffer *BufferArena::fresh(uint32_t size) +{ + if (entries_ == nullptr) + return nullptr; + uint32_t index = pop_entry(unused_entries_); + if (index == UINT32_MAX) { + index = next_entry_.load(std::memory_order_relaxed); + do { + if (index >= capacity) + return nullptr; + } while (!next_entry_.compare_exchange_weak(index, index + 1, std::memory_order_relaxed)); + } + uint64_t offset = next_offset.load(std::memory_order_relaxed); + uint64_t aligned; + do { + if (offset > bytes || offset > UINT64_MAX - 63U) { + push_entry(unused_entries_, index); + return nullptr; + } + aligned = (offset + 63U) & ~uint64_t{63}; + if (aligned > bytes || size > bytes - aligned) { + push_entry(unused_entries_, index); + return nullptr; + } + } while (!next_offset.compare_exchange_weak(offset, aligned + size, std::memory_order_relaxed)); + auto *buffer = new Buffer; + buffer->capacity = size; + buffer->type = BufferType::Registered; + buffer->pool = this; + buffer->pool_index = index; + buffer->ref = new std::atomic(1); + buffer->data = region->data + aligned; + buffer->allocation_data = buffer->data; + buffer->allocation_capacity = size; + buffer->registration_handle = region->registration_handle; + buffer->registration_key = region->registration_key; + buffer->registration_id = region->registration_id; + entries_[index].buffer.store(buffer, std::memory_order_release); + return buffer; +} + +Buffer *BufferArena::pop(size_t bucket) +{ + const auto index = pop_entry(free_heads_[bucket]); + return index == UINT32_MAX ? nullptr : entries_[index].buffer.load(std::memory_order_acquire); +} + +uint32_t BufferArena::pop_entry(std::atomic &head) +{ + uint64_t current = head.load(std::memory_order_acquire); + while (static_cast(current) != 0) { + const auto index = static_cast(current) - 1; + const auto next = entries_[index].next.load(std::memory_order_relaxed); + const auto replacement = ((current + (uint64_t{1} << 32)) & 0xffffffff00000000ULL) | next; + if (head.compare_exchange_weak(current, replacement, std::memory_order_acq_rel)) + return index; + } + return UINT32_MAX; +} + +void BufferArena::push(Buffer *buffer) +{ + push_entry(free_heads_[static_cast(buffer->capacity_class)], buffer->pool_index); +} + +void BufferArena::push_entry(std::atomic &head, uint32_t index) +{ + auto &entry = entries_[index]; + uint64_t current = head.load(std::memory_order_relaxed); + uint64_t replacement; + do { + entry.next.store(static_cast(current), std::memory_order_relaxed); + replacement = ((current + (uint64_t{1} << 32)) & 0xffffffff00000000ULL) | (index + 1); + } while (!head.compare_exchange_weak(current, replacement, std::memory_order_release, std::memory_order_relaxed)); +} + +Buffer *BufferArena::alloc(uint32_t requested) +{ + BufferCapacityClass cls; + uint32_t size; + if (!classify_rdma_capacity(requested, &cls, &size)) + return nullptr; + auto *buffer = pop(static_cast(cls)); + if (buffer == nullptr) + buffer = fresh(size); + if (buffer == nullptr) + return nullptr; + buffer->reset_view(); + buffer->len = 0; + buffer->ref->store(1, std::memory_order_relaxed); + buffer->ownership.store(BufferOwnership::Available, std::memory_order_release); + buffer->capacity_class = cls; + buffer->pool_owner = shared_from_this(); + outstanding.fetch_add(1, std::memory_order_relaxed); + return buffer; +} + +void BufferArena::recycle(Buffer *buffer) +{ + // Retain this receiver until after publication; another allocator may + // immediately replace buffer->pool_owner after push. + const auto owner = std::move(buffer->pool_owner); + buffer->reset_view(); + const auto state = buffer->ownership.load(std::memory_order_acquire); + if (state != BufferOwnership::Available && state != BufferOwnership::Quarantined) + (void)buffer->quarantine(); + outstanding.fetch_sub(1, std::memory_order_release); + if (buffer->ownership.load(std::memory_order_acquire) == BufferOwnership::Quarantined) { + quarantined.fetch_add(buffer->capacity, std::memory_order_relaxed); + return; + } + buffer->len = 0; + push(buffer); +} + +void BufferArena::quarantine_inflight() +{ + if (entries_ == nullptr) + return; + for (uint32_t index = 0; index < capacity; ++index) { + auto *buffer = entries_[index].buffer.load(std::memory_order_acquire); + if (buffer == nullptr) + continue; + const auto state = buffer->ownership.load(std::memory_order_acquire); + if (state != BufferOwnership::Available && state != BufferOwnership::Quarantined) + (void)buffer->quarantine(); + } +} + +bool BufferArena::detach(Buffer *buffer) +{ + if (buffer == nullptr || buffer->pool != this || buffer->last_release_cb != nullptr || buffer->ref == nullptr || + buffer->ref->load(std::memory_order_acquire) != 1) + return false; + const auto state = buffer->ownership.load(std::memory_order_acquire); + if (state != BufferOwnership::Available && state != BufferOwnership::CallerOwned) + return false; + auto context = std::make_unique(DetachedRelease{this, buffer->pool_index}); + entries_[buffer->pool_index].buffer.store(nullptr, std::memory_order_release); + buffer->pool = nullptr; + buffer->free_ctx = context.release(); + buffer->free_cb = &BufferArena::release_detached; + return true; +} + +void BufferArena::release_detached(void *context) +{ + const auto release = std::unique_ptr(static_cast(context)); + release->arena->outstanding.fetch_sub(1, std::memory_order_release); + release->arena->push_entry(release->arena->unused_entries_, release->index); +} + +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/buffer_arena.h b/lib/crowdb-rpc/src/buffer_arena.h new file mode 100644 index 000000000..12c8d915c --- /dev/null +++ b/lib/crowdb-rpc/src/buffer_arena.h @@ -0,0 +1,65 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/buffer.h" + +namespace crowdb::rpc +{ +struct BufferRegion +{ + uint8_t *data = nullptr; + uint64_t bytes = 0; + uintptr_t registration_handle = 0; + uint32_t registration_key = 0; + uint32_t registration_id = 0; + std::shared_ptr owner; + + static std::shared_ptr allocate(uint64_t bytes, uint32_t registration_id = 0); +}; + +class BufferArena : public BufferPool, public std::enable_shared_from_this +{ + public: + struct Entry + { + std::atomic buffer{nullptr}; + std::atomic next{0}; + }; + + const std::shared_ptr region; + const uint64_t bytes; + const uint32_t capacity; + std::atomic outstanding{0}; + std::atomic next_offset{0}; + std::atomic quarantined{0}; + + BufferArena(std::shared_ptr region, uint32_t count); + ~BufferArena() override; + Buffer *alloc(uint32_t requested) override; + void recycle(Buffer *buffer) override; + bool detach(Buffer *buffer) override; + void quarantine_inflight(); + + private: + std::unique_ptr entries_; + std::array, 4> free_heads_{}; + std::atomic unused_entries_{0}; + std::atomic next_entry_{0}; + + Buffer *fresh(uint32_t size); + Buffer *pop(size_t bucket); + void push(Buffer *buffer); + uint32_t pop_entry(std::atomic &head); + void push_entry(std::atomic &head, uint32_t index); + + struct DetachedRelease + { + BufferArena *arena; + uint32_t index; + }; + + static void release_detached(void *context); +}; +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/registered_buffer_pool.cpp b/lib/crowdb-rpc/src/registered_buffer_pool.cpp new file mode 100644 index 000000000..dcd1d8815 --- /dev/null +++ b/lib/crowdb-rpc/src/registered_buffer_pool.cpp @@ -0,0 +1,40 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "buffer_arena.h" + +namespace crowdb::rpc +{ +RegisteredBufferPool::RegisteredBufferPool(uint64_t max_bytes, uint32_t max_buffers, uint32_t registration_id) + : arena_(std::make_shared(BufferRegion::allocate(max_bytes, registration_id), max_buffers)), + registration_id_(registration_id) +{ +} + +RegisteredBufferPool::~RegisteredBufferPool() = default; + +Buffer *RegisteredBufferPool::alloc(uint32_t capacity) +{ + return arena_->alloc(capacity); +} + +void RegisteredBufferPool::recycle(Buffer *buffer) +{ + arena_->recycle(buffer); +} + +bool RegisteredBufferPool::detach(Buffer *buffer) +{ + return arena_->detach(buffer); +} + +uint64_t RegisteredBufferPool::committed_bytes() const +{ + return arena_->next_offset.load(std::memory_order_acquire); +} + +uint64_t RegisteredBufferPool::quarantined_bytes() const +{ + return arena_->quarantined.load(std::memory_order_acquire); +} +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_arena.h b/lib/crowdb-rpc/src/transport/rdma/rdma_arena.h deleted file mode 100644 index f8532fb6e..000000000 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_arena.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -#pragma once - -#include "crowdb-rpc/transport/rdma/rdma_transport.h" - -#ifdef CROWDB_RPC_HAVE_RDMA -namespace crowdb::rpc -{ -struct RdmaBufferPool::Arena : BufferPool, std::enable_shared_from_this -{ - struct Entry - { - std::atomic buffer{nullptr}; - std::atomic next{0}; - }; - - std::shared_ptr registration_owner; - ibv_mr *mr = nullptr; - void *region = nullptr; - uint64_t bytes; - uint32_t capacity; - std::unique_ptr entries; - std::array, 4> free_heads{}; - std::atomic unused_entries{0}; - std::atomic next_entry{0}; - std::atomic outstanding{0}; - std::atomic next_offset{0}; - std::atomic quarantined{0}; - - Arena(ibv_pd *pd, uint64_t size, uint32_t count, std::shared_ptr owner); - ~Arena() override; - Buffer *alloc(uint32_t requested) override; - void recycle(Buffer *buffer) override; - Buffer *fresh(uint32_t size); - Buffer *pop(size_t bucket); - void push(Buffer *buffer); - uint32_t pop_entry(std::atomic &head); - void push_entry(std::atomic &head, uint32_t index); - void quarantine_inflight(); -}; -} // namespace crowdb::rpc -#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp index d05989907..c67d0a868 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp @@ -1,19 +1,58 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -#include "rdma_arena.h" +#include "../../buffer_arena.h" +#include "crowdb-rpc/transport/rdma/rdma_transport.h" #ifdef CROWDB_RPC_HAVE_RDMA # include "crowdb-common/log.h" -# include # include +# include namespace crowdb::rpc { +namespace +{ +struct RdmaRegion +{ + std::shared_ptr device_owner; + std::shared_ptr allocation; + ibv_mr *mr = nullptr; + + ~RdmaRegion() + { + const int error = mr == nullptr ? 0 : ibv_dereg_mr(mr); + if (error != 0) + CRB_LOG_ERROR("rdma MR deregistration failed error={}", error); + } +}; + +std::shared_ptr register_region(ibv_pd *pd, uint64_t size, std::shared_ptr owner) +{ + if (pd == nullptr) + return nullptr; + auto region = BufferRegion::allocate(size); + if (region->data == nullptr) + return nullptr; + auto registration = std::make_shared(); + registration->device_owner = std::move(owner); + registration->mr = ibv_reg_mr(pd, region->data, size, IBV_ACCESS_LOCAL_WRITE); + if (registration->mr == nullptr) + return nullptr; + registration->allocation = std::move(region->owner); + auto *mr = registration->mr; + region->owner = std::move(registration); + region->registration_handle = reinterpret_cast(mr); + region->registration_key = mr->lkey; + region->registration_id = mr->lkey; + return region; +} +} // namespace + RdmaBufferPool::RdmaBufferPool(ibv_pd *pd, uint64_t mr_size, uint32_t max_buffers, std::shared_ptr registration_owner) - : arena_(std::make_shared(pd, mr_size, max_buffers, std::move(registration_owner))) + : arena_(std::make_shared(register_region(pd, mr_size, std::move(registration_owner)), max_buffers)) { } @@ -44,171 +83,5 @@ void RdmaBufferPool::quarantine_inflight() arena_->quarantine_inflight(); } -RdmaBufferPool::Arena::Arena(ibv_pd *pd, uint64_t size, uint32_t count, std::shared_ptr owner) - : registration_owner(std::move(owner)), - bytes(size), - capacity(count) -{ - if (pd == nullptr || size == 0 || count == 0 || count == UINT32_MAX) - return; - entries = std::make_unique(count); - if (posix_memalign(®ion, 64, size) != 0) { - region = nullptr; - return; - } - mr = ibv_reg_mr(pd, region, size, IBV_ACCESS_LOCAL_WRITE); - if (mr == nullptr) { - std::free(region); - region = nullptr; - } -} - -RdmaBufferPool::Arena::~Arena() -{ - assert(outstanding.load(std::memory_order_acquire) == 0); - if (entries != nullptr) { - for (uint32_t index = 0; index < capacity; ++index) { - if (auto *buffer = entries[index].buffer.load(std::memory_order_relaxed); buffer != nullptr) { - delete buffer->ref; - delete buffer; - } - } - } - if (mr != nullptr) { - const int error = ibv_dereg_mr(mr); - if (error != 0) - CRB_LOG_ERROR("rdma MR deregistration failed error={}", error); - } - std::free(region); -} - -Buffer *RdmaBufferPool::Arena::fresh(uint32_t size) -{ - if (mr == nullptr) - return nullptr; - uint32_t index = pop_entry(unused_entries); - if (index == UINT32_MAX) { - index = next_entry.load(std::memory_order_relaxed); - do { - if (index >= capacity) - return nullptr; - } while (!next_entry.compare_exchange_weak(index, index + 1, std::memory_order_relaxed)); - } - uint64_t offset = next_offset.load(std::memory_order_relaxed); - uint64_t aligned; - do { - if (offset > bytes || offset > UINT64_MAX - 63U) { - push_entry(unused_entries, index); - return nullptr; - } - aligned = (offset + 63U) & ~uint64_t{63}; - if (aligned > bytes || size > bytes - aligned) { - push_entry(unused_entries, index); - return nullptr; - } - } while (!next_offset.compare_exchange_weak(offset, aligned + size, std::memory_order_relaxed)); - auto *buffer = new Buffer; - buffer->capacity = size; - buffer->type = BufferType::Registered; - buffer->pool = this; - buffer->pool_index = index; - buffer->ref = new std::atomic(1); - buffer->data = static_cast(region) + aligned; - buffer->allocation_data = buffer->data; - buffer->allocation_capacity = size; - buffer->registration_handle = reinterpret_cast(mr); - buffer->registration_key = mr->lkey; - buffer->registration_id = mr->lkey; - entries[index].buffer.store(buffer, std::memory_order_release); - return buffer; -} - -Buffer *RdmaBufferPool::Arena::pop(size_t bucket) -{ - const auto index = pop_entry(free_heads[bucket]); - return index == UINT32_MAX ? nullptr : entries[index].buffer.load(std::memory_order_acquire); -} - -uint32_t RdmaBufferPool::Arena::pop_entry(std::atomic &head) -{ - uint64_t current = head.load(std::memory_order_acquire); - while (static_cast(current) != 0) { - const auto index = static_cast(current) - 1; - const auto next = entries[index].next.load(std::memory_order_relaxed); - const auto replacement = ((current + (uint64_t{1} << 32)) & 0xffffffff00000000ULL) | next; - if (head.compare_exchange_weak(current, replacement, std::memory_order_acq_rel)) - return index; - } - return UINT32_MAX; -} - -void RdmaBufferPool::Arena::push(Buffer *buffer) -{ - push_entry(free_heads[static_cast(buffer->capacity_class)], buffer->pool_index); -} - -void RdmaBufferPool::Arena::push_entry(std::atomic &head, uint32_t index) -{ - auto &entry = entries[index]; - uint64_t current = head.load(std::memory_order_relaxed); - uint64_t replacement; - do { - entry.next.store(static_cast(current), std::memory_order_relaxed); - replacement = ((current + (uint64_t{1} << 32)) & 0xffffffff00000000ULL) | (index + 1); - } while (!head.compare_exchange_weak(current, replacement, std::memory_order_release, std::memory_order_relaxed)); -} - -Buffer *RdmaBufferPool::Arena::alloc(uint32_t requested) -{ - BufferCapacityClass cls; - uint32_t size; - if (!classify_rdma_capacity(requested, &cls, &size)) - return nullptr; - auto *buffer = pop(static_cast(cls)); - if (buffer == nullptr) - buffer = fresh(size); - if (buffer == nullptr) - return nullptr; - buffer->reset_view(); - buffer->len = 0; - buffer->ref->store(1, std::memory_order_relaxed); - buffer->ownership.store(BufferOwnership::Available, std::memory_order_release); - buffer->capacity_class = cls; - buffer->pool_owner = shared_from_this(); - outstanding.fetch_add(1, std::memory_order_relaxed); - return buffer; -} - -void RdmaBufferPool::Arena::recycle(Buffer *buffer) -{ - // Retain this receiver until after publication; another allocator may - // immediately replace buffer->pool_owner after push. - const auto owner = std::move(buffer->pool_owner); - buffer->reset_view(); - const auto state = buffer->ownership.load(std::memory_order_acquire); - if (state != BufferOwnership::Available && state != BufferOwnership::Quarantined) - (void)buffer->quarantine(); - outstanding.fetch_sub(1, std::memory_order_release); - if (buffer->ownership.load(std::memory_order_acquire) == BufferOwnership::Quarantined) { - quarantined.fetch_add(buffer->capacity, std::memory_order_relaxed); - return; - } - buffer->len = 0; - push(buffer); -} - -void RdmaBufferPool::Arena::quarantine_inflight() -{ - if (entries == nullptr) - return; - for (uint32_t index = 0; index < capacity; ++index) { - auto *buffer = entries[index].buffer.load(std::memory_order_acquire); - if (buffer == nullptr) - continue; - const auto state = buffer->ownership.load(std::memory_order_acquire); - if (state != BufferOwnership::Available && state != BufferOwnership::Quarantined) - (void)buffer->quarantine(); - } -} } // namespace crowdb::rpc #endif diff --git a/lib/crowdb-rpc/tests/registered_buffer_test.cpp b/lib/crowdb-rpc/tests/registered_buffer_test.cpp new file mode 100644 index 000000000..938156812 --- /dev/null +++ b/lib/crowdb-rpc/tests/registered_buffer_test.cpp @@ -0,0 +1,142 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/buffer.h" + +#include + +#include +#include +#include +#include +#include + +using namespace crowdb::rpc; + +TEST(RegisteredBufferTest, HeldLeaseRetainsArenaAfterFacadeDestruction) +{ + auto pool = std::make_unique(4096, 4, 19); + auto *buffer = pool->alloc(32); + ASSERT_NE(buffer, nullptr); + buffer->write("portable lease", 14); + const auto *data = buffer->payload_data(); + auto *clone = buffer->ref_clone(); + std::weak_ptr arena = buffer->pool_owner; + pool.reset(); + buffer->release(); + EXPECT_FALSE(arena.expired()); + EXPECT_EQ(clone->payload_data(), data); + EXPECT_EQ(clone->registration_id, 19U); + EXPECT_EQ(std::memcmp(data, "portable lease", 14), 0); + clone->release(); + EXPECT_TRUE(arena.expired()); +} + +TEST(RegisteredBufferTest, DetachedLeaseRetainsRegionAndCanReleaseAfterFacade) +{ + auto pool = std::make_unique(4096, 4); + auto *buffer = pool->alloc(32); + ASSERT_NE(buffer, nullptr); + buffer->write("detached lease", 14); + ASSERT_TRUE(pool->detach(buffer)); + EXPECT_EQ(buffer->pool, nullptr); + auto *clone = buffer->ref_clone(); + std::weak_ptr arena = buffer->pool_owner; + pool.reset(); + buffer->release(); + EXPECT_FALSE(arena.expired()); + EXPECT_EQ(std::memcmp(clone->payload_data(), "detached lease", 14), 0); + clone->release(); + EXPECT_TRUE(arena.expired()); +} + +TEST(RegisteredBufferTest, DetachedFinalReleaseReturnsDescriptorAdmission) +{ + RegisteredBufferPool pool(4096, 1); + auto *buffer = pool.alloc(1); + ASSERT_NE(buffer, nullptr); + ASSERT_TRUE(pool.detach(buffer)); + EXPECT_EQ(pool.alloc(1), nullptr); + buffer->release(); + auto *next = pool.alloc(1); + ASSERT_NE(next, nullptr); + EXPECT_EQ(pool.committed_bytes(), 2048U); + next->release(); +} + +TEST(RegisteredBufferTest, CachedAndQuarantinedDescriptorsStayWithinTotalLimit) +{ + RegisteredBufferPool pool(2 * RDMA_DATAPATH_CAPACITY, 2); + auto *control = pool.alloc(1); + auto *metadata = pool.alloc(RDMA_METADATA_CAPACITY); + ASSERT_NE(control, nullptr); + ASSERT_NE(metadata, nullptr); + control->release(); + EXPECT_TRUE(metadata->quarantine()); + metadata->release(); + EXPECT_EQ(pool.alloc(RDMA_DATAPATH_CAPACITY), nullptr); + EXPECT_EQ(pool.quarantined_bytes(), RDMA_METADATA_CAPACITY); + auto *reused = pool.alloc(1); + ASSERT_EQ(reused, control); + reused->release(); +} + +TEST(RegisteredBufferTest, ConcurrentRecyclingKeepsSlotOwnershipExclusive) +{ + RegisteredBufferPool pool(32 * 1024, 32); + std::array, 32> occupied{}; + std::atomic errors{0}; + std::vector threads; + for (uint8_t producer = 1; producer <= 8; ++producer) { + threads.emplace_back([&, producer] { + for (uint32_t iteration = 0; iteration < 20000; ++iteration) { + auto *buffer = pool.alloc(32); + if (buffer == nullptr) { + errors.fetch_add(1); + continue; + } + if (occupied[buffer->pool_index].exchange(true)) + errors.fetch_add(1); + std::memset(buffer->data, producer, 32); + auto *clone = buffer->ref_clone(); + for (uint32_t index = 0; index < 32; ++index) + if (clone->data[index] != producer) + errors.fetch_add(1); + buffer->release(); + occupied[clone->pool_index].store(false); + clone->release(); + } + }); + } + for (auto &thread : threads) + thread.join(); + EXPECT_EQ(errors.load(), 0U); + EXPECT_LE(pool.committed_bytes(), 32U * 1024); +} + +TEST(RegisteredBufferTest, SystemAdmissionNeverExceedsConcurrentCapacity) +{ + SystemBufferPool pool(1); + std::barrier start(8); + std::atomic live{0}; + std::atomic errors{0}; + std::vector threads; + for (uint32_t producer = 0; producer < 8; ++producer) { + threads.emplace_back([&] { + start.arrive_and_wait(); + for (uint32_t iteration = 0; iteration < 20000; ++iteration) { + if (auto *buffer = pool.alloc(32); buffer != nullptr) { + if (live.fetch_add(1) != 0) + errors.fetch_add(1); + std::this_thread::yield(); + live.fetch_sub(1); + buffer->release(); + } + } + }); + } + for (auto &thread : threads) + thread.join(); + EXPECT_EQ(live.load(), 0U); + EXPECT_EQ(errors.load(), 0U); +} From 0cdfc52495f6fdcfb5fc006f80de3394045b53d7 Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 22 Oct 2026 22:00:06 +0800 Subject: [PATCH 027/258] Keep retired registrations charged to bounded device budgets --- doc/design/rpc/design-crowdb-rpc-rdma.md | 14 ++ doc/working/plan-rdma-rpc-transport.md | 38 +++++- doc/working/plan-rdma.md | 5 +- .../transport/rdma/rdma_transport.h | 59 +++++---- lib/crowdb-rpc/src/buffer_arena.cpp | 5 +- lib/crowdb-rpc/src/buffer_arena.h | 1 + .../src/transport/rdma/rdma_buffer_pool.cpp | 5 + .../src/transport/rdma/rdma_config.cpp | 22 ++- .../src/transport/rdma/rdma_connection.cpp | 8 +- lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp | 9 ++ .../src/transport/rdma/rdma_device.cpp | 32 ++++- .../src/transport/rdma/rdma_device.h | 28 +++- .../src/transport/rdma/rdma_transport.cpp | 26 +++- .../tests/common/test_rdma_transport.h | 13 ++ lib/crowdb-rpc/tests/rdma_completion_test.cpp | 1 + lib/crowdb-rpc/tests/rdma_config_test.cpp | 32 +++++ lib/crowdb-rpc/tests/rdma_memory_test.cpp | 125 ++++++++++++++++++ .../tests/registered_buffer_test.cpp | 16 +++ 18 files changed, 396 insertions(+), 43 deletions(-) create mode 100644 lib/crowdb-rpc/tests/rdma_memory_test.cpp diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index 8434f8663..63c2bcc5b 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -134,6 +134,20 @@ release. Their detached ranges remain committed and are not reused by another stream. Cached and quarantined descriptors both count toward the hard total. Application-pool count admission also uses one atomic reservation. +The device owner charges MR creation against +`registered_memory_budget_bytes`. Zero derives a checked bound from all +CQ-admitted QPs, their send/receive arenas and the two compatibility copy +arenas. An explicit budget must admit those compatibility arenas and at least +one QP. Invalid and overflowed limits fail configuration. Reservation and MR +creation failure reject setup before connection publication and return the +temporary reservation. Releasing CQ admission does not release registered +bytes while a closed connection or held lease still owns an MR; its final +deregistration returns the charge. `registered_bytes`, +`registered_bytes_high_water`, `registration_rejects` and +`registration_failures` expose current reservations, peak reservations, +budget rejection and provider failure separately. Reservations include pending +MR creation and remain bounded during concurrent setup. + ## 5. Shared CQ and completion model Connections share a bounded CQ pool. Each QP uses one assigned CQ for both diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 69d644ebc..754de25ad 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -32,7 +32,8 @@ prove held registered/receive leases survive pool/transport/CM-ID destruction. The C ABI now returns independent handles for retained buffer references. Portable and verbs-backed pools now share the same arena/lifetime machinery. -Still pending: registration admission across retired connections, bounded +Registration admission remains charged while retired connections retain MRs. +Still pending: negotiated TCP/CM preflight, bounded transport aggregation, `bench rdma`/shared coroutine reporting, unified regression orchestration, negotiated TCP capability preflight, failure injection, and the two-container acceptance profiles. Capability serialization and automatic @@ -373,7 +374,7 @@ Verified histogram checkpoint: the detached byte range stays committed. Add `CROWDB_RPC_ENABLE_RDMA=OFF` as a supported CMake build switch so the no-verbs gate is reproducible even on this host with rdma-core installed. -- [~] **Bound registration across connection retirement**: add a byte +- [x] **Bound registration across connection retirement**: add a byte reservation to the existing `RdmaDevice` resource owner. A held lease must retain that reservation after its QP releases CQ admission; a new QP cannot bypass the registered-memory bound by replacing an arena that is still held. @@ -389,6 +390,28 @@ Verified histogram checkpoint: Include final detached-buffer quarantine accounting in the shared arena and verify it with a no-device case. +Registration budget checkpoint: + +- `RdmaDevice` retains atomic byte reservations through MR deregistration. + `RdmaConfig::registration_budget()` checks the derived bound and explicit + minimum; malformed 64-bit environment values and arithmetic overflow fail + configuration. Pool registration success is checked before connection + publication. Registration rejection and provider failure have separate + counters; failed construction returns the temporary byte charge. +- Forty focused buffer/configuration/completion cases pass (one pre-CM copy + compatibility case skips). New cases cover 160,000 concurrent reservations, + failed-MR rollback, explicit/derived/overflowed budgets, and a real retired + MR that blocks replacement until its final buffer release. +- The real HCA setup case admits one QP, rejects a second QP when the explicit + byte budget is exhausted despite spare CQ capacity, and preserves the first + connection and exact current/high-water charge. The held-receive/registered + copy test also passes ten repeated transport/CM teardown cycles with this + byte reservation owner in place. +- Full enabled C++/FFI gate, all 29 Rust RPC FFI tests, formatting, scoped + tree-lint and workspace clippy pass. Complete gate artifact: + `.crowdb-runtime/artifacts/unit/cpp-cecb4460f864486aa296056ea8209115/`. + The updated no-verbs build also passes all 124 CTest cases. + Portable pool verification: - `BufferArena`/`BufferRegion` back both registered pool facades; detach keeps @@ -481,7 +504,7 @@ CQ verification: ### C. Bounded aggregation -- [ ] **Negotiate receive admission before coalescing**: finish TCP capability +- [~] **Negotiate receive admission before coalescing**: finish TCP capability preflight before the CM attempt; reuse its TCP connection only for allowed pre-payload fallback. Publish actual CM address/port, per-instance generation, device/GID identity, receive slot capacity and supported aggregate frame count. @@ -492,6 +515,13 @@ CQ verification: strict rejection before opening a socket. Retire a successful preflight TCP connection through the common worker cleanup path. This is a prerequisite for the peer-slot bound below; the remaining OCI/failure matrix stays in E. + Keep the compact CM offer within the RC request's 56-byte private-data limit; + use a fixed 48-byte offer with explicit encoding rather than a native struct + dump. It carries protocol, receive capacity/frame limit, local generation, + expected peer generation and endpoint identity digests. The full TCP + capability carries the CM endpoint and identities. Copy event private data + before acknowledging the CM event. Direct CM diagnostics omit only the + preflight expectation, while still negotiating receive admission. Files: RDMA capability/CM sources, `socket_transport.cpp`, server readiness, capability/path and hardware tests. - [ ] **Coalesce complete frames on the CQ owner**: use bounded lock-free @@ -598,6 +628,8 @@ Provider references for the implementation choices: (`rdma_get_devices` increments device references; `rdma_free_devices` releases them). Retaining the matching CM context list avoids substituting an independently opened context for a CM-owned QP. +- [RDMA-CM request private-data limit](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/man/rdma_connect.3) + and [accept event lifetime](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/man/rdma_accept.3). ## Historical verification record diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index b61586018..4fa6d9004 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -25,8 +25,7 @@ and C/Rust lease metadata accessors. R239's transport path is implemented and passes the prepared HCA local-loop plus the real `RpcServer`/`RpcClient` worker round trip. Its requirement is still open because negotiated capability preflight, -pre-payload fallback, registration admission across retired connections, -aggregation/CLI benchmarking and +pre-payload fallback, aggregation/CLI benchmarking and multi-container acceptance have not all passed. Live HCA tests now verify held registered/receive leases after pool, transport and CM-ID destruction; the final release deregisters MR before PD release. Application reply allocation @@ -158,7 +157,7 @@ TCP-only profile and build/mocked tests. tagged-index freelist machinery with the no-verbs registered pool, then verify concurrent bounds, held leases, quarantine and detach without an HCA. The detailed transport plan owns this continuation before aggregation. -- [ ] **Bound retired registration admission**: keep registered bytes charged +- [x] **Bound retired registration admission**: keep registered bytes charged to the existing device owner while old caller leases retain their MRs, reject failed MR setup before publishing a connection, and verify the current/high water budget and rejection counters. The transport plan owns this final diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index 7249d79c8..8e0d9b391 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -27,33 +27,36 @@ namespace crowdb::rpc class SocketEngine; struct RdmaDevice; +struct RdmaMemoryStats; constexpr uint32_t RDMA_WIRE_SLOT_CAPACITY = RDMA_DATAPATH_CAPACITY; struct RdmaConfig { std::string device; - uint8_t port = 1; - uint32_t cq_depth = 8192; - uint32_t cq_count = 1; - uint32_t wire_slot_capacity = RDMA_WIRE_SLOT_CAPACITY; - uint64_t send_arena_bytes = 64ULL << 20; - uint64_t receive_arena_bytes = 1ULL << 30; - uint32_t sq_depth = 1024; - uint32_t rq_depth = 1024; - uint32_t qp_max_send_sge = 2; - uint32_t qp_max_recv_sge = 2; - uint32_t poll_batch = 256; - uint32_t verbs_poll_batch = 32; - uint32_t poll_empty_budget = 256; - uint32_t shutdown_drain_ms = 3000; - uint32_t posted_receive_target = 1024; - uint8_t min_rnr_timer = 1; - uint8_t retry_count = 7; - uint8_t rnr_retry_count = 7; - - [[nodiscard]] bool apply_environment(); - [[nodiscard]] bool valid() const; + uint8_t port = 1; + uint32_t cq_depth = 8192; + uint32_t cq_count = 1; + uint32_t wire_slot_capacity = RDMA_WIRE_SLOT_CAPACITY; + uint64_t send_arena_bytes = 64ULL << 20; + uint64_t receive_arena_bytes = 1ULL << 30; + uint64_t registered_memory_budget_bytes = 0; + uint32_t sq_depth = 1024; + uint32_t rq_depth = 1024; + uint32_t qp_max_send_sge = 2; + uint32_t qp_max_recv_sge = 2; + uint32_t poll_batch = 256; + uint32_t verbs_poll_batch = 32; + uint32_t poll_empty_budget = 256; + uint32_t shutdown_drain_ms = 3000; + uint32_t posted_receive_target = 1024; + uint8_t min_rnr_timer = 1; + uint8_t retry_count = 7; + uint8_t rnr_retry_count = 7; + + [[nodiscard]] bool apply_environment(); + [[nodiscard]] bool valid() const; + [[nodiscard]] uint64_t registration_budget() const; }; // RdmaBufferPool: allocates buffers from ibv_mr-registered memory. @@ -75,6 +78,7 @@ class RdmaBufferPool : public BufferPool [[nodiscard]] uint64_t quarantined_bytes() const; [[nodiscard]] bool quiescent() const; + [[nodiscard]] bool ready() const; void quarantine_inflight(); private: @@ -109,6 +113,11 @@ class RdmaTransport : public Transport std::atomic cq_events{0}; std::atomic cq_wakeups{0}; std::atomic pending_deferred_receives{0}; + std::atomic registered_memory_budget_bytes{0}; + std::atomic registered_bytes{0}; + std::atomic registered_bytes_high_water{0}; + std::atomic registration_rejects{0}; + std::atomic registration_failures{0}; }; explicit RdmaTransport(RdmaConfig config = {}, BufferPool *application_pool = nullptr); @@ -170,6 +179,7 @@ class RdmaTransport : public Transport std::atomic capability_available_{false}; struct ibv_context *context_ = nullptr; std::shared_ptr device_; + std::shared_ptr memory_stats_; struct ibv_pd *pd_ = nullptr; struct rdma_event_channel *cm_channel_ = nullptr; struct rdma_cm_id *listen_id_ = nullptr; @@ -189,9 +199,10 @@ class RdmaTransport : public Transport std::atomic running_{false}; std::atomic_flag cm_polling_ = ATOMIC_FLAG_INIT; - bool prime_receive(const std::shared_ptr &connection); - mutable Stats stats_; - void refresh_poll_stats() const; + bool prime_receive(const std::shared_ptr &connection); + std::unique_ptr make_pool(uint64_t bytes, uint32_t buffers); + mutable Stats stats_; + void refresh_poll_stats() const; enum class WorkKind : uint8_t { Send, Receive }; struct ConnectionPools; diff --git a/lib/crowdb-rpc/src/buffer_arena.cpp b/lib/crowdb-rpc/src/buffer_arena.cpp index d9a75bead..1007ba1ef 100644 --- a/lib/crowdb-rpc/src/buffer_arena.cpp +++ b/lib/crowdb-rpc/src/buffer_arena.cpp @@ -180,7 +180,7 @@ bool BufferArena::detach(Buffer *buffer) const auto state = buffer->ownership.load(std::memory_order_acquire); if (state != BufferOwnership::Available && state != BufferOwnership::CallerOwned) return false; - auto context = std::make_unique(DetachedRelease{this, buffer->pool_index}); + auto context = std::make_unique(DetachedRelease{this, buffer->pool_index, buffer}); entries_[buffer->pool_index].buffer.store(nullptr, std::memory_order_release); buffer->pool = nullptr; buffer->free_ctx = context.release(); @@ -191,6 +191,9 @@ bool BufferArena::detach(Buffer *buffer) void BufferArena::release_detached(void *context) { const auto release = std::unique_ptr(static_cast(context)); + const auto state = release->buffer->ownership.load(std::memory_order_acquire); + if (state != BufferOwnership::Available && state != BufferOwnership::CallerOwned) + release->arena->quarantined.fetch_add(release->buffer->allocation_capacity, std::memory_order_relaxed); release->arena->outstanding.fetch_sub(1, std::memory_order_release); release->arena->push_entry(release->arena->unused_entries_, release->index); } diff --git a/lib/crowdb-rpc/src/buffer_arena.h b/lib/crowdb-rpc/src/buffer_arena.h index 12c8d915c..249fcd9c7 100644 --- a/lib/crowdb-rpc/src/buffer_arena.h +++ b/lib/crowdb-rpc/src/buffer_arena.h @@ -58,6 +58,7 @@ class BufferArena : public BufferPool, public std::enable_shared_from_thisoutstanding.load(std::memory_order_acquire) == 0; } +bool RdmaBufferPool::ready() const +{ + return arena_->region != nullptr && arena_->region->registration_handle != 0; +} + uint64_t RdmaBufferPool::quarantined_bytes() const { return arena_->quarantined.load(std::memory_order_relaxed); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp index 4d2815f60..fa5e479ff 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp @@ -37,6 +37,7 @@ bool RdmaConfig::apply_environment() read("CROWDB_RDMA_WIRE_SLOT_CAPACITY", wire_slot_capacity) && read("CROWDB_RDMA_SEND_ARENA_BYTES", send_arena_bytes) && read("CROWDB_RDMA_RECEIVE_ARENA_BYTES", receive_arena_bytes) && read("CROWDB_RDMA_CQ_DEPTH", cq_depth) && + read("CROWDB_RDMA_REGISTERED_MEMORY_BUDGET_BYTES", registered_memory_budget_bytes) && read("CROWDB_RDMA_CQ_COUNT", cq_count) && read("CROWDB_RDMA_POLL_BATCH", poll_batch) && read("CROWDB_RDMA_VERBS_POLL_BATCH", verbs_poll_batch) && read("CROWDB_RDMA_POLL_EMPTY_BUDGET", poll_empty_budget); @@ -54,7 +55,26 @@ bool RdmaConfig::valid() const poll_batch <= static_cast(INT32_MAX) && verbs_poll_batch > 0 && verbs_poll_batch <= 256 && poll_empty_budget > 0 && send_arena_bytes >= wire_slot_capacity && receive_arena_bytes >= static_cast(posted_receive_target) * wire_slot_capacity && - min_rnr_timer <= 31 && retry_count <= 7 && rnr_retry_count <= 7; + min_rnr_timer <= 31 && retry_count <= 7 && rnr_retry_count <= 7 && registration_budget() != 0; +} + +uint64_t RdmaConfig::registration_budget() const +{ + const uint64_t entries = static_cast(sq_depth) + posted_receive_target; + const uint64_t maximum = std::numeric_limits::max(); + if (entries == 0 || cq_depth < entries || send_arena_bytes > maximum / 2 || + receive_arena_bytes > maximum - send_arena_bytes) + return 0; + const uint64_t compatibility = 2 * send_arena_bytes; + const uint64_t per_qp = send_arena_bytes + receive_arena_bytes; + if (per_qp == 0 || per_qp > maximum - compatibility) + return 0; + if (registered_memory_budget_bytes != 0) + return registered_memory_budget_bytes >= compatibility + per_qp ? registered_memory_budget_bytes : 0; + const uint64_t connections = (cq_depth / entries) * cq_count; + if (connections == 0 || connections > (maximum - compatibility) / per_qp) + return 0; + return compatibility + connections * per_qp; } } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp index 6d476ca62..89e51a908 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp @@ -145,8 +145,8 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int pools->effective_capacity = qp_attr.cap; pools->cq_index = reservation.queue->index; pools->reserved_capacity = reservation.capacity; - pools->send = std::make_unique(id->pd, config_.send_arena_bytes, config_.sq_depth, device_); - pools->recv = std::make_unique(id->pd, config_.receive_arena_bytes, config_.rq_depth, device_); + pools->send = make_pool(config_.send_arena_bytes, config_.sq_depth); + pools->recv = make_pool(config_.receive_arena_bytes, config_.rq_depth); if (pools->send == nullptr || pools->recv == nullptr) { rdma_destroy_qp(id); rdma_destroy_id(id); @@ -324,8 +324,8 @@ std::shared_ptr RdmaTransport::accept() pools->effective_capacity = qp_attr.cap; pools->cq_index = reservation.queue->index; pools->reserved_capacity = reservation.capacity; - pools->send = std::make_unique(id->pd, config_.send_arena_bytes, config_.sq_depth, device_); - pools->recv = std::make_unique(id->pd, config_.receive_arena_bytes, config_.rq_depth, device_); + pools->send = make_pool(config_.send_arena_bytes, config_.sq_depth); + pools->recv = make_pool(config_.receive_arena_bytes, config_.rq_depth); if (pools->send == nullptr || pools->recv == nullptr) { rdma_destroy_qp(id); rdma_destroy_id(id); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp index ee3e97844..38e897025 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp @@ -4,6 +4,7 @@ #include "rdma_cq.h" #include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "rdma_device.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -72,6 +73,14 @@ void RdmaTransport::refresh_poll_stats() const } stats_.cq_polls.store(polls, std::memory_order_relaxed); stats_.cq_empty_polls.store(empty, std::memory_order_relaxed); + stats_.registered_memory_budget_bytes.store(memory_stats_->budget, std::memory_order_relaxed); + stats_.registered_bytes.store(memory_stats_->current.load(std::memory_order_acquire), std::memory_order_relaxed); + stats_.registered_bytes_high_water.store(memory_stats_->high_water.load(std::memory_order_relaxed), + std::memory_order_relaxed); + stats_.registration_rejects.store(memory_stats_->rejects.load(std::memory_order_relaxed), + std::memory_order_relaxed); + stats_.registration_failures.store(memory_stats_->failures.load(std::memory_order_relaxed), + std::memory_order_relaxed); } void RdmaTransport::destroy_completion_queues() diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_device.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_device.cpp index 9bfc92f97..097195994 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_device.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_device.cpp @@ -10,10 +10,11 @@ namespace crowdb::rpc { -std::shared_ptr RdmaDevice::retain(ibv_context *context) +std::shared_ptr RdmaDevice::retain(ibv_context *context, std::shared_ptr memory) { auto owner = std::make_shared(); - int count = 0; + owner->memory = std::move(memory); + int count = 0; owner->devices = rdma_get_devices(&count); if (owner->devices == nullptr || std::find(owner->devices, owner->devices + count, context) == owner->devices + count) @@ -22,6 +23,33 @@ std::shared_ptr RdmaDevice::retain(ibv_context *context) return owner->pd == nullptr ? nullptr : owner; } +std::shared_ptr RdmaDevice::reserve_bytes(uint64_t bytes) +{ + if (memory == nullptr) + return nullptr; + auto reservation = std::make_shared(); + reservation->device = shared_from_this(); + uint64_t current = memory->current.load(std::memory_order_relaxed); + do { + if (bytes == 0 || current > memory->budget || bytes > memory->budget - current) { + memory->rejects.fetch_add(1, std::memory_order_relaxed); + return nullptr; + } + } while (!memory->current.compare_exchange_weak(current, current + bytes, std::memory_order_acq_rel)); + reservation->bytes = bytes; + uint64_t high = memory->high_water.load(std::memory_order_relaxed); + while (high < current + bytes && + !memory->high_water.compare_exchange_weak(high, current + bytes, std::memory_order_relaxed)) { + } + return reservation; +} + +RdmaDevice::Reservation::~Reservation() +{ + if (bytes != 0) + device->memory->current.fetch_sub(bytes, std::memory_order_release); +} + RdmaDevice::~RdmaDevice() { if (pd != nullptr) { diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_device.h b/lib/crowdb-rpc/src/transport/rdma/rdma_device.h index 378added9..472b0f39c 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_device.h +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_device.h @@ -7,17 +7,41 @@ # include # include +# include # include namespace crowdb::rpc { -struct RdmaDevice +struct RdmaMemoryStats +{ + explicit RdmaMemoryStats(uint64_t bytes) : budget(bytes) + { + } + + const uint64_t budget; + std::atomic current{0}; + std::atomic high_water{0}; + std::atomic rejects{0}; + std::atomic failures{0}; +}; + +struct RdmaDevice : std::enable_shared_from_this { ibv_context **devices = nullptr; ibv_pd *pd = nullptr; - static std::shared_ptr retain(ibv_context *context); + std::shared_ptr memory; + static std::shared_ptr retain(ibv_context *context, std::shared_ptr memory); + std::shared_ptr reserve_bytes(uint64_t bytes); ~RdmaDevice(); + + private: + struct Reservation + { + std::shared_ptr device; + uint64_t bytes = 0; + ~Reservation(); + }; }; } // namespace crowdb::rpc #endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index f9e91761b..de5497516 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -39,6 +39,7 @@ RdmaTransport::RdmaTransport(RdmaConfig config, BufferPool *application_pool) config_.cq_depth = 1; config_.cq_count = 1; } + memory_stats_ = std::make_shared(valid_config ? config_.registration_budget() : 0); for (uint32_t index = 0; index < config_.cq_count; ++index) { auto queue = std::make_unique(); queue->index = index; @@ -112,7 +113,7 @@ bool RdmaTransport::adopt_cm_context(struct ibv_context *context) // arenas. Do not replace live QPs/registrations when another HCA appears. if (context_ != nullptr) return false; - auto device = RdmaDevice::retain(context); + auto device = RdmaDevice::retain(context, memory_stats_); if (device == nullptr) return false; if (!create_completion_queues(context)) { @@ -121,12 +122,31 @@ bool RdmaTransport::adopt_cm_context(struct ibv_context *context) context_ = context; device_ = std::move(device); pd_ = device_->pd; - send_pool_ = std::make_unique(pd_, config_.send_arena_bytes, config_.sq_depth, device_); - recv_pool_ = std::make_unique(pd_, config_.send_arena_bytes, config_.sq_depth, device_); + send_pool_ = make_pool(config_.send_arena_bytes, config_.sq_depth); + recv_pool_ = make_pool(config_.send_arena_bytes, config_.sq_depth); + if (send_pool_ == nullptr || recv_pool_ == nullptr) { + send_pool_.reset(); + recv_pool_.reset(); + capability_available_.store(false, std::memory_order_release); + return false; + } register_events(event_engine_); return ready(); } +std::unique_ptr RdmaTransport::make_pool(uint64_t bytes, uint32_t buffers) +{ + auto reservation = device_->reserve_bytes(bytes); + if (reservation == nullptr) + return nullptr; + auto pool = std::make_unique(pd_, bytes, buffers, std::move(reservation)); + if (!pool->ready()) { + memory_stats_->failures.fetch_add(1, std::memory_order_relaxed); + return nullptr; + } + return pool; +} + bool RdmaTransport::submit(Connection *conn, OutFrame *frame) { Operation operation(repost_state_); diff --git a/lib/crowdb-rpc/tests/common/test_rdma_transport.h b/lib/crowdb-rpc/tests/common/test_rdma_transport.h index 2a263dbd1..8cad0209b 100644 --- a/lib/crowdb-rpc/tests/common/test_rdma_transport.h +++ b/lib/crowdb-rpc/tests/common/test_rdma_transport.h @@ -4,6 +4,7 @@ #pragma once #include "../../src/transport/rdma/rdma_cq.h" +#include "../../src/transport/rdma/rdma_device.h" #include "crowdb-rpc/transport/rdma/rdma_transport.h" #if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) @@ -16,6 +17,18 @@ namespace crowdb::rpc struct TestRdmaTransport { + static void install_device_for_tests(RdmaTransport &transport, const std::shared_ptr &device) + { + transport.device_ = device; + transport.memory_stats_ = device->memory; + transport.pd_ = device->pd; + } + + static std::unique_ptr make_pool_for_tests(RdmaTransport &transport, uint64_t bytes, uint32_t count) + { + return transport.make_pool(bytes, count); + } + static std::weak_ptr device_owner(RdmaTransport &transport) { return std::shared_ptr(transport.device_); diff --git a/lib/crowdb-rpc/tests/rdma_completion_test.cpp b/lib/crowdb-rpc/tests/rdma_completion_test.cpp index 2d39722f4..a66301241 100644 --- a/lib/crowdb-rpc/tests/rdma_completion_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_completion_test.cpp @@ -22,6 +22,7 @@ RdmaConfig test_config() config.cq_depth = 8; config.cq_count = 2; config.rq_depth = 4; + config.sq_depth = 4; config.posted_receive_target = 4; config.poll_empty_budget = 1; return config; diff --git a/lib/crowdb-rpc/tests/rdma_config_test.cpp b/lib/crowdb-rpc/tests/rdma_config_test.cpp index 6bba7f83c..196c0473a 100644 --- a/lib/crowdb-rpc/tests/rdma_config_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_config_test.cpp @@ -87,4 +87,36 @@ TEST(RdmaConfigTest, EnvironmentParsesCqAndPollingKnobsStrictly) EXPECT_FALSE(config.apply_environment()); } +TEST(RdmaConfigTest, RegistrationBudgetIncludesAdmittedQpsAndCopyPools) +{ + RdmaConfig config; + config.sq_depth = config.rq_depth = config.posted_receive_target = 8; + config.wire_slot_capacity = 1024; + config.send_arena_bytes = config.receive_arena_bytes = 8192; + config.cq_depth = 16; + EXPECT_EQ(config.registration_budget(), 32768U); + config.cq_depth = 32; + config.cq_count = 2; + EXPECT_EQ(config.registration_budget(), 81920U); + config.registered_memory_budget_bytes = 32767; + EXPECT_FALSE(config.valid()); + config.registered_memory_budget_bytes = 32768; + EXPECT_TRUE(config.valid()); + config.receive_arena_bytes = UINT64_MAX; + EXPECT_FALSE(config.valid()); + config.registered_memory_budget_bytes = 0; + EXPECT_EQ(config.registration_budget(), 0U); +} + +TEST(RdmaConfigTest, RegistrationBudgetParses64BitValuesStrictly) +{ + const TestEnvironment budget("CROWDB_RDMA_REGISTERED_MEMORY_BUDGET_BYTES", "8589934592"); + RdmaConfig config; + ASSERT_TRUE(config.apply_environment()); + EXPECT_EQ(config.registered_memory_budget_bytes, uint64_t{8} << 30); + EXPECT_TRUE(config.valid()); + const TestEnvironment invalid("CROWDB_RDMA_REGISTERED_MEMORY_BUDGET_BYTES", "-1"); + EXPECT_FALSE(config.apply_environment()); +} + #endif diff --git a/lib/crowdb-rpc/tests/rdma_memory_test.cpp b/lib/crowdb-rpc/tests/rdma_memory_test.cpp new file mode 100644 index 000000000..10faed4ed --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_memory_test.cpp @@ -0,0 +1,125 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "common/test_rdma_device.h" +#include "common/test_rdma_transport.h" + +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) +# include + +# include +# include +# include + +using namespace crowdb::rpc; + +TEST(RdmaMemoryTest, ConcurrentReservationsStayWithinBudget) +{ + auto device = std::make_shared(); + device->memory = std::make_shared(4096); + std::atomic errors{0}; + std::vector threads; + for (uint32_t producer = 0; producer < 8; ++producer) { + threads.emplace_back([&] { + for (uint32_t iteration = 0; iteration < 20000; ++iteration) { + auto lease = device->reserve_bytes(1024); + if (device->memory->current.load() > 4096) + errors.fetch_add(1); + if (lease != nullptr) + std::this_thread::yield(); + } + }); + } + for (auto &thread : threads) + thread.join(); + EXPECT_EQ(errors.load(), 0U); + EXPECT_EQ(device->memory->current.load(), 0U); + EXPECT_GT(device->memory->high_water.load(), 0U); + EXPECT_LE(device->memory->high_water.load(), 4096U); +} + +TEST(RdmaMemoryTest, MrFailureRollsBackBytesAndReportsFailure) +{ + RdmaConfig config; + config.device = "test-no-device"; + RdmaTransport transport(config); + auto device = std::make_shared(); + device->memory = std::make_shared(4096); + TestRdmaTransport::install_device_for_tests(transport, device); + EXPECT_EQ(TestRdmaTransport::make_pool_for_tests(transport, 1024, 1), nullptr); + EXPECT_EQ(transport.stats().registered_bytes.load(), 0U); + EXPECT_EQ(transport.stats().registered_bytes_high_water.load(), 1024U); + EXPECT_EQ(transport.stats().registration_failures.load(), 1U); + EXPECT_EQ(transport.stats().registration_rejects.load(), 0U); + EXPECT_EQ(TestRdmaTransport::make_pool_for_tests(transport, 8192, 1), nullptr); + EXPECT_EQ(transport.stats().registered_bytes.load(), 0U); + EXPECT_EQ(transport.stats().registration_rejects.load(), 1U); +} + +TEST(RdmaMemoryTest, RetiredRealMrBlocksReplacementUntilFinalLeaseRelease) +{ + auto real = TestRdmaDevice::open(); + if (real == nullptr) + GTEST_SKIP() << "no accessible RDMA device"; + auto release_result = real->release_result; + auto device = std::make_shared(); + device->memory = std::make_shared(2048); + auto reservation = device->reserve_bytes(2048); + ASSERT_NE(reservation, nullptr); + auto owner = std::make_shared, std::shared_ptr>>( + real, std::move(reservation)); + auto pool = std::make_unique(real->pd, 2048, 2, owner); + ASSERT_TRUE(pool->ready()); + auto *buffer = pool->alloc(1); + ASSERT_NE(buffer, nullptr); + buffer->write("held", 4); + owner.reset(); + real.reset(); + pool.reset(); + EXPECT_EQ(device->memory->current.load(), 2048U); + EXPECT_EQ(device->reserve_bytes(1024), nullptr); + EXPECT_EQ(device->memory->rejects.load(), 1U); + EXPECT_EQ(release_result->load(), -1); + EXPECT_EQ(buffer->payload_data()[0], 'h'); + buffer->release(); + EXPECT_EQ(device->memory->current.load(), 0U); + EXPECT_EQ(release_result->load(), 0); + EXPECT_NE(device->reserve_bytes(2048), nullptr); +} + +TEST(RdmaHardwareTest, QpSetupRejectsRemainingRegistrationBudgetExhaustion) +{ + const char *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) + GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; + RdmaConfig config; + config.sq_depth = config.rq_depth = config.posted_receive_target = 8; + config.wire_slot_capacity = 1024; + config.send_arena_bytes = config.receive_arena_bytes = 8192; + config.cq_depth = 32; + config.registered_memory_budget_bytes = 32768; + RdmaTransport server(config); + RdmaTransport client(config); + ASSERT_TRUE(server.listen(address, 19389)); + std::shared_ptr incoming; + std::thread acceptor([&] { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (incoming == nullptr && std::chrono::steady_clock::now() < deadline) { + incoming = server.accept(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + }); + auto outgoing = client.connect(address, 19389); + acceptor.join(); + ASSERT_NE(outgoing, nullptr); + ASSERT_NE(incoming, nullptr); + EXPECT_EQ(client.stats().registered_bytes.load(), 32768U); + EXPECT_EQ(client.connect(address, 19389), nullptr); + EXPECT_TRUE(outgoing->is_open()); + EXPECT_TRUE(incoming->is_open()); + EXPECT_EQ(client.stats().registered_bytes.load(), 32768U); + EXPECT_EQ(client.stats().registered_bytes_high_water.load(), 32768U); + EXPECT_GT(client.stats().registration_rejects.load(), 0U); + EXPECT_EQ(client.stats().registration_failures.load(), 0U); +} +#endif diff --git a/lib/crowdb-rpc/tests/registered_buffer_test.cpp b/lib/crowdb-rpc/tests/registered_buffer_test.cpp index 938156812..b7bc1b9b5 100644 --- a/lib/crowdb-rpc/tests/registered_buffer_test.cpp +++ b/lib/crowdb-rpc/tests/registered_buffer_test.cpp @@ -81,6 +81,22 @@ TEST(RegisteredBufferTest, CachedAndQuarantinedDescriptorsStayWithinTotalLimit) reused->release(); } +TEST(RegisteredBufferTest, DetachedQuarantineRemainsChargedAndRangeIsNotReused) +{ + RegisteredBufferPool pool(4096, 1); + auto *buffer = pool.alloc(1); + ASSERT_NE(buffer, nullptr); + const auto *old_range = buffer->data; + ASSERT_TRUE(pool.detach(buffer)); + ASSERT_TRUE(buffer->quarantine()); + buffer->release(); + EXPECT_EQ(pool.quarantined_bytes(), 1024U); + auto *next = pool.alloc(1); + ASSERT_NE(next, nullptr); + EXPECT_NE(next->data, old_range); + next->release(); +} + TEST(RegisteredBufferTest, ConcurrentRecyclingKeepsSlotOwnershipExclusive) { RegisteredBufferPool pool(32 * 1024, 32); From 00e60299871edcb2631a3ce6b86218dc8ec779fe Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 22 Oct 2026 22:37:15 +0800 Subject: [PATCH 028/258] Bind RDMA connection admission to TCP capability preflight --- doc/design/rpc/design-crowdb-rpc-rdma.md | 15 + doc/working/plan-rdma-rpc-transport.md | 45 ++- .../include/crowdb-rpc/connection.h | 5 + .../transport/rdma/rdma_capability.h | 13 +- .../transport/rdma/rdma_transport.h | 24 +- .../crowdb-rpc/transport/socket_transport.h | 11 +- lib/crowdb-rpc/src/server/server.cpp | 8 - .../src/transport/rdma/rdma_capability.cpp | 45 ++- .../src/transport/rdma/rdma_connection.cpp | 60 +++- .../src/transport/rdma/rdma_offer.cpp | 86 ++++++ .../src/transport/rdma/rdma_offer.h | 29 ++ .../src/transport/rdma/rdma_preflight.cpp | 134 +++++++++ .../src/transport/rdma/rdma_preflight.h | 26 ++ .../src/transport/rdma/rdma_transport.cpp | 43 ++- .../src/transport/socket_connect.cpp | 124 +++++++++ .../src/transport/socket_transport.cpp | 133 ++------- lib/crowdb-rpc/tests/rdma_offer_test.cpp | 258 ++++++++++++++++++ 17 files changed, 903 insertions(+), 156 deletions(-) create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_offer.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_offer.h create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_preflight.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_preflight.h create mode 100644 lib/crowdb-rpc/src/transport/socket_connect.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_offer_test.cpp diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index 63c2bcc5b..aba159b61 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -48,6 +48,21 @@ RDMA CM address and port, GID/device identity, capability generation, protocol version, and supported data classes. The client verifies that the generation and peer identity are current before creating the RDMA connection. +`SocketTransport` performs this control exchange on a temporary nonblocking +TCP socket with one deadline. Version 2 also advertises receive slot capacity +and the maximum number of frames accepted in one SEND. A 48-byte CM offer +echoes the expected peer generation and identity digest, and declares the +sender's generation, digest and receive admission. Both CM sides validate it +before publishing the logical connection. The digest detects inconsistent or +stale discovery; it does not authenticate a peer. Low-level CM diagnostics may +omit the TCP expectation but still negotiate receive admission. + +The control socket closes on successful RDMA setup. Allowed TCP fallback may +transfer the fully consumed control stream to the ordinary worker; a failed or +partial control exchange closes it and uses a fresh socket. Strict RDMA policy +permits this TCP control traffic and rejects TCP workload fallback. Disabled or +unavailable local RDMA under strict policy fails before opening a socket. + One logical RPC connection chooses RDMA as its preferred path and TCP as its one-way fallback. The path is selected before the first payload. TCP is chosen when RDMA setup fails before payload submission; an established TCP connection diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 754de25ad..c6acd3dde 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -33,12 +33,12 @@ The C ABI now returns independent handles for retained buffer references. Portable and verbs-backed pools now share the same arena/lifetime machinery. Registration admission remains charged while retired connections retain MRs. -Still pending: negotiated TCP/CM preflight, bounded -transport aggregation, `bench rdma`/shared coroutine reporting, unified regression -orchestration, negotiated TCP capability preflight, failure injection, and the -two-container acceptance profiles. Capability serialization and automatic -consumption at the connection seam do not establish negotiated preflight: -`connect` still attempts CM before the required TCP preflight. Do not treat the +TCP control preflight now precedes CM setup; the compact CM offer binds the +preflight identity/generation and peer receive admission. Strict policy has +passed a two-QP real-HCA RPC round trip and a TCP-only peer rejection case. +Still pending: bounded transport aggregation, `bench rdma`/shared coroutine +reporting, unified regression orchestration, failure injection, and the +two-container acceptance profiles. Do not treat the historical benchmark comments or older subtask ledger as completed acceptance. Physical multi-node validation remains outside this scope in [the follow-up plan](plan-rdma-multi-node.md). @@ -504,7 +504,7 @@ CQ verification: ### C. Bounded aggregation -- [~] **Negotiate receive admission before coalescing**: finish TCP capability +- [x] **Negotiate receive admission before coalescing**: finish TCP capability preflight before the CM attempt; reuse its TCP connection only for allowed pre-payload fallback. Publish actual CM address/port, per-instance generation, device/GID identity, receive slot capacity and supported aggregate frame count. @@ -512,8 +512,10 @@ CQ verification: reject stale/mismatched peers before publishing an RDMA connection. Direct low-level CM diagnostics also negotiate receive slot capacities. Strict RDMA permits the TCP control preflight but never TCP workload bytes. Keep no-device - strict rejection before opening a socket. Retire a successful preflight TCP - connection through the common worker cleanup path. This is a prerequisite + strict rejection before opening a socket. Use an unregistered RAII TCP + control socket: close it on RDMA success and transfer its aligned stream to + the TCP worker only for allowed fallback. This avoids registering a short + lived probe in a data worker and racing descriptor reuse. This is a prerequisite for the peer-slot bound below; the remaining OCI/failure matrix stays in E. Keep the compact CM offer within the RC request's 56-byte private-data limit; use a fixed 48-byte offer with explicit encoding rather than a native struct @@ -524,7 +526,7 @@ CQ verification: preflight expectation, while still negotiating receive admission. Files: RDMA capability/CM sources, `socket_transport.cpp`, server readiness, capability/path and hardware tests. -- [ ] **Coalesce complete frames on the CQ owner**: use bounded lock-free +- [~] **Coalesce complete frames on the CQ owner**: use bounded lock-free admission and per-QP queues; drain available frames without adding a waiting timer. Keep per-QP ordering and common frame/request IDs. Limit a SEND to `min(aggregation_max_bytes, local send slot capacity, peer receive slot @@ -532,7 +534,10 @@ CQ verification: `aggregation_max_frames`. Disabled mode sends one frame. Reject oversized single frames before admission; do not fragment or silently fall back. Files: RDMA submit/owner loop, `transport.h`, config/capability validation. - Reuse `Connection::send_queue` for MPSC frame admission. Keep a bounded + Reuse the common `SendQueue` MPSC primitive in `ConnectionPools` for RDMA + admission. Keep it separate from `Connection::send_queue`: TCP close drains + that queue under `in_send_`, which cannot also act as the RDMA CQ consumer. + Keep a bounded per-CQ connection directory, changed only under its existing CQ ownership gate during cold setup/retirement. The CQ owner drains per-QP queues with a fixed SEND budget and retains one overflow frame for the next slot; do not @@ -560,6 +565,24 @@ CQ verification: SEND rate; compare aggregation disabled/enabled for both TCP and RDMA. Files: transport statistics, C API/Rust FFI and benchmark reports. +Preflight verification: + +- Version 2 TCP capability carries CM address/port, generation/device/GID, + receive slot and frame limit. The 48-byte CM offer binds both generations + and explicit identity digests; the digest is a consistency check, not peer + authentication. Event private data is copied before acknowledgement. +- The raw nonblocking control socket uses one absolute deadline for connect, + write, header and capability reads. Truncated and stalled replies fail + closed. An allowed TCP-only peer reuses the aligned socket for a validated + echo; strict policy records no CM attempt or fallback. +- All 169 enabled native CTest cases pass (opt-in hardware cases skip there). + The no-verbs build passes all 130 cases. Full C++/FFI gate, including all + 29 Rust RPC FFI tests, passes at + `.crowdb-runtime/artifacts/unit/cpp-8022af25b2b643cea3d99b3bc35b35d2/`. + A separate real-HCA two-process, two-QP worker case passes 1000 validated + requests with zero request/CQ errors and fallback disabled. This is + functional evidence, not a sustained benchmark. + ### D. CLI coroutine workload and canonical results - [ ] **Add `bench rdma` with shared echo workload**: reuse `bench rpc` diff --git a/lib/crowdb-rpc/include/crowdb-rpc/connection.h b/lib/crowdb-rpc/include/crowdb-rpc/connection.h index 8d3109085..1e3f13a44 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/connection.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/connection.h @@ -73,6 +73,11 @@ class Connection : public std::enable_shared_from_this void set_peer_capability(const RdmaEndpointCapability &capability) { + if (peer_handshake_received_.load(std::memory_order_acquire)) { + if (rdma_capability_identity(peer_capability_) != rdma_capability_identity(capability)) + close(); + return; + } peer_capability_ = capability; peer_handshake_received_.store(true, std::memory_order_release); } diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h index e2044dcba..49f8f8d3b 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h @@ -32,11 +32,16 @@ struct RdmaEndpointCapability std::string device; std::string gid; std::array data_classes{}; + uint32_t receive_slot_capacity = 0; + uint32_t receive_frame_limit = 1; + std::string cm_address{}; }; -constexpr uint16_t RDMA_HANDSHAKE_MAGIC = 0x5244; -constexpr uint16_t RDMA_HANDSHAKE_MSG_TYPE = 0xFFFE; -constexpr size_t RDMA_HANDSHAKE_SIZE = 128; +constexpr uint16_t RDMA_HANDSHAKE_MAGIC = 0x5244; +constexpr uint16_t RDMA_HANDSHAKE_MSG_TYPE = 0xFFFE; +constexpr uint16_t RDMA_PROTOCOL_VERSION = 2; +constexpr size_t RDMA_LEGACY_HANDSHAKE_SIZE = 128; +constexpr size_t RDMA_HANDSHAKE_SIZE = 160; [[nodiscard]] std::array serialize_rdma_handshake(const RdmaEndpointCapability &capability); @@ -46,6 +51,8 @@ serialize_rdma_handshake(const RdmaEndpointCapability &capability); [[nodiscard]] bool validate_rdma_capability(const RdmaEndpointCapability &capability, uint16_t expected_protocol, uint64_t expected_generation, const std::string &expected_device); +[[nodiscard]] uint64_t rdma_capability_identity(const RdmaEndpointCapability &capability); + [[nodiscard]] bool can_fallback_to_tcp(RdmaPathState state, bool payload_submitted); class RdmaConnectionState diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index 8e0d9b391..9d00c0695 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -34,6 +34,9 @@ constexpr uint32_t RDMA_WIRE_SLOT_CAPACITY = RDMA_DATAPATH_CAPACITY; struct RdmaConfig { std::string device; + std::string source_address{}; + std::string gid{}; + bool require_tcp_preflight = false; uint8_t port = 1; uint32_t cq_depth = 8192; uint32_t cq_count = 1; @@ -118,6 +121,9 @@ class RdmaTransport : public Transport std::atomic registered_bytes_high_water{0}; std::atomic registration_rejects{0}; std::atomic registration_failures{0}; + std::atomic cm_attempts{0}; + std::atomic preflight_attempts{0}; + std::atomic preflight_failures{0}; }; explicit RdmaTransport(RdmaConfig config = {}, BufferPool *application_pool = nullptr); @@ -130,7 +136,14 @@ class RdmaTransport : public Transport // RDMA-specific: connect to a peer endpoint. // Returns a new Connection on success, nullptr on failure. - std::shared_ptr connect(const std::string &addr, int port); + std::shared_ptr connect(const std::string &addr, int port, + const RdmaEndpointCapability *expected_peer = nullptr); + [[nodiscard]] RdmaEndpointCapability local_capability() const; + + [[nodiscard]] const std::string &source_address() const + { + return config_.source_address; + } // RDMA-specific: listen on an endpoint (server side). bool listen(const std::string &addr, int port); @@ -176,6 +189,8 @@ class RdmaTransport : public Transport friend struct TestRdmaTransport; # endif RdmaConfig config_; + RdmaEndpointCapability local_capability_; + std::atomic listen_active_{false}; std::atomic capability_available_{false}; struct ibv_context *context_ = nullptr; std::shared_ptr device_; @@ -313,8 +328,11 @@ class RdmaTransport : public Transport std::shared_ptr posting_state = std::make_shared(); std::unique_ptr send; std::unique_ptr recv; - uint32_t cq_index = 0; - uint32_t reserved_capacity = 0; + uint32_t cq_index = 0; + uint32_t reserved_capacity = 0; + uint32_t peer_receive_slot_capacity = 0; + uint32_t peer_receive_frame_limit = 1; + uint32_t qp_number = 0; }; struct RepostContext diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h index a2874f491..d8a103eb1 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h @@ -507,8 +507,9 @@ class SocketTransport : public Transport external_handler_(tag); } - bool send_rdma_handshake(const std::shared_ptr &conn); - bool listen_rdma(const std::string &addr, int port); + bool send_rdma_handshake(const std::shared_ptr &conn); + [[nodiscard]] RdmaEndpointCapability local_rdma_capability() const; + bool listen_rdma(const std::string &addr, int port); #if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) const RdmaTransport::Stats *rdma_stats_for_tests() const @@ -519,8 +520,10 @@ class SocketTransport : public Transport #endif private: - BufferPool *pool_; - bool allow_tcp_fallback_{true}; + std::shared_ptr connect_tcp(const std::string &addr, int port); + void prepare_tcp_socket(int fd) const; + BufferPool *pool_; + bool allow_tcp_fallback_{true}; #ifdef CROWDB_RPC_HAVE_RDMA std::unique_ptr rdma_transport_; RdmaConfig rdma_config_; diff --git a/lib/crowdb-rpc/src/server/server.cpp b/lib/crowdb-rpc/src/server/server.cpp index 44fd6d9e0..423321cc4 100644 --- a/lib/crowdb-rpc/src/server/server.cpp +++ b/lib/crowdb-rpc/src/server/server.cpp @@ -342,14 +342,6 @@ void RpcServer::dispatch(Frame *frame, Connection *conn) uint16_t msg_type = frame->header.msg_type; bool is_one_way = (frame->header.flags & FLAG_ONE_WAY) != 0; - if (msg_type == RDMA_HANDSHAKE_MSG_TYPE) { - RdmaEndpointCapability capability; - if (parse_rdma_handshake(frame->control, &capability)) - conn->set_peer_capability(capability); - delete frame; - return; - } - // Try request dispatch first — if a handler is registered for // this msg_type, dispatch as a request. This ensures request // frames are not intercepted by on_response (which matches by diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp index 3f306fce5..96e2f5f2d 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp @@ -3,6 +3,8 @@ #include "crowdb-rpc/transport/rdma/rdma_capability.h" +#include + #include #include @@ -35,12 +37,17 @@ std::array serialize_rdma_handshake(const RdmaEndp bytes[offset + 2] = static_cast(value >> 16); bytes[offset + 3] = static_cast(value >> 24); } + for (size_t i = 0; i < 4; ++i) { + bytes[128 + i] = static_cast(capability.receive_slot_capacity >> (i * 8)); + bytes[132 + i] = static_cast(capability.receive_frame_limit >> (i * 8)); + } + std::memcpy(bytes.data() + 136, capability.cm_address.data(), std::min(16, capability.cm_address.size())); return bytes; } bool parse_rdma_handshake(std::span bytes, RdmaEndpointCapability *out) { - if (out == nullptr || bytes.size() < RDMA_HANDSHAKE_SIZE) + if (out == nullptr || bytes.size() < RDMA_LEGACY_HANDSHAKE_SIZE) return false; auto get16 = [&](size_t offset) { return static_cast(bytes[offset]) | static_cast(bytes[offset + 1] << 8); @@ -53,6 +60,9 @@ bool parse_rdma_handshake(std::span bytes, RdmaEndpointCapability }; if (get16(0) != RDMA_HANDSHAKE_MAGIC) return false; + if (get16(2) >= RDMA_PROTOCOL_VERSION && bytes.size() < RDMA_HANDSHAKE_SIZE) + return false; + *out = RdmaEndpointCapability{}; out->protocol_version = get16(2); out->generation = get64(4); out->port = get16(12); @@ -67,6 +77,17 @@ bool parse_rdma_handshake(std::span bytes, RdmaEndpointCapability (static_cast(bytes[offset + 2]) << 16) | (static_cast(bytes[offset + 3]) << 24); } + if (out->protocol_version >= RDMA_PROTOCOL_VERSION) { + for (size_t i = 0; i < 4; ++i) { + out->receive_slot_capacity |= static_cast(bytes[128 + i]) << (i * 8); + if (i == 0) + out->receive_frame_limit = 0; + out->receive_frame_limit |= static_cast(bytes[132 + i]) << (i * 8); + } + out->cm_address.assign(reinterpret_cast(bytes.data() + 136), 16); + out->cm_address.resize(out->cm_address.find('\0') == std::string::npos ? out->cm_address.size() + : out->cm_address.find('\0')); + } return true; } @@ -78,12 +99,34 @@ bool validate_rdma_capability(const RdmaEndpointCapability &capability, uint16_t capability.gid.empty() || (!expected_device.empty() && capability.device != expected_device)) { return false; } + if (expected_protocol >= RDMA_PROTOCOL_VERSION) { + BufferCapacityClass cls; + uint32_t size = 0; + in_addr address{}; + in6_addr gid{}; + if (::inet_pton(AF_INET, capability.cm_address.c_str(), &address) != 1 || + ::inet_pton(AF_INET6, capability.gid.c_str(), &gid) != 1 || capability.generation == 0 || + !classify_rdma_capacity(capability.receive_slot_capacity, &cls, &size) || + size != capability.receive_slot_capacity || capability.receive_frame_limit == 0 || + capability.receive_frame_limit > 256) + return false; + } return capability.data_classes[0] == RDMA_CONTROL_CAPACITY && capability.data_classes[1] == RDMA_METADATA_CAPACITY && capability.data_classes[2] == RDMA_AGGREGATE_CAPACITY && capability.data_classes[3] == RDMA_DATAPATH_CAPACITY; } +uint64_t rdma_capability_identity(const RdmaEndpointCapability &capability) +{ + uint64_t digest = 14695981039346656037ULL; + for (const uint8_t byte : serialize_rdma_handshake(capability)) { + digest ^= byte; + digest *= 1099511628211ULL; + } + return digest; +} + bool can_fallback_to_tcp(RdmaPathState state, bool payload_submitted) { if (payload_submitted) diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp index 89e51a908..a70c9cac5 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp @@ -5,6 +5,7 @@ #include "crowdb-rpc/transport/rdma/rdma_transport.h" #include "rdma_channel.h" #include "rdma_cq.h" +#include "rdma_offer.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -17,13 +18,16 @@ namespace crowdb::rpc { -std::shared_ptr RdmaTransport::connect(const std::string &addr, int port) +std::shared_ptr RdmaTransport::connect(const std::string &addr, int port, + const RdmaEndpointCapability *expected_peer) { - if (!ready() || addr.empty() || port <= 0 || port > 65535) { + if (!ready() || addr.empty() || port <= 0 || port > 65535 || + (config_.require_tcp_preflight && expected_peer == nullptr)) { CRB_LOG_WARN("rdma connect: transport unavailable addr={} port={}", addr, port); return nullptr; } + stats_.cm_attempts.fetch_add(1, std::memory_order_relaxed); { std::scoped_lock lock(cm_mu_); if (cm_channel_ == nullptr) @@ -52,7 +56,7 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int } sockaddr_in source{}; sockaddr *source_address = nullptr; - if (const char *source_env = std::getenv("CROWDB_RDMA_SOURCE_ADDR"); source_env != nullptr && *source_env != '\0') { + if (const char *source_env = config_.source_address.c_str(); *source_env != '\0') { source.sin_family = AF_INET; if (::inet_pton(AF_INET, source_env, &source.sin_addr) != 1) { CRB_LOG_WARN("rdma connect: invalid source address {}", source_env); @@ -164,6 +168,10 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int return nullptr; } rdma_conn_param params{}; + const auto local = local_capability(); + const auto offer = serialize_rdma_offer(local, expected_peer); + params.private_data = offer.data(); + params.private_data_len = static_cast(offer.size()); params.initiator_depth = 1; params.responder_resources = 1; params.retry_count = config_.retry_count; @@ -186,7 +194,21 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int rdma_destroy_id(id); return nullptr; } + RdmaCmOffer peer_offer; + const bool valid_peer = event->param.conn.private_data != nullptr && + parse_rdma_offer(std::span(static_cast(event->param.conn.private_data), + event->param.conn.private_data_len), + &peer_offer) && + validate_rdma_offer(peer_offer, local, expected_peer, true); rdma_ack_cm_event(event); + if (!valid_peer) { + destroy_connection_qp(connection.get(), id); + rdma_destroy_id(id); + return nullptr; + } + pools->peer_receive_slot_capacity = peer_offer.receive_slot_capacity; + pools->peer_receive_frame_limit = peer_offer.receive_frame_limit; + pools->qp_number = id->qp->qp_num; ibv_qp_attr receive_attributes{}; receive_attributes.min_rnr_timer = config_.min_rnr_timer; if (ibv_modify_qp(id->qp, &receive_attributes, IBV_QP_MIN_RNR_TIMER) != 0) { @@ -232,6 +254,9 @@ bool RdmaTransport::listen(const std::string &addr, int port) listen_id_ = nullptr; return false; } + local_capability_.cm_address = addr; + local_capability_.port = static_cast(port); + listen_active_.store(true, std::memory_order_release); return true; } @@ -279,7 +304,18 @@ std::shared_ptr RdmaTransport::accept() if (event == nullptr) return nullptr; rdma_cm_id *id = event->id; + RdmaCmOffer peer_offer; + const bool valid_peer = event->param.conn.private_data != nullptr && + parse_rdma_offer(std::span(static_cast(event->param.conn.private_data), + event->param.conn.private_data_len), + &peer_offer) && + validate_rdma_offer(peer_offer, local_capability(), nullptr, config_.require_tcp_preflight); rdma_ack_cm_event(event); + if (!valid_peer) { + rdma_reject(id, nullptr, 0); + rdma_destroy_id(id); + return nullptr; + } // A new CONNECT_REQUEST may precede this connection's ESTABLISHED on // the listener channel. Keep setup events private until readiness. @@ -320,12 +356,15 @@ std::shared_ptr RdmaTransport::accept() rdma_destroy_id(id); return nullptr; } - auto pools = std::make_shared(); - pools->effective_capacity = qp_attr.cap; - pools->cq_index = reservation.queue->index; - pools->reserved_capacity = reservation.capacity; - pools->send = make_pool(config_.send_arena_bytes, config_.sq_depth); - pools->recv = make_pool(config_.receive_arena_bytes, config_.rq_depth); + auto pools = std::make_shared(); + pools->peer_receive_slot_capacity = peer_offer.receive_slot_capacity; + pools->peer_receive_frame_limit = peer_offer.receive_frame_limit; + pools->qp_number = id->qp->qp_num; + pools->effective_capacity = qp_attr.cap; + pools->cq_index = reservation.queue->index; + pools->reserved_capacity = reservation.capacity; + pools->send = make_pool(config_.send_arena_bytes, config_.sq_depth); + pools->recv = make_pool(config_.receive_arena_bytes, config_.rq_depth); if (pools->send == nullptr || pools->recv == nullptr) { rdma_destroy_qp(id); rdma_destroy_id(id); @@ -344,6 +383,9 @@ std::shared_ptr RdmaTransport::accept() } rdma_conn_param params{}; + const auto reply = serialize_rdma_reply(local_capability(), peer_offer); + params.private_data = reply.data(); + params.private_data_len = static_cast(reply.size()); params.initiator_depth = 1; params.responder_resources = 1; params.retry_count = config_.retry_count; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_offer.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_offer.cpp new file mode 100644 index 000000000..f8ae11608 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_offer.cpp @@ -0,0 +1,86 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_offer.h" + +namespace crowdb::rpc +{ +namespace +{ +constexpr uint32_t OFFER_MAGIC = 0x434d5244; + +void put(std::span bytes, size_t offset, uint64_t value, size_t size) +{ + for (size_t index = 0; index < size; ++index) + bytes[offset + index] = static_cast(value >> (index * 8)); +} + +uint64_t get(std::span bytes, size_t offset, size_t size) +{ + uint64_t value = 0; + for (size_t index = 0; index < size; ++index) + value |= static_cast(bytes[offset + index]) << (index * 8); + return value; +} +} // namespace + +std::array serialize_rdma_offer(const RdmaEndpointCapability &local, + const RdmaEndpointCapability *expected_peer) +{ + std::array bytes{}; + put(bytes, 0, OFFER_MAGIC, 4); + put(bytes, 4, RDMA_PROTOCOL_VERSION, 2); + put(bytes, 6, local.receive_frame_limit, 2); + put(bytes, 8, local.receive_slot_capacity, 4); + put(bytes, 16, local.generation, 8); + put(bytes, 24, expected_peer == nullptr ? 0 : expected_peer->generation, 8); + put(bytes, 32, rdma_capability_identity(local), 8); + put(bytes, 40, expected_peer == nullptr ? 0 : rdma_capability_identity(*expected_peer), 8); + return bytes; +} + +bool parse_rdma_offer(std::span bytes, RdmaCmOffer *out) +{ + if (out == nullptr || bytes.size() < RDMA_CM_OFFER_SIZE || get(bytes, 0, 4) != OFFER_MAGIC || + get(bytes, 4, 2) != RDMA_PROTOCOL_VERSION || get(bytes, 12, 4) != 0) + return false; + RdmaCmOffer offer; + offer.receive_frame_limit = static_cast(get(bytes, 6, 2)); + offer.receive_slot_capacity = static_cast(get(bytes, 8, 4)); + offer.generation = get(bytes, 16, 8); + offer.expected_generation = get(bytes, 24, 8); + offer.identity = get(bytes, 32, 8); + offer.expected_identity = get(bytes, 40, 8); + BufferCapacityClass cls; + uint32_t size = 0; + if (!classify_rdma_capacity(offer.receive_slot_capacity, &cls, &size) || size != offer.receive_slot_capacity || + offer.receive_frame_limit == 0 || offer.receive_frame_limit > 256 || offer.generation == 0 || + offer.identity == 0 || (offer.expected_generation == 0) != (offer.expected_identity == 0)) + return false; + *out = offer; + return true; +} + +std::array serialize_rdma_reply(const RdmaEndpointCapability &local, + const RdmaCmOffer &peer) +{ + auto bytes = serialize_rdma_offer(local); + put(bytes, 24, peer.generation, 8); + put(bytes, 40, peer.identity, 8); + return bytes; +} + +bool validate_rdma_offer(const RdmaCmOffer &offer, const RdmaEndpointCapability &local, + const RdmaEndpointCapability *expected_peer, bool require_preflight) +{ + if (require_preflight && offer.expected_generation == 0) + return false; + if (offer.expected_generation != 0 && + (offer.expected_generation != local.generation || offer.expected_identity != rdma_capability_identity(local))) + return false; + return expected_peer == nullptr || (offer.generation == expected_peer->generation && + offer.identity == rdma_capability_identity(*expected_peer) && + offer.receive_slot_capacity == expected_peer->receive_slot_capacity && + offer.receive_frame_limit == expected_peer->receive_frame_limit); +} +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_offer.h b/lib/crowdb-rpc/src/transport/rdma/rdma_offer.h new file mode 100644 index 000000000..21263421f --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_offer.h @@ -0,0 +1,29 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/transport/rdma/rdma_capability.h" + +namespace crowdb::rpc +{ +constexpr size_t RDMA_CM_OFFER_SIZE = 48; + +struct RdmaCmOffer +{ + uint16_t receive_frame_limit = 0; + uint32_t receive_slot_capacity = 0; + uint64_t generation = 0; + uint64_t expected_generation = 0; + uint64_t identity = 0; + uint64_t expected_identity = 0; +}; + +std::array serialize_rdma_offer(const RdmaEndpointCapability &local, + const RdmaEndpointCapability *expected_peer = nullptr); +std::array serialize_rdma_reply(const RdmaEndpointCapability &local, + const RdmaCmOffer &peer); +bool parse_rdma_offer(std::span bytes, RdmaCmOffer *out); +bool validate_rdma_offer(const RdmaCmOffer &offer, const RdmaEndpointCapability &local, + const RdmaEndpointCapability *expected_peer, bool require_preflight); +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_preflight.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_preflight.cpp new file mode 100644 index 000000000..73675e490 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_preflight.cpp @@ -0,0 +1,134 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_preflight.h" + +#include "crowdb-rpc/framing.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace crowdb::rpc +{ +namespace +{ +using Clock = std::chrono::steady_clock; + +bool wait_for(int fd, short events, Clock::time_point deadline) +{ + for (;;) { + const auto remaining = std::chrono::duration_cast(deadline - Clock::now()).count(); + if (remaining <= 0) + return false; + pollfd descriptor{fd, events, 0}; + const int result = ::poll(&descriptor, 1, static_cast(std::min(remaining, INT32_MAX))); + if (result > 0) + return (descriptor.revents & events) != 0; + if (result == 0 || errno != EINTR) + return false; + } +} + +bool transfer(int fd, std::span bytes, bool writing, Clock::time_point deadline) +{ + while (!bytes.empty()) { + int flags = 0; +#ifdef MSG_NOSIGNAL + flags = MSG_NOSIGNAL; +#endif + const auto count = + writing ? ::send(fd, bytes.data(), bytes.size(), flags) : ::recv(fd, bytes.data(), bytes.size(), 0); + if (count > 0) { + bytes = bytes.subspan(static_cast(count)); + continue; + } + if (count == 0) + return false; + if (errno == EINTR) + continue; + if ((errno != EAGAIN && errno != EWOULDBLOCK) || !wait_for(fd, writing ? POLLOUT : POLLIN, deadline)) + return false; + } + return true; +} +} // namespace + +RdmaPreflight::~RdmaPreflight() +{ + if (fd >= 0) + ::close(fd); +} + +int RdmaPreflight::release_fd() +{ + return std::exchange(fd, -1); +} + +std::unique_ptr RdmaPreflight::connect(const std::string &address, int port, + const RdmaEndpointCapability &local, + const std::string &source_address, uint32_t timeout_ms) +{ + if (port <= 0 || port > 65535 || timeout_ms == 0) + return nullptr; + auto connection = std::make_unique(); + connection->fd = ::socket(AF_INET, SOCK_STREAM, 0); + if (connection->fd < 0 || ::fcntl(connection->fd, F_SETFL, O_NONBLOCK) != 0) + return nullptr; +#ifdef SO_NOSIGPIPE + int nosigpipe = 1; + ::setsockopt(connection->fd, SOL_SOCKET, SO_NOSIGPIPE, &nosigpipe, sizeof(nosigpipe)); +#endif + sockaddr_in peer{}; + peer.sin_family = AF_INET; + peer.sin_port = htons(static_cast(port)); + if (::inet_pton(AF_INET, address.c_str(), &peer.sin_addr) != 1) + return nullptr; + if (!source_address.empty()) { + sockaddr_in source{}; + source.sin_family = AF_INET; + if (::inet_pton(AF_INET, source_address.c_str(), &source.sin_addr) != 1 || + ::bind(connection->fd, reinterpret_cast(&source), sizeof(source)) != 0) + return nullptr; + } + const auto deadline = Clock::now() + std::chrono::milliseconds(timeout_ms); + const int result = ::connect(connection->fd, reinterpret_cast(&peer), sizeof(peer)); + if (result != 0 && (errno != EINPROGRESS || !wait_for(connection->fd, POLLOUT, deadline))) + return nullptr; + int error = 0; + socklen_t size = sizeof(error); + if (::getsockopt(connection->fd, SOL_SOCKET, SO_ERROR, &error, &size) != 0 || error != 0) + return nullptr; + + Header header; + header.msg_type = RDMA_HANDSHAKE_MSG_TYPE; + header.msg_size = RDMA_HANDSHAKE_SIZE; + header.flags = FLAG_ONE_WAY; + std::array request{}; + serialize_header(request.data(), header); + const auto capability = serialize_rdma_handshake(local); + std::copy(capability.begin(), capability.end(), request.begin() + HEADER_SIZE); + if (!transfer(connection->fd, request, true, deadline)) + return nullptr; + + std::array response_header{}; + if (!transfer(connection->fd, response_header, false, deadline)) + return nullptr; + header = parse_header(response_header.data()); + if (header.magic != MAGIC || header.msg_type != RDMA_HANDSHAKE_MSG_TYPE || header.msg_offset != HEADER_SIZE || + header.data_size != 0 || header.msg_size < RDMA_LEGACY_HANDSHAKE_SIZE || header.msg_size > RDMA_HANDSHAKE_SIZE) + return nullptr; + std::array response{}; + const auto bytes = std::span(response).first(header.msg_size); + if (!transfer(connection->fd, bytes, false, deadline) || !parse_rdma_handshake(bytes, &connection->peer)) + return nullptr; + return connection; +} +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_preflight.h b/lib/crowdb-rpc/src/transport/rdma/rdma_preflight.h new file mode 100644 index 000000000..93d30672d --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_preflight.h @@ -0,0 +1,26 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/transport/rdma/rdma_capability.h" + +#include + +namespace crowdb::rpc +{ +struct RdmaPreflight +{ + RdmaPreflight() = default; + RdmaPreflight(const RdmaPreflight &) = delete; + RdmaPreflight &operator=(const RdmaPreflight &) = delete; + int fd = -1; + RdmaEndpointCapability peer; + ~RdmaPreflight(); + int release_fd(); + + static std::unique_ptr connect(const std::string &address, int port, + const RdmaEndpointCapability &local, + const std::string &source_address, uint32_t timeout_ms = 2000); +}; +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index de5497516..20f366c13 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -23,6 +23,7 @@ # include # include # include +# include # include namespace crowdb::rpc @@ -60,6 +61,24 @@ RdmaTransport::RdmaTransport(RdmaConfig config, BufferPool *application_pool) } requested_device_ = config_.device; port_ = config_.port; + if (config_.source_address.empty()) { + if (const char *source = std::getenv("CROWDB_RDMA_SOURCE_ADDR"); source != nullptr) + config_.source_address = source; + } + if (config_.gid.empty()) { + if (const char *gid = std::getenv("CROWDB_RDMA_GID"); gid != nullptr) + config_.gid = gid; + } + std::random_device random; + local_capability_.protocol_version = RDMA_PROTOCOL_VERSION; + local_capability_.generation = (static_cast(random()) << 32) | random(); + if (local_capability_.generation == 0) + local_capability_.generation = 1; + local_capability_.data_classes = {RDMA_CONTROL_CAPACITY, RDMA_METADATA_CAPACITY, RDMA_AGGREGATE_CAPACITY, + RDMA_DATAPATH_CAPACITY}; + local_capability_.receive_slot_capacity = config_.wire_slot_capacity; + local_capability_.receive_frame_limit = 1; + local_capability_.cm_address = config_.source_address; int device_count = 0; ibv_device **devices = ibv_get_device_list(&device_count); @@ -80,12 +99,28 @@ RdmaTransport::RdmaTransport(RdmaConfig config, BufferPool *application_pool) ibv_port_attr attributes{}; capability_available_ = ibv_query_port(probe, port_, &attributes) == 0 && attributes.state == IBV_PORT_ACTIVE; + local_capability_.device = ibv_get_device_name(selected); + local_capability_.gid = config_.gid; + if (local_capability_.gid.empty()) { + ibv_gid gid{}; + char text[INET6_ADDRSTRLEN]{}; + if (ibv_query_gid(probe, port_, 0, &gid) == 0 && + ::inet_ntop(AF_INET6, &gid, text, sizeof(text)) != nullptr) + local_capability_.gid = text; + } ibv_close_device(probe); } } ibv_free_device_list(devices); } +RdmaEndpointCapability RdmaTransport::local_capability() const +{ + auto capability = local_capability_; + capability.rdma_available = ready() && (capability.port == 0 || listen_active_.load(std::memory_order_acquire)); + return capability; +} + RdmaTransport::~RdmaTransport() { RdmaTransport::shutdown(); @@ -113,6 +148,8 @@ bool RdmaTransport::adopt_cm_context(struct ibv_context *context) // arenas. Do not replace live QPs/registrations when another HCA appears. if (context_ != nullptr) return false; + if (local_capability_.device != ibv_get_device_name(context->device)) + return false; auto device = RdmaDevice::retain(context, memory_stats_); if (device == nullptr) return false; @@ -188,11 +225,12 @@ bool RdmaTransport::submit(Connection *conn, OutFrame *frame) } const uint32_t data_len = frame->data == nullptr ? 0 : frame->data->len; const uint64_t wire_len = static_cast(HEADER_SIZE) + control_len + data_len; - if (wire_len > slot->capacity) { + if (wire_len > slot->capacity || wire_len > pools->peer_receive_slot_capacity) { + const auto capacity = slot->capacity; (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); slot->release(); CRB_LOG_WARN("rdma send: message exceeds wire slot control_len={} data_len={} capacity={}", control_len, - data_len, slot->capacity); + data_len, capacity); return false; } frame->header.msg_size = control_len; @@ -330,6 +368,7 @@ Buffer *RdmaTransport::register_buffer(Buffer *buf) void RdmaTransport::shutdown() { + listen_active_.store(false, std::memory_order_release); repost_state_->users.fetch_or(RepostState::STOPPED, std::memory_order_acq_rel); while ((repost_state_->users.load(std::memory_order_acquire) & ~RepostState::STOPPED) != 0) std::this_thread::yield(); diff --git a/lib/crowdb-rpc/src/transport/socket_connect.cpp b/lib/crowdb-rpc/src/transport/socket_connect.cpp new file mode 100644 index 000000000..df4ddb3a1 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/socket_connect.cpp @@ -0,0 +1,124 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-common/log.h" +#include "crowdb-rpc/transport/socket_transport.h" +#include "rdma/rdma_preflight.h" + +#include +#include +#include +#include +#include + +#include +#include + +namespace crowdb::rpc +{ +std::shared_ptr SocketTransport::connect(const std::string &addr, int port) +{ +#ifdef CROWDB_RPC_HAVE_RDMA + if (rdma_transport_ != nullptr && rdma_transport_->ready()) { + auto &stats = rdma_transport_->stats(); + stats.preflight_attempts.fetch_add(1, std::memory_order_relaxed); + auto probe = RdmaPreflight::connect(addr, port, local_rdma_capability(), rdma_transport_->source_address()); + if (probe != nullptr && + validate_rdma_capability(probe->peer, RDMA_PROTOCOL_VERSION, probe->peer.generation, {})) { + const auto &cm_address = probe->peer.cm_address; + const auto destination = cm_address == "0.0.0.0" ? addr : cm_address; + auto connection = rdma_transport_->connect(destination, probe->peer.port, &probe->peer); + if (connection != nullptr) { + for (auto &engine : engines_) + rdma_transport_->register_events(engine.get()); + attach_rdma_connection(connection); + return connection; + } + } + else { + stats.preflight_failures.fetch_add(1, std::memory_order_relaxed); + } + if (!allow_tcp_fallback_) { + CRB_LOG_WARN("RDMA preflight/setup failed; TCP fallback disabled addr={} port={}", addr, port); + return nullptr; + } + stats.fallback_count.fetch_add(1, std::memory_order_relaxed); + if (probe != nullptr) { + const int fd = probe->release_fd(); + prepare_tcp_socket(fd); + auto connection = create_connection(fd, addr + ":" + std::to_string(port)); + connection->set_peer_capability(probe->peer); + return connection; + } + return connect_tcp(addr, port); + } +#endif + if (!allow_tcp_fallback_) { + CRB_LOG_WARN("RDMA connection unavailable; TCP fallback disabled addr={} port={}", addr, port); + return nullptr; + } +#ifdef CROWDB_RPC_HAVE_RDMA + if (rdma_transport_ != nullptr) + rdma_transport_->stats().fallback_count.fetch_add(1, std::memory_order_relaxed); +#endif + return connect_tcp(addr, port); +} + +void SocketTransport::prepare_tcp_socket(int fd) const +{ + int nodelay = tcp_nodelay_ ? 1 : 0; + ::setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &nodelay, sizeof(nodelay)); +#ifdef __linux__ + if (quickack_) { + int quickack = 1; + ::setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, &quickack, sizeof(quickack)); + } +#endif +} + +std::shared_ptr SocketTransport::connect_tcp(const std::string &addr, int port) +{ + int fd = ::socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) { + CRB_LOG_WARN("connect: socket() failed addr={} port={} errno={} ({})", addr, port, errno, std::strerror(errno)); + return nullptr; + } + + struct sockaddr_in sa{}; + sa.sin_family = AF_INET; + sa.sin_port = htons(static_cast(port)); + if (::inet_pton(AF_INET, addr.c_str(), &sa.sin_addr) <= 0) { + CRB_LOG_WARN("connect: inet_pton failed addr={} port={} errno={} ({})", addr, port, errno, + std::strerror(errno)); + ::close(fd); + return nullptr; + } + + const int flags = fcntl(fd, F_GETFL, 0); + if (flags < 0 || fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) { + CRB_LOG_WARN("connect: nonblocking setup failed addr={} port={} errno={} ({})", addr, port, errno, + std::strerror(errno)); + ::close(fd); + return nullptr; + } + + // A missing peer must not block the caller's executor while TCP retries + // its handshake. Queued writes wait for writability in the worker; + // handshake failures use the same error/close path as established peers. + if (::connect(fd, reinterpret_cast(&sa), sizeof(sa)) < 0 && errno != EINPROGRESS) { + CRB_LOG_WARN("connect: connect() failed addr={} port={} errno={} ({})", addr, port, errno, + std::strerror(errno)); + ::close(fd); + return nullptr; + } + + prepare_tcp_socket(fd); + + auto conn = create_connection(fd, addr + ":" + std::to_string(port)); + send_rdma_handshake(conn); + CRB_LOG_INFO("rpc transport: connection initiated -> {}:{} conn_id={}", addr, port, + static_cast(conn->id())); + return conn; +} + +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/socket_transport.cpp b/lib/crowdb-rpc/src/transport/socket_transport.cpp index cba359c80..d1962fc99 100644 --- a/lib/crowdb-rpc/src/transport/socket_transport.cpp +++ b/lib/crowdb-rpc/src/transport/socket_transport.cpp @@ -559,7 +559,8 @@ SocketTransport::SocketTransport(uint32_t io_engines, uint32_t io_workers, Buffe pool_ = new SystemBufferPool(); // own a default pool } #ifdef CROWDB_RPC_HAVE_RDMA - rdma_config_ = std::move(rdma_config); + rdma_config_ = std::move(rdma_config); + rdma_config_.require_tcp_preflight = true; if (const char *enabled = std::getenv("CROWDB_RDMA_ENABLED"); enabled != nullptr && std::strcmp(enabled, "1") == 0) { rdma_transport_ = std::make_unique(rdma_config_, pool_); @@ -883,106 +884,6 @@ size_t SocketTransport::connection_count() const return live_conns_.size(); } -std::shared_ptr SocketTransport::connect(const std::string &addr, int port) -{ -#ifdef CROWDB_RPC_HAVE_RDMA - if (rdma_transport_ != nullptr) { - int rdma_port = port; - bool explicit_rdma_port = false; - if (const char *port_env = std::getenv("CROWDB_RDMA_E2E_PORT"); port_env != nullptr && *port_env != '\0') { - char *end = nullptr; - errno = 0; - const long configured = std::strtol(port_env, &end, 10); - if (errno == 0 && end != port_env && *end == '\0' && configured > 0 && configured <= 65535) { - rdma_port = static_cast(configured); - explicit_rdma_port = true; - } - else { - CRB_LOG_WARN("connect: invalid RDMA CM port value={}", port_env); - } - } - if (!explicit_rdma_port) { - const char *offset_env = std::getenv("CROWDB_PORT_OFFSET"); - if (offset_env == nullptr || *offset_env == '\0') - offset_env = nullptr; - if (offset_env != nullptr) { - char *end = nullptr; - errno = 0; - const long offset = std::strtol(offset_env, &end, 10); - if (errno == 0 && end != offset_env && *end == '\0' && offset >= 0 && offset <= 65535 - port) - rdma_port += static_cast(offset); - else - CRB_LOG_WARN("connect: invalid RDMA port offset value={}", offset_env); - } - } - if (auto rdma_conn = rdma_transport_->connect(addr, rdma_port); rdma_conn != nullptr) { - for (auto &engine : engines_) - rdma_transport_->register_events(engine.get()); - register_conn(rdma_conn); - attach_rdma_connection(rdma_conn); - return rdma_conn; - } - } -#endif - if (!allow_tcp_fallback_) { - CRB_LOG_WARN("RDMA connection unavailable; TCP fallback disabled addr={} port={}", addr, port); - return nullptr; - } -#ifdef CROWDB_RPC_HAVE_RDMA - if (rdma_transport_ != nullptr) { - rdma_transport_->stats().fallback_count.fetch_add(1, std::memory_order_relaxed); - } -#endif - int fd = ::socket(AF_INET, SOCK_STREAM, 0); - if (fd < 0) { - CRB_LOG_WARN("connect: socket() failed addr={} port={} errno={} ({})", addr, port, errno, std::strerror(errno)); - return nullptr; - } - - struct sockaddr_in sa{}; - sa.sin_family = AF_INET; - sa.sin_port = htons(static_cast(port)); - if (::inet_pton(AF_INET, addr.c_str(), &sa.sin_addr) <= 0) { - CRB_LOG_WARN("connect: inet_pton failed addr={} port={} errno={} ({})", addr, port, errno, - std::strerror(errno)); - ::close(fd); - return nullptr; - } - - const int flags = fcntl(fd, F_GETFL, 0); - if (flags < 0 || fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) { - CRB_LOG_WARN("connect: nonblocking setup failed addr={} port={} errno={} ({})", addr, port, errno, - std::strerror(errno)); - ::close(fd); - return nullptr; - } - - // A missing peer must not block the caller's executor while TCP retries - // its handshake. Queued writes wait for writability in the worker; - // handshake failures use the same error/close path as established peers. - if (::connect(fd, reinterpret_cast(&sa), sizeof(sa)) < 0 && errno != EINPROGRESS) { - CRB_LOG_WARN("connect: connect() failed addr={} port={} errno={} ({})", addr, port, errno, - std::strerror(errno)); - ::close(fd); - return nullptr; - } - - int nodelay = tcp_nodelay_ ? 1 : 0; - ::setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &nodelay, sizeof(nodelay)); -#ifdef __linux__ - if (quickack_) { - int quickack = 1; - ::setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, &quickack, sizeof(quickack)); - } -#endif - - auto conn = create_connection(fd, addr + ":" + std::to_string(port)); - send_rdma_handshake(conn); - CRB_LOG_INFO("rpc transport: connection initiated -> {}:{} conn_id={}", addr, port, - static_cast(conn->id())); - return conn; -} - void SocketTransport::attach_rdma_connection(const std::shared_ptr &conn, Connection::OnFrameCallback on_frame, Connection::OnCloseCallback on_close) { @@ -1003,24 +904,26 @@ void SocketTransport::set_rdma_accept_handler(RdmaAcceptHandler handler) rdma_accept_handler_ = std::move(handler); } +RdmaEndpointCapability SocketTransport::local_rdma_capability() const +{ +#ifdef CROWDB_RPC_HAVE_RDMA + if (rdma_transport_ != nullptr) + return rdma_transport_->local_capability(); +#endif + RdmaEndpointCapability capability; + capability.protocol_version = RDMA_PROTOCOL_VERSION; + capability.data_classes = {RDMA_CONTROL_CAPACITY, RDMA_METADATA_CAPACITY, RDMA_AGGREGATE_CAPACITY, + RDMA_DATAPATH_CAPACITY}; + return capability; +} + bool SocketTransport::send_rdma_handshake(const std::shared_ptr &conn) { if (!conn || pool_ == nullptr) return false; - RdmaEndpointCapability capability; - const char *enabled = std::getenv("CROWDB_RDMA_ENABLED"); - capability.rdma_available = enabled != nullptr && std::strcmp(enabled, "1") == 0; - capability.protocol_version = 1; - capability.generation = 1; - capability.port = 1; - if (const char *device = std::getenv("CROWDB_RDMA_DEVICE"); device != nullptr) - capability.device = device; - if (const char *gid = std::getenv("CROWDB_RDMA_GID"); gid != nullptr) - capability.gid = gid; - capability.data_classes = {RDMA_CONTROL_CAPACITY, RDMA_METADATA_CAPACITY, RDMA_AGGREGATE_CAPACITY, - RDMA_DATAPATH_CAPACITY}; - const auto wire = serialize_rdma_handshake(capability); - Buffer *control = build_raw_control(pool_, wire.data(), static_cast(wire.size())); + const auto capability = local_rdma_capability(); + const auto wire = serialize_rdma_handshake(capability); + Buffer *control = build_raw_control(pool_, wire.data(), static_cast(wire.size())); if (control == nullptr) return false; OutFrame *frame = build_out_frame(0, RDMA_HANDSHAKE_MSG_TYPE, control, nullptr, FLAG_ONE_WAY); diff --git a/lib/crowdb-rpc/tests/rdma_offer_test.cpp b/lib/crowdb-rpc/tests/rdma_offer_test.cpp new file mode 100644 index 000000000..83d0a648d --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_offer_test.cpp @@ -0,0 +1,258 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "../src/transport/rdma/rdma_offer.h" +#include "../src/transport/rdma/rdma_preflight.h" +#include "common/test_environment.h" +#include "crowdb-rpc/server/message.h" +#include "crowdb-rpc/server/server.h" + +#include +#include +#include +#include +#include + +#include +#include + +using namespace crowdb::rpc; + +namespace +{ +RdmaEndpointCapability test_capability(uint64_t generation) +{ + return { + .rdma_available = true, + .protocol_version = RDMA_PROTOCOL_VERSION, + .generation = generation, + .port = 19101, + .device = "test-device", + .gid = "fe80::1", + .data_classes = {1024, 65536, 262144, 1048576}, + .receive_slot_capacity = 1024, + .receive_frame_limit = 1, + .cm_address = "10.200.1.1" + }; +} + +void test_echo_round_trip(SocketTransport &transport, const std::shared_ptr &connection) +{ + struct TestEchoResult + { + std::atomic complete{false}; + std::atomic valid{false}; + }; + + auto result = std::make_shared(); + connection->set_on_frame([result](Frame *frame, Connection *) { + result->valid.store(frame->request_id == 91 && frame->header.msg_type == 100 && frame->data_buf != nullptr && + frame->data_buf->len == 16 && + std::memcmp(frame->data_buf->payload_data(), "fallback payload", 16) == 0); + delete frame; + result->complete.store(true, std::memory_order_release); + }); + auto *control = build_ping_request(transport.pool(), 91, 0); + auto *data = transport.pool()->alloc(16); + ASSERT_NE(control, nullptr); + ASSERT_NE(data, nullptr); + data->write("fallback payload", 16); + auto *frame = build_out_frame(91, 100, control, data); + const bool accepted = transport.submit(connection.get(), frame); + if (!accepted) { + control->release(); + frame->release_data(); + delete frame; + } + ASSERT_TRUE(accepted); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (!result->complete.load(std::memory_order_acquire) && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + EXPECT_TRUE(result->complete.load()); + EXPECT_TRUE(result->valid.load()); +} +} // namespace + +TEST(RdmaOfferTest, PrivateDataBindsPreflightIdentityAndReceiveAdmission) +{ + const auto client = test_capability(11); + auto server = test_capability(17); + server.receive_slot_capacity = 65536; + RdmaCmOffer decoded; + const auto wire = serialize_rdma_offer(client, &server); + ASSERT_TRUE(parse_rdma_offer(wire, &decoded)); + EXPECT_EQ(wire.size(), 48U); + EXPECT_EQ(decoded.receive_slot_capacity, 1024U); + EXPECT_TRUE(validate_rdma_offer(decoded, server, &client, true)); + auto stale = server; + ++stale.generation; + EXPECT_FALSE(validate_rdma_offer(decoded, stale, &client, true)); + stale = server; + stale.gid = "fe80::2"; + EXPECT_FALSE(validate_rdma_offer(decoded, stale, &client, true)); + RdmaCmOffer reply; + ASSERT_TRUE(parse_rdma_offer(serialize_rdma_reply(server, decoded), &reply)); + EXPECT_TRUE(validate_rdma_offer(reply, client, &server, true)); + server.receive_slot_capacity = 1024; + EXPECT_FALSE(validate_rdma_offer(reply, client, &server, true)); +} + +TEST(RdmaOfferTest, MissingPreflightAndMalformedAdmissionFailClosed) +{ + const auto client = test_capability(1); + const auto server = test_capability(2); + auto wire = serialize_rdma_offer(client); + RdmaCmOffer decoded; + ASSERT_TRUE(parse_rdma_offer(wire, &decoded)); + EXPECT_TRUE(validate_rdma_offer(decoded, server, nullptr, false)); + EXPECT_FALSE(validate_rdma_offer(decoded, server, nullptr, true)); + for (size_t size = 0; size < RDMA_CM_OFFER_SIZE; ++size) + EXPECT_FALSE(parse_rdma_offer(std::span(wire).first(size), &decoded)); + wire[8] = 1; + EXPECT_FALSE(parse_rdma_offer(wire, &decoded)); + wire = serialize_rdma_offer(client); + wire[12] = 1; + EXPECT_FALSE(parse_rdma_offer(wire, &decoded)); +} + +TEST(RdmaOfferTest, TcpCapabilityCarriesCmAddressAndRetainsLegacyFallbackParsing) +{ + const auto source = test_capability(27); + auto wire = serialize_rdma_handshake(source); + RdmaEndpointCapability decoded; + ASSERT_TRUE(parse_rdma_handshake(wire, &decoded)); + EXPECT_EQ(decoded.cm_address, source.cm_address); + EXPECT_EQ(decoded.receive_slot_capacity, source.receive_slot_capacity); + EXPECT_EQ(decoded.receive_frame_limit, source.receive_frame_limit); + EXPECT_TRUE(validate_rdma_capability(decoded, RDMA_PROTOCOL_VERSION, 27, "test-device")); + EXPECT_FALSE(parse_rdma_handshake(std::span(wire).first(RDMA_LEGACY_HANDSHAKE_SIZE), &decoded)); + wire[2] = 1; + ASSERT_TRUE(parse_rdma_handshake(std::span(wire).first(RDMA_LEGACY_HANDSHAKE_SIZE), &decoded)); + EXPECT_EQ(decoded.protocol_version, 1U); + EXPECT_EQ(decoded.receive_slot_capacity, 0U); +} + +TEST(RdmaOfferTest, RawTcpPreflightConsumesOnlyCapabilityAndLeavesFallbackUsable) +{ + const TestEnvironment disabled("CROWDB_RDMA_ENABLED", "0"); + RpcServer server; + ASSERT_TRUE(server.listen("127.0.0.1", 0)); + server.register_handler(100, handle_echo); + server.start(); + const auto client = test_capability(91); + auto probe = RdmaPreflight::connect("127.0.0.1", server.listen_port(), client, {}); + ASSERT_NE(probe, nullptr); + EXPECT_FALSE(probe->peer.rdma_available); + EXPECT_EQ(probe->peer.protocol_version, RDMA_PROTOCOL_VERSION); + SocketTransport transport(1, 1); + transport.start(); + auto connection = transport.create_connection(probe->release_fd(), "fallback"); + ASSERT_NE(connection, nullptr); + connection->set_peer_capability(probe->peer); + EXPECT_TRUE(connection->is_open()); + EXPECT_EQ(connection->path(), ConnectionPath::Tcp); + test_echo_round_trip(transport, connection); + transport.stop(); + server.stop(); +} + +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) +TEST(RdmaHardwareTest, TcpOnlyPeerSkipsCmAndHonorsFallbackPolicy) +{ + const char *address_env = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address_env == nullptr) + GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; + const std::string address(address_env); + const TestEnvironment disabled("CROWDB_RDMA_ENABLED", "0"); + RpcServer server; + ASSERT_TRUE(server.listen(address, 0)); + server.register_handler(100, handle_echo); + server.start(); + const TestEnvironment enabled("CROWDB_RDMA_ENABLED", "1"); + { + const TestEnvironment forbidden("CROWDB_RDMA_ALLOW_TCP_FALLBACK", "0"); + SocketTransport transport(1, 1); + ASSERT_TRUE(transport.local_rdma_capability().rdma_available); + transport.start(); + EXPECT_EQ(transport.connect(address, server.listen_port()), nullptr); + const auto *stats = transport.rdma_stats_for_tests(); + ASSERT_NE(stats, nullptr); + EXPECT_EQ(stats->preflight_attempts.load(), 1U); + EXPECT_EQ(stats->cm_attempts.load(), 0U); + EXPECT_EQ(stats->fallback_count.load(), 0U); + } + { + const TestEnvironment allowed("CROWDB_RDMA_ALLOW_TCP_FALLBACK", "1"); + SocketTransport transport(1, 1); + transport.start(); + auto connection = transport.connect(address, server.listen_port()); + ASSERT_NE(connection, nullptr); + EXPECT_EQ(connection->path(), ConnectionPath::Tcp); + const auto *stats = transport.rdma_stats_for_tests(); + ASSERT_NE(stats, nullptr); + EXPECT_EQ(stats->preflight_attempts.load(), 1U); + EXPECT_EQ(stats->cm_attempts.load(), 0U); + EXPECT_EQ(stats->fallback_count.load(), 1U); + test_echo_round_trip(transport, connection); + transport.stop(); + } + server.stop(); +} +#endif + +TEST(RdmaOfferTest, PublishedPeerCapabilityRejectsIdentityReplacement) +{ + SystemBufferPool pool; + Connection connection(1, "capability", &pool); + auto peer = test_capability(19); + connection.set_peer_capability(peer); + connection.set_peer_capability(peer); + EXPECT_TRUE(connection.is_open()); + peer.generation++; + connection.set_peer_capability(peer); + EXPECT_FALSE(connection.is_open()); + EXPECT_EQ(connection.peer_capability().generation, 19U); +} + +TEST(RdmaOfferTest, TruncatedAndStalledControlRepliesFailWithinOneDeadline) +{ + for (bool truncated : {true, false}) { + const int listener = ::socket(AF_INET, SOCK_STREAM, 0); + ASSERT_GE(listener, 0); + sockaddr_in address{}; + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + ASSERT_EQ(::bind(listener, reinterpret_cast(&address), sizeof(address)), 0); + ASSERT_EQ(::listen(listener, 1), 0); + socklen_t length = sizeof(address); + ASSERT_EQ(::getsockname(listener, reinterpret_cast(&address), &length), 0); + std::atomic done{false}; + std::thread peer([&] { + pollfd descriptor{listener, POLLIN, 0}; + if (::poll(&descriptor, 1, 1000) <= 0) + return; + const int fd = ::accept(listener, nullptr, nullptr); + if (fd < 0) + return; + if (truncated) { + const uint8_t partial[] = {0x01, 0x02}; + (void)::send(fd, partial, sizeof(partial), 0); + (void)::shutdown(fd, SHUT_WR); + } + while (!done.load(std::memory_order_acquire)) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + ::close(fd); + }); + const auto begin = std::chrono::steady_clock::now(); + auto probe = RdmaPreflight::connect("127.0.0.1", ntohs(address.sin_port), test_capability(1), {}, 30); + const auto elapsed = std::chrono::steady_clock::now() - begin; + done.store(true, std::memory_order_release); + peer.join(); + ::close(listener); + EXPECT_EQ(probe, nullptr); + EXPECT_LT(elapsed, std::chrono::seconds(1)); + if (!truncated) { + EXPECT_GE(elapsed, std::chrono::milliseconds(20)); + } + } +} From d5c3b7bfbf4ad3eb4962db62c3fc6273ccccbf97 Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 22 Oct 2026 22:38:13 +0800 Subject: [PATCH 029/258] Compile RPC native FFI sources with Cargo parallel jobs --- doc/working/plan-rdma-rpc-transport.md | 7 +++++++ lib/crowdb-rpc/ffi/Cargo.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index c6acd3dde..ab9fac7f8 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -583,6 +583,13 @@ Preflight verification: requests with zero request/CQ errors and fallback disabled. This is functional evidence, not a sustained benchmark. +Build parallelism checkpoint: + +- Native CMake builds use `-j32`. The RPC FFI build dependency enables `cc`'s + `parallel` feature so standalone Cargo builds also share Cargo's jobserver + budget instead of compiling native sources serially. `cargo check -j32` + passes; the FFI suite validates the resulting library separately. + ### D. CLI coroutine workload and canonical results - [ ] **Add `bench rdma` with shared echo workload**: reuse `bench rpc` diff --git a/lib/crowdb-rpc/ffi/Cargo.toml b/lib/crowdb-rpc/ffi/Cargo.toml index 1bdb20aba..9cf0b993e 100644 --- a/lib/crowdb-rpc/ffi/Cargo.toml +++ b/lib/crowdb-rpc/ffi/Cargo.toml @@ -34,4 +34,4 @@ flatbuffers = { workspace = true } crowdb-rpc-ffi = { path = ".", features = ["test-util"] } [build-dependencies] -cc = "1" +cc = { version = "1", features = ["parallel"] } From ca932ea3b4a2f12d56971e4b2d6203a8bef15458 Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 22 Oct 2026 22:59:31 +0800 Subject: [PATCH 030/258] Coalesce bounded RPC frames on their RDMA completion owner --- doc/design/rpc/design-crowdb-rpc-rdma.md | 35 +++- doc/working/plan-rdma-rpc-transport.md | 55 +++++- lib/crowdb-rpc/include/crowdb-rpc/buffer.h | 4 + lib/crowdb-rpc/include/crowdb-rpc/transport.h | 1 + .../transport/rdma/rdma_transport.h | 44 ++++- lib/crowdb-rpc/src/buffer.cpp | 21 ++ .../src/transport/rdma/rdma_aggregate.cpp | 83 ++++++++ .../src/transport/rdma/rdma_aggregate.h | 32 +++ .../src/transport/rdma/rdma_completion.cpp | 112 ++--------- .../src/transport/rdma/rdma_config.cpp | 11 +- .../src/transport/rdma/rdma_connection.cpp | 4 +- lib/crowdb-rpc/src/transport/rdma/rdma_cq.h | 16 +- .../src/transport/rdma/rdma_poller.cpp | 12 +- lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp | 8 +- .../src/transport/rdma/rdma_receive.cpp | 82 ++++++++ .../src/transport/rdma/rdma_send.cpp | 182 ++++++++++++++++++ .../src/transport/rdma/rdma_transport.cpp | 105 +--------- .../src/transport/rdma/rdma_work_request.cpp | 20 +- .../tests/common/test_rdma_transport.h | 19 ++ lib/crowdb-rpc/tests/rdma_aggregate_test.cpp | 84 ++++++++ lib/crowdb-rpc/tests/rdma_completion_test.cpp | 122 +++++++++++- lib/crowdb-rpc/tests/rdma_config_test.cpp | 26 +++ lib/crowdb-rpc/tests/rdma_send_test.cpp | 163 ++++++++++++++++ 23 files changed, 1004 insertions(+), 237 deletions(-) create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_aggregate.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_aggregate.h create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_aggregate_test.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_send_test.cpp diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index aba159b61..ed1d3b65f 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -70,17 +70,21 @@ is never upgraded to RDMA. ## 3. Message and wire-slot model -ChunkIO and Chunk-KV retain ownership of aggregation and flush policy. RDMA -receives an already aggregated buffer and transmits it without additional -waiting or coalescing. The RDMA wire representation is contiguous inside one -fixed-size wire slot: +ChunkIO and Chunk-KV retain ownership of application aggregation and flush +policy. The RDMA CQ owner may also pack available complete RPC frames into +one SEND without a waiting timer. The wire representation is contiguous inside +one fixed-size wire slot: ```text -[common RPC header][control metadata][payload][unused slot bytes] +[RPC header][control][payload] ... [RPC header][control][payload][unused bytes] ``` The common header carries the control and payload lengths. The complete wire -length must not exceed the slot capacity. A logical result larger than the +length must fit `min(aggregation_max_bytes, wire_slot_capacity, +peer_receive_slot_capacity)`, including every frame's header and control. +`aggregation_max_frames` and the peer's advertised frame limit also bound each +SEND; disabled aggregation sends one frame. Oversized individual frames fail +admission without fragmentation or fallback. A logical result larger than the selected capacity is an ordered sequence identified by request ID, stream ID, sequence, logical offset, and START/DATA/END/ERROR flags. @@ -92,8 +96,23 @@ SGE on each side keeps the wire layout independent of sender SGE boundaries. The transport may copy caller-owned control and payload buffers into the slot before posting SEND. This copy is the transport boundary needed to make the -wire message contiguous; it is not an additional application-level message or -aggregation step. +wire message contiguous. RPC correlation and logical operation counts remain +per frame, independently of SEND count. + +Each QP has a bounded `SendQueue` in `ConnectionPools`. Producers validate and +transfer immutable frame ownership through its MPSC admission; the CQ owner +alone drains it, carries one next-slot frame and posts SENDs within effective +SQ admission. QPs on a CQ rotate through the SEND budget. A linked frame chain +keeps every source owner alive until that SEND completes. Failure closes the +uncertain stream; no suffix moves to TCP. Queue reservations, high-water and +rejection counters include admitted frames awaiting publication. + +Receive parsing validates the entire aggregate and its bounded frame count +before publishing any frame. Independent immutable `Buffer::slice` descriptors +retain one parent receive-slot lease. Releasing one frame cannot change another +frame's payload offset; the final view release alone allows repost. The parent +also remains retained across dispatch of all frames. An early aggregate that +contains application data waits for handler publication as a whole. ## 4. Registered arenas and slot ownership diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index ab9fac7f8..b9a4da8df 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -526,7 +526,7 @@ CQ verification: preflight expectation, while still negotiating receive admission. Files: RDMA capability/CM sources, `socket_transport.cpp`, server readiness, capability/path and hardware tests. -- [~] **Coalesce complete frames on the CQ owner**: use bounded lock-free +- [x] **Coalesce complete frames on the CQ owner**: use bounded lock-free admission and per-QP queues; drain available frames without adding a waiting timer. Keep per-QP ordering and common frame/request IDs. Limit a SEND to `min(aggregation_max_bytes, local send slot capacity, peer receive slot @@ -545,7 +545,7 @@ CQ verification: atomic-shared-pointer load with hidden locking on every poll. Wake a sleeping owner with a pending-publication/idle handshake rather than an eventfd write for every RPC. Retire admitted/pending frames on every failure path. -- [ ] **Retain aggregate owners through completion**: one SEND WR retains all +- [x] **Retain aggregate owners through completion**: one SEND WR retains all constituent frames until CQE; failure closes the uncertain stream and quarantines its slot once. Validate the entire received aggregate before publishing any frame. Give each frame an independent immutable buffer view @@ -560,11 +560,44 @@ CQ verification: alone cannot represent different immutable offsets safely. Keep a parent reference until every frame has been dispatched. Defer a mixed capability/data aggregate if the application handler is not yet published. -- [ ] **Expose aggregation observations**: export actual frames/SEND, wire +- [~] **Expose aggregation observations**: export actual frames/SEND, wire bytes, queue high-water and rejects. Keep RPC TPS distinct from transport SEND rate; compare aggregation disabled/enabled for both TCP and RDMA. Files: transport statistics, C API/Rust FFI and benchmark reports. +Aggregation checkpoint: + +- `rdma_aggregate.{h,cpp}` validates all header/control/payload ranges and a + maximum of 256 frames before dispatch. `Buffer::slice` creates independent + descriptors retaining one parent receive lease. `rdma_receive.cpp` holds the + parent across all callbacks and postpones mixed early aggregates as a whole. +- `rdma_send.cpp` owns per-QP bounded MPSC admission and one pending next-slot + frame. The assigned CQ rotates QPs through its SEND budget, bounds posts by + effective SQ capacity, and retains the linked source chain until completion. + Queue retirement waits for producer publication. CQ cancellation uses an + atomic connection identity before reading retained owners from a WR that + may concurrently be reused by another QP. +- Producers wake an idle CQ using one atomic IDLE/PENDING handshake; active + traffic performs no eventfd write per RPC, and busy polling no longer reads + the continuation fd on every iteration. Config exposes + `aggregation_enabled`, `aggregation_max_bytes`, `aggregation_max_frames`, + and `send_queue_capacity`; receive capabilities advertise parser support. +- All 37 focused aggregate/completion/configuration/send cases pass. These + include exact fit and one byte over, malformed suffix, independent held + middle frame, capability/data deferral, chain release after CQ error, + frame/byte limits, disabled aggregation, queue exhaustion and failed post. + Send-post fixtures use real registered slots and a controlled QP post hook. +- Real two-process strict RDMA, two QPs, window 32 completes 1000 validated + requests with zero RPC/CQ errors; aggregation uses 225 client SENDs and + 248 server SENDs (including control traffic). Held receive/MR lifetime and + registration-budget HCA cases also pass. This fixed-count result is + functional evidence only; phase E owns sustained comparisons. +- Full C++/FFI gate, all 29 Rust RPC FFI tests, formatting, scoped tree-lint + and workspace clippy pass. Full gate artifact: + `.crowdb-runtime/artifacts/unit/cpp-00a90cd3faa7480f894c80dee7ab03bc/`. + The no-verbs build passes all 133 CTest cases. Native statistics are present; + the status/FFI/CLI report exposure is the active continuation. + Preflight verification: - Version 2 TCP capability carries CM address/port, generation/device/GID, @@ -599,13 +632,25 @@ Build parallelism checkpoint: existing TCP command behavior; no OS thread per loader. Reject inconsistent settings before starting. Files: CLI `bench.rs`, `bench/verb.rs`, `bench/rpc.rs`, shared workload/config modules. + Add cold, typed C ABI configuration before listen/start, rather than changing + process environment from a multithreaded Rust runtime. Explicit configuration + overrides environment defaults. Capture QP number/CQ/effective queue/peer-slot + information before publishing each connection; status must read immutable + snapshots rather than dereferencing a QP during teardown. Return versioned + transport status JSON in an owned C ABI buffer for FFI, and expose the same + status through an fb-server benchmark control RPC while workload QPs remain + live. Server readiness follows actual CM listen/start. - [ ] **Validate and measure each completion**: expose response data to `CoBenchHandler`, verify request ID, length and payload, count submission rejects/retries/timeouts/connection/payload/duplicate errors, and reconcile C++ coroutine statistics with Rust results. Warm up for one second, measure completions for ten seconds using a monotonic deadline, then drain separately; - exclude setup/warmup/drain from TPS. Define latency from submission to - validated response in nanoseconds. Never destroy a coroutine while a late + exclude setup/warmup/drain from TPS. Record native monotonic submission-to- + response-callback RTT in nanoseconds, only for validated responses. State + explicitly that request construction and Rust validation are outside the + latency sample; validation remains inside throughput work. Do not combine + C++ and Rust clock values to approximate a validation-inclusive interval. + Never destroy a coroutine while a late completion can resume it (the lifecycle prerequisite is in phase A). Files: `co_client.cpp`, FFI `co_bench.rs`, CLI echo handler and lifecycle tests. diff --git a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h index c867261e3..73da3acd3 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h @@ -114,6 +114,10 @@ struct Buffer // allocation. Each consumer that needs to hold the buffer calls ref(). Buffer *ref_clone(); + // Independent immutable descriptor retaining this buffer's final-release + // lease. Offsets are relative to payload_data(); the parent is not mutated. + Buffer *slice(uint32_t offset, uint32_t size); + // Decrement refcount. On ref == 0, recycle to pool or call free_cb. void release(); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport.h index 2a7b379d0..bec46d740 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport.h @@ -35,6 +35,7 @@ struct OutFrame uint8_t data_view_count = 0; // zero preserves legacy direct `data` assignment uint32_t sent_offset = 0; // bytes already sent (partial write tracking) uint64_t create_nano = 0; // steady_clock ns at submit/submit_inline + OutFrame *next_aggregate = nullptr; // SEND completion owns the full chain [[nodiscard]] uint8_t data_views() const { diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index 9d00c0695..c38520b5c 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -56,6 +56,10 @@ struct RdmaConfig uint8_t min_rnr_timer = 1; uint8_t retry_count = 7; uint8_t rnr_retry_count = 7; + bool aggregation_enabled = false; + uint32_t aggregation_max_bytes = RDMA_WIRE_SLOT_CAPACITY; + uint32_t aggregation_max_frames = 256; + uint32_t send_queue_capacity = 4096; [[nodiscard]] bool apply_environment(); [[nodiscard]] bool valid() const; @@ -124,6 +128,14 @@ class RdmaTransport : public Transport std::atomic cm_attempts{0}; std::atomic preflight_attempts{0}; std::atomic preflight_failures{0}; + std::atomic protocol_errors{0}; + std::atomic rpc_frames_sent{0}; + std::atomic rpc_frames_received{0}; + std::atomic wire_bytes_sent{0}; + std::atomic wire_bytes_received{0}; + std::atomic submit_rejects{0}; + std::atomic send_queue_high_water{0}; + std::atomic send_errors{0}; }; explicit RdmaTransport(RdmaConfig config = {}, BufferPool *application_pool = nullptr); @@ -227,6 +239,7 @@ class RdmaTransport : public Transport std::atomic token{0}; WorkKind kind; std::shared_ptr conn; + std::atomic connection_identity{nullptr}; std::shared_ptr pools; Buffer *slot = nullptr; OutFrame *frame = nullptr; @@ -260,13 +273,18 @@ class RdmaTransport : public Transport CompletionQueue *reserve_cq(uint32_t capacity); bool create_completion_queues(ibv_context *context); void destroy_completion_queues(); - int drain_queue(uint32_t index, uint32_t max_completions, bool wait_for_owner = false); - uint32_t process_deferred(CompletionQueue &queue, uint32_t max_completions); - void cancel_deferred(CompletionQueue &queue, Connection *connection); - void start_pollers(); - void run_poller(CompletionQueue *queue); - void notify_queue(uint32_t index); - void install_frame_notifier(const std::shared_ptr &connection, uint32_t index); + int drain_queue(uint32_t index, uint32_t max_completions, bool wait_for_owner = false); + uint32_t process_deferred(CompletionQueue &queue, uint32_t max_completions); + void cancel_deferred(CompletionQueue &queue, Connection *connection); + void start_pollers(); + void run_poller(CompletionQueue *queue); + void notify_queue(uint32_t index); + void install_frame_notifier(const std::shared_ptr &connection, uint32_t index); + uint32_t flush_sends(CompletionQueue &queue, uint32_t budget); + uint32_t flush_connection(Connection *connection, ConnectionPools &pools, uint32_t budget); + void retire_sends(CompletionQueue &queue, Connection *connection); + bool post_aggregate(Connection *connection, ConnectionPools &pools, Buffer *slot, OutFrame *frames, + uint32_t frame_count); static constexpr uint64_t CM_EVENT_TAG = 1; static constexpr uint64_t CQ_EVENT_BASE = 2; @@ -324,6 +342,12 @@ class RdmaTransport : public Transport struct ConnectionPools : std::enable_shared_from_this { + explicit ConnectionPools(uint32_t queue_capacity = 4096) : send_queue(queue_capacity) + { + } + + ~ConnectionPools(); + ibv_qp_cap effective_capacity{}; std::shared_ptr posting_state = std::make_shared(); std::unique_ptr send; @@ -333,6 +357,10 @@ class RdmaTransport : public Transport uint32_t peer_receive_slot_capacity = 0; uint32_t peer_receive_frame_limit = 1; uint32_t qp_number = 0; + SendQueue send_queue; + std::atomic queued_frames{0}; + OutFrame *pending_frame = nullptr; + uint32_t sends_in_flight = 0; // CQ owner only }; struct RepostContext @@ -347,7 +375,7 @@ class RdmaTransport : public Transport void release_completed_frame(WorkRequest *work, const struct ibv_wc &wc); void release_completed_receive(WorkRequest *work, const struct ibv_wc &wc); void record_completion_error(const struct ibv_wc &wc, bool expected_flush = false); - static void repost_frame_release(void *context, Buffer *data); + static void repost_frame_release(void *context, Buffer *slot); bool adopt_cm_context(struct ibv_context *context); }; diff --git a/lib/crowdb-rpc/src/buffer.cpp b/lib/crowdb-rpc/src/buffer.cpp index acc4680e5..0ddabade9 100644 --- a/lib/crowdb-rpc/src/buffer.cpp +++ b/lib/crowdb-rpc/src/buffer.cpp @@ -105,6 +105,27 @@ void Buffer::release() } } +Buffer *Buffer::slice(uint32_t offset, uint32_t size) +{ + if (offset > capacity || size > capacity - offset || data == nullptr || ref_clone() == nullptr) + return nullptr; + auto *view = new Buffer; + view->data = payload_data() + offset; + view->len = size; + view->capacity = size; + view->type = type; + view->ref = new std::atomic(1); + view->ownership.store(BufferOwnership::CallerOwned, std::memory_order_relaxed); + view->capacity_class = capacity_class; + view->registration_id = registration_id; + view->registration_key = registration_key; + view->registration_handle = registration_handle; + view->set_metadata(request_id, stream_id, sequence, logical_offset, registration_id); + view->free_cb = [](void *parent) { static_cast(parent)->release(); }; + view->free_ctx = this; + return view; +} + bool Buffer::transition(BufferOwnership from, BufferOwnership to) { return ownership.compare_exchange_strong(from, to, std::memory_order_acq_rel, std::memory_order_acquire); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_aggregate.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_aggregate.cpp new file mode 100644 index 000000000..96809892c --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_aggregate.cpp @@ -0,0 +1,83 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_aggregate.h" + +#include + +namespace crowdb::rpc +{ +bool parse_rdma_aggregate(std::span bytes, RdmaAggregate &result) +{ + result.count = 0; + size_t offset = 0; + while (offset < bytes.size()) { + if (result.count == RDMA_MAX_AGGREGATE_FRAMES || bytes.size() - offset < HEADER_SIZE) + return false; + const auto header = parse_header(bytes.data() + offset); + const uint64_t control_end = static_cast(header.msg_offset) + header.msg_size; + const uint64_t frame_size = control_end + header.data_size; + if (header.magic != MAGIC || header.msg_offset != HEADER_SIZE || control_end > RDMA_CONTROL_CAPACITY || + frame_size > bytes.size() - offset) + return false; + result.frames[result.count++] = {header, static_cast(offset), + static_cast(offset + control_end)}; + offset += static_cast(frame_size); + } + return result.count != 0; +} + +bool rdma_frame_size(const OutFrame &frame, uint32_t limit, uint32_t &size) +{ + if (frame.control == nullptr || frame.control->data == nullptr || frame.control->len > frame.control->capacity || + frame.control->len > RDMA_CONTROL_CAPACITY - HEADER_SIZE || frame.header.magic != MAGIC || + frame.header.msg_offset != HEADER_SIZE || frame.data_views() > MAX_DATA_VIEWS) + return false; + uint64_t bytes = static_cast(HEADER_SIZE) + frame.control->len; + for (uint8_t index = 0; index < frame.data_views(); ++index) { + const auto *view = frame.data_view(index); + if (view == nullptr || view->len > view->capacity || (view->len != 0 && view->payload_data() == nullptr)) + return false; + bytes += view->len; + } + if (bytes > limit) + return false; + size = static_cast(bytes); + return true; +} + +void copy_rdma_frame(uint8_t *destination, const OutFrame &frame) +{ + auto header = frame.header; + header.msg_size = frame.control->len; + header.data_size = 0; + for (uint8_t index = 0; index < frame.data_views(); ++index) + header.data_size += frame.data_view(index)->len; + serialize_header(destination, header); + uint32_t offset = HEADER_SIZE; + std::memcpy(destination + offset, frame.control->data, header.msg_size); + offset += header.msg_size; + for (uint8_t index = 0; index < frame.data_views(); ++index) { + const auto *view = frame.data_view(index); + if (view->len != 0) + std::memcpy(destination + offset, view->payload_data(), view->len); + offset += view->len; + } +} + +void release_rdma_frames(OutFrame *frame, bool quarantine) +{ + while (frame != nullptr) { + auto *next = frame->next_aggregate; + if (quarantine) { + for (uint8_t index = 0; index < frame->data_views(); ++index) + (void)frame->data_view(index)->quarantine(); + } + if (frame->control != nullptr) + frame->control->release(); + frame->release_data(); + delete frame; + frame = next; + } +} +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_aggregate.h b/lib/crowdb-rpc/src/transport/rdma/rdma_aggregate.h new file mode 100644 index 000000000..bf7918211 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_aggregate.h @@ -0,0 +1,32 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/transport.h" + +#include + +namespace crowdb::rpc +{ +constexpr uint32_t RDMA_MAX_AGGREGATE_FRAMES = 256; + +struct RdmaFrameView +{ + Header header; + uint32_t offset; + uint32_t data_offset; +}; + +struct RdmaAggregate +{ + std::array frames; + uint32_t count = 0; +}; + +// The complete aggregate must validate before any application callback. +bool parse_rdma_aggregate(std::span bytes, RdmaAggregate &result); +bool rdma_frame_size(const OutFrame &frame, uint32_t limit, uint32_t &size); +void copy_rdma_frame(uint8_t *destination, const OutFrame &frame); +void release_rdma_frames(OutFrame *frame, bool quarantine = false); +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp index 79812851c..2b9964efb 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp @@ -4,6 +4,7 @@ #include "crowdb-common/log.h" #include "crowdb-rpc/transport/rdma/rdma_transport.h" #include "crowdb-rpc/transport/socket_transport.h" +#include "rdma_aggregate.h" #include "rdma_cq.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -37,12 +38,15 @@ int RdmaTransport::drain_queue(uint32_t index, uint32_t max_completions, bool wa } // Only the CQ consumer may clear its continuation. A competing worker // must leave the wakeup readable when it cannot acquire the CQ. - if (queue.continuation_fd >= 0) { + const bool managed = queue.managed.load(std::memory_order_acquire); + const bool woke = !managed || queue.event_wakeup.exchange(false, std::memory_order_acq_rel); + if (woke && queue.continuation_fd >= 0) { uint64_t value = 0; (void)::read(queue.continuation_fd, &value, sizeof(value)); } if (max_completions == 0) max_completions = std::max(1U, config_.poll_batch); + const uint32_t sent = flush_sends(queue, max_completions); uint32_t drained = process_deferred(queue, max_completions); uint32_t empty_polls = 0; const uint32_t kEmptyPollBudget = config_.poll_empty_budget; @@ -68,8 +72,7 @@ int RdmaTransport::drain_queue(uint32_t index, uint32_t max_completions, bool wa drained += static_cast(batch); } }; - const bool managed = queue.managed.load(std::memory_order_acquire); - if (queue.channel != nullptr && (!managed || queue.event_wakeup.exchange(false))) { + if (queue.channel != nullptr && woke) { ibv_cq *event_cq = nullptr; void *context = nullptr; while (ibv_get_cq_event(queue.channel, &event_cq, &context) == 0) { @@ -78,9 +81,9 @@ int RdmaTransport::drain_queue(uint32_t index, uint32_t max_completions, bool wa } } poll_one(queue.cq); - if (managed && drained > 0) { + if (managed && (drained > 0 || sent > 0)) { queue.polling.clear(std::memory_order_release); - return static_cast(drained); + return static_cast(drained + sent); } (void)ibv_req_notify_cq(queue.cq, 0); // Completions arriving between the empty poll and notification arming @@ -92,7 +95,7 @@ int RdmaTransport::drain_queue(uint32_t index, uint32_t max_completions, bool wa (void)::write(queue.continuation_fd, &value, sizeof(value)); } queue.polling.clear(std::memory_order_release); - return static_cast(drained); + return static_cast(drained + sent); } void RdmaTransport::register_events(SocketEngine *engine) @@ -200,100 +203,14 @@ void RdmaTransport::record_completion_error(const ibv_wc &wc, bool expected_flus } } -void RdmaTransport::release_completed_receive(WorkRequest *work, const ibv_wc &wc) -{ - - auto quarantine_and_release = [&] { - record_completion_error(wc, !work->conn->is_open()); - work->conn->close(); - (void)work->slot->quarantine(); - work->slot->release(); - release_work(work); - }; - if (wc.status != IBV_WC_SUCCESS) { - if (wc.status != IBV_WC_WR_FLUSH_ERR || work->conn->is_open()) - CRB_LOG_WARN("rdma receive: completion failed status={} vendor_err={} opcode={}", - static_cast(wc.status), static_cast(wc.vendor_err), - static_cast(wc.opcode)); - quarantine_and_release(); - return; - } - if (!work->slot->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight)) { - quarantine_and_release(); - return; - } - const uint32_t slot_capacity = work->slot->capacity; - if (slot_capacity < HEADER_SIZE || wc.byte_len < HEADER_SIZE || !work->conn->is_open()) { - CRB_LOG_WARN("rdma receive: slot capacity too small capacity={} header={}", slot_capacity, HEADER_SIZE); - quarantine_and_release(); - return; - } - const Header header = parse_header(work->slot->data); - const uint64_t data_offset = static_cast(header.msg_offset) + header.msg_size; - const uint64_t expected_bytes = data_offset + header.data_size; - if (header.magic != MAGIC || header.msg_offset != HEADER_SIZE || data_offset > RDMA_CONTROL_CAPACITY || - expected_bytes != wc.byte_len || expected_bytes > slot_capacity) { - CRB_LOG_WARN( - "rdma receive: invalid frame magic={} type={} msg_size={} data_size={} offset={} capacity={} byte_len={}", - header.magic, header.msg_type, header.msg_size, header.data_size, header.msg_offset, slot_capacity, - wc.byte_len); - quarantine_and_release(); - return; - } - - auto *frame = new Frame; - frame->header = header; - if (header.msg_size != 0) { - frame->control.assign(work->slot->data + header.msg_offset, - work->slot->data + header.msg_offset + header.msg_size); - extract_control_fields(work->slot->data + header.msg_offset, header.msg_size, frame->request_id, - frame->rpc_create_nano, frame->response_create_nano); - } - work->request_id = frame->request_id; - work->stream_id = work->slot->stream_id; - work->sequence = work->slot->sequence; - work->logical_offset = work->slot->logical_offset; - work->slot->view_offset = static_cast(data_offset); - work->slot->capacity = slot_capacity - static_cast(data_offset); - work->slot->len = header.data_size; - if (!work->slot->transition(BufferOwnership::ReceiveInFlight, BufferOwnership::CallerOwned)) { - quarantine_and_release(); - delete frame; - return; - } - auto connection = std::move(work->conn); - auto pools = std::move(work->pools); - auto *repost = new RepostContext{repost_state_, connection, std::move(pools)}; - frame->data_buf = work->slot; - work->slot->last_release_cb = &RdmaTransport::repost_frame_release; - work->slot->last_release_ctx = repost; - release_work(work); - // A callback may immediately release/repost the slot or submit a reply. - // Return this completed WR entry before either can need bounded admission. - stats_.receives.fetch_add(1, std::memory_order_relaxed); - connection->on_frame(frame); -} - -void RdmaTransport::repost_frame_release(void *context, Buffer *slot) -{ - auto repost = std::unique_ptr(static_cast(context)); - slot->reset_view(); - if (!slot->transition(BufferOwnership::CallerOwned, BufferOwnership::Available)) - (void)slot->quarantine(); - { - Operation operation(repost->state); - if (operation && slot->ownership.load(std::memory_order_acquire) == BufferOwnership::Available) - (void)repost->state->transport->post_receive(repost->connection.get(), slot); - } - slot->release(); -} - void RdmaTransport::release_completed_frame(WorkRequest *work, const ibv_wc &wc) { OutFrame *frame = work->frame; auto connection = std::move(work->conn); const auto pools = std::move(work->pools); Buffer *slot = work->slot; + if (pools != nullptr && pools->sends_in_flight != 0) + --pools->sends_in_flight; if (wc.status != IBV_WC_SUCCESS) { record_completion_error(wc, connection != nullptr && !connection->is_open()); @@ -301,8 +218,6 @@ void RdmaTransport::release_completed_frame(WorkRequest *work, const ibv_wc &wc) // suffix to be appended to this RDMA connection. if (work->slot != nullptr) (void)work->slot->quarantine(); - for (uint8_t i = 0; i < frame->data_views(); ++i) - (void)frame->data_view(i)->quarantine(); if (connection != nullptr) connection->close(); } @@ -315,10 +230,7 @@ void RdmaTransport::release_completed_frame(WorkRequest *work, const ibv_wc &wc) release_work(work); if (slot != nullptr) slot->release(); - if (frame->control != nullptr) - frame->control->release(); - frame->release_data(); - delete frame; + release_rdma_frames(frame, wc.status != IBV_WC_SUCCESS); } } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp index fa5e479ff..9fd2c5a17 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp @@ -40,7 +40,11 @@ bool RdmaConfig::apply_environment() read("CROWDB_RDMA_REGISTERED_MEMORY_BUDGET_BYTES", registered_memory_budget_bytes) && read("CROWDB_RDMA_CQ_COUNT", cq_count) && read("CROWDB_RDMA_POLL_BATCH", poll_batch) && read("CROWDB_RDMA_VERBS_POLL_BATCH", verbs_poll_batch) && - read("CROWDB_RDMA_POLL_EMPTY_BUDGET", poll_empty_budget); + read("CROWDB_RDMA_POLL_EMPTY_BUDGET", poll_empty_budget) && + read("CROWDB_RDMA_AGGREGATION_ENABLED", aggregation_enabled) && + read("CROWDB_RDMA_AGGREGATION_MAX_BYTES", aggregation_max_bytes) && + read("CROWDB_RDMA_AGGREGATION_MAX_FRAMES", aggregation_max_frames) && + read("CROWDB_RDMA_SEND_QUEUE_CAPACITY", send_queue_capacity); } bool RdmaConfig::valid() const @@ -55,7 +59,10 @@ bool RdmaConfig::valid() const poll_batch <= static_cast(INT32_MAX) && verbs_poll_batch > 0 && verbs_poll_batch <= 256 && poll_empty_budget > 0 && send_arena_bytes >= wire_slot_capacity && receive_arena_bytes >= static_cast(posted_receive_target) * wire_slot_capacity && - min_rnr_timer <= 31 && retry_count <= 7 && rnr_retry_count <= 7 && registration_budget() != 0; + min_rnr_timer <= 31 && retry_count <= 7 && rnr_retry_count <= 7 && registration_budget() != 0 && + aggregation_max_bytes >= HEADER_SIZE && aggregation_max_bytes <= RDMA_WIRE_SLOT_CAPACITY && + aggregation_max_frames > 0 && aggregation_max_frames <= 256 && send_queue_capacity >= 2 && + send_queue_capacity <= (1U << 20) && (send_queue_capacity & (send_queue_capacity - 1)) == 0; } uint64_t RdmaConfig::registration_budget() const diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp index a70c9cac5..41154930e 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp @@ -145,7 +145,7 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int rdma_destroy_id(id); return nullptr; } - auto pools = std::make_shared(); + auto pools = std::make_shared(config_.send_queue_capacity); pools->effective_capacity = qp_attr.cap; pools->cq_index = reservation.queue->index; pools->reserved_capacity = reservation.capacity; @@ -356,7 +356,7 @@ std::shared_ptr RdmaTransport::accept() rdma_destroy_id(id); return nullptr; } - auto pools = std::make_shared(); + auto pools = std::make_shared(config_.send_queue_capacity); pools->peer_receive_slot_capacity = peer_offer.receive_slot_capacity; pools->peer_receive_frame_limit = peer_offer.receive_frame_limit; pools->qp_number = id->qp->qp_num; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h index f2a3b0732..31745c96b 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h @@ -25,7 +25,21 @@ struct RdmaTransport::CompletionQueue ibv_wc completion; }; - std::vector deferred; + std::vector deferred; + + struct ConnectionBinding + { + std::shared_ptr connection; + std::shared_ptr pools; + }; + + std::vector connections; // protected by CQ polling ownership + size_t next_connection = 0; + // Producers publish PENDING and clear IDLE atomically. Only the first + // producer observing IDLE writes the wake fd; active traffic needs none. + static constexpr uint32_t IDLE = 1; + static constexpr uint32_t PENDING = 2; + std::atomic wake_state{0}; std::unique_ptr engine; std::thread worker; std::atomic managed{false}; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp index cfe9714b3..1dd70e72f 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp @@ -57,8 +57,14 @@ void RdmaTransport::run_poller(CompletionQueue *queue) crowdb::common::set_current_thread_name(name.c_str()); std::array events{}; while (running_.load(std::memory_order_acquire)) { + queue->wake_state.exchange(0, std::memory_order_acq_rel); if (drain_queue(queue->index, config_.poll_batch) > 0) continue; + const auto pending = queue->wake_state.fetch_or(CompletionQueue::IDLE, std::memory_order_acq_rel); + if ((pending & CompletionQueue::PENDING) != 0) { + queue->wake_state.fetch_and(~CompletionQueue::IDLE, std::memory_order_acq_rel); + continue; + } const int count = queue->engine->wait(events.data(), events.size(), 1000); if (count > 0) { queue->event_wakeup.store(true, std::memory_order_release); @@ -69,8 +75,12 @@ void RdmaTransport::run_poller(CompletionQueue *queue) void RdmaTransport::notify_queue(uint32_t index) { + auto &queue = *completion_queues_[index]; + const auto previous = queue.wake_state.exchange(CompletionQueue::PENDING, std::memory_order_acq_rel); + if (queue.managed.load(std::memory_order_acquire) && (previous & CompletionQueue::IDLE) == 0) + return; const uint64_t value = 1; - (void)::write(completion_queues_[index]->continuation_fd, &value, sizeof(value)); + (void)::write(queue.continuation_fd, &value, sizeof(value)); } void RdmaTransport::install_frame_notifier(const std::shared_ptr &connection, uint32_t index) diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp index b78973dac..09b268060 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp @@ -28,7 +28,13 @@ bool RdmaTransport::publish_cm_connection(rdma_cm_id *id, const std::shared_ptr< return false; cm_ids_.push_back(id); cm_connections_.emplace(id, connection); - cm_pools_.emplace(id, std::move(pools)); + cm_pools_.emplace(id, pools); + auto &queue = *completion_queues_[pools->cq_index]; + while (queue.polling.test_and_set(std::memory_order_acquire)) + std::this_thread::yield(); + queue.connections.push_back({connection, std::move(pools)}); + queue.polling.clear(std::memory_order_release); + notify_queue(queue.index); return true; } diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp new file mode 100644 index 000000000..ca26bb2e7 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp @@ -0,0 +1,82 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "rdma_aggregate.h" + +#ifdef CROWDB_RPC_HAVE_RDMA + +namespace crowdb::rpc +{ +void RdmaTransport::release_completed_receive(WorkRequest *work, const ibv_wc &wc) +{ + auto fail = [&] { + if (wc.status != IBV_WC_SUCCESS) + record_completion_error(wc, !work->conn->is_open()); + else if (work->conn->is_open()) + stats_.protocol_errors.fetch_add(1, std::memory_order_relaxed); + auto connection = work->conn; + auto *slot = work->slot; + (void)slot->quarantine(); + release_work(work); + connection->close(); + slot->release(); + }; + if (wc.status != IBV_WC_SUCCESS || !work->conn->is_open() || + !work->slot->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight)) { + fail(); + return; + } + RdmaAggregate aggregate; + if (wc.byte_len > work->slot->capacity || + !parse_rdma_aggregate(std::span(work->slot->data, wc.byte_len), aggregate) || + !work->slot->transition(BufferOwnership::ReceiveInFlight, BufferOwnership::CallerOwned)) { + fail(); + return; + } + + auto connection = std::move(work->conn); + auto pools = std::move(work->pools); + auto *slot = work->slot; + slot->last_release_cb = &RdmaTransport::repost_frame_release; + slot->last_release_ctx = new RepostContext{repost_state_, connection, std::move(pools)}; + release_work(work); + stats_.receives.fetch_add(1, std::memory_order_relaxed); + stats_.rpc_frames_received.fetch_add(aggregate.count, std::memory_order_relaxed); + stats_.wire_bytes_received.fetch_add(wc.byte_len, std::memory_order_relaxed); + // The parent reference spans all dispatches. A callback may release its + // own view immediately; another frame may retain its view past teardown. + for (uint32_t index = 0; index < aggregate.count; ++index) { + const auto &view = aggregate.frames[index]; + auto *frame = new Frame; + frame->header = view.header; + if (view.header.msg_size != 0) { + const auto *control = slot->data + view.offset + HEADER_SIZE; + frame->control.assign(control, control + view.header.msg_size); + extract_control_fields(control, view.header.msg_size, frame->request_id, frame->rpc_create_nano, + frame->response_create_nano); + } + frame->data_buf = slot->slice(view.data_offset, view.header.data_size); + if (connection->is_open()) + connection->on_frame(frame); + else + delete frame; + } + slot->release(); +} + +void RdmaTransport::repost_frame_release(void *context, Buffer *slot) +{ + auto repost = std::unique_ptr(static_cast(context)); + slot->reset_view(); + if (!slot->transition(BufferOwnership::CallerOwned, BufferOwnership::Available)) + (void)slot->quarantine(); + { + Operation operation(repost->state); + if (operation && slot->ownership.load(std::memory_order_acquire) == BufferOwnership::Available) + (void)repost->state->transport->post_receive(repost->connection.get(), slot); + } + slot->release(); +} +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp new file mode 100644 index 000000000..135fd88e3 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp @@ -0,0 +1,182 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_aggregate.h" +#include "rdma_cq.h" + +#ifdef CROWDB_RPC_HAVE_RDMA +# include + +namespace crowdb::rpc +{ +RdmaTransport::ConnectionPools::~ConnectionPools() +{ + release_rdma_frames(pending_frame); + OutFrame *frame = nullptr; + while (send_queue.drain(&frame, 1) != 0) + release_rdma_frames(frame); +} + +bool RdmaTransport::submit(Connection *connection, OutFrame *frame) +{ + auto reject = [&] { + stats_.submit_rejects.fetch_add(1, std::memory_order_relaxed); + return false; + }; + Operation operation(repost_state_); + if (!operation || connection == nullptr || frame == nullptr || !connection->is_open()) + return reject(); + auto *pools = pools_for(connection); + if (pools == nullptr || pools->send == nullptr || pools->cq_index >= completion_queues_.size()) + return reject(); + Operation posting(pools->posting_state); + uint32_t bytes = 0; + const auto limit = + std::min({config_.wire_slot_capacity, pools->peer_receive_slot_capacity, config_.aggregation_max_bytes}); + if (!posting || !connection->is_open() || !rdma_frame_size(*frame, limit, bytes)) + return reject(); + uint32_t queued = pools->queued_frames.load(std::memory_order_relaxed); + do { + if (queued >= config_.send_queue_capacity) + return reject(); + } while (!pools->queued_frames.compare_exchange_weak(queued, queued + 1, std::memory_order_acq_rel)); + if (!pools->send_queue.try_push(frame)) { + pools->queued_frames.fetch_sub(1, std::memory_order_release); + return reject(); + } + uint64_t peak = stats_.send_queue_high_water.load(std::memory_order_relaxed); + while (peak < queued + 1 && + !stats_.send_queue_high_water.compare_exchange_weak(peak, queued + 1, std::memory_order_relaxed)) { + } + notify_queue(pools->cq_index); + return true; +} + +uint32_t RdmaTransport::flush_connection(Connection *connection, ConnectionPools &pools, uint32_t budget) +{ + const auto byte_limit = + std::min({config_.wire_slot_capacity, pools.peer_receive_slot_capacity, config_.aggregation_max_bytes}); + const auto frame_limit = + config_.aggregation_enabled ? std::min(config_.aggregation_max_frames, pools.peer_receive_frame_limit) : 1U; + uint32_t sent = 0; + while (sent < budget && connection->is_open() && + pools.sends_in_flight < std::min(config_.sq_depth, pools.effective_capacity.max_send_wr)) { + if (pools.pending_frame == nullptr && pools.send_queue.drain(&pools.pending_frame, 1) == 0) + break; + auto *slot = pools.send->alloc(config_.wire_slot_capacity); + if (slot == nullptr) + break; + slot->len = 0; + OutFrame *first = nullptr; + OutFrame **tail = &first; + uint32_t count = 0; + while (pools.pending_frame != nullptr && count < frame_limit) { + auto *frame = pools.pending_frame; + uint32_t bytes = 0; + // submit validated the immutable frame against the same bound. + if (!rdma_frame_size(*frame, byte_limit, bytes)) { + connection->close(); + break; + } + if (bytes > byte_limit - slot->len) + break; + copy_rdma_frame(slot->data + slot->len, *frame); + slot->len += bytes; + frame->next_aggregate = nullptr; + *tail = frame; + tail = &frame->next_aggregate; + pools.pending_frame = nullptr; + pools.queued_frames.fetch_sub(1, std::memory_order_release); + ++count; + if (count < frame_limit) + (void)pools.send_queue.drain(&pools.pending_frame, 1); + } + if (count == 0 || !connection->is_open()) { + release_rdma_frames(first); + slot->release(); + break; + } + if (!post_aggregate(connection, pools, slot, first, count)) + break; + ++sent; + } + return sent; +} + +bool RdmaTransport::post_aggregate(Connection *connection, ConnectionPools &pools, Buffer *slot, OutFrame *frames, + uint32_t frame_count) +{ + Operation posting(pools.posting_state); + WorkRequest *work = nullptr; + if (posting && connection->is_open() && connection->transport_handle != 0 && + slot->transition(BufferOwnership::Available, BufferOwnership::SendInFlight)) + work = acquire_work(WorkKind::Send, connection, slot, frames); + if (work != nullptr) { + ibv_sge sge{}; + sge.addr = reinterpret_cast(slot->data); + sge.length = slot->len; + sge.lkey = slot->registration_key; + ibv_send_wr wr{}; + wr.wr_id = work->token.load(std::memory_order_acquire); + wr.sg_list = &sge; + wr.num_sge = 1; + wr.opcode = IBV_WR_SEND; + wr.send_flags = IBV_SEND_SIGNALED; + ibv_send_wr *bad = nullptr; + if (ibv_post_send(reinterpret_cast(connection->transport_handle), &wr, &bad) == 0) { + ++pools.sends_in_flight; + stats_.sends.fetch_add(1, std::memory_order_relaxed); + stats_.rpc_frames_sent.fetch_add(frame_count, std::memory_order_relaxed); + stats_.wire_bytes_sent.fetch_add(slot->len, std::memory_order_relaxed); + return true; + } + release_work(work); + } + stats_.send_errors.fetch_add(1, std::memory_order_relaxed); + connection->close(); + (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); + slot->release(); + release_rdma_frames(frames); + return false; +} + +uint32_t RdmaTransport::flush_sends(CompletionQueue &queue, uint32_t budget) +{ + uint32_t sent = 0; + for (size_t visited = 0; visited < queue.connections.size() && sent < budget; ++visited) { + const auto &entry = queue.connections[queue.next_connection % queue.connections.size()]; + queue.next_connection = (queue.next_connection + 1) % queue.connections.size(); + if (entry.connection->is_open()) + sent += flush_connection(entry.connection.get(), *entry.pools, budget - sent); + } + // Closed producers finish publishing before their bounded queue retires. + for (size_t index = 0; index < queue.connections.size();) { + auto &entry = queue.connections[index]; + if (!entry.connection->is_open() && + (entry.pools->posting_state->users.fetch_or(RepostState::STOPPED, std::memory_order_acq_rel) & + ~RepostState::STOPPED) == 0) { + retire_sends(queue, entry.connection.get()); + } + else { + ++index; + } + } + return sent; +} + +void RdmaTransport::retire_sends(CompletionQueue &queue, Connection *connection) +{ + std::erase_if(queue.connections, [&](const auto &entry) { + if (connection != nullptr && entry.connection.get() != connection) + return false; + auto &pools = *entry.pools; + release_rdma_frames(std::exchange(pools.pending_frame, nullptr)); + OutFrame *frame = nullptr; + while (pools.send_queue.drain(&frame, 1) != 0) + release_rdma_frames(frame); + pools.queued_frames.store(0, std::memory_order_release); + return true; + }); +} +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index 20f366c13..6d0733b4b 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -5,6 +5,7 @@ #include "crowdb-common/log.h" #include "crowdb-rpc/transport/socket_transport.h" +#include "rdma_aggregate.h" #include "rdma_channel.h" #include "rdma_cq.h" #include "rdma_device.h" @@ -46,6 +47,7 @@ RdmaTransport::RdmaTransport(RdmaConfig config, BufferPool *application_pool) queue->index = index; queue->work_requests = std::make_unique(config_.cq_depth); queue->deferred.reserve(config_.cq_depth); + queue->connections.reserve(config_.cq_depth / std::max(1U, config_.sq_depth + config_.posted_receive_target)); queue->continuation_fd = ::eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); completion_queues_.push_back(std::move(queue)); } @@ -77,7 +79,7 @@ RdmaTransport::RdmaTransport(RdmaConfig config, BufferPool *application_pool) local_capability_.data_classes = {RDMA_CONTROL_CAPACITY, RDMA_METADATA_CAPACITY, RDMA_AGGREGATE_CAPACITY, RDMA_DATAPATH_CAPACITY}; local_capability_.receive_slot_capacity = config_.wire_slot_capacity; - local_capability_.receive_frame_limit = 1; + local_capability_.receive_frame_limit = RDMA_MAX_AGGREGATE_FRAMES; local_capability_.cm_address = config_.source_address; int device_count = 0; @@ -184,107 +186,6 @@ std::unique_ptr RdmaTransport::make_pool(uint64_t bytes, uint32_ return pool; } -bool RdmaTransport::submit(Connection *conn, OutFrame *frame) -{ - Operation operation(repost_state_); - if (!operation) - return false; - if (conn == nullptr || !conn->is_open() || frame == nullptr || frame->control == nullptr) { - return false; - } - if (frame->data_views() > 1) { - return false; - } - auto *pools = pools_for(conn); - if (pools == nullptr || pools->send == nullptr || pools->cq_index >= completion_queues_.size() || - completion_queues_[pools->cq_index]->cq == nullptr) { - return false; - } - - Operation posting(pools->posting_state); - if (!posting || !conn->is_open() || conn->transport_handle == 0) - return false; - - // TCP serializes the header and buffers independently. RDMA materializes - // the complete wire message in one registered slot so the receiver can - // post and recycle one contiguous buffer. - Buffer *source_control = frame->control; - const uint32_t control_len = source_control->len; - if (source_control->data == nullptr || control_len > source_control->capacity || - control_len > RDMA_CONTROL_CAPACITY - HEADER_SIZE) { - CRB_LOG_WARN("rdma send: control buffer exceeds wire capacity len={} capacity={}", control_len, - source_control->capacity); - return false; - } - Buffer *slot = pools->send->alloc(config_.wire_slot_capacity); - if (slot == nullptr) - return false; - if (!slot->transition(BufferOwnership::Available, BufferOwnership::SendInFlight)) { - slot->release(); - return false; - } - const uint32_t data_len = frame->data == nullptr ? 0 : frame->data->len; - const uint64_t wire_len = static_cast(HEADER_SIZE) + control_len + data_len; - if (wire_len > slot->capacity || wire_len > pools->peer_receive_slot_capacity) { - const auto capacity = slot->capacity; - (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); - slot->release(); - CRB_LOG_WARN("rdma send: message exceeds wire slot control_len={} data_len={} capacity={}", control_len, - data_len, capacity); - return false; - } - frame->header.msg_size = control_len; - frame->header.data_size = data_len; - serialize_header(slot->data, frame->header); - if (control_len != 0) - std::memcpy(slot->data + HEADER_SIZE, source_control->data, control_len); - if (data_len != 0) { - if (frame->data->payload_data() == nullptr || data_len > frame->data->capacity) { - (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); - slot->release(); - CRB_LOG_WARN("rdma send: data buffer exceeds capacity len={} capacity={}", data_len, frame->data->capacity); - return false; - } - std::memcpy(slot->data + HEADER_SIZE + control_len, frame->data->payload_data(), data_len); - } - slot->len = static_cast(wire_len); - slot->request_id = source_control->request_id; - slot->stream_id = source_control->stream_id; - slot->sequence = source_control->sequence; - slot->logical_offset = source_control->logical_offset; - - WorkRequest *work = acquire_work(WorkKind::Send, conn, slot, frame); - if (work == nullptr) { - (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); - slot->release(); - return false; - } - - ibv_sge sge{}; - sge.addr = reinterpret_cast(slot->data); - sge.length = slot->len; - sge.lkey = slot->registration_key; - ibv_send_wr wr{}; - wr.wr_id = work->token.load(std::memory_order_acquire); - wr.sg_list = &sge; - wr.num_sge = 1; - wr.opcode = IBV_WR_SEND; - wr.send_flags = IBV_SEND_SIGNALED; - ibv_send_wr *bad = nullptr; - if (ibv_post_send(reinterpret_cast(conn->transport_handle), &wr, &bad) != 0) { - release_work(work); - CRB_LOG_WARN("rdma send: post failed qp={} errno={}", reinterpret_cast(conn->transport_handle), errno); - (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); - slot->release(); - return false; - } - - // All work and frame ownership is published before posting. A CQ worker - // may already have released both by the time ibv_post_send returns. - stats_.sends.fetch_add(1, std::memory_order_relaxed); - return true; -} - bool RdmaTransport::post_receive(Connection *conn, Buffer *slot) { Operation operation(repost_state_); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp index 49fdc8180..00f18e711 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. #include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "rdma_aggregate.h" #include "rdma_cq.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -25,8 +26,9 @@ RdmaTransport::WorkRequest *RdmaTransport::acquire_work(WorkKind kind, Connectio uint64_t empty = 0; if (!work.token.compare_exchange_strong(empty, WORK_RESERVED, std::memory_order_acq_rel)) continue; - work.kind = kind; - work.conn = std::move(owner); + work.kind = kind; + work.conn = std::move(owner); + work.connection_identity.store(conn, std::memory_order_relaxed); work.pools = pools == nullptr ? nullptr : pools->shared_from_this(); work.slot = slot; work.frame = frame; @@ -71,9 +73,13 @@ void RdmaTransport::dispatch_completion(const ibv_wc &wc) // Error completions do not provide a trustworthy opcode. The retained // work record owns dispatch, even after disconnect or QP destruction. if (work->kind == WorkKind::Receive && wc.status == IBV_WC_SUCCESS && !work->conn->has_frame_handler()) { - const bool handshake = wc.byte_len >= HEADER_SIZE && work->slot->capacity >= HEADER_SIZE && - parse_header(work->slot->data).msg_type == RDMA_HANDSHAKE_MSG_TYPE; - if (!handshake) { + RdmaAggregate aggregate; + const bool valid = wc.byte_len <= work->slot->capacity && + parse_rdma_aggregate(std::span(work->slot->data, wc.byte_len), aggregate); + bool application = false; + for (uint32_t index = 0; valid && index < aggregate.count; ++index) + application |= aggregate.frames[index].header.msg_type != RDMA_HANDSHAKE_MSG_TYPE; + if (application) { const uint32_t index = static_cast(wc.wr_id) / config_.cq_depth; completion_queues_[index]->deferred.push_back({work, wc}); stats_.pending_deferred_receives.fetch_add(1, std::memory_order_relaxed); @@ -93,11 +99,13 @@ void RdmaTransport::cancel_work(Connection *connection) for (const auto &queue : completion_queues_) { while (queue->polling.test_and_set(std::memory_order_acquire)) std::this_thread::yield(); + retire_sends(*queue, connection); cancel_deferred(*queue, connection); for (uint32_t index = 0; index < config_.cq_depth; ++index) { auto &work = queue->work_requests[index]; const uint64_t token = work.token.load(std::memory_order_acquire); - if (token == 0 || token == WORK_RESERVED || (connection != nullptr && work.conn.get() != connection)) + if (token == 0 || token == WORK_RESERVED || + (connection != nullptr && work.connection_identity.load(std::memory_order_acquire) != connection)) continue; ibv_wc completion{}; completion.wr_id = token; diff --git a/lib/crowdb-rpc/tests/common/test_rdma_transport.h b/lib/crowdb-rpc/tests/common/test_rdma_transport.h index 8cad0209b..36a8a1b33 100644 --- a/lib/crowdb-rpc/tests/common/test_rdma_transport.h +++ b/lib/crowdb-rpc/tests/common/test_rdma_transport.h @@ -17,6 +17,25 @@ namespace crowdb::rpc struct TestRdmaTransport { + static void set_aggregation_for_tests(RdmaTransport &transport, bool enabled) + { + transport.config_.aggregation_enabled = enabled; + } + + static void bind_sender_for_tests(RdmaTransport &transport, const std::shared_ptr &connection, + std::unique_ptr pool, ibv_qp *qp, uint32_t peer_slot, + uint32_t peer_frames) + { + auto pools = std::make_shared(transport.config_.send_queue_capacity); + pools->send = std::move(pool); + pools->peer_receive_slot_capacity = peer_slot; + pools->peer_receive_frame_limit = peer_frames; + pools->effective_capacity.max_send_wr = transport.config_.sq_depth; + RdmaTransport::set_connection_pools(connection.get(), pools); + connection->transport_handle = reinterpret_cast(qp); + transport.completion_queues_[0]->connections.push_back({connection, std::move(pools)}); + } + static void install_device_for_tests(RdmaTransport &transport, const std::shared_ptr &device) { transport.device_ = device; diff --git a/lib/crowdb-rpc/tests/rdma_aggregate_test.cpp b/lib/crowdb-rpc/tests/rdma_aggregate_test.cpp new file mode 100644 index 000000000..2ad680201 --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_aggregate_test.cpp @@ -0,0 +1,84 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "../src/transport/rdma/rdma_aggregate.h" +#include "crowdb-rpc/server/message.h" + +#include + +using namespace crowdb::rpc; + +TEST(RdmaAggregateTest, ExactWireLimitIncludesEveryHeaderAndControlByte) +{ + SystemBufferPool pool; + auto *control = build_ping_request(&pool, 17, 0); + auto *data = pool.alloc(128); + ASSERT_NE(control, nullptr); + ASSERT_NE(data, nullptr); + data->write("data", 4); + auto *frame = build_out_frame(17, 100, control, data); + const uint32_t expected = HEADER_SIZE + control->len + data->len; + uint32_t size = 0; + EXPECT_FALSE(rdma_frame_size(*frame, expected - 1, size)); + ASSERT_TRUE(rdma_frame_size(*frame, expected, size)); + EXPECT_EQ(size, expected); + std::vector wire(size * 2); + copy_rdma_frame(wire.data(), *frame); + copy_rdma_frame(wire.data() + size, *frame); + RdmaAggregate decoded; + ASSERT_TRUE(parse_rdma_aggregate(wire, decoded)); + EXPECT_EQ(decoded.count, 2U); + EXPECT_EQ(decoded.frames[1].offset, expected); + EXPECT_EQ(std::memcmp(wire.data() + decoded.frames[1].data_offset, "data", 4), 0); + wire.push_back(0); + EXPECT_FALSE(parse_rdma_aggregate(wire, decoded)); + wire.resize(size * 2); + wire[size] ^= 1; + EXPECT_FALSE(parse_rdma_aggregate(wire, decoded)); + release_rdma_frames(frame); +} + +TEST(RdmaAggregateTest, FrameCountAndControlArithmeticAreBounded) +{ + Header header; + std::vector wire(HEADER_SIZE * RDMA_MAX_AGGREGATE_FRAMES); + for (size_t offset = 0; offset < wire.size(); offset += HEADER_SIZE) + serialize_header(wire.data() + offset, header); + RdmaAggregate decoded; + ASSERT_TRUE(parse_rdma_aggregate(wire, decoded)); + EXPECT_EQ(decoded.count, RDMA_MAX_AGGREGATE_FRAMES); + wire.resize(wire.size() + HEADER_SIZE); + serialize_header(wire.data() + wire.size() - HEADER_SIZE, header); + EXPECT_FALSE(parse_rdma_aggregate(wire, decoded)); + header.msg_size = UINT32_MAX; + header.data_size = UINT32_MAX; + serialize_header(wire.data(), header); + EXPECT_FALSE(parse_rdma_aggregate(wire, decoded)); +} + +TEST(RdmaAggregateTest, IndependentSlicesKeepOneParentLeaseUntilLastRelease) +{ + RegisteredBufferPool pool(1024, 1); + auto *parent = pool.alloc(1024); + ASSERT_NE(parent, nullptr); + std::memcpy(parent->data, "firstmiddlelast", 15); + auto *first = parent->slice(0, 5); + auto *middle = parent->slice(5, 6); + auto *last = parent->slice(11, 4); + ASSERT_NE(first, nullptr); + ASSERT_NE(middle, nullptr); + ASSERT_NE(last, nullptr); + EXPECT_EQ(parent->slice(1023, 2), nullptr); + auto *held = middle->ref_clone(); + parent->release(); + middle->release(); + first->release(); + last->release(); + EXPECT_EQ(pool.alloc(1024), nullptr); + EXPECT_EQ(std::memcmp(held->payload_data(), "middle", 6), 0); + EXPECT_EQ(held->capacity, 6U); + held->release(); + auto *recycled = pool.alloc(1024); + ASSERT_NE(recycled, nullptr); + recycled->release(); +} diff --git a/lib/crowdb-rpc/tests/rdma_completion_test.cpp b/lib/crowdb-rpc/tests/rdma_completion_test.cpp index a66301241..ef013f820 100644 --- a/lib/crowdb-rpc/tests/rdma_completion_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_completion_test.cpp @@ -339,7 +339,7 @@ TEST_F(RdmaCompletionTest, CallerLeaseKeepsPayloadAfterFrameAndTransportRelease) ASSERT_NE(lease, nullptr); delete received; EXPECT_EQ(lease->ownership.load(), BufferOwnership::CallerOwned); - EXPECT_EQ(lease->view_offset, HEADER_SIZE); + EXPECT_EQ(lease->payload_data(), slot->data + HEADER_SIZE); source.reset(); EXPECT_EQ(std::memcmp(lease->payload_data(), "held", 4), 0); lease->release(); @@ -370,6 +370,126 @@ TEST_F(RdmaCompletionTest, RejectsTruncatedOverflowedAndExtraWireBytes) EXPECT_EQ(slots.quarantined_bytes(), 3U * RDMA_WIRE_SLOT_CAPACITY); } +TEST_F(RdmaCompletionTest, MalformedAggregateSuffixPublishesNoPrefix) +{ + uint32_t published = 0; + connection->set_on_frame([&](Frame *frame, Connection *) { + ++published; + delete frame; + }); + auto *slot = slots.alloc(RDMA_WIRE_SLOT_CAPACITY); + ASSERT_NE(slot, nullptr); + serialize_header(slot->data, Header{}); + Header bad; + bad.data_size = 100; + serialize_header(slot->data + HEADER_SIZE, bad); + ASSERT_TRUE(slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive)); + const auto token = TestRdmaTransport::receive(transport, connection, slot); + TestRdmaTransport::complete(transport, token, IBV_WC_SUCCESS, 2 * HEADER_SIZE); + EXPECT_EQ(published, 0U); + EXPECT_FALSE(connection->is_open()); + EXPECT_EQ(transport.stats().protocol_errors.load(), 1U); + EXPECT_EQ(transport.stats().cq_errors.load(), 0U); +} + +TEST_F(RdmaCompletionTest, AggregateViewsKeepMiddleFrameImmutableAndDelayRecycling) +{ + std::vector frames; + connection->set_on_frame([&](Frame *frame, Connection *) { frames.push_back(frame); }); + auto *slot = slots.alloc(RDMA_WIRE_SLOT_CAPACITY); + ASSERT_NE(slot, nullptr); + Header header; + header.data_size = 1; + for (uint8_t index = 0; index < 3; ++index) { + const auto offset = index * (HEADER_SIZE + 1); + serialize_header(slot->data + offset, header); + slot->data[offset + HEADER_SIZE] = index + 41; + } + ASSERT_TRUE(slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive)); + const auto token = TestRdmaTransport::receive(transport, connection, slot); + TestRdmaTransport::complete(transport, token, IBV_WC_SUCCESS, 3 * (HEADER_SIZE + 1)); + ASSERT_EQ(frames.size(), 3U); + EXPECT_NE(frames[0]->data_buf, frames[1]->data_buf); + auto *held = frames[1]->data_buf->ref_clone(); + for (auto *frame : frames) + delete frame; + EXPECT_EQ(slot->ref->load(), 1); + EXPECT_EQ(slot->ownership.load(), BufferOwnership::CallerOwned); + EXPECT_EQ(held->payload_data()[0], 42); + held->release(); + EXPECT_EQ(slot->ref->load(), 0); + EXPECT_EQ(slot->ownership.load(), BufferOwnership::Available); +} + +TEST_F(RdmaCompletionTest, MixedCapabilityAndApplicationAggregateWaitsForHandler) +{ + auto *slot = slots.alloc(RDMA_WIRE_SLOT_CAPACITY); + ASSERT_NE(slot, nullptr); + RdmaEndpointCapability capability; + capability.protocol_version = RDMA_PROTOCOL_VERSION; + capability.generation = 123; + const auto bytes = serialize_rdma_handshake(capability); + Header handshake; + handshake.msg_type = RDMA_HANDSHAKE_MSG_TYPE; + handshake.msg_size = bytes.size(); + serialize_header(slot->data, handshake); + std::memcpy(slot->data + HEADER_SIZE, bytes.data(), bytes.size()); + const uint32_t offset = HEADER_SIZE + bytes.size(); + Header application; + application.data_size = 1; + serialize_header(slot->data + offset, application); + slot->data[offset + HEADER_SIZE] = 42; + ASSERT_TRUE(slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive)); + const auto token = TestRdmaTransport::receive(transport, connection, slot); + TestRdmaTransport::complete(transport, token, IBV_WC_SUCCESS, offset + HEADER_SIZE + 1); + EXPECT_FALSE(connection->peer_handshake_received()); + EXPECT_EQ(transport.stats().pending_deferred_receives.load(), 1U); + uint32_t published = 0; + connection->set_on_frame([&](Frame *frame, Connection *) { + EXPECT_TRUE(connection->peer_handshake_received()); + EXPECT_EQ(frame->data_buf->payload_data()[0], 42); + ++published; + delete frame; + }); + TestRdmaTransport::set_cq(transport, &queue.cq); + EXPECT_EQ(transport.drain_cq(), 1); + EXPECT_EQ(published, 1U); + EXPECT_EQ(connection->peer_capability().generation, 123U); + EXPECT_EQ(transport.stats().pending_deferred_receives.load(), 0U); + EXPECT_EQ(slot->ref->load(), 0); +} + +TEST_F(RdmaCompletionTest, FailedSendReleasesEveryAggregateOwnerExactlyOnce) +{ + uint32_t released = 0; + auto external = [&] { + auto *buffer = new Buffer; + buffer->ref = new std::atomic(1); + buffer->free_cb = [](void *counter) { ++*static_cast(counter); }; + buffer->free_ctx = &released; + return buffer; + }; + auto *first = new OutFrame; + first->data = external(); + auto *second = new OutFrame; + second->data = external(); + first->next_aggregate = second; + auto *held = second->data->ref_clone(); + auto *slot = slots.alloc(1024); + ASSERT_NE(slot, nullptr); + ASSERT_TRUE(slot->transition(BufferOwnership::Available, BufferOwnership::SendInFlight)); + const auto token = TestRdmaTransport::send(transport, connection, slot, first); + ASSERT_NE(token, 0U); + TestRdmaTransport::complete(transport, token, IBV_WC_LOC_PROT_ERR); + EXPECT_EQ(released, 1U); + EXPECT_EQ(held->ownership.load(), BufferOwnership::Quarantined); + TestRdmaTransport::complete(transport, token, IBV_WC_LOC_PROT_ERR); + EXPECT_EQ(released, 1U); + held->release(); + EXPECT_EQ(released, 2U); + EXPECT_EQ(slots.quarantined_bytes(), 1024U); +} + TEST_F(RdmaCompletionTest, PollAfterArmingClosesNotificationRace) { const auto token = send(); diff --git a/lib/crowdb-rpc/tests/rdma_config_test.cpp b/lib/crowdb-rpc/tests/rdma_config_test.cpp index 196c0473a..80a86ce09 100644 --- a/lib/crowdb-rpc/tests/rdma_config_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_config_test.cpp @@ -119,4 +119,30 @@ TEST(RdmaConfigTest, RegistrationBudgetParses64BitValuesStrictly) EXPECT_FALSE(config.apply_environment()); } +TEST(RdmaConfigTest, AggregationAndQueueLimitsRejectInvalidBounds) +{ + RdmaConfig config; + config.aggregation_enabled = true; + config.aggregation_max_frames = 257; + EXPECT_FALSE(config.valid()); + config.aggregation_max_frames = 0; + EXPECT_FALSE(config.valid()); + config.aggregation_max_frames = 256; + config.aggregation_max_bytes = 0; + EXPECT_FALSE(config.valid()); + config.aggregation_max_bytes = 1024; + EXPECT_TRUE(config.valid()); + config.send_queue_capacity = 3; + EXPECT_FALSE(config.valid()); + config.send_queue_capacity = 1U << 31; + EXPECT_FALSE(config.valid()); + config.send_queue_capacity = 4096; + EXPECT_TRUE(config.valid()); + const TestEnvironment enabled("CROWDB_RDMA_AGGREGATION_ENABLED", "1"); + EXPECT_TRUE(config.apply_environment()); + EXPECT_TRUE(config.aggregation_enabled); + const TestEnvironment invalid("CROWDB_RDMA_AGGREGATION_ENABLED", "yes"); + EXPECT_FALSE(config.apply_environment()); +} + #endif diff --git a/lib/crowdb-rpc/tests/rdma_send_test.cpp b/lib/crowdb-rpc/tests/rdma_send_test.cpp new file mode 100644 index 000000000..bf90e4bd8 --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_send_test.cpp @@ -0,0 +1,163 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "../src/transport/rdma/rdma_aggregate.h" +#include "common/test_rdma_device.h" +#include "common/test_rdma_transport.h" +#include "crowdb-rpc/server/message.h" + +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) +# include + +using namespace crowdb::rpc; + +namespace +{ +RdmaConfig test_send_config() +{ + RdmaConfig config; + config.device = "test-no-device"; + config.wire_slot_capacity = 1024; + config.sq_depth = config.rq_depth = config.posted_receive_target = 4; + config.cq_depth = 8; + config.send_arena_bytes = config.receive_arena_bytes = 4096; + config.poll_empty_budget = 1; + config.aggregation_enabled = true; + config.send_queue_capacity = 4; + return config; +} + +class RdmaSendTest : public ::testing::Test +{ + protected: + TestCompletionQueue queue; + ibv_context context{}; + ibv_qp qp{}; + SystemBufferPool buffers; + RdmaTransport transport{test_send_config(), &buffers}; + std::shared_ptr connection = std::make_shared(1, "sender", &buffers); + std::vector> sends; + bool fail_send = false; + + void SetUp() override + { + auto device = TestRdmaDevice::open(); + if (device == nullptr) { + GTEST_SKIP() << "no accessible HCA for registered send slots"; + } + auto pool = std::make_unique(device->pd, 4096, 4, device); + ASSERT_TRUE(pool->ready()); + qp.context = &context; + qp.qp_context = this; + context.ops.post_send = [](ibv_qp *qp, ibv_send_wr *wr, ibv_send_wr **bad) { + auto &self = *static_cast(qp->qp_context); + if (self.fail_send) { + *bad = wr; + return EIO; + } + const auto *data = reinterpret_cast(wr->sg_list[0].addr); + self.sends.emplace_back(data, data + wr->sg_list[0].length); + self.queue.ready.push_back(TestCompletionQueue::completion(wr->wr_id)); + return 0; + }; + TestRdmaTransport::bind_sender_for_tests(transport, connection, std::move(pool), &qp, 1024, 3); + TestRdmaTransport::set_cq(transport, &queue.cq); + } + + void TearDown() override + { + TestRdmaTransport::set_cq(transport, nullptr); + TestRdmaTransport::cancel(transport); + } + + OutFrame *frame(uint64_t id, uint32_t bytes = 128) + { + auto *data = buffers.alloc(bytes); + if (data == nullptr) + return nullptr; + std::memset(data->data, static_cast(id), bytes); + data->len = bytes; + return build_out_frame(id, 100, build_ping_request(&buffers, id, 0), data); + } +}; + +TEST_F(RdmaSendTest, BoundedQueueCoalescesInOrderWithinPeerFrameLimit) +{ + for (uint64_t id = 1; id <= 4; ++id) + ASSERT_TRUE(transport.submit(connection.get(), frame(id))); + auto *rejected = frame(5); + EXPECT_FALSE(transport.submit(connection.get(), rejected)); + release_rdma_frames(rejected); + EXPECT_TRUE(sends.empty()) << "submit transfers ownership; the CQ owner performs SEND"; + EXPECT_GT(transport.drain_cq(8), 0); + ASSERT_EQ(sends.size(), 2U); + RdmaAggregate first, second; + ASSERT_TRUE(parse_rdma_aggregate(sends[0], first)); + ASSERT_TRUE(parse_rdma_aggregate(sends[1], second)); + EXPECT_EQ(first.count, 3U); + EXPECT_EQ(second.count, 1U); + for (uint32_t index = 0; index < first.count; ++index) + EXPECT_EQ(sends[0][first.frames[index].data_offset], index + 1); + EXPECT_EQ(sends[1][second.frames[0].data_offset], 4); + EXPECT_EQ(transport.stats().rpc_frames_sent.load(), 4U); + EXPECT_EQ(transport.stats().sends.load(), 2U); + EXPECT_EQ(transport.stats().submit_rejects.load(), 1U); + EXPECT_EQ(transport.stats().send_queue_high_water.load(), 4U); + EXPECT_EQ(transport.stats().send_errors.load(), 0U); +} + +TEST_F(RdmaSendTest, PeerSlotOversizeRejectsBeforeQueueOrSend) +{ + auto *oversized = frame(1, 1024); + EXPECT_FALSE(transport.submit(connection.get(), oversized)); + release_rdma_frames(oversized); + EXPECT_EQ(transport.drain_cq(), 0); + EXPECT_TRUE(sends.empty()); + EXPECT_EQ(transport.stats().send_queue_high_water.load(), 0U); +} + +TEST_F(RdmaSendTest, SlotByteLimitSplitsWithoutReorderingPendingFrame) +{ + for (uint64_t id = 1; id <= 4; ++id) + ASSERT_TRUE(transport.submit(connection.get(), frame(id, 400))); + EXPECT_GT(transport.drain_cq(8), 0); + ASSERT_EQ(sends.size(), 2U); + uint32_t expected = 1; + for (const auto &wire : sends) { + EXPECT_LE(wire.size(), 1024U); + RdmaAggregate decoded; + ASSERT_TRUE(parse_rdma_aggregate(wire, decoded)); + EXPECT_EQ(decoded.count, 2U); + for (uint32_t index = 0; index < decoded.count; ++index) + EXPECT_EQ(wire[decoded.frames[index].data_offset], expected++); + } +} + +TEST_F(RdmaSendTest, DisabledAggregationUsesOneSendPerFrame) +{ + TestRdmaTransport::set_aggregation_for_tests(transport, false); + for (uint64_t id = 1; id <= 4; ++id) + ASSERT_TRUE(transport.submit(connection.get(), frame(id))); + EXPECT_GT(transport.drain_cq(8), 0); + ASSERT_EQ(sends.size(), 4U); + for (const auto &wire : sends) { + RdmaAggregate decoded; + ASSERT_TRUE(parse_rdma_aggregate(wire, decoded)); + EXPECT_EQ(decoded.count, 1U); + } +} + +TEST_F(RdmaSendTest, FailedPostClosesAndRetiresAllAdmittedFrames) +{ + fail_send = true; + for (uint64_t id = 1; id <= 4; ++id) + ASSERT_TRUE(transport.submit(connection.get(), frame(id))); + (void)transport.drain_cq(); + EXPECT_FALSE(connection->is_open()); + EXPECT_EQ(transport.stats().send_errors.load(), 1U); + EXPECT_EQ(transport.stats().sends.load(), 0U); + EXPECT_TRUE(sends.empty()); + EXPECT_EQ(transport.drain_cq(), 0); +} +} // namespace +#endif From 0eb1dc2e048eab785515fb40ba5bf8b8038c6852 Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 22 Oct 2026 23:31:38 +0800 Subject: [PATCH 031/258] Expose explicit transport configuration and versioned status snapshots --- Cargo.lock | 1 + doc/working/plan-rdma-rpc-transport.md | 33 +++- doc/working/plan-rdma.md | 11 +- lib/crowdb-rpc/examples/fb_server.cpp | 34 +++- lib/crowdb-rpc/ffi/Cargo.toml | 1 + lib/crowdb-rpc/ffi/src/lib.rs | 2 + lib/crowdb-rpc/ffi/src/sys.rs | 38 +++++ lib/crowdb-rpc/ffi/src/transport.rs | 156 ++++++++++++++++++ .../ffi/tests/transport_config_test.rs | 130 +++++++++++++++ lib/crowdb-rpc/include/crowdb-rpc/buffer.h | 10 ++ lib/crowdb-rpc/include/crowdb-rpc/c_api.h | 40 +++++ .../include/crowdb-rpc/connection.h | 3 + .../include/crowdb-rpc/server/server.h | 6 + .../crowdb-rpc/transport/rdma/rdma_config.h | 48 ++++++ .../transport/rdma/rdma_transport.h | 43 +---- .../crowdb-rpc/transport/socket_transport.h | 26 ++- lib/crowdb-rpc/src/c_api.cpp | 9 +- lib/crowdb-rpc/src/c_api/transport.cpp | 143 ++++++++++++++++ lib/crowdb-rpc/src/connection.cpp | 8 + lib/crowdb-rpc/src/server/server.cpp | 51 +----- lib/crowdb-rpc/src/status_json.h | 75 +++++++++ .../src/transport/rdma/rdma_config.cpp | 21 +-- lib/crowdb-rpc/src/transport/rdma/rdma_cq.h | 4 +- lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp | 10 ++ .../src/transport/rdma/rdma_settings.cpp | 99 +++++++++++ .../src/transport/rdma/rdma_status.cpp | 87 ++++++++++ .../src/transport/rdma/rdma_transport.cpp | 8 +- .../src/transport/socket_transport.cpp | 2 + lib/crowdb-rpc/src/transport/status.cpp | 98 +++++++++++ .../tests/transport_config_test.cpp | 74 +++++++++ 30 files changed, 1157 insertions(+), 114 deletions(-) create mode 100644 lib/crowdb-rpc/ffi/src/transport.rs create mode 100644 lib/crowdb-rpc/ffi/tests/transport_config_test.rs create mode 100644 lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h create mode 100644 lib/crowdb-rpc/src/c_api/transport.cpp create mode 100644 lib/crowdb-rpc/src/status_json.h create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_settings.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp create mode 100644 lib/crowdb-rpc/src/transport/status.cpp create mode 100644 lib/crowdb-rpc/tests/transport_config_test.cpp diff --git a/Cargo.lock b/Cargo.lock index eb2b412a7..76c1fc35c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1248,6 +1248,7 @@ dependencies = [ "crowdb-rpc-ffi", "ctor", "flatbuffers", + "serde_json", "tokio", ] diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index b9a4da8df..14e4f7ef8 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -36,7 +36,8 @@ Registration admission remains charged while retired connections retain MRs. TCP control preflight now precedes CM setup; the compact CM offer binds the preflight identity/generation and peer receive admission. Strict policy has passed a two-QP real-HCA RPC round trip and a TCP-only peer rejection case. -Still pending: bounded transport aggregation, `bench rdma`/shared coroutine +Bounded transport aggregation is implemented and verified below. +Still pending: `bench rdma`/shared coroutine reporting, unified regression orchestration, failure injection, and the two-container acceptance profiles. Do not treat the historical benchmark comments or older subtask ledger as completed acceptance. @@ -623,6 +624,36 @@ Build parallelism checkpoint: budget instead of compiling native sources serially. `cargo check -j32` passes; the FFI suite validates the resulting library separately. +Status/configuration checkpoint: + +- Portable `RdmaConfig` now supports explicit cold configuration, with a + versioned C ABI and safe Rust wrapper. Configure before listen; explicit + values bypass process environment. Client configuration disables its CM + listener, while the fb-server starts its listener before readiness output. +- Versioned owned JSON snapshots expose actual path, cached QP/CQ/provider + capacities, requested queue/slot/arena/aggregation settings, native errors, + memory budgets, CPU time, context switches and affinity. Unsupported fields + remain null; `quarantined_bytes` has no global retained-arena snapshot yet + and must not be represented by the old unused zero-valued transport counter. +- Benchmark status control RPC uses message type 101. The CLI will query it + through an explicit short-lived TCP management connection before/after + traffic, since the complete status JSON is larger than small 1 KiB wire + slots. These are control exchanges outside the measurement interval, not + workload fallback; report `control_path=tcp` separately from verified RDMA + workload QPs and strict fallback count. Keep no extra management worker + active during measured traffic. Reuse the same status schema as local FFI. +- Native client error reads now include flushed + current metric windows. + Stop periodic flush and join callbacks before exact final totals; a flush + must not erase evidence of earlier submit/retry/timeout/missed responses. +- All 186 enabled native CTest cases and all 137 no-verbs cases pass; opt-in + HCA cases skip in the ordinary gate. All 33 Rust FFI tests pass, including + four new status/configuration cases. The opt-in real-HCA typed configuration + test separately proves immutable provider/QP status survives shutdown. + Formatting, scoped tree-lint and FFI clippy pass. The legacy two-process + strict two-QP worker round trip also passes after extracting listener port + resolution. Readiness/runner acceptance and CLI integration remain pending; + this checkpoint makes no new performance claim. + ### D. CLI coroutine workload and canonical results - [ ] **Add `bench rdma` with shared echo workload**: reuse `bench rpc` diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index 4fa6d9004..979be972c 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -102,16 +102,23 @@ R242 fat client SDK and FFI lease ## Immediate implementation order +User continuation on 2026-10-22: complete R238 and R239, then continue through +R240 and R241 in this same task. Finish transport/benchmark/OCI acceptance +before advancing; implement R240, then R241, using their existing +[datapath stream plan](plan-rdma-datapath-stream.md) and +[Chunk-KV/access plan](plan-rdma-chunk-kv-access.md), with their own gates. +Do not create duplicate plan files. R242 remains outside this requested continuation. + 1. Finish the R238 foundation contract and its gates. Do not add new R239 transport behavior until the common and verbs-backed pools agree on class, ownership, quarantine, and shutdown semantics. 2. Close R239 acceptance gaps: injectable CM/CQ failure coverage, stable `wr_id` ownership mapping, pre-payload retry on TCP, partial-transfer isolation, and the single-node multi-container OCI profile. -3. Run the R238/R239 handoff gates and then start R240 and R241 in parallel. +3. Run the R238/R239 handoff gates, complete R240, then complete R241. R242 remains downstream of R241 and the stable FFI lease surface. -R240 and R241 may run in parallel after R239 passes its transport acceptance. +R240 and R241 start after R239 passes its transport acceptance. R242 starts only after R241 establishes the access semantics and after the FFI lease contract from R238 is stable. diff --git a/lib/crowdb-rpc/examples/fb_server.cpp b/lib/crowdb-rpc/examples/fb_server.cpp index 20124be73..ba48b5f8b 100644 --- a/lib/crowdb-rpc/examples/fb_server.cpp +++ b/lib/crowdb-rpc/examples/fb_server.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include static std::atomic g_running{true}; @@ -47,6 +48,8 @@ DEFINE_bool(n, false, "Alias for --enable_nagle"); DEFINE_bool(quickack, false, "Enable TCP_QUICKACK on connections (Linux only). Breaks Nagle + delayed-ACK deadlock."); DEFINE_bool(event_write, false, "Event-write mode: submit() enqueues to I/O worker for coalesced writev."); DEFINE_uint32(send_queue_capacity, 4096, "Per-connection send queue capacity (backpressure bound)."); +DEFINE_bool(require_rdma, false, "Fail startup unless the RDMA CM listener is active."); +DEFINE_string(status_file, "", "Write startup transport status JSON after listener readiness."); DEFINE_string(logdir, "", "Log directory for server + metrics logs. Default: workspace runtime root."); DEFINE_string(l, "", "Alias for --logdir"); DEFINE_uint32(metrics_interval, 5, "Metrics flush interval in seconds."); @@ -117,15 +120,38 @@ int main(int argc, char *argv[]) } int actual_port = crowdb_rpc_server_port(server); - // Print the port so the parent process can read it. - std::printf("listening port=%d\n", actual_port); - std::fflush(stdout); - // Register the built-in echo handler for msg_type 100. const uint16_t ECHO_MSG_TYPE = 100; crowdb_rpc_server_register_echo_handler(server, ECHO_MSG_TYPE); + crowdb_rpc_server_register_status_handler(server, 101); crowdb_rpc_server_start(server); + if (FLAGS_require_rdma && crowdb_rpc_server_rdma_listening(server) == 0) { + std::fprintf(stderr, "error: required RDMA CM listener is unavailable\n"); + crowdb_rpc_server_stop(server); + crowdb_rpc_server_destroy(server); + return 1; + } + auto status = crowdb_rpc_server_status_json(server); + if (status == nullptr) { + std::fprintf(stderr, "error: startup transport status allocation failed\n"); + crowdb_rpc_server_destroy(server); + return 1; + } + if (!FLAGS_status_file.empty()) { + std::ofstream file(FLAGS_status_file); + file.write(reinterpret_cast(crowdb_rpc_buffer_data(status)), crowdb_rpc_buffer_len(status)); + file.close(); + if (!file) { + std::fprintf(stderr, "error: cannot write startup status file %s\n", FLAGS_status_file.c_str()); + crowdb_rpc_buffer_release(status); + crowdb_rpc_server_destroy(server); + return 1; + } + } + crowdb_rpc_buffer_release(status); + std::printf("listening port=%d\n", actual_port); + std::fflush(stdout); // Start metrics flush — the RPC lib registers all histograms, // bandwidths, and counters with MetricsRegistry::global() via diff --git a/lib/crowdb-rpc/ffi/Cargo.toml b/lib/crowdb-rpc/ffi/Cargo.toml index 9cf0b993e..c9c895a37 100644 --- a/lib/crowdb-rpc/ffi/Cargo.toml +++ b/lib/crowdb-rpc/ffi/Cargo.toml @@ -29,6 +29,7 @@ ctor = { version = "1.0.13", optional = true } crowdb-common = { path = "../../crowdb-common/rust" } crowdb-protocol = { path = "../../crowdb-protocol" } flatbuffers = { workspace = true } +serde_json = "1" # Self dev-dependency: turns on `test-util` so integration tests auto-init # C++ logging to a temp dir on first RpcServer::new. crowdb-rpc-ffi = { path = ".", features = ["test-util"] } diff --git a/lib/crowdb-rpc/ffi/src/lib.rs b/lib/crowdb-rpc/ffi/src/lib.rs index 41534cb5d..64a97ce7b 100644 --- a/lib/crowdb-rpc/ffi/src/lib.rs +++ b/lib/crowdb-rpc/ffi/src/lib.rs @@ -20,6 +20,7 @@ pub mod logging; mod owned_route; mod server; pub mod sys; +mod transport; pub use buffer::{Buffer, BufferChain, BufferChainError, BufferLeaseMetadata, BufferOwnership, BufferPool}; pub use client::{noop_completion, CallFuture, ClientRequest, Response, RpcClient}; @@ -31,6 +32,7 @@ pub use logging::{ }; pub use owned_route::{OwnedClientRoute, RpcClientHandle}; pub use server::{Connection, RpcError, RpcServer, ServerRequest}; +pub use transport::RdmaConfig; pub use sys::{CrowdbRpcLatencyStats, CrowdbRpcTransportStats}; diff --git a/lib/crowdb-rpc/ffi/src/sys.rs b/lib/crowdb-rpc/ffi/src/sys.rs index 1bfff75e1..6a92e8cec 100644 --- a/lib/crowdb-rpc/ffi/src/sys.rs +++ b/lib/crowdb-rpc/ffi/src/sys.rs @@ -50,6 +50,36 @@ pub struct CrowdbRpcTransportStats { pub send_queue_rejects: u64, } +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub struct CrowdbRpcRdmaConfig { + pub struct_size: u32, + pub version: u32, + pub enabled: u32, + pub allow_tcp_fallback: u32, + pub listen_enabled: u32, + pub port: u32, + pub cq_count: u32, + pub cq_depth: u32, + pub sq_depth: u32, + pub rq_depth: u32, + pub posted_receive_target: u32, + pub wire_slot_capacity: u32, + pub poll_batch: u32, + pub verbs_poll_batch: u32, + pub poll_empty_budget: u32, + pub send_queue_capacity: u32, + pub aggregation_enabled: u32, + pub aggregation_max_bytes: u32, + pub aggregation_max_frames: u32, + pub send_arena_bytes: u64, + pub receive_arena_bytes: u64, + pub registered_memory_budget_bytes: u64, + pub device: *const c_char, + pub source_address: *const c_char, + pub gid: *const c_char, +} + #[repr(C)] #[derive(Default, Debug)] pub struct CrowdbRpcClientCounters { @@ -120,6 +150,14 @@ pub type crowdb_rpc_co_on_response = Option< >; extern "C" { + pub fn crowdb_rpc_rdma_config_init(config: *mut CrowdbRpcRdmaConfig); + pub fn crowdb_rpc_server_configure_rdma( + server: crowdb_rpc_server_t, + config: *const CrowdbRpcRdmaConfig, + ) -> crowdb_rpc_status; + pub fn crowdb_rpc_server_status_json(server: crowdb_rpc_server_t) -> crowdb_rpc_buffer_t; + pub fn crowdb_rpc_conn_status_json(connection: crowdb_rpc_conn_t) -> crowdb_rpc_buffer_t; + pub fn crowdb_rpc_server_register_status_handler(server: crowdb_rpc_server_t, msg_type: u16); pub fn crowdb_rpc_buffer_alloc(pool: crowdb_rpc_pool_t, capacity: u32) -> crowdb_rpc_buffer_t; pub fn crowdb_rpc_buffer_write(buf: crowdb_rpc_buffer_t, data: *const u8, len: u32); pub fn crowdb_rpc_buffer_data(buf: crowdb_rpc_buffer_t) -> *const u8; diff --git a/lib/crowdb-rpc/ffi/src/transport.rs b/lib/crowdb-rpc/ffi/src/transport.rs new file mode 100644 index 000000000..ab4c71021 --- /dev/null +++ b/lib/crowdb-rpc/ffi/src/transport.rs @@ -0,0 +1,156 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Cold transport configuration and owned status snapshots. + +use std::ffi::CString; +use std::mem::MaybeUninit; + +use crate::{sys, Buffer, Connection, RpcError, RpcServer}; + +/// Explicit configuration overrides native environment defaults before listen. +#[derive(Debug, Clone)] +#[allow(clippy::struct_excessive_bools)] +pub struct RdmaConfig { + pub enabled: bool, + pub allow_tcp_fallback: bool, + pub listen_enabled: bool, + pub port: u32, + pub cq_count: u32, + pub cq_depth: u32, + pub sq_depth: u32, + pub rq_depth: u32, + pub posted_receive_target: u32, + pub wire_slot_capacity: u32, + pub poll_batch: u32, + pub verbs_poll_batch: u32, + pub poll_empty_budget: u32, + pub send_queue_capacity: u32, + pub aggregation_enabled: bool, + pub aggregation_max_bytes: u32, + pub aggregation_max_frames: u32, + pub send_arena_bytes: u64, + pub receive_arena_bytes: u64, + pub registered_memory_budget_bytes: u64, + pub device: String, + pub source_address: String, + pub gid: String, +} + +fn native_defaults() -> sys::CrowdbRpcRdmaConfig { + let mut config = MaybeUninit::uninit(); + // SAFETY: initializer fills every field of the versioned C ABI struct. + unsafe { + sys::crowdb_rpc_rdma_config_init(config.as_mut_ptr()); + config.assume_init() + } +} + +impl Default for RdmaConfig { + fn default() -> Self { + let raw = native_defaults(); + Self { + enabled: raw.enabled != 0, + allow_tcp_fallback: raw.allow_tcp_fallback != 0, + listen_enabled: raw.listen_enabled != 0, + port: raw.port, + cq_count: raw.cq_count, + cq_depth: raw.cq_depth, + sq_depth: raw.sq_depth, + rq_depth: raw.rq_depth, + posted_receive_target: raw.posted_receive_target, + wire_slot_capacity: raw.wire_slot_capacity, + poll_batch: raw.poll_batch, + verbs_poll_batch: raw.verbs_poll_batch, + poll_empty_budget: raw.poll_empty_budget, + send_queue_capacity: raw.send_queue_capacity, + aggregation_enabled: raw.aggregation_enabled != 0, + aggregation_max_bytes: raw.aggregation_max_bytes, + aggregation_max_frames: raw.aggregation_max_frames, + send_arena_bytes: raw.send_arena_bytes, + receive_arena_bytes: raw.receive_arena_bytes, + registered_memory_budget_bytes: raw.registered_memory_budget_bytes, + device: String::new(), + source_address: String::new(), + gid: String::new(), + } + } +} + +impl RpcServer { + /// Configure before listen/start; strings are copied by the native layer. + /// + /// # Errors + /// Rejects invalid bounds, an unavailable requested HCA, or late changes. + pub fn configure_rdma(&self, config: &RdmaConfig) -> Result<(), RpcError> { + let device = CString::new(config.device.as_str()).map_err(|_| RpcError::InvalidArg)?; + let source = CString::new(config.source_address.as_str()).map_err(|_| RpcError::InvalidArg)?; + let gid = CString::new(config.gid.as_str()).map_err(|_| RpcError::InvalidArg)?; + let mut raw = native_defaults(); + raw.enabled = u32::from(config.enabled); + raw.allow_tcp_fallback = u32::from(config.allow_tcp_fallback); + raw.listen_enabled = u32::from(config.listen_enabled); + raw.port = config.port; + raw.cq_count = config.cq_count; + raw.cq_depth = config.cq_depth; + raw.sq_depth = config.sq_depth; + raw.rq_depth = config.rq_depth; + raw.posted_receive_target = config.posted_receive_target; + raw.wire_slot_capacity = config.wire_slot_capacity; + raw.poll_batch = config.poll_batch; + raw.verbs_poll_batch = config.verbs_poll_batch; + raw.poll_empty_budget = config.poll_empty_budget; + raw.send_queue_capacity = config.send_queue_capacity; + raw.aggregation_enabled = u32::from(config.aggregation_enabled); + raw.aggregation_max_bytes = config.aggregation_max_bytes; + raw.aggregation_max_frames = config.aggregation_max_frames; + raw.send_arena_bytes = config.send_arena_bytes; + raw.receive_arena_bytes = config.receive_arena_bytes; + raw.registered_memory_budget_bytes = config.registered_memory_budget_bytes; + raw.device = device.as_ptr(); + raw.source_address = source.as_ptr(); + raw.gid = gid.as_ptr(); + // SAFETY: handle is borrowed, struct and CStrings live through the call. + let status = unsafe { sys::crowdb_rpc_server_configure_rdma(self.handle(), &raw) }; + if status == sys::CROWDB_RPC_OK { + Ok(()) + } else { + Err(RpcError::from_status(status)) + } + } + + /// Return an owned UTF-8 JSON snapshot with a versioned schema. + /// + /// # Errors + /// Returns an error if native status allocation fails. + pub fn transport_status_json(&self) -> Result, RpcError> { + // SAFETY: server stays borrowed until the owned buffer is copied. + copy_status(unsafe { sys::crowdb_rpc_server_status_json(self.handle()) }) + } + + /// Register a benchmark control RPC returning the same status schema. + pub fn register_status_handler(&self, msg_type: u16) { + // SAFETY: registration copies the handler into the borrowed server. + unsafe { sys::crowdb_rpc_server_register_status_handler(self.handle(), msg_type) }; + } +} + +impl Connection { + /// Actual path and immutable QP/CQ/provider capacities, without reading a + /// live QP pointer during teardown. + /// + /// # Errors + /// Returns an error if native status allocation fails. + pub fn transport_status_json(&self) -> Result, RpcError> { + // SAFETY: connection wrapper owns/borrows the native handle. + copy_status(unsafe { sys::crowdb_rpc_conn_status_json(self.handle()) }) + } +} + +fn copy_status(handle: sys::crowdb_rpc_buffer_t) -> Result, RpcError> { + if handle.is_null() { + return Err(RpcError::RegistrationFailed); + } + let buffer = Buffer::from_raw(handle); + Ok(buffer.as_slice().to_vec()) +} diff --git a/lib/crowdb-rpc/ffi/tests/transport_config_test.rs b/lib/crowdb-rpc/ffi/tests/transport_config_test.rs new file mode 100644 index 000000000..9caa52c80 --- /dev/null +++ b/lib/crowdb-rpc/ffi/tests/transport_config_test.rs @@ -0,0 +1,130 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::fb::{ConnectionPingRequest, ConnectionPingRequestArgs}; +use crowdb_rpc_ffi::{Buffer, RdmaConfig, RpcClient, RpcServer}; +use flatbuffers::FlatBufferBuilder; +use serde_json::Value; + +#[test] +fn explicit_tcp_configuration_is_typed_and_cold() { + let server = RpcServer::new(None); + let config = RdmaConfig::default(); + assert_eq!(config.sq_depth, 1024); + server.configure_rdma(&config).unwrap(); + let status: Value = serde_json::from_slice(&server.transport_status_json().unwrap()).unwrap(); + assert_eq!(status["schema_version"], 1); + assert_eq!(status["allow_tcp_fallback"], true); + assert!(status["rdma"].is_null()); + assert!(status["cpu_user_ns"].is_u64()); + server.listen("127.0.0.1", 0).unwrap(); + assert!(server.configure_rdma(&config).is_err()); + server.start(); + assert!(server.configure_rdma(&config).is_err()); + server.stop(); +} + +#[test] +fn invalid_queue_bounds_fail_before_start() { + let server = RpcServer::new(None); + let config = RdmaConfig { + cq_count: 0, + ..RdmaConfig::default() + }; + assert!(server.configure_rdma(&config).is_err()); + let config = RdmaConfig { + send_queue_capacity: 3, + ..RdmaConfig::default() + }; + assert!(server.configure_rdma(&config).is_err()); +} + +#[tokio::test] +async fn status_control_rpc_returns_owned_versioned_json() { + let server = RpcServer::new(None); + server.configure_rdma(&RdmaConfig::default()).unwrap(); + server.listen("127.0.0.1", 0).unwrap(); + server.register_status_handler(101); + server.start(); + let connection = server.connect("127.0.0.1", server.port()).unwrap(); + let client = RpcClient::new(); + client.attach(&connection); + let mut builder = FlatBufferBuilder::new(); + let request = ConnectionPingRequest::create( + &mut builder, + &ConnectionPingRequestArgs { + id: 137, + rpc_create_nano: 0, + }, + ); + builder.finish(request, None); + let reply = client + .call( + &server, + &connection, + 137, + Buffer::from_bytes(builder.finished_data()), + None, + 101, + ) + .unwrap() + .await + .unwrap(); + let status: Value = serde_json::from_slice(reply.data.as_ref().unwrap().as_slice()).unwrap(); + assert_eq!(reply.request_id, 137); + assert_eq!(status["schema_version"], 1); + assert!(status["rdma"].is_null()); + let connection_status: Value = + serde_json::from_slice(&connection.transport_status_json().unwrap()).unwrap(); + assert_eq!(connection_status["path"], "tcp"); + server.stop(); + let closed: Value = serde_json::from_slice(&connection.transport_status_json().unwrap()).unwrap(); + assert_eq!(closed["open"], false); +} + +#[test] +fn hardware_qpair_status_retains_immutable_provider_values_after_close() { + if std::env::var_os("CROWDB_RDMA_E2E").is_none() { + return; + } + let address = std::env::var("CROWDB_RDMA_E2E_LISTEN_ADDR").unwrap(); + let config = RdmaConfig { + enabled: true, + allow_tcp_fallback: false, + listen_enabled: true, + device: std::env::var("CROWDB_RDMA_DEVICE").unwrap_or_default(), + source_address: address.clone(), + wire_slot_capacity: 1024, + sq_depth: 8, + rq_depth: 8, + posted_receive_target: 8, + cq_depth: 16, + send_arena_bytes: 8192, + receive_arena_bytes: 8192, + ..RdmaConfig::default() + }; + let server = RpcServer::new(None); + server.configure_rdma(&config).unwrap(); + server.listen(&address, 0).unwrap(); + server.start(); + let client = RpcServer::new(None); + client + .configure_rdma(&RdmaConfig { + listen_enabled: false, + ..config + }) + .unwrap(); + client.listen("127.0.0.1", 0).unwrap(); + client.start(); + let connection = client.connect(&address, server.port()).unwrap(); + let info: Value = serde_json::from_slice(&connection.transport_status_json().unwrap()).unwrap(); + assert_eq!(info["path"], "rdma"); + assert!(info["info"]["qpair_number"].as_u64().unwrap() > 0); + assert!(info["info"]["effective_sq_depth"].as_u64().unwrap() >= 8); + assert_eq!(info["info"]["peer_receive_slot_capacity"], 1024); + client.stop(); + server.stop(); + let closed: Value = serde_json::from_slice(&connection.transport_status_json().unwrap()).unwrap(); + assert_eq!(closed["open"], false); + assert_eq!(closed["info"], info["info"]); +} diff --git a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h index 73da3acd3..59be719c8 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h @@ -167,6 +167,16 @@ class SystemBufferPool : public BufferPool void recycle(Buffer *buf) override; bool detach(Buffer *buf) override; + [[nodiscard]] uint32_t max_buffers() const + { + return max_buffers_; + } + + [[nodiscard]] uint32_t outstanding() const + { + return outstanding_.load(std::memory_order_relaxed); + } + private: uint32_t max_buffers_; std::atomic outstanding_{0}; // in-use buffers (allocated, not yet freed) diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h index 85a2cc0e2..e90455ec2 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h @@ -70,6 +70,44 @@ crowdb_rpc_pool_t crowdb_rpc_pool_create(uint32_t max_buffers); void crowdb_rpc_pool_destroy(crowdb_rpc_pool_t pool); // ── Server ──────────────────────────────────────────────────────── +typedef struct crowdb_rpc_rdma_config +{ + uint32_t struct_size; + uint32_t version; + uint32_t enabled; + uint32_t allow_tcp_fallback; + uint32_t listen_enabled; + uint32_t port; + uint32_t cq_count; + uint32_t cq_depth; + uint32_t sq_depth; + uint32_t rq_depth; + uint32_t posted_receive_target; + uint32_t wire_slot_capacity; + uint32_t poll_batch; + uint32_t verbs_poll_batch; + uint32_t poll_empty_budget; + uint32_t send_queue_capacity; + uint32_t aggregation_enabled; + uint32_t aggregation_max_bytes; + uint32_t aggregation_max_frames; + uint64_t send_arena_bytes; + uint64_t receive_arena_bytes; + uint64_t registered_memory_budget_bytes; + const char *device; + const char *source_address; + const char *gid; +} crowdb_rpc_rdma_config_t; + +void crowdb_rpc_rdma_config_init(crowdb_rpc_rdma_config_t *config); +// Explicit cold configuration overrides environment; rejected after listen. +crowdb_rpc_status crowdb_rpc_server_configure_rdma(crowdb_rpc_server_t server, const crowdb_rpc_rdma_config_t *config); +// Owned, versioned UTF-8 JSON snapshots. Release using buffer_release. +crowdb_rpc_buffer_t crowdb_rpc_server_status_json(crowdb_rpc_server_t server); +crowdb_rpc_buffer_t crowdb_rpc_conn_status_json(crowdb_rpc_conn_t connection); +void crowdb_rpc_server_register_status_handler(crowdb_rpc_server_t server, uint16_t msg_type); +int crowdb_rpc_server_rdma_listening(crowdb_rpc_server_t server); + crowdb_rpc_server_t crowdb_rpc_server_create(crowdb_rpc_pool_t pool); crowdb_rpc_server_t crowdb_rpc_server_create_with_workers(crowdb_rpc_pool_t pool, uint32_t num_workers); crowdb_rpc_server_t crowdb_rpc_server_create_with_engines(crowdb_rpc_pool_t pool, uint32_t io_engines, @@ -121,6 +159,8 @@ void crowdb_rpc_server_transport_stats(crowdb_rpc_server_t server, crowdb_rpc_tr // Global (static) — shared across all RpcClient instances. Read via // crowdb_rpc_client_get_counters (the client param is ignored, kept for // ABI compatibility). +// Includes flushed and current windows. Exact after writers and metric flush +// join; live reads can overlap a flush and are approximate. typedef struct crowdb_rpc_client_counters { uint64_t submit_fail; // send() submit failed diff --git a/lib/crowdb-rpc/include/crowdb-rpc/connection.h b/lib/crowdb-rpc/include/crowdb-rpc/connection.h index 1e3f13a44..500d6b52c 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/connection.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/connection.h @@ -66,6 +66,8 @@ class Connection : public std::enable_shared_from_this path_ = path; } + [[nodiscard]] std::string status_json() const; + [[nodiscard]] bool peer_handshake_received() const { return peer_handshake_received_.load(std::memory_order_acquire); @@ -273,6 +275,7 @@ class Connection : public std::enable_shared_from_this } private: + std::string transport_info_json_ = "{}"; // immutable after connection publication friend class RdmaTransport; std::shared_ptr transport_context_; int64_t id_; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/server/server.h b/lib/crowdb-rpc/include/crowdb-rpc/server/server.h index 5cc8d70e5..4df3f8f23 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/server/server.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/server/server.h @@ -33,6 +33,12 @@ class RpcServer // If port is 0, the OS assigns an ephemeral port. bool listen(const std::string &addr, int port); + bool configure_rdma(RdmaConfig config, bool enabled, bool allow_tcp_fallback, bool listen_enabled) + { + return listen_fd_ < 0 && !running_.load(std::memory_order_acquire) && + transport_->configure_rdma(std::move(config), enabled, allow_tcp_fallback, listen_enabled); + } + int listen_port() const { return listen_port_; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h new file mode 100644 index 000000000..d73b27a90 --- /dev/null +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h @@ -0,0 +1,48 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/buffer.h" + +#include + +namespace crowdb::rpc +{ +struct RdmaConfig +{ + bool use_environment = true; + std::string device; + std::string source_address{}; + std::string gid{}; + bool require_tcp_preflight = false; + uint8_t port = 1; + uint32_t cq_depth = 8192; + uint32_t cq_count = 1; + uint32_t wire_slot_capacity = RDMA_DATAPATH_CAPACITY; + uint64_t send_arena_bytes = 64ULL << 20; + uint64_t receive_arena_bytes = 1ULL << 30; + uint64_t registered_memory_budget_bytes = 0; + uint32_t sq_depth = 1024; + uint32_t rq_depth = 1024; + uint32_t qp_max_send_sge = 2; + uint32_t qp_max_recv_sge = 2; + uint32_t poll_batch = 256; + uint32_t verbs_poll_batch = 32; + uint32_t poll_empty_budget = 256; + uint32_t shutdown_drain_ms = 3000; + uint32_t posted_receive_target = 1024; + uint8_t min_rnr_timer = 1; + uint8_t retry_count = 7; + uint8_t rnr_retry_count = 7; + bool aggregation_enabled = false; + uint32_t aggregation_max_bytes = RDMA_DATAPATH_CAPACITY; + uint32_t aggregation_max_frames = 256; + uint32_t send_queue_capacity = 4096; + + [[nodiscard]] bool apply_environment(); + [[nodiscard]] bool valid() const; + [[nodiscard]] uint64_t registration_budget() const; +}; + +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index c38520b5c..687c99956 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -7,6 +7,7 @@ #include "crowdb-rpc/connection.h" #include "crowdb-rpc/transport.h" #include "crowdb-rpc/transport/rdma/rdma_capability.h" +#include "crowdb-rpc/transport/rdma/rdma_config.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -31,41 +32,6 @@ struct RdmaMemoryStats; constexpr uint32_t RDMA_WIRE_SLOT_CAPACITY = RDMA_DATAPATH_CAPACITY; -struct RdmaConfig -{ - std::string device; - std::string source_address{}; - std::string gid{}; - bool require_tcp_preflight = false; - uint8_t port = 1; - uint32_t cq_depth = 8192; - uint32_t cq_count = 1; - uint32_t wire_slot_capacity = RDMA_WIRE_SLOT_CAPACITY; - uint64_t send_arena_bytes = 64ULL << 20; - uint64_t receive_arena_bytes = 1ULL << 30; - uint64_t registered_memory_budget_bytes = 0; - uint32_t sq_depth = 1024; - uint32_t rq_depth = 1024; - uint32_t qp_max_send_sge = 2; - uint32_t qp_max_recv_sge = 2; - uint32_t poll_batch = 256; - uint32_t verbs_poll_batch = 32; - uint32_t poll_empty_budget = 256; - uint32_t shutdown_drain_ms = 3000; - uint32_t posted_receive_target = 1024; - uint8_t min_rnr_timer = 1; - uint8_t retry_count = 7; - uint8_t rnr_retry_count = 7; - bool aggregation_enabled = false; - uint32_t aggregation_max_bytes = RDMA_WIRE_SLOT_CAPACITY; - uint32_t aggregation_max_frames = 256; - uint32_t send_queue_capacity = 4096; - - [[nodiscard]] bool apply_environment(); - [[nodiscard]] bool valid() const; - [[nodiscard]] uint64_t registration_budget() const; -}; - // RdmaBufferPool: allocates buffers from ibv_mr-registered memory. // The pool pre-registers a large memory region and carves out buffers // from it, avoiding per-buffer ibv_reg_mr calls (which are expensive). @@ -152,6 +118,13 @@ class RdmaTransport : public Transport const RdmaEndpointCapability *expected_peer = nullptr); [[nodiscard]] RdmaEndpointCapability local_capability() const; + [[nodiscard]] const RdmaConfig &config() const + { + return config_; + } + + [[nodiscard]] std::string status_json() const; + [[nodiscard]] const std::string &source_address() const { return config_.source_address; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h index d8a103eb1..c000b0532 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h @@ -5,6 +5,7 @@ #include "crowdb-rpc/connection.h" #include "crowdb-rpc/transport.h" +#include "crowdb-rpc/transport/rdma/rdma_config.h" #ifdef CROWDB_RPC_HAVE_RDMA # include "crowdb-rpc/transport/rdma/rdma_transport.h" #endif @@ -86,6 +87,10 @@ struct TransportStats LatencyHistogram submit_to_writev; // Total enqueue_send rejections (queue full or connection closed). std::atomic send_queue_rejects{0}; + std::atomic writev_calls{0}; + std::atomic writev_progress_calls{0}; + std::atomic wire_bytes_sent{0}; + std::atomic rpc_frames_sent{0}; }; // ── SocketEngine: platform-specific event loop primitives ───────── @@ -510,6 +515,22 @@ class SocketTransport : public Transport bool send_rdma_handshake(const std::shared_ptr &conn); [[nodiscard]] RdmaEndpointCapability local_rdma_capability() const; bool listen_rdma(const std::string &addr, int port); + bool listen_rdma_service(const std::string &address, int tcp_port); + // Cold configuration, before any listener, worker or connection exists. + bool configure_rdma(RdmaConfig config, bool enabled, bool allow_tcp_fallback, bool listen_enabled); + [[nodiscard]] bool rdma_listener_enabled() const; + + [[nodiscard]] bool explicit_rdma_configuration() const + { + return rdma_explicit_configuration_; + } + + [[nodiscard]] const RdmaConfig &rdma_config() const + { + return rdma_config_; + } + + [[nodiscard]] std::string status_json() const; #if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) const RdmaTransport::Stats *rdma_stats_for_tests() const @@ -524,9 +545,12 @@ class SocketTransport : public Transport void prepare_tcp_socket(int fd) const; BufferPool *pool_; bool allow_tcp_fallback_{true}; + RdmaConfig rdma_config_; + bool rdma_explicit_configuration_ = false; + bool rdma_listen_enabled_ = false; + std::atomic started_{false}; #ifdef CROWDB_RPC_HAVE_RDMA std::unique_ptr rdma_transport_; - RdmaConfig rdma_config_; #endif std::vector> workers_; std::vector> engines_; // transport owns all engines diff --git a/lib/crowdb-rpc/src/c_api.cpp b/lib/crowdb-rpc/src/c_api.cpp index 9cc1f9085..cfe432dec 100644 --- a/lib/crowdb-rpc/src/c_api.cpp +++ b/lib/crowdb-rpc/src/c_api.cpp @@ -437,10 +437,11 @@ void crowdb_rpc_client_get_counters(crowdb_rpc_client_t /*client*/, crowdb_rpc_c if (out == nullptr) { return; } - out->submit_fail = crowdb::rpc::cnt_send_queue_full().window(); - out->submit_retry = crowdb::rpc::rpc_submit_retry().window(); - out->resp_missed = crowdb::rpc::rpc_resp_missed().window(); - out->reaped = crowdb::rpc::rpc_reaped().window(); + const auto cumulative = [](const auto &counter) { return counter.total() + counter.window(); }; + out->submit_fail = cumulative(crowdb::rpc::cnt_send_queue_full()); + out->submit_retry = cumulative(crowdb::rpc::rpc_submit_retry()); + out->resp_missed = cumulative(crowdb::rpc::rpc_resp_missed()); + out->reaped = cumulative(crowdb::rpc::rpc_reaped()); } catch (...) { } diff --git a/lib/crowdb-rpc/src/c_api/transport.cpp b/lib/crowdb-rpc/src/c_api/transport.cpp new file mode 100644 index 000000000..0ad2db010 --- /dev/null +++ b/lib/crowdb-rpc/src/c_api/transport.cpp @@ -0,0 +1,143 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/c_api_internal.h" +#include "crowdb-rpc/server/message.h" + +using namespace crowdb::rpc; + +#define CONFIG_NUMBERS(X) \ + X(cq_count) \ + X(cq_depth) \ + X(sq_depth) \ + X(rq_depth) \ + X(posted_receive_target) X(wire_slot_capacity) X(poll_batch) X(verbs_poll_batch) X(poll_empty_budget) \ + X(send_queue_capacity) X(aggregation_max_bytes) X(aggregation_max_frames) X(send_arena_bytes) \ + X(receive_arena_bytes) X(registered_memory_budget_bytes) + +extern "C" void crowdb_rpc_rdma_config_init(crowdb_rpc_rdma_config_t *config) +{ + if (config == nullptr) + return; + *config = {}; + config->struct_size = sizeof(*config); + config->version = 1; + config->allow_tcp_fallback = 1; + const RdmaConfig defaults; + config->port = defaults.port; + config->aggregation_enabled = defaults.aggregation_enabled; +#define COPY_DEFAULT(name) config->name = defaults.name; + CONFIG_NUMBERS(COPY_DEFAULT) +#undef COPY_DEFAULT +} + +extern "C" crowdb_rpc_status crowdb_rpc_server_configure_rdma(crowdb_rpc_server_t server, + const crowdb_rpc_rdma_config_t *config) +{ + if (server == nullptr || config == nullptr || config->struct_size != sizeof(*config) || config->version != 1 || + config->enabled > 1 || config->allow_tcp_fallback > 1 || config->listen_enabled > 1 || + config->aggregation_enabled > 1 || config->port > UINT8_MAX) + return CROWDB_RPC_ERR_INVALID_ARG; + try { + RdmaConfig native; + native.use_environment = false; + native.port = static_cast(config->port); + native.aggregation_enabled = config->aggregation_enabled != 0; + native.device = config->device == nullptr ? "" : config->device; + native.source_address = config->source_address == nullptr ? "" : config->source_address; + native.gid = config->gid == nullptr ? "" : config->gid; +#define COPY_VALUE(name) native.name = config->name; + CONFIG_NUMBERS(COPY_VALUE) +#undef COPY_VALUE + if (!native.valid()) + return CROWDB_RPC_ERR_INVALID_ARG; + return server->server->configure_rdma(std::move(native), config->enabled != 0, config->allow_tcp_fallback != 0, + config->listen_enabled != 0) + ? CROWDB_RPC_OK + : CROWDB_RPC_ERR_CONN_ERROR; + } + catch (...) { + return CROWDB_RPC_ERR_REGISTRATION; + } +} + +#undef CONFIG_NUMBERS + +extern "C" int crowdb_rpc_server_rdma_listening(crowdb_rpc_server_t server) +{ + if (server == nullptr) + return 0; + try { + const auto capability = server->server->transport()->local_rdma_capability(); + return capability.rdma_available && capability.port != 0 ? 1 : 0; + } + catch (...) { + return 0; + } +} + +extern "C" crowdb_rpc_buffer_t crowdb_rpc_server_status_json(crowdb_rpc_server_t server) +{ + if (server == nullptr) + return nullptr; + try { + const auto json = server->server->transport()->status_json(); + return crowdb_rpc_buffer_create(reinterpret_cast(json.data()), json.size()); + } + catch (...) { + return nullptr; + } +} + +extern "C" crowdb_rpc_buffer_t crowdb_rpc_conn_status_json(crowdb_rpc_conn_t connection) +{ + if (connection == nullptr || connection->conn == nullptr) + return nullptr; + try { + const auto json = connection->conn->status_json(); + return crowdb_rpc_buffer_create(reinterpret_cast(json.data()), json.size()); + } + catch (...) { + return nullptr; + } +} + +namespace +{ +OutFrame *status_reply(RpcServer *server, Frame *request, Connection *connection) +{ + const std::unique_ptr owner(request); + try { + const auto json = server->transport()->status_json(); + auto release = [](Buffer *buffer) { buffer->release(); }; + std::unique_ptr control( + build_ping_response(server->pool(), request->request_id, request->rpc_create_nano, 0), release); + std::unique_ptr data(server->pool()->alloc(static_cast(json.size())), + release); + if (control != nullptr && data != nullptr) { + data->write(json.data(), static_cast(json.size())); + auto *reply = build_out_frame(request->request_id, request->header.msg_type, control.get(), data.get()); + (void)control.release(); + (void)data.release(); + return reply; + } + } + catch (...) { + } + connection->close(); + return nullptr; +} +} // namespace + +extern "C" void crowdb_rpc_server_register_status_handler(crowdb_rpc_server_t server, uint16_t msg_type) +{ + if (server == nullptr) + return; + try { + server->server->register_handler(msg_type, [native = server->server](Frame *request, Connection *connection) { + return status_reply(native, request, connection); + }); + } + catch (...) { + } +} diff --git a/lib/crowdb-rpc/src/connection.cpp b/lib/crowdb-rpc/src/connection.cpp index c4c1be418..e50d6d451 100644 --- a/lib/crowdb-rpc/src/connection.cpp +++ b/lib/crowdb-rpc/src/connection.cpp @@ -282,8 +282,14 @@ bool Connection::try_send(int fd, TransportStats *stats) } ssize_t written = ::writev(fd, iovs, iov_count); + if (stats != nullptr) + stats->writev_calls.fetch_add(1, std::memory_order_relaxed); if (written > 0) { bw_writev().observe(static_cast(written)); + if (stats != nullptr) { + stats->writev_progress_calls.fetch_add(1, std::memory_order_relaxed); + stats->wire_bytes_sent.fetch_add(static_cast(written), std::memory_order_relaxed); + } } if (written < 0) { @@ -329,6 +335,8 @@ bool Connection::try_send(int fd, TransportStats *stats) pending_count_ = 0; for (int i = 0; i < frame_count; i++) { if (frames[i]->sent_offset >= frame_totals[i]) { + if (stats != nullptr) + stats->rpc_frames_sent.fetch_add(1, std::memory_order_relaxed); release_frame(frames[i]); } else { diff --git a/lib/crowdb-rpc/src/server/server.cpp b/lib/crowdb-rpc/src/server/server.cpp index 423321cc4..44958b85e 100644 --- a/lib/crowdb-rpc/src/server/server.cpp +++ b/lib/crowdb-rpc/src/server/server.cpp @@ -180,9 +180,7 @@ void RpcServer::start() return; } CRB_LOG_INFO("rpc server: starting"); - if (const char *enabled = std::getenv("CROWDB_RDMA_ENABLED"); - enabled != nullptr && std::strcmp(enabled, "1") == 0 && - !(std::getenv("CROWDB_RDMA_LISTEN") != nullptr && std::strcmp(std::getenv("CROWDB_RDMA_LISTEN"), "0") == 0)) { + if (transport_->rdma_listener_enabled()) { transport_->set_rdma_accept_handler([this](const std::shared_ptr &conn) { transport_->attach_rdma_connection( conn, [this](Frame *frame, Connection *c) { dispatch(frame, c); }, @@ -192,52 +190,7 @@ void RpcServer::start() }); transport_->send_rdma_handshake(conn); }); - int rdma_port = 0; - if (const char *rdma_port_env = std::getenv("CROWDB_RDMA_E2E_PORT"); - rdma_port_env != nullptr && *rdma_port_env != '\0') { - char *end = nullptr; - errno = 0; - const long configured_port = std::strtol(rdma_port_env, &end, 10); - if (errno == 0 && end != rdma_port_env && *end == '\0' && configured_port > 0 && configured_port <= 65535) { - rdma_port = static_cast(configured_port); - } - else { - CRB_LOG_WARN("rpc server: invalid RDMA CM port value={}", rdma_port_env); - } - } - else { - rdma_port = listen_port_; - if (const char *offset_env = std::getenv("CROWDB_PORT_OFFSET"); - offset_env != nullptr && *offset_env != '\0') { - char *end = nullptr; - errno = 0; - const long offset = std::strtol(offset_env, &end, 10); - if (errno != 0 || end == offset_env || *end != '\0' || offset < 0 || offset > 65535 - rdma_port) { - CRB_LOG_WARN("rpc server: invalid RDMA port offset value={}", offset_env); - rdma_port = 0; - } - else { - rdma_port += static_cast(offset); - } - } - } - if (rdma_port != 0) { - if (const char *slot_env = std::getenv("CROWDB_RDMA_SERVICE_SLOT"); - slot_env != nullptr && *slot_env != '\0') { - char *end = nullptr; - errno = 0; - const long slot = std::strtol(slot_env, &end, 10); - if (errno != 0 || end == slot_env || *end != '\0' || slot < 0 || slot > 65535 - rdma_port) { - CRB_LOG_WARN("rpc server: invalid RDMA service slot value={}", slot_env); - rdma_port = 0; - } - else { - rdma_port += static_cast(slot); - } - } - } - const char *rdma_addr = std::getenv("CROWDB_RDMA_SOURCE_ADDR"); - if (rdma_port == 0 || !transport_->listen_rdma(rdma_addr == nullptr ? listen_addr_ : rdma_addr, rdma_port)) + if (!transport_->listen_rdma_service(listen_addr_, listen_port_)) CRB_LOG_WARN("rpc server: RDMA listener unavailable; TCP remains active"); } transport_->start(); diff --git a/lib/crowdb-rpc/src/status_json.h b/lib/crowdb-rpc/src/status_json.h new file mode 100644 index 000000000..7df14197d --- /dev/null +++ b/lib/crowdb-rpc/src/status_json.h @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include +#include +#include + +namespace crowdb::rpc +{ +// Cold status serialization; keys and strings are escaped without a runtime +// JSON dependency in the portable transport library. +class StatusJson +{ + public: + void number(std::string_view key, uint64_t value) + { + raw(key, std::to_string(value)); + } + + void boolean(std::string_view key, bool value) + { + raw(key, value ? "true" : "false"); + } + + void string(std::string_view key, std::string_view value) + { + raw(key, quote(value)); + } + + void null(std::string_view key) + { + raw(key, "null"); + } + + void raw(std::string_view key, std::string_view value) + { + if (text_.size() != 1) + text_ += ','; + text_ += quote(key); + text_ += ':'; + text_ += value; + } + + [[nodiscard]] std::string finish() const + { + return text_ + '}'; + } + + private: + std::string text_ = "{"; + + static std::string quote(std::string_view value) + { + constexpr char hex[] = "0123456789abcdef"; + std::string text = "\""; + for (const unsigned char byte : value) { + if (byte == '"' || byte == '\\') { + text += '\\'; + text += static_cast(byte); + } + else if (byte < 32) { + text += "\\u00"; + text += hex[byte >> 4]; + text += hex[byte & 15]; + } + else { + text += static_cast(byte); + } + } + return text + '"'; + } +}; +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp index 9fd2c5a17..c861343aa 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp @@ -1,16 +1,15 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -#include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "crowdb-rpc/transport/rdma/rdma_config.h" -#ifdef CROWDB_RPC_HAVE_RDMA +#include "crowdb-common/log.h" +#include "crowdb-rpc/framing.h" -# include "crowdb-common/log.h" - -# include -# include -# include -# include +#include +#include +#include +#include namespace crowdb::rpc { @@ -34,7 +33,7 @@ bool RdmaConfig::apply_environment() }; return (port != 1 || read("CROWDB_RDMA_PORT", port)) && read("CROWDB_RDMA_SQ_DEPTH", sq_depth) && read("CROWDB_RDMA_RQ_DEPTH", rq_depth) && read("CROWDB_RDMA_POSTED_RECEIVE_TARGET", posted_receive_target) && - read("CROWDB_RDMA_WIRE_SLOT_CAPACITY", wire_slot_capacity) && + read("CROWDB_RDMA_DATAPATH_CAPACITY", wire_slot_capacity) && read("CROWDB_RDMA_SEND_ARENA_BYTES", send_arena_bytes) && read("CROWDB_RDMA_RECEIVE_ARENA_BYTES", receive_arena_bytes) && read("CROWDB_RDMA_CQ_DEPTH", cq_depth) && read("CROWDB_RDMA_REGISTERED_MEMORY_BUDGET_BYTES", registered_memory_budget_bytes) && @@ -60,7 +59,7 @@ bool RdmaConfig::valid() const poll_empty_budget > 0 && send_arena_bytes >= wire_slot_capacity && receive_arena_bytes >= static_cast(posted_receive_target) * wire_slot_capacity && min_rnr_timer <= 31 && retry_count <= 7 && rnr_retry_count <= 7 && registration_budget() != 0 && - aggregation_max_bytes >= HEADER_SIZE && aggregation_max_bytes <= RDMA_WIRE_SLOT_CAPACITY && + aggregation_max_bytes >= HEADER_SIZE && aggregation_max_bytes <= RDMA_DATAPATH_CAPACITY && aggregation_max_frames > 0 && aggregation_max_frames <= 256 && send_queue_capacity >= 2 && send_queue_capacity <= (1U << 20) && (send_queue_capacity & (send_queue_capacity - 1)) == 0; } @@ -85,5 +84,3 @@ uint64_t RdmaConfig::registration_budget() const } } // namespace crowdb::rpc - -#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h index 31745c96b..713e63b98 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h @@ -12,8 +12,8 @@ namespace crowdb::rpc { struct RdmaTransport::CompletionQueue { - uint32_t index = 0; - uint32_t effective_depth = 0; + uint32_t index = 0; + std::atomic effective_depth{0}; ibv_cq *cq = nullptr; ibv_comp_channel *channel = nullptr; int continuation_fd = -1; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp index 09b268060..a8f206c8b 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp @@ -1,6 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#include "../../status_json.h" #include "rdma_cq.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -23,6 +24,15 @@ bool RdmaTransport::publish_cm_connection(rdma_cm_id *id, const std::shared_ptr< { // The shared CM consumer must see the owner before a migrated terminal // event can retire this ID. Setup events stay private until this point. + StatusJson info; + info.number("qpair_number", pools->qp_number); + info.number("cq_index", pools->cq_index); + info.number("effective_sq_depth", pools->effective_capacity.max_send_wr); + info.number("effective_rq_depth", pools->effective_capacity.max_recv_wr); + info.number("effective_cq_depth", completion_queues_[pools->cq_index]->effective_depth); + info.number("peer_receive_slot_capacity", pools->peer_receive_slot_capacity); + info.number("peer_receive_frame_limit", pools->peer_receive_frame_limit); + connection->transport_info_json_ = info.finish(); std::scoped_lock lock(cm_mu_); if (rdma_migrate_id(id, cm_channel_) != 0) return false; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_settings.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_settings.cpp new file mode 100644 index 000000000..fe2e564f5 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_settings.cpp @@ -0,0 +1,99 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/transport/socket_transport.h" + +#include +#include +#include +#include + +namespace crowdb::rpc +{ +namespace +{ +bool read_service_port(const char *name, uint32_t &value) +{ + const char *text = std::getenv(name); + if (text == nullptr || *text == '\0') + return true; + const std::string_view input(text); + const auto result = std::from_chars(input.data(), input.data() + input.size(), value); + return result.ec == std::errc{} && result.ptr == input.data() + input.size() && value <= 65535; +} +} // namespace + +bool SocketTransport::listen_rdma_service(const std::string &address, int tcp_port) +{ + uint32_t port = static_cast(tcp_port); + const char *source = nullptr; + if (rdma_explicit_configuration_) { + source = rdma_config_.source_address.empty() ? nullptr : rdma_config_.source_address.c_str(); + } + else { + const char *override_port = std::getenv("CROWDB_RDMA_E2E_PORT"); + uint32_t offset = 0; + uint32_t service_slot = 0; + if (override_port != nullptr && *override_port != '\0') { + if (!read_service_port("CROWDB_RDMA_E2E_PORT", port)) + return false; + } + else if (!read_service_port("CROWDB_PORT_OFFSET", offset)) { + return false; + } + if (!read_service_port("CROWDB_RDMA_SERVICE_SLOT", service_slot) || + static_cast(port) + offset + service_slot > 65535) + return false; + port += offset + service_slot; + source = std::getenv("CROWDB_RDMA_SOURCE_ADDR"); + } + return port != 0 && port <= 65535 && listen_rdma(source == nullptr ? address : source, static_cast(port)); +} + +bool SocketTransport::configure_rdma(RdmaConfig config, bool enabled, bool allow_tcp_fallback, bool listen_enabled) +{ + if (started_.load(std::memory_order_acquire) || connection_count() != 0 || !config.valid()) + return false; + config.use_environment = false; + config.require_tcp_preflight = true; +#ifdef CROWDB_RPC_HAVE_RDMA + std::unique_ptr replacement; + if (enabled) { + replacement = std::make_unique(config, pool_); + if (!replacement->ready()) + return false; + } + rdma_transport_ = std::move(replacement); + if (rdma_transport_ != nullptr) { + for (const auto &engine : engines_) + rdma_transport_->register_events(engine.get()); + rdma_transport_->set_connection_handler([this](const std::shared_ptr &connection) { + attach_rdma_connection(connection); + if (rdma_accept_handler_) + rdma_accept_handler_(connection); + }); + set_external_handler([this](uint64_t tag) { rdma_transport_->handle_external_event(tag); }); + } + else { + set_external_handler({}); + } +#else + if (enabled) + return false; +#endif + rdma_config_ = std::move(config); + allow_tcp_fallback_ = allow_tcp_fallback; + rdma_listen_enabled_ = enabled && listen_enabled; + rdma_explicit_configuration_ = true; + return true; +} + +bool SocketTransport::rdma_listener_enabled() const +{ + if (rdma_explicit_configuration_) + return rdma_listen_enabled_; + const auto *enabled = std::getenv("CROWDB_RDMA_ENABLED"); + const auto *listen = std::getenv("CROWDB_RDMA_LISTEN"); + return enabled != nullptr && std::strcmp(enabled, "1") == 0 && (listen == nullptr || std::strcmp(listen, "0") != 0); +} +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp new file mode 100644 index 000000000..b46a59ef4 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp @@ -0,0 +1,87 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "../../status_json.h" +#include "rdma_cq.h" + +#ifdef CROWDB_RPC_HAVE_RDMA +namespace crowdb::rpc +{ +std::string RdmaTransport::status_json() const +{ + const auto &counters = stats(); + const auto capability = local_capability(); + StatusJson status; + status.boolean("ready", ready()); + status.boolean("listen_active", listen_active_.load(std::memory_order_acquire)); + status.string("device", capability.device); + status.string("gid", capability.gid); + status.string("source_address", config_.source_address); + status.string("cm_address", capability.cm_address); + status.number("cm_port", capability.port); + status.number("generation", capability.generation); + status.boolean("aggregation_enabled", config_.aggregation_enabled); +# define CONFIG_FIELD(name) status.number(#name, config_.name) + CONFIG_FIELD(port); + CONFIG_FIELD(cq_count); + CONFIG_FIELD(cq_depth); + CONFIG_FIELD(sq_depth); + CONFIG_FIELD(rq_depth); + CONFIG_FIELD(posted_receive_target); + CONFIG_FIELD(wire_slot_capacity); + CONFIG_FIELD(send_arena_bytes); + CONFIG_FIELD(receive_arena_bytes); + CONFIG_FIELD(poll_batch); + CONFIG_FIELD(verbs_poll_batch); + CONFIG_FIELD(poll_empty_budget); + CONFIG_FIELD(send_queue_capacity); + CONFIG_FIELD(aggregation_max_bytes); + CONFIG_FIELD(aggregation_max_frames); +# undef CONFIG_FIELD +# define COUNTER_FIELD(name) status.number(#name, counters.name.load(std::memory_order_relaxed)) + COUNTER_FIELD(sends); + COUNTER_FIELD(receives); + COUNTER_FIELD(rpc_frames_sent); + COUNTER_FIELD(rpc_frames_received); + COUNTER_FIELD(wire_bytes_sent); + COUNTER_FIELD(wire_bytes_received); + COUNTER_FIELD(cq_errors); + COUNTER_FIELD(protocol_errors); + COUNTER_FIELD(send_errors); + COUNTER_FIELD(submit_rejects); + COUNTER_FIELD(send_queue_high_water); + COUNTER_FIELD(fallback_count); + COUNTER_FIELD(posted_receives); + COUNTER_FIELD(cq_polls); + COUNTER_FIELD(cq_empty_polls); + COUNTER_FIELD(cm_disconnects); + COUNTER_FIELD(flush_errors); + COUNTER_FIELD(shutdown_flushes); + COUNTER_FIELD(local_protection_errors); + COUNTER_FIELD(retry_exceeded_errors); + COUNTER_FIELD(remote_access_errors); + COUNTER_FIELD(poller_count); + COUNTER_FIELD(cq_events); + COUNTER_FIELD(cq_wakeups); + COUNTER_FIELD(pending_deferred_receives); + COUNTER_FIELD(registered_memory_budget_bytes); + COUNTER_FIELD(registered_bytes); + COUNTER_FIELD(registered_bytes_high_water); + COUNTER_FIELD(registration_rejects); + COUNTER_FIELD(registration_failures); + COUNTER_FIELD(cm_attempts); + COUNTER_FIELD(preflight_attempts); + COUNTER_FIELD(preflight_failures); +# undef COUNTER_FIELD + status.null("quarantined_bytes"); + std::string depths = "["; + for (const auto &queue : completion_queues_) { + if (depths.size() > 1) + depths += ','; + depths += std::to_string(queue->effective_depth.load(std::memory_order_acquire)); + } + status.raw("effective_cq_depths", depths + ']'); + return status.finish(); +} +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index 6d0733b4b..f3f08738f 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -36,7 +36,7 @@ RdmaTransport::RdmaTransport(RdmaConfig config, BufferPool *application_pool) : config_(std::move(config)), application_pool_(application_pool == nullptr ? &default_application_pool_ : application_pool) { - const bool valid_config = config_.apply_environment() && config_.valid(); + const bool valid_config = (!config_.use_environment || config_.apply_environment()) && config_.valid(); if (!valid_config) { config_.cq_depth = 1; config_.cq_count = 1; @@ -57,17 +57,17 @@ RdmaTransport::RdmaTransport(RdmaConfig config, BufferPool *application_pool) CRB_LOG_WARN("rdma configuration: inconsistent queue depths, slot capacity, or arena budget; using TCP"); return; } - if (config_.device.empty()) { + if (config_.use_environment && config_.device.empty()) { if (const char *requested_device = std::getenv("CROWDB_RDMA_DEVICE"); requested_device != nullptr) config_.device = requested_device; } requested_device_ = config_.device; port_ = config_.port; - if (config_.source_address.empty()) { + if (config_.use_environment && config_.source_address.empty()) { if (const char *source = std::getenv("CROWDB_RDMA_SOURCE_ADDR"); source != nullptr) config_.source_address = source; } - if (config_.gid.empty()) { + if (config_.use_environment && config_.gid.empty()) { if (const char *gid = std::getenv("CROWDB_RDMA_GID"); gid != nullptr) config_.gid = gid; } diff --git a/lib/crowdb-rpc/src/transport/socket_transport.cpp b/lib/crowdb-rpc/src/transport/socket_transport.cpp index d1962fc99..ab5309f50 100644 --- a/lib/crowdb-rpc/src/transport/socket_transport.cpp +++ b/lib/crowdb-rpc/src/transport/socket_transport.cpp @@ -616,6 +616,7 @@ SocketTransport::~SocketTransport() void SocketTransport::start() { + started_.store(true, std::memory_order_release); #ifdef CROWDB_RPC_HAVE_RDMA if (rdma_transport_ != nullptr) rdma_transport_->start(); @@ -643,6 +644,7 @@ void SocketTransport::stop() if (rdma_transport_ != nullptr) rdma_transport_->shutdown(); #endif + started_.store(false, std::memory_order_release); } void SocketTransport::add_listen_fd(int fd) diff --git a/lib/crowdb-rpc/src/transport/status.cpp b/lib/crowdb-rpc/src/transport/status.cpp new file mode 100644 index 000000000..008dafa3c --- /dev/null +++ b/lib/crowdb-rpc/src/transport/status.cpp @@ -0,0 +1,98 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "../status_json.h" + +#include +#ifdef __linux__ +# include +#endif +#include "crowdb-rpc/transport/socket_transport.h" + +namespace crowdb::rpc +{ +std::string Connection::status_json() const +{ + StatusJson status; + status.number("schema_version", 1); + status.string("path", path() == ConnectionPath::Rdma ? "rdma" : "tcp"); + status.boolean("open", is_open()); + status.raw("info", transport_info_json_); + return status.finish(); +} + +std::string SocketTransport::status_json() const +{ + StatusJson status; + status.number("schema_version", 1); + status.number("io_engines", engines_.size()); + status.number("io_workers", workers_.size()); + status.number("connection_count", connection_count()); + status.boolean("tcp_nodelay", tcp_nodelay_); + status.boolean("tcp_quickack", quickack_); + status.boolean("event_write", event_write_); + status.boolean("allow_tcp_fallback", allow_tcp_fallback_); + status.number("send_queue_capacity", send_queue_capacity_); + status.number("send_queue_rejects", stats_.send_queue_rejects.load(std::memory_order_relaxed)); + status.number("writev_calls", stats_.writev_calls.load(std::memory_order_relaxed)); + status.number("writev_progress_calls", stats_.writev_progress_calls.load(std::memory_order_relaxed)); + status.number("wire_bytes_sent", stats_.wire_bytes_sent.load(std::memory_order_relaxed)); + status.number("rpc_frames_sent", stats_.rpc_frames_sent.load(std::memory_order_relaxed)); + if (const auto *system_pool = dynamic_cast(pool_); system_pool != nullptr) { + status.number("application_buffer_capacity", system_pool->max_buffers()); + status.number("application_buffers_in_use", system_pool->outstanding()); + } + else { + status.null("application_buffer_capacity"); + status.null("application_buffers_in_use"); + } + rusage usage{}; + if (::getrusage(RUSAGE_SELF, &usage) == 0) { + const auto nanos = [](const timeval &time) { + return static_cast(time.tv_sec) * 1'000'000'000ULL + static_cast(time.tv_usec) * 1000; + }; + status.number("cpu_user_ns", nanos(usage.ru_utime)); + status.number("cpu_system_ns", nanos(usage.ru_stime)); + status.number("voluntary_context_switches", usage.ru_nvcsw); + status.number("involuntary_context_switches", usage.ru_nivcsw); + } + else { + status.null("cpu_user_ns"); + status.null("cpu_system_ns"); + status.null("voluntary_context_switches"); + status.null("involuntary_context_switches"); + } + status.null("cpu_migrations"); +#ifdef __linux__ + cpu_set_t affinity; + CPU_ZERO(&affinity); + if (::sched_getaffinity(0, sizeof(affinity), &affinity) == 0) { + std::string cpus; + for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { + if (CPU_ISSET(cpu, &affinity)) { + if (!cpus.empty()) + cpus += ','; + cpus += std::to_string(cpu); + } + } + status.string("cpu_affinity", cpus); + } + else { + status.null("cpu_affinity"); + } +#else + status.null("cpu_affinity"); +#endif +#ifdef CROWDB_RPC_HAVE_RDMA + status.boolean("rdma_compiled", true); + if (rdma_transport_ != nullptr) + status.raw("rdma", rdma_transport_->status_json()); + else + status.null("rdma"); +#else + status.boolean("rdma_compiled", false); + status.null("rdma"); +#endif + return status.finish(); +} +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/tests/transport_config_test.cpp b/lib/crowdb-rpc/tests/transport_config_test.cpp new file mode 100644 index 000000000..f4e0d33dd --- /dev/null +++ b/lib/crowdb-rpc/tests/transport_config_test.cpp @@ -0,0 +1,74 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "../src/status_json.h" +#include "common/test_environment.h" +#include "crowdb-rpc/c_api_internal.h" +#include "crowdb-rpc/client/rpc_client_metrics.h" +#include "crowdb-rpc/rpc_metrics.h" + +#include + +using namespace crowdb::rpc; + +TEST(TransportConfigTest, ExplicitTcpOverridesEnvironmentAndRejectsLateChanges) +{ + const TestEnvironment enabled("CROWDB_RDMA_ENABLED", "1"); + const TestEnvironment invalid_device("CROWDB_RDMA_DEVICE", "test-no-device"); + const TestEnvironment strict("CROWDB_RDMA_ALLOW_TCP_FALLBACK", "0"); + RpcServer server; + RdmaConfig config; + ASSERT_TRUE(server.configure_rdma(config, false, true, false)); + EXPECT_NE(server.transport()->status_json().find("\"allow_tcp_fallback\":true"), std::string::npos); + EXPECT_NE(server.transport()->status_json().find("\"rdma\":null"), std::string::npos); + ASSERT_TRUE(server.listen("127.0.0.1", 0)); + EXPECT_FALSE(server.configure_rdma(config, false, true, false)); + server.start(); + auto connection = server.transport()->connect("127.0.0.1", server.listen_port()); + ASSERT_NE(connection, nullptr); + EXPECT_EQ(connection->path(), ConnectionPath::Tcp); + server.stop(); +} + +TEST(TransportConfigTest, VersionedCAbiRejectsMalformedBoundsBeforePublication) +{ + const auto server = crowdb_rpc_server_create(nullptr); + ASSERT_NE(server, nullptr); + crowdb_rpc_rdma_config_t config; + crowdb_rpc_rdma_config_init(&config); + EXPECT_EQ(config.struct_size, sizeof(config)); + config.version = 2; + EXPECT_EQ(crowdb_rpc_server_configure_rdma(server, &config), CROWDB_RPC_ERR_INVALID_ARG); + config.version = 1; + config.port = 256; + EXPECT_EQ(crowdb_rpc_server_configure_rdma(server, &config), CROWDB_RPC_ERR_INVALID_ARG); + config.port = 1; + config.cq_count = 0; + EXPECT_EQ(crowdb_rpc_server_configure_rdma(server, &config), CROWDB_RPC_ERR_INVALID_ARG); + config.cq_count = 1; + EXPECT_EQ(crowdb_rpc_server_configure_rdma(server, &config), CROWDB_RPC_OK); + crowdb_rpc_server_destroy(server); +} + +TEST(TransportConfigTest, ClientErrorCountersSurviveMetricWindowFlush) +{ + crowdb_rpc_client_counters_t before{}, pending{}, flushed{}; + crowdb_rpc_client_get_counters(nullptr, &before); + rpc_submit_retry().inc_by(2); + rpc_resp_missed().inc(); + crowdb_rpc_client_get_counters(nullptr, &pending); + rpc_submit_retry().flush(); + rpc_resp_missed().flush(); + crowdb_rpc_client_get_counters(nullptr, &flushed); + EXPECT_EQ(pending.submit_retry, before.submit_retry + 2); + EXPECT_EQ(pending.resp_missed, before.resp_missed + 1); + EXPECT_EQ(flushed.submit_retry, pending.submit_retry); + EXPECT_EQ(flushed.resp_missed, pending.resp_missed); +} + +TEST(TransportConfigTest, StatusEscapesConfiguredStrings) +{ + StatusJson status; + status.string("key", "a\"b\\c\n"); + EXPECT_EQ(status.finish(), "{\"key\":\"a\\\"b\\\\c\\u000a\"}"); +} From becdcaeb7c4a4c1b138a64be57b9f79028579fdb Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 22 Oct 2026 23:40:42 +0800 Subject: [PATCH 032/258] Preserve the RDMA wire slot environment configuration contract --- doc/working/plan-rdma-rpc-transport.md | 14 ++++++++++++-- lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp | 2 +- lib/crowdb-rpc/tests/rdma_config_test.cpp | 10 ++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 14e4f7ef8..aafe3d2fb 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -650,9 +650,19 @@ Status/configuration checkpoint: four new status/configuration cases. The opt-in real-HCA typed configuration test separately proves immutable provider/QP status survives shutdown. Formatting, scoped tree-lint and FFI clippy pass. The legacy two-process - strict two-QP worker round trip also passes after extracting listener port - resolution. Readiness/runner acceptance and CLI integration remain pending; + strict two-QP worker round trip failed at its first client connection + (`rdma_worker_test.cpp:165`, null connection); diagnose this separately from + the passing typed-configuration HCA test before proceeding. The preceding + checkpoint prematurely recorded that last run as passing. + Readiness/runner acceptance and CLI integration remain pending; this checkpoint makes no new performance claim. +- The first divergence was an accidental environment-key rename while moving + portable configuration: the runner's `CROWDB_RDMA_WIRE_SLOT_CAPACITY=1024` + was ignored, leaving 1 MiB slots and rejecting its 1 MiB receive arena. + Restore the established key and add an exact environment-contract regression. + The corrected two-process/two-QP aggregate diagnostic passes 1000 requests + with zero RPC/CQ errors, and all 187 enabled native tests pass. Scoped + tree-lint passes; this is functional verification only. ### D. CLI coroutine workload and canonical results diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp index c861343aa..e41054195 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp @@ -33,7 +33,7 @@ bool RdmaConfig::apply_environment() }; return (port != 1 || read("CROWDB_RDMA_PORT", port)) && read("CROWDB_RDMA_SQ_DEPTH", sq_depth) && read("CROWDB_RDMA_RQ_DEPTH", rq_depth) && read("CROWDB_RDMA_POSTED_RECEIVE_TARGET", posted_receive_target) && - read("CROWDB_RDMA_DATAPATH_CAPACITY", wire_slot_capacity) && + read("CROWDB_RDMA_WIRE_SLOT_CAPACITY", wire_slot_capacity) && read("CROWDB_RDMA_SEND_ARENA_BYTES", send_arena_bytes) && read("CROWDB_RDMA_RECEIVE_ARENA_BYTES", receive_arena_bytes) && read("CROWDB_RDMA_CQ_DEPTH", cq_depth) && read("CROWDB_RDMA_REGISTERED_MEMORY_BUDGET_BYTES", registered_memory_budget_bytes) && diff --git a/lib/crowdb-rpc/tests/rdma_config_test.cpp b/lib/crowdb-rpc/tests/rdma_config_test.cpp index 80a86ce09..4fb9b7711 100644 --- a/lib/crowdb-rpc/tests/rdma_config_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_config_test.cpp @@ -34,6 +34,16 @@ TEST(RdmaConfigTest, SmallWireSlotsAllowDeepRqWithinSmallArena) EXPECT_FALSE(config.valid()); } +TEST(RdmaConfigTest, EnvironmentWireSlotCapacityMatchesRunnerContract) +{ + const TestEnvironment slot("CROWDB_RDMA_WIRE_SLOT_CAPACITY", "1024"); + const TestEnvironment receive_bytes("CROWDB_RDMA_RECEIVE_ARENA_BYTES", "1048576"); + RdmaConfig config; + ASSERT_TRUE(config.apply_environment()); + EXPECT_EQ(config.wire_slot_capacity, 1024U); + EXPECT_TRUE(config.valid()); +} + TEST(RdmaConfigTest, LargeWireSlotsAnd4096ReceivesUse64BitArenaBudget) { RdmaConfig config; From dcbc936068c51d4f582301e82e41aea148b45fea Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 00:05:43 +0800 Subject: [PATCH 033/258] Measure validated coroutine RPC completions on explicit TCP and RDMA paths --- doc/design/rpc/design-crowdb-rpc-rdma.md | 21 + doc/working/plan-rdma-rpc-transport.md | 50 ++- .../rust/src/metrics/histogram.rs | 6 + .../rust/tests/latency_histogram_test.rs | 7 + lib/crowdb-rpc/ffi/src/lib.rs | 2 + lib/crowdb-rpc/ffi/src/test_support.rs | 43 ++ .../crowdb-rpc/transport/socket_transport.h | 1 + .../src/transport/rdma/rdma_settings.cpp | 1 + .../src/transport/socket_connect.cpp | 2 + lib/crowdb-rpc/src/transport/status.cpp | 2 + .../tests/transport_config_test.cpp | 4 +- tools/rpc-test/src/args.rs | 179 +++++++- tools/rpc-test/src/workload.rs | 398 ++++++------------ tools/rpc-test/src/workload/control.rs | 59 +++ tools/rpc-test/src/workload/report.rs | 243 +++++++++++ tools/rpc-test/src/workload/runtime.rs | 172 ++++++++ tools/rpc-test/src/workload/status.rs | 58 +++ tools/rpc-test/src/workload/workload.rs | 228 ++++++++++ tools/rpc-test/tests/validated_echo_test.rs | 156 +++++++ 19 files changed, 1330 insertions(+), 302 deletions(-) create mode 100644 lib/crowdb-rpc/ffi/src/test_support.rs create mode 100644 tools/rpc-test/src/workload/control.rs create mode 100644 tools/rpc-test/src/workload/report.rs create mode 100644 tools/rpc-test/src/workload/runtime.rs create mode 100644 tools/rpc-test/src/workload/status.rs create mode 100644 tools/rpc-test/src/workload/workload.rs create mode 100644 tools/rpc-test/tests/validated_echo_test.rs diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index ed1d3b65f..35f3f01fc 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -63,6 +63,11 @@ partial control exchange closes it and uses a fresh socket. Strict RDMA policy permits this TCP control traffic and rejects TCP workload fallback. Disabled or unavailable local RDMA under strict policy fails before opening a socket. +Cold explicit configuration distinguishes a deliberately selected TCP transport +(`rdma_enabled_ = false`) from a failed RDMA attempt. That selection connects +directly over TCP even when `allow_tcp_fallback_ = false`. It does not count as +fallback. Explicit values override environment defaults before listen/start. + One logical RPC connection chooses RDMA as its preferred path and TCP as its one-way fallback. The path is selected before the first payload. TCP is chosen when RDMA setup fails before payload submission; an established TCP connection @@ -281,6 +286,22 @@ one 8192-entry CQ admits four QPs with the default send and posted-receive limits. `poller_count` follows the number of initialized CQs. Small-message benchmarks select 1024-byte slots and matching arena budgets explicitly. +The versioned C ABI and Rust `RdmaConfig` expose cold configuration. Owned +status JSON contains requested settings, actual QP/CQ/provider capacities, +transport errors, registration budgets and process resource counters. Connection +status caches immutable provider values before publication and retains them +after close; it never dereferences a QP during teardown. Unsupported resource +measurements are null. + +`bench rdma` requires strict RDMA workload connections and unique QPs. +`bench rpc` explicitly selects TCP. Both use the same coroutine echo workload, +nanosecond histogram and canonical result file. Each loader has one outstanding +RPC; one startup thread creates the coroutines, and completion workers resume +them. TPS counts validated response callbacks inside the measurement window; +warmup and drain are separate. Native RTT starts after request construction, +before submission, and ends before Rust validation. TCP status management +connections run outside the traffic interval and close before measurement. + The effective slot counts are bounded by registered bytes, QP send/receive depth, DiskIO queue bytes, and SDK-held slots. A configured slot size is a capacity limit, not a required wire length; SEND uses the actual message diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index aafe3d2fb..1fa7d042e 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -561,9 +561,10 @@ CQ verification: alone cannot represent different immutable offsets safely. Keep a parent reference until every frame has been dispatched. Defer a mixed capability/data aggregate if the application handler is not yet published. -- [~] **Expose aggregation observations**: export actual frames/SEND, wire +- [x] **Expose aggregation observations**: export actual frames/SEND, wire bytes, queue high-water and rejects. Keep RPC TPS distinct from transport - SEND rate; compare aggregation disabled/enabled for both TCP and RDMA. + SEND rate. Export TCP frames/writev and Nagle settings alongside RDMA SEND + aggregation; phase E owns the sustained comparisons. Files: transport statistics, C API/Rust FFI and benchmark reports. Aggregation checkpoint: @@ -666,7 +667,44 @@ Status/configuration checkpoint: ### D. CLI coroutine workload and canonical results -- [ ] **Add `bench rdma` with shared echo workload**: reuse `bench rpc` +Implementation checkpoint: + +- `bench rdma` and `bench rpc` now share the timed coroutine echo workload, + bounded loader correlation slots and per-completion-owner histogram shards. + `LatencyHistogram::capture_total` merges those shards after callbacks join. + Input uses the agreed queue/slot variables; CLI aliases preserve existing + `--loader-num`, `--connections` and `--value-size` invocations. +- Canonical JSON goes directly to `--output`; it records warmup/window/drain, + native submission failures and correlation counters, verified payloads, + unique QPs and native client/server status. Missing required status counters + invalidate a run. Resource/transport deltas explicitly cover warmup/drain; + only validated response callbacks inside the duration contribute to TPS. +- Four CLI integration cases pass for TCP coroutine/Tokio timing, corrupted + payload rejection, timeout reporting, unavailable strict RDMA and invalid + input. The initial integration run exposed explicit TCP being rejected by + the strict fallback guard. `rdma_enabled_` now distinguishes a deliberately + selected TCP connection from a failed RDMA attempt; legacy strict RDMA still + fails closed. The native regression tests this combination with environment + RDMA enabled and fallback disabled. All 187 enabled CTest cases and scoped + tree-lint pass. Common histogram merge tests and CLI clippy also pass. +- All four CLI integration cases pass on the final source; the four shared + histogram tests, workspace clippy, formatting and scoped tree-lint pass. + Full C++/FFI gate passes at + `.crowdb-runtime/artifacts/unit/cpp-b17dc30f9af94d00ad083d4062a3c79e/`. + The no-verbs build passes all 137 CTest cases and all 33 FFI tests pass. +- The primary regression script now launches the fb-server and release CLI, + consumes startup/status/result JSON files, and verifies errors, actual path + and unique QPs. Its first strict native HCA run uses one CQ/poller per peer, + one generic I/O worker per peer, 128-byte payload, 1024-byte wire slots, + SQ/RQ/post target 1024 and 1 MiB send/receive arenas per QP. Each row has + 1 s warmup and 10 s measured traffic, with zero errors/fallback: + one loader gives 58,570 TPS (avg 16.54 us, P90 16.13 us); 32 loaders give + 337,553 TPS with aggregation disabled and 436,374 TPS enabled. These are + initial single trials, not a maximum or repeatability claim. Wide canonical + tables, complete settings/counters and binary identities are retained at + `.crowdb-runtime/rdma-check/artifacts/bench/rdma-rpc-20261022T155651.230011Z/`. + +- [x] **Add `bench rdma` with shared echo workload**: reuse `bench rpc` coroutine machinery; expose `qpair_count`, loader count, warmup/duration, slot/aggregation and CQ/queue knobs using the same code variable names as the report. RDMA command requires RDMA and disables TCP fallback. Preserve @@ -681,7 +719,7 @@ Status/configuration checkpoint: transport status JSON in an owned C ABI buffer for FFI, and expose the same status through an fb-server benchmark control RPC while workload QPs remain live. Server readiness follows actual CM listen/start. -- [ ] **Validate and measure each completion**: expose response data to +- [x] **Validate and measure each completion**: expose response data to `CoBenchHandler`, verify request ID, length and payload, count submission rejects/retries/timeouts/connection/payload/duplicate errors, and reconcile C++ coroutine statistics with Rust results. Warm up for one second, measure @@ -695,7 +733,7 @@ Status/configuration checkpoint: completion can resume it (the lifecycle prerequisite is in phase A). Files: `co_client.cpp`, FFI `co_bench.rs`, CLI echo handler and lifecycle tests. -- [ ] **Write the canonical report**: emit a versioned JSON result directly +- [x] **Write the canonical report**: emit a versioned JSON result directly to a requested file, plus human output; serialize all schema columns above. Use explicit null for unavailable counters. Record measured errors and mark invalid on setup/client/server/report failure or RDMA path mismatch. Include @@ -714,7 +752,7 @@ Status/configuration checkpoint: failed/early-exit server, disabled fallback and actual argument propagation. Files: `tools/benchmark/bench-{rdma-,}rpc-regression.sh`, common runner and `tools/benchmark/tests/rdma_regression_test.py`. -- [ ] **Run gradual native hardware profiles**: first one coroutine/QP/CQ, +- [~] **Run gradual native hardware profiles**: first one coroutine/QP/CQ, then loader sweep 1/2/4/8/16/32 and higher as needed, then QP and CQ sweeps; compare aggregation off/on, SQ/RQ 1024 and 4096, small slot classes and larger payload/slot pairs. Increase depth only when counters show it limits load. diff --git a/lib/crowdb-common/rust/src/metrics/histogram.rs b/lib/crowdb-common/rust/src/metrics/histogram.rs index 8081ec8e6..d8752a704 100644 --- a/lib/crowdb-common/rust/src/metrics/histogram.rs +++ b/lib/crowdb-common/rust/src/metrics/histogram.rs @@ -106,6 +106,12 @@ impl LatencyHistogram { self.total.capture(false).snapshot(0) } + /// Capture a mergeable cumulative shard. Join writers for an exact result. + #[must_use] + pub fn capture_total(&self) -> HistogramData { + self.total.capture(false) + } + #[must_use] pub fn name(&self) -> &str { &self.name diff --git a/lib/crowdb-common/rust/tests/latency_histogram_test.rs b/lib/crowdb-common/rust/tests/latency_histogram_test.rs index baac8fe57..af6d6a1b2 100644 --- a/lib/crowdb-common/rust/tests/latency_histogram_test.rs +++ b/lib/crowdb-common/rust/tests/latency_histogram_test.rs @@ -77,4 +77,11 @@ fn concurrent_writers_have_exact_final_counts_and_sum() { let s = h.snapshot_total(); assert_eq!((s.count, s.sum, s.min, s.max), (40_000, 199_980_000, 0, 9999)); assert_eq!(s.zero_count, 4); + let captured = h.capture_total(); + let mut merged = HistogramData::default(); + merged.merge(&captured); + merged.merge(&captured); + let merged = merged.snapshot(0); + assert_eq!((merged.count, merged.sum), (80_000, 399_960_000)); + assert_eq!((merged.p90, merged.min, merged.max), (s.p90, 0, 9999)); } diff --git a/lib/crowdb-rpc/ffi/src/lib.rs b/lib/crowdb-rpc/ffi/src/lib.rs index 64a97ce7b..5821bdb95 100644 --- a/lib/crowdb-rpc/ffi/src/lib.rs +++ b/lib/crowdb-rpc/ffi/src/lib.rs @@ -20,6 +20,8 @@ pub mod logging; mod owned_route; mod server; pub mod sys; +#[cfg(feature = "test-util")] +mod test_support; mod transport; pub use buffer::{Buffer, BufferChain, BufferChainError, BufferLeaseMetadata, BufferOwnership, BufferPool}; diff --git a/lib/crowdb-rpc/ffi/src/test_support.rs b/lib/crowdb-rpc/ffi/src/test_support.rs new file mode 100644 index 000000000..32e8d2710 --- /dev/null +++ b/lib/crowdb-rpc/ffi/src/test_support.rs @@ -0,0 +1,43 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Controlled replies for callers testing response validation. + +use crate::{sys, RpcServer, ServerRequest}; + +impl RpcServer { + /// Register a synchronous reply fixture, keeping raw handles in the FFI. + pub fn register_reply_for_tests(&self, msg_type: u16, reply: F) + where + F: Fn(&ServerRequest) -> (Vec, Vec) + Send + Sync + 'static, + { + let server = self.handle() as usize; + self.register_handler(msg_type, move |request| { + let (control, data) = reply(&request); + let (Ok(control_len), Ok(data_len)) = (u32::try_from(control.len()), u32::try_from(data.len())) + else { + return; + }; + // SAFETY: synchronous dispatch keeps the server and connection + // alive; byte owners outlive the C ABI call, which copies them. + let status = unsafe { + sys::crowdb_rpc_server_submit_response( + server as sys::crowdb_rpc_server_t, + request.conn_handle, + control.as_ptr(), + control_len, + data.as_ptr(), + data_len, + msg_type, + request.request_id, + ) + }; + if status != sys::CROWDB_RPC_OK { + eprintln!( + "test reply submit failed: request={} status={status}", + request.request_id + ); + } + }); + } +} diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h index c000b0532..21de20e87 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h @@ -547,6 +547,7 @@ class SocketTransport : public Transport bool allow_tcp_fallback_{true}; RdmaConfig rdma_config_; bool rdma_explicit_configuration_ = false; + bool rdma_enabled_ = false; bool rdma_listen_enabled_ = false; std::atomic started_{false}; #ifdef CROWDB_RPC_HAVE_RDMA diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_settings.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_settings.cpp index fe2e564f5..ccb35786e 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_settings.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_settings.cpp @@ -82,6 +82,7 @@ bool SocketTransport::configure_rdma(RdmaConfig config, bool enabled, bool allow return false; #endif rdma_config_ = std::move(config); + rdma_enabled_ = enabled; allow_tcp_fallback_ = allow_tcp_fallback; rdma_listen_enabled_ = enabled && listen_enabled; rdma_explicit_configuration_ = true; diff --git a/lib/crowdb-rpc/src/transport/socket_connect.cpp b/lib/crowdb-rpc/src/transport/socket_connect.cpp index df4ddb3a1..fa6cd13d5 100644 --- a/lib/crowdb-rpc/src/transport/socket_connect.cpp +++ b/lib/crowdb-rpc/src/transport/socket_connect.cpp @@ -18,6 +18,8 @@ namespace crowdb::rpc { std::shared_ptr SocketTransport::connect(const std::string &addr, int port) { + if (rdma_explicit_configuration_ && !rdma_enabled_) + return connect_tcp(addr, port); #ifdef CROWDB_RPC_HAVE_RDMA if (rdma_transport_ != nullptr && rdma_transport_->ready()) { auto &stats = rdma_transport_->stats(); diff --git a/lib/crowdb-rpc/src/transport/status.cpp b/lib/crowdb-rpc/src/transport/status.cpp index 008dafa3c..2101e575f 100644 --- a/lib/crowdb-rpc/src/transport/status.cpp +++ b/lib/crowdb-rpc/src/transport/status.cpp @@ -4,6 +4,7 @@ #include "../status_json.h" #include +#include #ifdef __linux__ # include #endif @@ -25,6 +26,7 @@ std::string SocketTransport::status_json() const { StatusJson status; status.number("schema_version", 1); + status.number("process_id", static_cast(::getpid())); status.number("io_engines", engines_.size()); status.number("io_workers", workers_.size()); status.number("connection_count", connection_count()); diff --git a/lib/crowdb-rpc/tests/transport_config_test.cpp b/lib/crowdb-rpc/tests/transport_config_test.cpp index f4e0d33dd..d78ee08ac 100644 --- a/lib/crowdb-rpc/tests/transport_config_test.cpp +++ b/lib/crowdb-rpc/tests/transport_config_test.cpp @@ -18,8 +18,8 @@ TEST(TransportConfigTest, ExplicitTcpOverridesEnvironmentAndRejectsLateChanges) const TestEnvironment strict("CROWDB_RDMA_ALLOW_TCP_FALLBACK", "0"); RpcServer server; RdmaConfig config; - ASSERT_TRUE(server.configure_rdma(config, false, true, false)); - EXPECT_NE(server.transport()->status_json().find("\"allow_tcp_fallback\":true"), std::string::npos); + ASSERT_TRUE(server.configure_rdma(config, false, false, false)); + EXPECT_NE(server.transport()->status_json().find("\"allow_tcp_fallback\":false"), std::string::npos); EXPECT_NE(server.transport()->status_json().find("\"rdma\":null"), std::string::npos); ASSERT_TRUE(server.listen("127.0.0.1", 0)); EXPECT_FALSE(server.configure_rdma(config, false, true, false)); diff --git a/tools/rpc-test/src/args.rs b/tools/rpc-test/src/args.rs index 8f2c08598..35ec177e9 100644 --- a/tools/rpc-test/src/args.rs +++ b/tools/rpc-test/src/args.rs @@ -1,51 +1,186 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! Explicit test transport parameters; no application deployment inputs. +//! Shared TCP/RDMA workload parameters and explicit native configuration. + +use std::path::PathBuf; + +use crowdb_rpc_ffi::RdmaConfig; +use serde::Serialize; use clap::ValueEnum; -#[derive(clap::Args, Debug)] +#[derive(clap::Args, Debug, Serialize)] pub struct RpcArgs { - /// Emit the structured workload result on stdout. + /// Emit the structured result on stdout. #[arg(long)] pub json: bool, - /// Run duration in seconds. - #[arg(long, default_value_t = 10)] + /// Require RDMA and disable TCP fallback. + #[arg(long)] + pub rdma: bool, + /// Measured completion window, excluding warmup and drain. + #[arg(long, visible_alias = "measure-secs", default_value_t = 10)] pub duration_secs: u64, - /// Number of loader coroutines / tasks. - #[arg(long, default_value_t = 1)] - pub loader_num: usize, - /// Number of client connections to the fb-server. #[arg(long, default_value_t = 1)] - pub connections: usize, - /// Echo payload size in bytes. - #[arg(long, default_value_t = 128)] - pub value_size: usize, - /// Client transport I/O engines (must match the fb-server). + pub warmup_secs: u64, + /// One outstanding RPC per coroutine loader. + #[arg(long, visible_alias = "loader-num", default_value_t = 1)] + pub loader_count: u32, + /// Independent TCP connections or RDMA queue pairs. + #[arg(long, visible_aliases = ["connections", "qpair-count"], default_value_t = 1)] + pub connection_count: u32, + #[arg(long, visible_alias = "value-size", default_value_t = 128)] + pub payload_bytes: usize, #[arg(long, default_value_t = 1)] pub io_engines: u32, - /// Client transport I/O worker threads (must match the fb-server). #[arg(long, default_value_t = 1)] pub io_workers: u32, - /// Concurrency model: `coroutine` (C++ coroutines) or `tokio`. #[arg(long, value_enum, default_value_t = BenchMode::Coroutine)] pub mode: BenchMode, - /// fb-server port to connect to. #[arg(long)] pub server_port: u16, - /// Address used for the TCP handshake and RDMA CM connection. #[arg(long, default_value = "127.0.0.1")] pub server_host: String, - /// Enable TCP Nagle (coalesce small frames per writev). - #[arg(long, default_value_t = false)] + #[arg(long)] pub enable_nagle: bool, - /// Metrics flush interval in seconds. 0 disables the metrics log. #[arg(long, default_value_t = 1)] pub metrics_interval: u64, + #[arg(long, default_value_t = 5)] + pub request_timeout_secs: u64, + /// Owned canonical JSON file; stdout remains human readable. + #[arg(long)] + pub output: Option, + #[arg(long, default_value = "manual")] + pub case_id: String, + #[arg(long, default_value_t = 1)] + pub repeat_index: u32, + #[command(flatten)] + #[serde(flatten)] + pub queues: QueueArgs, +} + +#[derive(clap::Args, Debug, Serialize)] +pub struct QueueArgs { + #[arg(long, default_value = "")] + pub device: String, + #[arg(long, default_value = "")] + pub source_address: String, + #[arg(long, default_value = "")] + pub gid: String, + #[arg(long, default_value_t = 1)] + pub hca_port: u32, + #[arg(long, default_value_t = 1)] + pub cq_count: u32, + /// Zero derives enough entries for every QP's SQ and posted RQ. + #[arg(long, default_value_t = 0)] + pub cq_depth: u32, + #[arg(long, default_value_t = 1024)] + pub sq_depth: u32, + #[arg(long, default_value_t = 1024)] + pub rq_depth: u32, + #[arg(long, default_value_t = 1024)] + pub posted_receive_target: u32, + #[arg(long, default_value_t = 1024)] + pub wire_slot_capacity: u32, + /// Zero derives `sq_depth * wire_slot_capacity` per QP. + #[arg(long, default_value_t = 0)] + pub send_arena_bytes: u64, + /// Zero derives `posted_receive_target * wire_slot_capacity` per QP. + #[arg(long, default_value_t = 0)] + pub receive_arena_bytes: u64, + #[arg(long, default_value_t = 0)] + pub registered_memory_budget_bytes: u64, + #[arg(long, default_value_t = 256)] + pub poll_batch: u32, + #[arg(long, default_value_t = 32)] + pub verbs_poll_batch: u32, + #[arg(long, default_value_t = 256)] + pub poll_empty_budget: u32, + #[arg(long, default_value_t = 4096)] + pub send_queue_capacity: u32, + #[arg(long)] + pub aggregation_enabled: bool, + /// Zero uses `wire_slot_capacity`; peer admission may lower this further. + #[arg(long, default_value_t = 0)] + pub aggregation_max_bytes: u32, + #[arg(long, default_value_t = 256)] + pub aggregation_max_frames: u32, +} + +impl RpcArgs { + pub(super) fn resolve(&mut self, rdma: bool) -> Result { + if self.duration_secs == 0 + || self.duration_secs > 3600 + || self.warmup_secs > 3600 + || self.loader_count == 0 + || self.loader_count > (1 << 20) + || self.connection_count == 0 + || self.connection_count > self.loader_count + || self.io_engines == 0 + || self.io_engines > 32 + || self.io_workers == 0 + || self.io_workers > 32 + || self.io_workers % self.io_engines != 0 + || self.request_timeout_secs == 0 + || self.request_timeout_secs > 60 + || self.payload_bytes > (1 << 20) + { + return Err("invalid duration, loader/connection count, workers, timeout or payload size".into()); + } + let q = &mut self.queues; + if q.cq_count == 0 || q.cq_count > 256 { + return Err("cq_count must be in 1..=256".into()); + } + if q.cq_depth == 0 { + q.cq_depth = self + .connection_count + .div_ceil(q.cq_count) + .checked_mul( + q.sq_depth + .checked_add(q.posted_receive_target) + .ok_or("queue entry overflow")?, + ) + .ok_or("cq_depth overflow")?; + } + if q.send_arena_bytes == 0 { + q.send_arena_bytes = u64::from(q.sq_depth) * u64::from(q.wire_slot_capacity); + } + if q.receive_arena_bytes == 0 { + q.receive_arena_bytes = u64::from(q.posted_receive_target) * u64::from(q.wire_slot_capacity); + } + if q.aggregation_max_bytes == 0 { + q.aggregation_max_bytes = q.wire_slot_capacity; + } + Ok(RdmaConfig { + enabled: rdma, + allow_tcp_fallback: false, + listen_enabled: false, + device: q.device.clone(), + source_address: q.source_address.clone(), + gid: q.gid.clone(), + port: q.hca_port, + cq_count: q.cq_count, + cq_depth: q.cq_depth, + sq_depth: q.sq_depth, + rq_depth: q.rq_depth, + posted_receive_target: q.posted_receive_target, + wire_slot_capacity: q.wire_slot_capacity, + send_arena_bytes: q.send_arena_bytes, + receive_arena_bytes: q.receive_arena_bytes, + registered_memory_budget_bytes: q.registered_memory_budget_bytes, + poll_batch: q.poll_batch, + verbs_poll_batch: q.verbs_poll_batch, + poll_empty_budget: q.poll_empty_budget, + send_queue_capacity: q.send_queue_capacity, + aggregation_enabled: q.aggregation_enabled, + aggregation_max_bytes: q.aggregation_max_bytes, + aggregation_max_frames: q.aggregation_max_frames, + }) + } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum, Serialize)] +#[serde(rename_all = "lowercase")] pub enum BenchMode { Coroutine, Tokio, diff --git a/tools/rpc-test/src/workload.rs b/tools/rpc-test/src/workload.rs index 24cfb72aa..e455df431 100644 --- a/tools/rpc-test/src/workload.rs +++ b/tools/rpc-test/src/workload.rs @@ -1,303 +1,157 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! Standalone test transport measurement over the real crowdb-rpc echo server. -//! -//! Connects to a standalone `crowdb-rpc-fb-server` (deployed via -//! the owning test fixture) and fires `msg_type=100` echo -//! requests at it for `--duration-secs`. Two concurrency models: -//! -//! - `coroutine` (default): C++ coroutines via -//! `crowdb_rpc_ffi::co_spawn`. One heap-allocated coroutine frame per -//! loader, zero per-op tokio scheduling — the high-throughput path. -//! Latency is recorded from the C++ `on_response` callback directly -//! into the shared [`BenchRecorder`]. -//! - `tokio`: `run_workload` spawns one tokio task per loader, each -//! looping `RpcClient::call` + `.await`. Lower throughput (tokio -//! scheduler round-trip per op) but exercises the Rust async path. -//! -//! The client creates its own `RpcServer` (for connection management) -//! with I/O config matching the fb-server, then connects -//! `--connections` connections to it. +//! Shared timed echo benchmark for explicit TCP and strict RDMA transports. -use std::process::ExitCode; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use crowdb_protocol::fb::{ConnectionPingRequest, ConnectionPingRequestArgs}; -use crowdb_rpc_ffi::co_bench::{co_spawn, CoBenchHandler}; -use crowdb_rpc_ffi::{Buffer, Connection, RpcClient, RpcError, RpcServer}; -use flatbuffers::FlatBufferBuilder; +mod control; +mod report; +mod runtime; +mod status; +mod workload; use super::args::{BenchMode, RpcArgs}; -use super::loader::{run_workload, BenchRecorder}; -use super::metrics::BenchMetrics; -use super::result::{BenchOps, BenchResult}; -use std::path::Path; - -/// Echo `msg_type` registered on the fb-server (`fb_server.cpp`). -const ECHO_MSG_TYPE: u16 = 100; -pub async fn run(log_dir: &Path, args: RpcArgs) -> ExitCode { - let data_bytes = if args.value_size == 0 { - Vec::new() - } else { - vec![0xAB; args.value_size] - }; +use std::process::ExitCode; +use std::sync::Arc; +use std::time::Duration; - // Client-side server for connection management. - let server = Arc::new(RpcServer::with_engines(None, args.io_engines, args.io_workers)); - server.set_tcp_nodelay(!args.enable_nagle); - if server.listen("127.0.0.1", 0).is_err() { - eprintln!("error: client rpc server listen failed"); - return ExitCode::from(2); - } - server.start(); +use crowdb_common::logging::open_named_log; +use crowdb_common::metrics::{MetricsRegistry, MetricsRunner}; +use crowdb_rpc_ffi::RdmaConfig; +use serde_json::{json, Value}; - // Connect `connections` connections to the fb-server. - let mut conns: Vec = Vec::with_capacity(args.connections.max(1)); - for _ in 0..args.connections.max(1) { - match server.connect(&args.server_host, i32::from(args.server_port)) { - Ok(c) => conns.push(c), - Err(e) => { - eprintln!( - "error: connect to fb-server {}:{} failed: {e}", - args.server_host, args.server_port - ); - server.stop(); - return ExitCode::from(2); +use std::path::Path; +use workload::EchoWorkload; + +pub async fn run(log_dir: &Path, mut args: RpcArgs) -> ExitCode { + let rdma = args.rdma; + let config = args.resolve(rdma); + let mut report = report::base(&args, rdma); + match config { + Ok(config) => { + if let Err(error) = run_case(log_dir, &args, config, rdma, &mut report).await { + report::fail(&mut report, &error); } } + Err(error) => report::fail(&mut report, &error), } - - let client = Arc::new(RpcClient::new()); - for c in &conns { - client.attach(c); - } - // Size the slab completion pool >= max in-flight (loaders). - let loaders_u32 = u32::try_from(args.loader_num.max(1)).unwrap_or(u32::MAX); - client.set_completion_pool_size(next_pow2(loaders_u32)); - client.start_reaper(30_000_000_000, 1_000_000_000); - - let mut metrics = BenchMetrics::new(log_dir, args.metrics_interval); - metrics.start(); - - let duration = Duration::from_secs(args.duration_secs); - let start = Instant::now(); - let recorder = Arc::clone(&metrics.recorder); - let recorder = match args.mode { - BenchMode::Coroutine => { - run_coroutine( - recorder, - Arc::clone(&client), - Arc::clone(&server), - conns, - args.loader_num, - duration, - data_bytes, - ) - .await + report["client_exit"] = json!(u8::from(report["valid"] != true)); + if let Some(path) = &args.output { + let result = serde_json::to_vec_pretty(&report) + .map_err(std::io::Error::other) + .and_then(|bytes| std::fs::write(path, bytes)); + if let Err(error) = result { + report::fail(&mut report, &format!("write result {}: {error}", path.display())); + report["report_errors"] = json!(1); } - BenchMode::Tokio => { - run_tokio( - recorder, - Arc::clone(&client), - Arc::clone(&server), - conns, - args.loader_num, - duration, - data_bytes, - ) - .await - } - }; - let duration_ms: u64 = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX); - - client.stop_reaper(); - server.stop(); - - metrics.stop().await; - - let snap = recorder.hist_snapshot(); - eprintln!( - "bench rpc: {} ops, {} errors, {}ms ({} mode)", - recorder.ops(), - recorder.errors(), - duration_ms, - match args.mode { - BenchMode::Coroutine => "coroutine", - BenchMode::Tokio => "tokio", - } - ); - - let result = BenchResult { - total_ops: recorder.ops(), - duration_ms, - total_errors: recorder.errors(), - correctness_errors: None, - by_op: BenchOps::default().write(snap), - client_transport_stats: None, - server_metrics: None, - }; - let json = serde_json::to_value(&result).unwrap_or_default(); - tracing::info!(report = %json, "bench_report"); + } if args.json { - println!("{json}"); + println!("{report}"); } else { - println!("{result}"); + println!( + "bench {}: completed={} TPS={} avg_ns={} p90_ns={} p99_ns={} errors={} valid={}", + if rdma { "rdma" } else { "rpc" }, + report["completed_in_window"], + report["rpc_tps"], + report["avg_ns"], + report["p90_ns"], + report["p99_ns"], + report["errors_total"], + report["valid"] + ); } - if result.total_ops > 0 && result.total_errors == 0 { + if report["valid"] == true { ExitCode::SUCCESS } else { - eprintln!("error: RPC benchmark did not complete an error-free workload"); + eprintln!("error: invalid benchmark result: {report}"); ExitCode::FAILURE } } -/// Build a `ConnectionPingRequest` control flatbuffer with `id=request_id`. -/// The echo handler extracts `request_id` from this `id` field during -/// parse and echoes it back in the response control — the response is -/// correlated by this `id` against the slab slot's `request_id`. -fn build_control(request_id: u64) -> Vec { - let mut fbb = FlatBufferBuilder::new(); - let req = ConnectionPingRequest::create( - &mut fbb, - &ConnectionPingRequestArgs { - id: request_id, - rpc_create_nano: 0, - }, - ); - fbb.finish(req, None); - fbb.finished_data().to_vec() -} - -/// Coroutine mode: drive `crowdb_rpc_ffi::co_spawn` on a blocking -/// thread. The C++ call blocks until all coroutines finish (deadline -/// driven by the handler returning false). -async fn run_coroutine( - recorder: Arc, - client: Arc, - server: Arc, - conns: Vec, - loader_num: usize, - duration: Duration, - data_bytes: Vec, -) -> Arc { - let handler = Arc::new(EchoHandler::new(Arc::clone(&recorder), data_bytes, duration)); - let num_co = u32::try_from(loader_num.max(1)).unwrap_or(u32::MAX); - - // co_spawn blocks; run on a blocking thread so the tokio runtime - // is not stalled. The handler callbacks fire on C++ coroutine - // threads during the call. - let _ = tokio::task::spawn_blocking(move || { - co_spawn(&client, &server, &conns, num_co, ECHO_MSG_TYPE, handler); - }) - .await; - recorder -} - -/// Tokio mode: one task per loader, each looping `RpcClient::call`. -async fn run_tokio( - recorder: Arc, - client: Arc, - server: Arc, - conns: Vec, - loader_num: usize, - duration: Duration, - data_bytes: Vec, -) -> Arc { - let conns = Arc::new(conns); - let next_id = Arc::new(AtomicU64::new(1)); - let run = run_workload( - recorder, - loader_num, - duration, - move |rec: Arc| { - let client = Arc::clone(&client); - let server = Arc::clone(&server); - let conns = Arc::clone(&conns); - let next_id = Arc::clone(&next_id); - let data_bytes = data_bytes.clone(); - async move { - let id = next_id.fetch_add(1, Ordering::Relaxed); - let conn_idx = usize::try_from(id).map_or(0, |v| v % conns.len()); - let conn = &conns[conn_idx]; - let ctrl = Buffer::from_bytes(&build_control(id)); - let data = Buffer::from_bytes(&data_bytes); - let t0 = Instant::now(); - match client.call(&server, conn, id, ctrl, Some(data), ECHO_MSG_TYPE) { - Ok(fut) => match fut.await { - Ok(_) => { - rec.record_ok( - t0.elapsed().as_micros().try_into().unwrap_or(u64::MAX), - data_bytes.len() as u64, - ); - } - Err(_) => rec.record_err(), - }, - Err(_) => rec.record_err(), - } - } - }, - || {}, - ) - .await; - run.recorder -} - -/// `CoBenchHandler` for the echo workload: builds per-request control -/// (with `id=request_id`) + reused data buffer, records latency / errors -/// into the shared recorder, and stops each coroutine at the deadline. -struct EchoHandler { - recorder: Arc, - data_bytes: Vec, - deadline: Instant, -} - -impl EchoHandler { - fn new(recorder: Arc, data_bytes: Vec, duration: Duration) -> Self { - Self { - recorder, - data_bytes, - deadline: Instant::now() + duration, - } +async fn run_case( + log_dir: &Path, + args: &RpcArgs, + config: RdmaConfig, + rdma: bool, + report: &mut Value, +) -> Result<(), String> { + let session = runtime::Session::new(args, &config)?; + let connections = session + .conns + .iter() + .map(|conn| { + let bytes = conn.transport_status_json().map_err(|e| e.to_string())?; + serde_json::from_slice(&bytes).map_err(|e| e.to_string()) + }) + .collect::, String>>()?; + let owners = report::paths(report, &connections, rdma)?; + runtime::validate_wire_bound(args, &report["client_connections"], rdma)?; + report["server_before"] = control::remote_status(args).await?; + report["client_before"] = control::local_status(&session.server)?; + status::validate(&report["client_before"], rdma)?; + status::validate(&report["server_before"], rdma)?; + let before = session.client.counters(); + let mut runner = metrics_runner(log_dir, args)?; + let handler = create_workload(args, &owners, runner.as_ref()); + if let Some(runner) = &mut runner { + runner.start(); } - - fn before_deadline(&self) -> bool { - Instant::now() < self.deadline + let native = runtime::drive(args, &session, handler.clone()).await?; + session.client.stop_reaper(); + if let Some(runner) = &mut runner { + runner.stop().await; } -} - -impl CoBenchHandler for EchoHandler { - fn build_request(&self, request_id: u64) -> Option<(Buffer, Buffer)> { - if !self.before_deadline() { - return None; - } - let ctrl = Buffer::from_bytes(&build_control(request_id)); - let data = Buffer::from_bytes(&self.data_bytes); - Some((ctrl, data)) + let mut counters = session.client.counters(); + counters.submit_fail = counters.submit_fail.saturating_sub(before.submit_fail); + counters.submit_retry = counters.submit_retry.saturating_sub(before.submit_retry); + counters.resp_missed = counters.resp_missed.saturating_sub(before.resp_missed); + counters.reaped = counters.reaped.saturating_sub(before.reaped); + report["client_after"] = control::local_status(&session.server)?; + report["server_after"] = control::remote_status(args).await?; + report::finish(report, args, &handler, &native, &counters); + status::validate(&report["client_after"], rdma)?; + status::validate(&report["server_after"], rdma)?; + if session.conns.iter().any(|conn| !conn.is_open()) { + return Err("workload connection closed before final status collection".into()); } - - fn on_response(&self, _request_id: u64, status: Result<(), RpcError>, latency_ns: u64) -> bool { - if status.is_ok() { - self.recorder - .record_ok_ns(latency_ns, self.data_bytes.len() as u64); - } else { - self.recorder.record_err(); - } - self.before_deadline() + if args.mode == BenchMode::Tokio { + report["latency_scope"] = + json!("Rust pre-call to awaited response; Tokio scheduling included; validation excluded"); } + Ok(()) } -/// Next power of two >= n (min 1). -fn next_pow2(n: u32) -> u32 { - if n <= 1 { - return 1; +fn create_workload(args: &RpcArgs, owners: &[usize], runner: Option<&MetricsRunner>) -> Arc { + let create = |registry: &mut MetricsRegistry| { + Arc::new(EchoWorkload::new( + args.loader_count, + owners, + args.payload_bytes, + Duration::from_secs(args.warmup_secs), + Duration::from_secs(args.duration_secs), + registry, + )) + }; + if let Some(runner) = runner { + create(&mut runner.registry().lock().expect("metrics registry")) + } else { + create(&mut MetricsRegistry::new()) } - let mut p = 1u32; - while p < n { - p = p.saturating_mul(2); +} + +fn metrics_runner(log_dir: &Path, args: &RpcArgs) -> Result, String> { + if args.metrics_interval == 0 { + return Ok(None); } - p + let file = open_named_log(log_dir, "rpc-test-metrics", 50, 1)?; + let mut runner = MetricsRunner::new(file, args.metrics_interval); + runner.set_cpp_flush(|writer, seconds, stamp, width, count_w, tps_w| { + let width = width.max(crowdb_rpc_ffi::cpp_global_metrics_max_name_len()); + if let Some(text) = + crowdb_rpc_ffi::flush_cpp_global_metrics(seconds, stamp, "cpp-rpc", width, count_w, tps_w) + { + let _ = std::io::Write::write_all(writer, text.as_bytes()); + } + }); + Ok(Some(runner)) } diff --git a/tools/rpc-test/src/workload/control.rs b/tools/rpc-test/src/workload/control.rs new file mode 100644 index 000000000..570c77c3f --- /dev/null +++ b/tools/rpc-test/src/workload/control.rs @@ -0,0 +1,59 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Status exchanges run outside traffic windows on an explicit TCP connection. + +use std::time::Duration; + +use crowdb_rpc_ffi::{Buffer, RdmaConfig, RpcClient, RpcServer}; +use serde_json::Value; + +use super::{workload::build_control, RpcArgs}; + +pub(super) async fn remote_status(args: &RpcArgs) -> Result { + let server = RpcServer::with_engines(None, 1, 1); + server + .configure_rdma(&RdmaConfig { + allow_tcp_fallback: false, + ..RdmaConfig::default() + }) + .map_err(|e| format!("TCP control configuration: {e}"))?; + server + .listen("127.0.0.1", 0) + .map_err(|e| format!("TCP control listen: {e}"))?; + server.start(); + let client = RpcClient::new(); + client.set_completion_pool_size(1024); + client.start_reaper(2_000_000_000, 10_000_000); + let result = query(args, &server, &client).await; + client.stop_reaper(); + server.stop(); + result +} + +async fn query(args: &RpcArgs, server: &RpcServer, client: &RpcClient) -> Result { + let conn = server + .connect(&args.server_host, i32::from(args.server_port)) + .map_err(|e| format!("TCP control connect: {e}"))?; + client.attach(&conn); + let future = client + .call(server, &conn, 1, Buffer::from_vec(build_control(1)), None, 101) + .map_err(|e| format!("status submit: {e}"))?; + let response = tokio::time::timeout(Duration::from_secs(3), future) + .await + .map_err(|e| format!("status deadline: {e}"))? + .map_err(|e| format!("status response: {e}"))?; + let data = response.data.ok_or("status response has no data")?; + let status: Value = serde_json::from_slice(data.as_slice()).map_err(|e| format!("status JSON: {e}"))?; + if status["schema_version"] != 1 { + return Err("unsupported server status schema".into()); + } + Ok(status) +} + +pub(super) fn local_status(server: &RpcServer) -> Result { + let bytes = server + .transport_status_json() + .map_err(|e| format!("local status: {e}"))?; + serde_json::from_slice(&bytes).map_err(|e| format!("local status JSON: {e}")) +} diff --git a/tools/rpc-test/src/workload/report.rs b/tools/rpc-test/src/workload/report.rs new file mode 100644 index 000000000..67c1c66d3 --- /dev/null +++ b/tools/rpc-test/src/workload/report.rs @@ -0,0 +1,243 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Canonical machine result; unavailable measurements stay null. + +use std::collections::BTreeSet; + +use crowdb_rpc_ffi::co_bench::CoStats; +use crowdb_rpc_ffi::sys::CrowdbRpcClientCounters; +use serde_json::{json, Value}; + +use super::workload::{Count, EchoWorkload}; +use super::RpcArgs; + +pub(super) fn base(args: &RpcArgs, rdma: bool) -> Value { + let mut row = json!({ + "schema_version": 1, "case_id": args.case_id, "repeat_index": args.repeat_index, + "run_id": chrono::Utc::now().to_rfc3339(), "requested_path": if rdma {"rdma"} else {"tcp"}, + "control_path": "tcp", "client_mode": args.mode, "parameters": args, + "loader_count": args.loader_count, "connection_count": args.connection_count, + "producer_threads": 0, "coroutine_starter_threads": 1, "in_flight_per_loader": 1, "logical_ops_per_rpc": 1, + "payload_bytes": args.payload_bytes, "warmup_s": args.warmup_secs, "measure_s": args.duration_secs, + "allow_tcp_fallback": false, "valid": false, "client_connections": [], + "latency_scope": "native post-build/pre-submit to response callback; Rust validation excluded", + "transport_counter_scope": "all traffic including warmup/drain and capability frames", + "resource_counter_scope": "all traffic including warmup/drain", + }); + for name in [ + "actual_path", + "qpair_count", + "fallback_count", + "submitted", + "submitted_in_window", + "wire_bytes", + "avg_ns", + "p50_ns", + "p90_ns", + "p95_ns", + "p99_ns", + "p999_ns", + "min_ns", + "max_ns", + "submit_rejects", + "timeouts", + "connection_errors", + "cq_errors", + "payload_mismatches", + "duplicate_completions", + "retries", + "response_errors", + "client_exit", + "server_exit", + "failure", + "client_before", + "client_after", + "server_before", + "server_after", + "frames_per_send", + "server_frames_per_send", + "epoll_wakeups", + "rnr_provider_counters", + "topology", + "git_sha", + "binary_hashes", + "cpu_hca_identity", + ] { + row[name] = Value::Null; + } + for name in [ + "completed_in_window", + "warmup_completed", + "drain_completed", + "rpc_tps", + "logical_ops_s", + "payload_Bps", + "latency_count", + "histogram_zero_count", + "histogram_overflow_count", + "errors_total", + "setup_errors", + "report_errors", + ] { + row[name] = json!(0); + } + row +} + +pub(super) fn paths(report: &mut Value, connections: &[Value], rdma: bool) -> Result, String> { + let expected = if rdma { "rdma" } else { "tcp" }; + let mut qpairs = BTreeSet::new(); + let mut owners = Vec::new(); + for (index, connection) in connections.iter().enumerate() { + if connection["path"] != expected || connection["open"] != true { + return Err(format!("workload path mismatch: {connection}")); + } + if rdma { + let qp = connection["info"]["qpair_number"] + .as_u64() + .filter(|v| *v > 0) + .ok_or("missing actual QP number")?; + if !qpairs.insert(qp) { + return Err("RDMA connections share a QP".into()); + } + owners.push( + connection["info"]["cq_index"] + .as_u64() + .ok_or("missing CQ owner")? + .try_into() + .map_err(|_| "invalid CQ owner")?, + ); + } else { + owners.push(index); + } + } + report["actual_path"] = json!(expected); + report["qpair_count"] = json!(qpairs.len()); + report["client_connections"] = json!(connections); + Ok(owners) +} + +pub(super) fn finish( + report: &mut Value, + args: &RpcArgs, + workload: &EchoWorkload, + native: &CoStats, + counters: &CrowdbRpcClientCounters, +) { + let counts = workload.totals(); + let get = |c: Count| counts[c as usize]; + let histogram = workload.histogram(); + let rejects = native.total_errors.saturating_sub(get(Count::ResponseErrors)); + let accepted = get(Count::Attempts).saturating_sub(rejects); + let reconcile = native.total_ops != get(Count::Responses) + || native.total_errors < get(Count::ResponseErrors) + || accepted != native.total_ops + || workload.started_loaders() != args.loader_count as usize + || get(Count::LateLoaders) != 0 + || histogram.count == 0 + || histogram.overflow_count != 0; + let cq_errors = sum_rdma_delta(report, "cq_errors"); + let fallback = sum_rdma_delta(report, "fallback_count"); + let transport_errors: u64 = ["protocol_errors", "send_errors", "registration_failures"] + .into_iter() + .map(|field| sum_rdma_delta(report, field)) + .sum(); + let errors = native.total_errors + + get(Count::PayloadMismatches) + + get(Count::Duplicates) + + counters.resp_missed + + cq_errors + + fallback + + transport_errors + + u64::from(reconcile); + let tps = ratio(histogram.count, args.duration_secs); + let fields = json!({ + "submitted": accepted, "build_attempts": get(Count::Attempts), + "submitted_in_window": get(Count::SubmittedInWindow), + "completed_in_window": histogram.count, "window_callbacks": get(Count::Window), + "warmup_completed": get(Count::Warmup), "drain_completed": get(Count::Drain), + "callbacks_total": get(Count::Responses), "native_total_ops": native.total_ops, + "native_total_errors": native.total_errors, "started_loaders": workload.started_loaders(), + "late_loaders": get(Count::LateLoaders), "rpc_tps": tps, "logical_ops_s": tps, + "payload_Bps": tps * f64::from(u32::try_from(args.payload_bytes).unwrap_or(u32::MAX)), "latency_count": histogram.count, + "avg_ns": histogram.avg, "p50_ns": histogram.p50, "p90_ns": histogram.p90, + "p95_ns": histogram.p95, "p99_ns": histogram.p99, "p999_ns": histogram.p999, + "min_ns": histogram.min, "max_ns": histogram.max, "histogram_zero_count": histogram.zero_count, + "histogram_overflow_count": histogram.overflow_count, + }); + extend(report, &fields); + let fields = json!({ + "errors_total": errors, "submit_rejects": rejects, "response_errors": get(Count::ResponseErrors), + "timeouts": get(Count::Timeouts), "reaped": counters.reaped, + "connection_errors": get(Count::ConnectionErrors), "cq_errors": cq_errors, + "payload_mismatches": get(Count::PayloadMismatches), "duplicate_completions": get(Count::Duplicates), + "retries": counters.submit_retry, "native_submit_fail": counters.submit_fail, + "unmatched_responses": counters.resp_missed, "fallback_count": fallback, + "report_errors": u64::from(reconcile), "valid": errors == 0, + }); + extend(report, &fields); + for (side, output) in [ + ("client", "frames_per_send"), + ("server", "server_frames_per_send"), + ] { + let rdma = report["actual_path"] == "rdma"; + let frames = delta(report, side, "rpc_frames_sent", rdma); + let sends = delta( + report, + side, + if rdma { "sends" } else { "writev_progress_calls" }, + rdma, + ); + report[output] = if sends > 0 { + json!(ratio(frames, sends)) + } else { + Value::Null + }; + } + report["wire_bytes"] = json!(delta( + report, + "client", + "wire_bytes_sent", + report["actual_path"] == "rdma" + )); +} + +fn sum_rdma_delta(report: &Value, name: &str) -> u64 { + delta(report, "client", name, true) + delta(report, "server", name, true) +} + +fn delta(report: &Value, side: &str, name: &str, rdma: bool) -> u64 { + let read = |phase| { + let status = &report[format!("{side}_{phase}")]; + (if rdma { &status["rdma"] } else { status })[name] + .as_u64() + .unwrap_or(0) + }; + read("after").saturating_sub(read("before")) +} + +#[allow(clippy::cast_precision_loss)] +fn ratio(numerator: u64, denominator: u64) -> f64 { + numerator as f64 / denominator as f64 +} + +pub(super) fn fail(report: &mut Value, error: &str) { + report["failure"] = json!(error); + let category = if report["native_total_ops"].is_number() { + "report_errors" + } else { + "setup_errors" + }; + report[category] = json!(report[category].as_u64().unwrap_or(0) + 1); + report["errors_total"] = json!(report["errors_total"].as_u64().unwrap_or(0) + 1); + report["valid"] = json!(false); + report["client_exit"] = json!(1); +} + +fn extend(report: &mut Value, fields: &Value) { + report + .as_object_mut() + .expect("report object") + .extend(fields.as_object().expect("fields object").clone()); +} diff --git a/tools/rpc-test/src/workload/runtime.rs b/tools/rpc-test/src/workload/runtime.rs new file mode 100644 index 000000000..e21b84a81 --- /dev/null +++ b/tools/rpc-test/src/workload/runtime.rs @@ -0,0 +1,172 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Session ownership, admission and coroutine/Tokio scheduling. + +use std::sync::Arc; +use std::time::Instant; + +use crowdb_protocol::fb::{ConnectionPingResponse, ConnectionPingResponseArgs}; +use crowdb_rpc_ffi::co_bench::{co_spawn, CoBenchHandler, CoStats}; +use crowdb_rpc_ffi::{Connection, RdmaConfig, RpcClient, RpcServer}; +use flatbuffers::FlatBufferBuilder; +use serde_json::Value; + +use crate::args::BenchMode; +use super::workload::{build_control, Count, EchoWorkload}; +use super::RpcArgs; + +pub(super) struct Session { + pub server: Arc, + pub client: Arc, + pub conns: Arc>, +} + +impl Session { + pub fn new(args: &RpcArgs, config: &RdmaConfig) -> Result { + let server = Arc::new(RpcServer::with_engines(None, args.io_engines, args.io_workers)); + server + .configure_rdma(config) + .map_err(|e| format!("workload configuration: {e}"))?; + server.set_tcp_nodelay(!args.enable_nagle); + server.set_send_queue_capacity(args.queues.send_queue_capacity); + server + .listen("127.0.0.1", 0) + .map_err(|e| format!("workload listen: {e}"))?; + server.start(); + let mut conns = Vec::new(); + for _ in 0..args.connection_count { + conns.push( + server + .connect(&args.server_host, i32::from(args.server_port)) + .map_err(|e| format!("workload connect: {e}"))?, + ); + } + let client = Arc::new(RpcClient::new()); + client.set_completion_pool_size(args.loader_count.next_power_of_two()); + for conn in &conns { + client.attach(conn); + } + client.start_reaper(args.request_timeout_secs * 1_000_000_000, 10_000_000); + Ok(Self { + server, + client, + conns: Arc::new(conns), + }) + } +} + +impl Drop for Session { + fn drop(&mut self) { + self.client.stop_reaper(); + self.server.stop(); + } +} + +pub(super) fn validate_wire_bound(args: &RpcArgs, connections: &Value, rdma: bool) -> Result<(), String> { + if !rdma { + return Ok(()); + } + let mut builder = FlatBufferBuilder::new(); + let response = ConnectionPingResponse::create( + &mut builder, + &ConnectionPingResponseArgs { + id: u64::MAX, + rpc_create_nano: u64::MAX, + response_create_nano: u64::MAX, + ..ConnectionPingResponseArgs::default() + }, + ); + builder.finish(response, None); + let control_bytes = builder.finished_data().len().max(build_control(u64::MAX).len()); + let frame_bytes = 14 + control_bytes + args.payload_bytes; + for connection in connections.as_array().ok_or("missing connection statuses")? { + let peer = connection["info"]["peer_receive_slot_capacity"] + .as_u64() + .ok_or("missing peer receive admission")?; + let bound = peer + .min(u64::from(args.queues.wire_slot_capacity)) + .min(u64::from(args.queues.aggregation_max_bytes)); + if frame_bytes as u64 > bound { + return Err(format!( + "frame_bytes={frame_bytes} exceeds negotiated wire bound={bound} (header/control included)" + )); + } + } + Ok(()) +} + +pub(super) async fn drive( + args: &RpcArgs, + session: &Session, + handler: Arc, +) -> Result { + let client = session.client.clone(); + let server = session.server.clone(); + let conns = session.conns.clone(); + let loaders = args.loader_count; + if args.mode == BenchMode::Coroutine { + tokio::task::spawn_blocking(move || co_spawn(&client, &server, &conns, loaders, 100, handler)) + .await + .map_err(|e| format!("coroutine join: {e}")) + } else { + drive_tokio(session, loaders, handler).await + } +} + +async fn drive_tokio(session: &Session, loaders: u32, handler: Arc) -> Result { + let mut tasks = Vec::new(); + for loader in 0..loaders { + let client = session.client.clone(); + let server = session.server.clone(); + let conns = session.conns.clone(); + let workload = handler.clone(); + tasks.push(tokio::spawn(async move { + let mut id = u64::from(loader); + let mut rejects = 0; + while let Some((control, data)) = workload.build_request(id) { + let started = Instant::now(); + if let Ok(future) = client.call( + &server, + &conns[loader as usize % conns.len()], + id, + control, + Some(data), + 100, + ) { + let response = future.await; + let nanos = u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX); + match response { + Ok(reply) => { + workload.on_response_data( + id, + Ok(()), + nanos, + reply.control.as_ref().map_or(&[], |b| b.as_slice()), + reply.data.as_ref().map_or(&[], |b| b.as_slice()), + ); + } + Err(error) => { + workload.on_response(id, Err(error), nanos); + } + } + } else { + rejects += 1; + tokio::task::yield_now().await; + } + id += u64::from(loaders.next_power_of_two()); + } + rejects + })); + } + let mut rejects = 0; + for task in tasks { + rejects += task.await.map_err(|e| format!("Tokio loader join: {e}"))?; + } + let counts = handler.totals(); + Ok(CoStats { + total_ops: counts[Count::Responses as usize], + total_errors: rejects + counts[Count::ResponseErrors as usize], + ..CoStats::default() + }) +} diff --git a/tools/rpc-test/src/workload/status.rs b/tools/rpc-test/src/workload/status.rs new file mode 100644 index 000000000..1409c28b6 --- /dev/null +++ b/tools/rpc-test/src/workload/status.rs @@ -0,0 +1,58 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Required counters must be present before a benchmark can be valid. + +use serde_json::Value; + +pub(super) fn validate(status: &Value, rdma: bool) -> Result<(), String> { + if status["schema_version"] != 1 { + return Err("unsupported transport status schema".into()); + } + for field in [ + "io_engines", + "io_workers", + "send_queue_rejects", + "rpc_frames_sent", + "wire_bytes_sent", + "writev_calls", + "writev_progress_calls", + ] { + if !status[field].is_u64() { + return Err(format!("missing transport counter: {field}")); + } + } + if rdma { + let native = &status["rdma"]; + if status["allow_tcp_fallback"] != false || native["ready"] != true { + return Err( + "RDMA benchmark requires a ready transport and disabled TCP fallback on both peers".into(), + ); + } + for field in [ + "cq_errors", + "protocol_errors", + "send_errors", + "submit_rejects", + "fallback_count", + "registration_failures", + "sends", + "rpc_frames_sent", + "wire_bytes_sent", + "poller_count", + "cq_count", + "registered_bytes", + ] { + if !native[field].is_u64() { + return Err(format!("missing RDMA counter: {field}")); + } + } + if native["fallback_count"] != 0 { + return Err("RDMA transport recorded TCP fallback".into()); + } + if native["poller_count"] != native["cq_count"] { + return Err("CQ/poller ownership count mismatch".into()); + } + } + Ok(()) +} diff --git a/tools/rpc-test/src/workload/workload.rs b/tools/rpc-test/src/workload/workload.rs new file mode 100644 index 000000000..a10e56008 --- /dev/null +++ b/tools/rpc-test/src/workload/workload.rs @@ -0,0 +1,228 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Bounded per-loader correlation and per-completion-owner measurement. + +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crowdb_common::metrics::histogram::HistogramData; +use crowdb_common::metrics::{HistogramSnapshot, LatencyHistogram, MetricsRegistry}; +use crowdb_protocol::fb::{ConnectionPingRequest, ConnectionPingRequestArgs, ConnectionPingResponse}; +use crowdb_rpc_ffi::co_bench::CoBenchHandler; +use crowdb_rpc_ffi::{Buffer, RpcError}; +use flatbuffers::FlatBufferBuilder; + +#[derive(Clone, Copy)] +pub(super) enum Count { + Attempts, + Responses, + Warmup, + Window, + Drain, + ResponseErrors, + Timeouts, + ConnectionErrors, + PayloadMismatches, + Duplicates, + SubmittedInWindow, + LateLoaders, +} + +const COUNT: usize = 12; +const EMPTY: u64 = u64::MAX; + +struct Shard { + counts: [AtomicU64; COUNT], + latency: Arc, +} + +struct Slot { + expected: AtomicU64, + started: AtomicBool, + submitted_in_window: AtomicBool, + shard: usize, +} + +pub(super) struct EchoWorkload { + slots: Vec, + shards: Vec, + mask: u64, + payload_bytes: usize, + pub measure_start: Instant, + pub measure_end: Instant, +} + +impl EchoWorkload { + pub fn new( + loaders: u32, + owners: &[usize], + payload_bytes: usize, + warmup: Duration, + measure: Duration, + registry: &mut MetricsRegistry, + ) -> Self { + let shard_count = owners.iter().copied().max().unwrap_or(0) + 1; + let measure_start = Instant::now() + warmup; + Self { + slots: (0..loaders) + .map(|i| Slot { + expected: AtomicU64::new(EMPTY), + started: AtomicBool::new(false), + submitted_in_window: AtomicBool::new(false), + shard: owners[i as usize % owners.len()], + }) + .collect(), + shards: (0..shard_count) + .map(|i| Shard { + counts: std::array::from_fn(|_| AtomicU64::new(0)), + latency: registry.register_histogram(format!("bench.rpc.owner{i}.latency.lh")), + }) + .collect(), + mask: u64::from(loaders.next_power_of_two() - 1), + payload_bytes, + measure_start, + measure_end: measure_start + measure, + } + } + + fn slot(&self, id: u64) -> &Slot { + &self.slots[usize::try_from(id & self.mask).expect("bounded loader index")] + } + + fn inc(&self, slot: &Slot, count: Count) { + self.shards[slot.shard].counts[count as usize].fetch_add(1, Ordering::Relaxed); + } + + pub fn totals(&self) -> [u64; COUNT] { + std::array::from_fn(|i| { + self.shards + .iter() + .map(|s| s.counts[i].load(Ordering::Relaxed)) + .sum() + }) + } + + pub fn started_loaders(&self) -> usize { + self.slots + .iter() + .filter(|s| s.started.load(Ordering::Relaxed)) + .count() + } + + pub fn histogram(&self) -> HistogramSnapshot { + let mut data = HistogramData::default(); + for shard in &self.shards { + data.merge(&shard.latency.capture_total()); + } + data.snapshot(0) + } + + fn complete( + &self, + id: u64, + status: Result<(), RpcError>, + latency_ns: u64, + control: &[u8], + data: &[u8], + ) -> bool { + let now = Instant::now(); + let slot = self.slot(id); + self.inc(slot, Count::Responses); + self.inc( + slot, + if now < self.measure_start { + Count::Warmup + } else if now < self.measure_end { + Count::Window + } else { + Count::Drain + }, + ); + if slot.submitted_in_window.load(Ordering::Relaxed) { + self.inc(slot, Count::SubmittedInWindow); + } + let correlated = slot.expected.swap(EMPTY, Ordering::AcqRel) == id; + if !correlated { + self.inc(slot, Count::Duplicates); + } + if let Err(error) = status { + self.inc(slot, Count::ResponseErrors); + match error { + RpcError::Timeout => self.inc(slot, Count::Timeouts), + RpcError::ConnectionClosed | RpcError::ConnectionError => { + self.inc(slot, Count::ConnectionErrors); + } + _ => {} + } + } else { + let valid = correlated && valid_response(id, self.payload_bytes, control, data); + if !valid { + self.inc(slot, Count::PayloadMismatches); + } + if valid && now >= self.measure_start && now < self.measure_end { + self.shards[slot.shard].latency.observe(latency_ns); + } + } + now < self.measure_end + } +} + +impl CoBenchHandler for EchoWorkload { + fn build_request(&self, id: u64) -> Option<(Buffer, Buffer)> { + let now = Instant::now(); + if now >= self.measure_end { + return None; + } + let slot = self.slot(id); + if !slot.started.swap(true, Ordering::Relaxed) && now >= self.measure_start { + self.inc(slot, Count::LateLoaders); + } + slot.expected.store(id, Ordering::Release); + slot.submitted_in_window + .store(now >= self.measure_start, Ordering::Relaxed); + self.inc(slot, Count::Attempts); + let mut bytes = vec![0xAB; self.payload_bytes]; + let prefix = bytes.len().min(8); + bytes[..prefix].copy_from_slice(&id.to_le_bytes()[..prefix]); + Some((Buffer::from_vec(build_control(id)), Buffer::from_vec(bytes))) + } + + fn on_response(&self, id: u64, status: Result<(), RpcError>, latency_ns: u64) -> bool { + self.complete(id, status, latency_ns, &[], &[]) + } + + fn on_response_data( + &self, + id: u64, + status: Result<(), RpcError>, + latency_ns: u64, + control: &[u8], + data: &[u8], + ) -> bool { + self.complete(id, status, latency_ns, control, data) + } +} + +pub(super) fn build_control(id: u64) -> Vec { + let mut builder = FlatBufferBuilder::new(); + let request = ConnectionPingRequest::create( + &mut builder, + &ConnectionPingRequestArgs { + id, + rpc_create_nano: 0, + }, + ); + builder.finish(request, None); + builder.finished_data().to_vec() +} + +fn valid_response(id: u64, size: usize, control: &[u8], data: &[u8]) -> bool { + let prefix = size.min(8); + data.len() == size + && data[..prefix] == id.to_le_bytes()[..prefix] + && data[prefix..].iter().all(|byte| *byte == 0xAB) + && flatbuffers::root::>(control) + .is_ok_and(|response| response.id() == id && response.ret().0 == 0) +} diff --git a/tools/rpc-test/tests/validated_echo_test.rs b/tools/rpc-test/tests/validated_echo_test.rs new file mode 100644 index 000000000..e237e870f --- /dev/null +++ b/tools/rpc-test/tests/validated_echo_test.rs @@ -0,0 +1,156 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::path::PathBuf; +use std::process::{Command, Output}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crowdb_protocol::fb::{ConnectionPingResponse, ConnectionPingResponseArgs}; +use crowdb_rpc_ffi::{RdmaConfig, RpcServer}; +use flatbuffers::FlatBufferBuilder; +use serde_json::Value; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +struct TestServer(RpcServer); + +impl TestServer { + fn new() -> Self { + let server = RpcServer::new(None); + server.configure_rdma(&RdmaConfig::default()).unwrap(); + server.listen("127.0.0.1", 0).unwrap(); + server.register_status_handler(101); + Self(server) + } +} + +impl Drop for TestServer { + fn drop(&mut self) { + self.0.stop(); + } +} + +fn run(server: &TestServer, verb: &str, args: &[&str]) -> (Output, Value) { + let root = std::env::temp_dir().join(format!( + "crowdb-rpc-bench-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&root).unwrap(); + let result: PathBuf = root.join("result.json"); + let output = Command::new(env!("CARGO_BIN_EXE_crowdb-rpc-test")) + .env("CROWDB_LOG_ROOT", root.join("logs")) + .env("CROWDB_RUNTIME_ROOT", &root) + .args([ + "--log-dir", + root.join("logs").to_str().unwrap(), + "--server-port", + &server.0.port().to_string(), + "--warmup-secs", + "1", + "--duration-secs", + "1", + "--metrics-interval", + "0", + "--output", + result.to_str().unwrap(), + ]) + .args(if verb == "rdma" { vec!["--rdma"] } else { Vec::new() }) + .args(args) + .output() + .unwrap(); + let report = std::fs::read(&result).unwrap_or_else(|error| panic!("{error}: {output:?}")); + let report = serde_json::from_slice(&report).unwrap(); + std::fs::remove_dir_all(root).unwrap(); + (output, report) +} + +#[test] +fn tcp_coroutine_and_tokio_validate_timed_completions() { + let server = TestServer::new(); + server.0.register_echo_handler(100); + server.0.start(); + for mode in ["coroutine", "tokio"] { + let (output, report) = run( + &server, + "rpc", + &["--loader-count", "8", "--connections", "2", "--mode", mode], + ); + assert!(output.status.success(), "{output:?}"); + assert_eq!(report["valid"], true, "{report}"); + assert_eq!(report["actual_path"], "tcp"); + assert_eq!(report["qpair_count"], 0); + assert_eq!(report["errors_total"], 0); + assert_eq!(report["measure_s"], 1); + let completed = report["completed_in_window"].as_u64().unwrap(); + assert!(completed > 0); + assert_eq!(report["latency_count"], completed); + let expected = f64::from(u32::try_from(completed).unwrap()); + assert!((report["rpc_tps"].as_f64().unwrap() - expected).abs() < f64::EPSILON); + assert_eq!(report["submitted"], report["native_total_ops"]); + assert_eq!( + report["warmup_completed"].as_u64().unwrap() + + report["window_callbacks"].as_u64().unwrap() + + report["drain_completed"].as_u64().unwrap(), + report["native_total_ops"].as_u64().unwrap() + ); + assert!(report["warmup_completed"].as_u64().unwrap() > 0); + assert!(report["server_after"]["cpu_user_ns"].is_u64()); + assert!(report["p90_ns"].as_u64().unwrap() <= report["max_ns"].as_u64().unwrap()); + } +} + +#[test] +fn payload_corruption_is_reported_and_never_counted_as_success() { + let server = TestServer::new(); + server.0.register_reply_for_tests(100, |request| { + let mut builder = FlatBufferBuilder::new(); + let response = ConnectionPingResponse::create( + &mut builder, + &ConnectionPingResponseArgs { + id: request.request_id, + ..ConnectionPingResponseArgs::default() + }, + ); + builder.finish(response, None); + (builder.finished_data().to_vec(), vec![0xFF; 128]) + }); + server.0.start(); + let (output, report) = run(&server, "rpc", &[]); + assert!(!output.status.success(), "{output:?}"); + assert_eq!(report["valid"], false); + assert_eq!(report["completed_in_window"], 0); + assert!(report["payload_mismatches"].as_u64().unwrap() > 0, "{report}"); + assert!(report["errors_total"].as_u64().unwrap() > 0); +} + +#[test] +fn response_timeouts_survive_into_invalid_result() { + let server = TestServer::new(); + server.0.register_handler(100, |_request| {}); + server.0.start(); + let (output, report) = run(&server, "rpc", &["--request-timeout-secs", "1"]); + assert!(!output.status.success(), "{output:?}"); + assert_eq!(report["valid"], false); + assert!(report["timeouts"].as_u64().unwrap() > 0, "{report}"); + assert!(report["native_total_errors"].as_u64().unwrap() > 0); +} + +#[test] +fn strict_rdma_and_invalid_input_produce_setup_failure_reports() { + let server = TestServer::new(); + server.0.register_echo_handler(100); + server.0.start(); + for (verb, args) in [ + ("rdma", vec!["--device", "crowdb-missing-device"]), + ("rpc", vec!["--loader-count", "0"]), + ] { + let (output, report) = run(&server, verb, &args); + assert!(!output.status.success(), "{output:?}"); + assert_eq!(report["valid"], false); + assert_eq!(report["setup_errors"], 1); + assert_eq!(report["allow_tcp_fallback"], false); + assert!(report["actual_path"].is_null()); + assert!(report["submit_rejects"].is_null()); + } +} From 54af2e031a535e9fea832f241a9cb0b8baf7b480 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 00:06:53 +0800 Subject: [PATCH 034/258] Unify RPC regression manifests and retain canonical benchmark evidence --- doc/working/plan-rdma-rpc-transport.md | 26 ++- tools/benchmark/bench-rdma-rpc-regression.sh | 143 +------------ tools/benchmark/bench-rpc-regression.sh | 64 +----- tools/benchmark/rpc_bench_cases.py | 106 ++++++++++ tools/benchmark/rpc_bench_process.py | 105 ++++++++++ tools/benchmark/rpc_bench_report.py | 125 ++++++++++++ tools/benchmark/rpc_regression.py | 165 +++++++++++++++ tools/benchmark/tests/rdma_regression_test.py | 189 +++++++++++++----- 8 files changed, 669 insertions(+), 254 deletions(-) create mode 100644 tools/benchmark/rpc_bench_cases.py create mode 100644 tools/benchmark/rpc_bench_process.py create mode 100644 tools/benchmark/rpc_bench_report.py create mode 100644 tools/benchmark/rpc_regression.py diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 1fa7d042e..7a62856e9 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -37,9 +37,9 @@ TCP control preflight now precedes CM setup; the compact CM offer binds the preflight identity/generation and peer receive admission. Strict policy has passed a two-QP real-HCA RPC round trip and a TCP-only peer rejection case. Bounded transport aggregation is implemented and verified below. -Still pending: `bench rdma`/shared coroutine -reporting, unified regression orchestration, failure injection, and the -two-container acceptance profiles. Do not treat the +The CLI workload and shared regression orchestration are now implemented. +Still pending: expanded sustained comparisons, remaining failure injection, +and the two-container acceptance profiles. Do not treat the historical benchmark comments or older subtask ledger as completed acceptance. Physical multi-node validation remains outside this scope in [the follow-up plan](plan-rdma-multi-node.md). @@ -598,7 +598,8 @@ Aggregation checkpoint: and workspace clippy pass. Full gate artifact: `.crowdb-runtime/artifacts/unit/cpp-00a90cd3faa7480f894c80dee7ab03bc/`. The no-verbs build passes all 133 CTest cases. Native statistics are present; - the status/FFI/CLI report exposure is the active continuation. + status/FFI/CLI exposure is verified in phase D; sustained comparisons are + the active continuation in phase E. Preflight verification: @@ -742,7 +743,7 @@ Implementation checkpoint: ### E. One runner, staged evidence and requirement completion -- [ ] **Unify regression orchestration**: keep the named RDMA regression +- [x] **Unify regression orchestration**: keep the named RDMA regression script as the entry point and share common orchestration with TCP regression. Launch the RPC server then the CLI client from one case manifest; wait for actual RDMA readiness, not just the TCP listen message. Save complete logs, @@ -752,6 +753,21 @@ Implementation checkpoint: failed/early-exit server, disabled fallback and actual argument propagation. Files: `tools/benchmark/bench-{rdma-,}rpc-regression.sh`, common runner and `tools/benchmark/tests/rdma_regression_test.py`. +- Runner checkpoint: both shell entry points dispatch to `rpc_regression.py`; + `rpc_bench_cases.py` resolves one JSON manifest and derives queue/arena + capacities, `rpc_bench_process.py` owns readiness and process lifetimes, and + `rpc_bench_report.py` validates/exports canonical JSON plus wide CSV/TSV. + Every child executable runs through Pixi. A native server PID from its owned + startup status permits graceful shutdown while preserving the actual exit + code. Binary hashes, source revision/dirty state, CPU/HCA/affinity and readable + provider counters accompany every row. The runner stops at an invalid case + and prints its full server/client logs. Twelve fixture contract tests pass, + including both entry points, malformed/forged reports, missing readiness, + duplicate QPs and early server exit. Shell syntax and Python compilation pass. + Native/container command construction shares this manifest; actual OCI + execution remains pending in the task below. TCP uses writev/Nagle controls; + RDMA uses bounded complete-frame SEND aggregation, so the sweep does not + pretend these two transport mechanisms are the same switch. - [~] **Run gradual native hardware profiles**: first one coroutine/QP/CQ, then loader sweep 1/2/4/8/16/32 and higher as needed, then QP and CQ sweeps; compare aggregation off/on, SQ/RQ 1024 and 4096, small slot classes and larger diff --git a/tools/benchmark/bench-rdma-rpc-regression.sh b/tools/benchmark/bench-rdma-rpc-regression.sh index 8f0b22da5..b9ed4d33a 100755 --- a/tools/benchmark/bench-rdma-rpc-regression.sh +++ b/tools/benchmark/bench-rdma-rpc-regression.sh @@ -1,138 +1,7 @@ #!/usr/bin/env bash -# RDMA RPC echo benchmark. Mirrors bench-rpc-regression.sh with an HCA-backed -# connection so transport throughput and latency can be compared directly. -# -# Historical run (2026-10-18, 3s per case, reported 0 errors): -# Audit on 2026-10-22 found TCP socket traffic matching the 128B counts. -# These values are not a verified RDMA baseline; retain them for provenance. -# payload workers/connections ops/s avg-us p50-us p99-us -# 128 B 1/1 43,161 22.1 65 65 -# 128 B 4/4 441,779 143.6 124 215 -# 128 B 8/8 732,084 695.0 675 917 -# 128 B 16/32 1,186,876 832.9 419 5,439 -# 1 KiB 1/1 43,003 22.2 65 65 -# 1 KiB 4/4 414,466 153.2 136 221 -# 1 KiB 8/8 707,931 718.3 704 954 -# 1 KiB 16/32 1,120,659 882.5 448 6,193 -# 4 KiB 1/1 39,625 24.0 65 65 -# 4 KiB 4/4 376,107 168.8 149 266 -# 4 KiB 8/8 582,940 872.2 811 1,810 -# 4 KiB 16/32 834,076 1,184.5 606 6,881 -# New RDMA results require strict path selection and a valid structured report. -# Record the exact HCA, binary, duration, payload, and concurrency configuration. -set -euo pipefail -cd "$(dirname "$0")/../.." - -RUN_STAMP=$(date +%Y%m%d-%H%M%S) -LOG_ROOT="${RDMA_BENCH_LOG_ROOT:-${CROWDB_RUNTIME_ROOT:-$(pwd)/.crowdb-runtime}/artifacts/bench/rdma-rpc-$RUN_STAMP}" -RESULTS_FILE="${RDMA_BENCH_RESULTS:-$LOG_ROOT/results.tsv}" -REGRESSION_LOG_ROOT="$LOG_ROOT" -source tools/benchmark/bench-regression-common.sh -export CROWDB_LOG_ROOT="$LOG_ROOT" -export CROWDB_RDMA_ALLOW_TCP_FALLBACK=0 -regression_init - -DURATION="${RDMA_BENCH_DURATION:-10}" -VALUE_SIZE="${RDMA_BENCH_VALUE_SIZE:-128}" -CASES="${RDMA_BENCH_CASES:-}" -ADDR="${CROWDB_RDMA_SOURCE_ADDR:-10.200.1.1}" -DEVICE="${CROWDB_RDMA_DEVICE:-rocep35s0f0}" -HCA_PORT="${CROWDB_RDMA_PORT:-1}" -PORT_OFFSET="${CROWDB_PORT_OFFSET:-0}" -PORT=$((19101 + PORT_OFFSET)) -SERVER_PID="" -SERVER_BIN="${RDMA_BENCH_SERVER_BIN:-lib/crowdb-rpc/build/crowdb-rpc-fb-server}" -CLI_BIN="${RDMA_BENCH_CLI_BIN:-target/release/crowdb-cli}" - -start_server() { - local workers="$1" - echo " [server] RDMA host=$ADDR port=$PORT device=$DEVICE workers=$workers" >&2 - CROWDB_RDMA_ENABLED=1 CROWDB_RDMA_E2E_PORT="$PORT" \ - CROWDB_RDMA_DEVICE="$DEVICE" CROWDB_RDMA_PORT="$HCA_PORT" \ - CROWDB_RDMA_SOURCE_ADDR="$ADDR" \ - "$SERVER_BIN" --host "$ADDR" --port "$PORT" \ - --io-engines 1 --io-workers "$workers" --logdir "$LOG_ROOT/server" \ - >"$LOG_ROOT/server.stdout" 2>&1 & - SERVER_PID=$! - for _ in $(seq 1 100); do - if rg -q '^listening port=' "$LOG_ROOT/server.stdout"; then return 0; fi - if ! kill -0 "$SERVER_PID" 2>/dev/null; then - cat "$LOG_ROOT/server.stdout" >&2 - return 1 - fi - sleep 0.1 - done - echo " [server] timeout waiting for readiness" >&2 - return 1 -} - -stop_server() { - if [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then - kill -TERM "$SERVER_PID" 2>/dev/null || true - wait "$SERVER_PID" 2>/dev/null || true - fi - SERVER_PID="" -} - -run_bench() { - local loaders="$1" connections="$2" label="$3" workers="$4" - if [[ -n "$CASES" && " $CASES " != *" $label "* ]]; then return; fi - echo ">>> $label (RDMA, workers=$workers) ..." - stop_server - mkdir -p "$LOG_ROOT/server" - start_server "$workers" - local output json client_exit=0 - output=$(CROWDB_RDMA_ENABLED=1 CROWDB_RDMA_LISTEN=0 CROWDB_RDMA_E2E_PORT="$PORT" \ - CROWDB_RDMA_DEVICE="$DEVICE" CROWDB_RDMA_PORT="$HCA_PORT" \ - CROWDB_RDMA_SOURCE_ADDR="$ADDR" CROWDB_PORT_OFFSET="$PORT_OFFSET" \ - pixi run -- "$CLI_BIN" bench rpc \ - --duration-secs "$DURATION" --loader-num "$loaders" --connections "$connections" \ - --mode coroutine \ - --value-size "$VALUE_SIZE" --io-engines 1 --io-workers "$workers" \ - --server-host "$ADDR" --server-port "$PORT" 2>&1) || client_exit=$? - printf '%s\n' "$output" >"$LOG_ROOT/$label.client.log" - json=$(echo "$output" | sed -n '/^{/,/^}/p') - if [[ -z "$json" ]] || ! jq -e '(.total_ops | type) == "number" and - (.total_errors | type) == "number" and (.duration_ms | type) == "number" and - (.by_op.write.latency_us.avg_us | type) == "number" and - (.by_op.write.latency_us.p50_us | type) == "number" and - (.by_op.write.latency_us.p99_us | type) == "number" and - .total_ops >= 0 and .total_errors >= 0 and .duration_ms > 0' <<<"$json" >/dev/null; then - printf '%s\n' "$output" >&2 - printf '%s\t%s\tNA\tNA\tNA\tNA\tNA\tNA\t%s\t1\t0\n' \ - "$label" "$workers" "$client_exit" >>"$RESULTS_FILE" - stop_server - echo " ERROR: missing or invalid structured benchmark report; client_exit=$client_exit" >&2 - return 1 - else - local ops avg p50 p99 p999 errors valid=0 - ops=$(echo "$json" | jq -r '.total_ops * 1000 / .duration_ms' | awk '{printf "%.0f", $1}') - avg=$(echo "$json" | jq -r '.by_op.write.latency_us.avg_us') - p50=$(echo "$json" | jq -r '.by_op.write.latency_us.p50_us') - p99=$(echo "$json" | jq -r '.by_op.write.latency_us.p99_us') - p999=$(echo "$json" | jq -r '.by_op.write.latency_us.p999_us') - errors=$(echo "$json" | jq -r '.total_errors') - if [[ "$client_exit" == 0 && "$errors" == 0 ]] && jq -e '.total_ops > 0' <<<"$json" >/dev/null; then - valid=1 - fi - echo " ops/s=$ops avg=${avg}us p50=${p50}us p99=${p99}us p999=${p999}us err=$errors" - printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t0\t%s\n' \ - "$label" "$workers" "$ops" "$avg" "$p50" "$p99" "$p999" "$errors" "$client_exit" "$valid" >>"$RESULTS_FILE" - stop_server - if [[ "$valid" != 1 ]]; then - echo " ERROR: failed benchmark profile; client_exit=$client_exit errors=$errors" >&2 - return 1 - fi - fi -} - -trap stop_server EXIT -mkdir -p "$LOG_ROOT" -printf 'label\twkr\tops_s\tavg_us\tp50_us\tp99_us\tp999_us\terrors\tclient_exit\treport_errors\tvalid\n' >"$RESULTS_FILE" -echo "=== RDMA RPC echo ($VALUE_SIZE B, ${DURATION}s, HCA $DEVICE:$HCA_PORT) ===" -run_bench 1 1 rdma_1w_1l_1c 1 -run_bench 64 4 rdma_4w_64l_4c 4 -run_bench 512 8 rdma_8w_512l_8c 8 -run_bench 1000 32 rdma_16w_1000l_32c 16 -echo "=== DONE ===" -echo "Results in $RESULTS_FILE" +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +# Shared coroutine workload, structured readiness and canonical result files. +set -Eeuo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/../.." +exec pixi run -- python tools/benchmark/rpc_regression.py --transport rdma "$@" diff --git a/tools/benchmark/bench-rpc-regression.sh b/tools/benchmark/bench-rpc-regression.sh index c5864addb..338793230 100755 --- a/tools/benchmark/bench-rpc-regression.sh +++ b/tools/benchmark/bench-rpc-regression.sh @@ -1,63 +1,7 @@ #!/usr/bin/env bash # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. -# Real echo transport regression; the test fixture owns each server lifetime. -set -euo pipefail -cd "$(dirname "$0")/../.." -unset CROWDB_ASAN -RUN_STAMP=$(date +%Y%m%d-%H%M%S) -LOG_ROOT="${RPC_BENCH_LOG_ROOT:-${CROWDB_RUNTIME_ROOT:-$(pwd)/.crowdb-runtime}/artifacts/bench/rpc-regression-$RUN_STAMP}" -RESULTS_FILE="${RPC_BENCH_RESULTS:-$LOG_ROOT/results.tsv}" -DURATION="${RPC_BENCH_DURATION:-20}" -VALUE_SIZE="${RPC_BENCH_VALUE_SIZE:-128}" -CASES="${RPC_BENCH_CASES:-}" -FAILURES=0 -mkdir -p "$LOG_ROOT" -pixi run cargo build --release -p crowdb-rpc-test -pixi run cmake --build lib/crowdb-rpc/build -j 4 --target crowdb-rpc-fb-server -printf 'label\twkr\tops_s\tavg_us\tp50_us\tp99_us\tnagle\terrors\n' > "$RESULTS_FILE" - -run_bench() { - local loaders="$1" connections="$2" label="$3" engines="${4:-1}" workers="${5:-1}" mode="${6:-coroutine}" nagle="${7:-0}" - if [ -n "$CASES" ] && [[ " $CASES " != *" $label "* ]]; then return; fi - local options=() code=0 output - if [ "$nagle" = 1 ]; then options+=(--enable-nagle); fi - echo ">>> $label" - output=$(pixi run python tools/rpc-test/run.py --log-dir "$LOG_ROOT/$label" \ - --duration-secs "$DURATION" --loader-num "$loaders" --connections "$connections" \ - --value-size "$VALUE_SIZE" --io-engines "$engines" --io-workers "$workers" \ - --mode "$mode" "${options[@]}") || code=$? - printf '%s\n' "$output" > "$LOG_ROOT/$label.json" - if [ "$code" -ne 0 ] || ! jq -e '.total_ops > 0 and .total_errors == 0' <<< "$output" >/dev/null; then - printf '%s\n' "$output" >&2 - echo "ERROR: $label failed (exit=$code)" >&2 - FAILURES=$((FAILURES + 1)) - return - fi - jq -r --arg label "$label" --arg workers "$workers" --arg nagle "$nagle" \ - '[$label, $workers, (.total_ops * 1000 / .duration_ms), .by_op.write.latency_us.avg_us, .by_op.write.latency_us.p50_us, .by_op.write.latency_us.p99_us, $nagle, .total_errors] | @tsv' \ - <<< "$output" >> "$RESULTS_FILE" -} - -# Standard regression sweep — single-engine, scale workers with load. -# Coroutine: all 4 configs. Tokio: only 1/64/1000 loaders (skip 512). -run_bench 1 1 "rpc_1e1w_1l_1c" 1 1 -run_bench 1 1 "rpc_1e1w_1l_1c_tokio" 1 1 tokio -run_bench 64 4 "rpc_1e4w_64l_4c" 1 4 -run_bench 64 4 "rpc_1e4w_64l_4c_tokio" 1 4 tokio -run_bench 512 8 "rpc_1e8w_512l_8c" 1 8 - -# High-concurrency: 1000T, single-engine 16 workers. -run_bench 1000 32 "rpc_1e16w_1000l_32c" 1 16 -run_bench 1000 32 "rpc_1e16w_1000l_32c_tokio" 1 16 tokio - -# Nagle-enabled (coroutine) — TCP coalescing batches multiple small -# frames per writev/read syscall. Run at 64/512/1000T to measure the -# nagle benefit across load levels. -run_bench 64 4 "rpc_1e4w_64l_4c_nagle" 1 4 coroutine 1 -run_bench 512 8 "rpc_1e8w_512l_8c_nagle" 1 8 coroutine 1 -run_bench 1000 32 "rpc_1e16w_1000l_32c_nagle" 1 16 coroutine 1 - -printf 'Results: %s\n' "$RESULTS_FILE" -cat "$RESULTS_FILE" -[ "$FAILURES" -eq 0 ] +# Shared coroutine workload, structured readiness and canonical result files. +set -Eeuo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/../.." +exec pixi run -- python tools/benchmark/rpc_regression.py --transport tcp "$@" diff --git a/tools/benchmark/rpc_bench_cases.py b/tools/benchmark/rpc_bench_cases.py new file mode 100644 index 000000000..6ce47e6a0 --- /dev/null +++ b/tools/benchmark/rpc_bench_cases.py @@ -0,0 +1,106 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +"""Shared case resolution for native and container TCP/RDMA benchmarks.""" + +import json +import os + + +DEFAULTS = dict(loader_count=1, connection_count=1, payload_bytes=128, + io_engines=1, io_workers=1, cq_count=1, sq_depth=1024, + rq_depth=1024, posted_receive_target=1024, wire_slot_capacity=1024, + poll_batch=256, verbs_poll_batch=32, poll_empty_budget=256, + send_queue_capacity=4096, aggregation_enabled=False, + aggregation_max_frames=256, enable_nagle=False, + warmup_secs=1, duration_secs=10, metrics_interval=1, + request_timeout_secs=5, registered_memory_budget_bytes=0) + +ENV_FIELDS = dict(cq_count="CQ_COUNT", cq_depth="CQ_DEPTH", sq_depth="SQ_DEPTH", + rq_depth="RQ_DEPTH", posted_receive_target="POSTED_RECEIVE_TARGET", + wire_slot_capacity="WIRE_SLOT_CAPACITY", send_arena_bytes="SEND_ARENA_BYTES", + receive_arena_bytes="RECEIVE_ARENA_BYTES", poll_batch="POLL_BATCH", + verbs_poll_batch="VERBS_POLL_BATCH", poll_empty_budget="POLL_EMPTY_BUDGET", + send_queue_capacity="SEND_QUEUE_CAPACITY", aggregation_enabled="AGGREGATION_ENABLED", + aggregation_max_bytes="AGGREGATION_MAX_BYTES", aggregation_max_frames="AGGREGATION_MAX_FRAMES", + registered_memory_budget_bytes="REGISTERED_MEMORY_BUDGET_BYTES") + + +def resolve(case, defaults=None): + resolved = DEFAULTS | (defaults or {}) | case + qps, cqs = resolved["connection_count"], resolved["cq_count"] + if not 0 < cqs <= 256 or not 0 < qps <= resolved["loader_count"]: + raise ValueError("invalid CQ, QP/connection or loader count") + slot = resolved["wire_slot_capacity"] + if not resolved.get("cq_depth"): + resolved["cq_depth"] = ((qps + cqs - 1) // cqs) * (resolved["sq_depth"] + resolved["posted_receive_target"]) + if not resolved.get("send_arena_bytes"): + resolved["send_arena_bytes"] = slot * resolved["sq_depth"] + if not resolved.get("receive_arena_bytes"): + resolved["receive_arena_bytes"] = slot * resolved["posted_receive_target"] + if not resolved.get("aggregation_max_bytes"): + resolved["aggregation_max_bytes"] = slot + return resolved + + +def load_cases(path, transport): + prefix = "RDMA" if transport == "rdma" else "RPC" + defaults = dict(duration_secs=int(os.environ.get(f"{prefix}_BENCH_DURATION", "10")), + payload_bytes=int(os.environ.get(f"{prefix}_BENCH_VALUE_SIZE", "128"))) + if path: + manifest = json.loads(path.read_text()) + if manifest.get("schema_version") != 1: + raise ValueError("unsupported case manifest schema") + defaults |= manifest.get("defaults", {}) + cases = manifest["cases"] + else: + cases = [dict(case_id=f"loaders-{n}", loader_count=n) for n in (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024)] + if transport == "rdma": + cases += [dict(case_id=f"qp-{n}-agg-{int(agg)}", loader_count=512, connection_count=n, + aggregation_enabled=agg) for agg in (False, True) for n in (1, 2, 4, 8)] + cases += [dict(case_id=f"cq-{n}", loader_count=512, connection_count=8, + cq_count=n, aggregation_enabled=True) for n in (2, 4, 8)] + cases += [dict(case_id="rq-4096", loader_count=512, connection_count=4, + cq_count=4, rq_depth=4096, posted_receive_target=4096, aggregation_enabled=True)] + cases += [dict(case_id=f"slot-{slot}", loader_count=512, aggregation_enabled=True, + wire_slot_capacity=slot) for slot in (65536, 262144)] + else: + cases += [dict(case_id=f"connections-{n}-nagle-{int(nagle)}", loader_count=512, + connection_count=n, enable_nagle=nagle) for nagle in (False, True) for n in (1, 2, 4, 8)] + cases += [dict(case_id=f"workers-{n}", loader_count=1000, connection_count=32, + io_workers=n, enable_nagle=True) for n in (2, 4, 8, 16, 32)] + selected = set(os.environ.get(f"{prefix}_BENCH_CASES", "").split()) + result = [resolve(case, defaults) for case in cases if not selected or case["case_id"] in selected] + names = [case["case_id"] for case in result] + if not result or len(set(names)) != len(names): + raise ValueError("case selection is empty or contains duplicate case_id values") + return result + + +def server_environment(case, transport, address, device, hca_port, port, runtime): + env = {key: value for key, value in os.environ.items() if not key.startswith("CROWDB_RDMA_")} + env.update(CROWDB_RUNTIME_ROOT=str(runtime), CROWDB_RDMA_ENABLED=str(int(transport == "rdma")), + CROWDB_LOG_ROOT=str(runtime / "logs"), + CROWDB_RDMA_ALLOW_TCP_FALLBACK="0", CROWDB_RDMA_E2E_PORT=str(port), + CROWDB_RDMA_DEVICE=device, CROWDB_RDMA_SOURCE_ADDR=address, + CROWDB_RDMA_PORT=str(hca_port), CROWDB_PORT_OFFSET="0") + for field, suffix in ENV_FIELDS.items(): + value = case[field] + env[f"CROWDB_RDMA_{suffix}"] = str(int(value)) if isinstance(value, bool) else str(value) + return env + + +def client_arguments(case): + keys = set(DEFAULTS) | set(ENV_FIELDS) | {"case_id", "repeat_index"} + args = [] + for key in sorted(keys): + if key not in case: + continue + value = case[key] + flag = "--" + key.replace("_", "-") + if isinstance(value, bool): + if value: + args.append(flag) + else: + args += [flag, str(value)] + return args diff --git a/tools/benchmark/rpc_bench_process.py b/tools/benchmark/rpc_bench_process.py new file mode 100644 index 000000000..4ad0efbea --- /dev/null +++ b/tools/benchmark/rpc_bench_process.py @@ -0,0 +1,105 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +"""Owned process groups and structured server readiness.""" + +import json +import os +from pathlib import Path +import signal +import subprocess +import time +import uuid + + +class Runtime: + def __init__(self, root, output, pixi, topology="native", image="crowdb-node:dev"): + self.root, self.output, self.pixi = root, output, pixi + self.topology, self.image = topology, image + self.names = {} + self.server_pid = None + + def command(self, argv, env, side, cpus=None): + command = (["taskset", "-c", cpus] if cpus else []) + [str(arg) for arg in argv] + if self.topology == "native": + return [self.pixi, "run", "--", *command] + name = "crowdb-rpc-bench-" + side + "-" + uuid.uuid4().hex[:12] + self.names[side] = name + lib = self.root / ".pixi/envs/default/lib" + docker = ["docker", "run", "--rm", "--init", "--name", name, "--network=host", + "--user", f"{os.getuid()}:{os.getgid()}", "--ulimit", "memlock=-1:-1", + "--volume", f"{self.root}:{self.root}:ro", "--volume", f"{self.output}:{self.output}:rw", + "--volume", f"{lib}:/tmp/crowdb-libs:ro", "--workdir", str(self.root)] + if self.topology == "containers": + docker += ["--device", "/dev/infiniband/rdma_cm"] + for device in sorted(Path("/dev/infiniband").glob("uverbs*")): + docker += ["--device", str(device)] + for key, value in env.items(): + if key.startswith("CROWDB_RDMA_") or key in ("CROWDB_RUNTIME_ROOT", "CROWDB_LOG_ROOT", "CROWDB_PORT_OFFSET"): + docker += ["--env", f"{key}={value}"] + docker += ["--env", "LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/opt/crowdb/lib:/tmp/crowdb-libs", + "--env", "LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libibverbs.so.1:/usr/lib/x86_64-linux-gnu/librdmacm.so.1", + "--env", "RDMAV_DRIVERS_DIR=/opt/crowdb/lib/libibverbs", "--entrypoint", command[0], + self.image, *command[1:]] + return [self.pixi, "run", "--", *docker] + + def stop(self, process, side): + if process.poll() is not None: + return process.returncode + if side in self.names: + subprocess.run([self.pixi, "run", "--", "docker", "stop", "--time", "5", self.names[side]], + check=False, timeout=15) + else: + pid = self.server_pid if side == "server" else None + try: + if pid is not None and os.getpgid(pid) == process.pid: + os.kill(pid, signal.SIGTERM) + else: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + return process.wait(timeout=6) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + if side in self.names: + subprocess.run([self.pixi, "run", "--", "docker", "rm", "-f", self.names[side]], + check=False, timeout=15) + return process.wait(timeout=5) + + +def start(command, env, log, root): + return subprocess.Popen(command, cwd=root, env=env, stdout=log, stderr=subprocess.STDOUT, + start_new_session=True) + + +def ready(process, status_path, transport, timeout): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError(f"server exited before readiness: {process.returncode}") + try: + status = json.loads(status_path.read_text()) + if status.get("schema_version") != 1: + raise RuntimeError("unsupported startup status schema") + native = status.get("rdma") or {} + if transport == "rdma" and not (native.get("listen_active") is True + and native.get("ready") is True + and native.get("cm_port", 0) > 0 + and status.get("allow_tcp_fallback") is False): + raise RuntimeError("server did not establish a strict RDMA listener") + return status + except (FileNotFoundError, json.JSONDecodeError): + time.sleep(0.05) + raise RuntimeError("server readiness deadline expired") + + +def wait_client(client, server, timeout): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if client.poll() is not None: + return client.returncode + if server.poll() is not None: + raise RuntimeError(f"server exited during workload: {server.returncode}") + time.sleep(0.05) + raise RuntimeError("client deadline expired") diff --git a/tools/benchmark/rpc_bench_report.py b/tools/benchmark/rpc_bench_report.py new file mode 100644 index 000000000..3dfabb353 --- /dev/null +++ b/tools/benchmark/rpc_bench_report.py @@ -0,0 +1,125 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +"""Validate canonical rows and retain complete comparison tables.""" + +import csv +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import subprocess + + +def file_hash(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def identity(root, binaries, device, hca_port): + cpuinfo = Path("/proc/cpuinfo").read_text() if Path("/proc/cpuinfo").exists() else "" + cpu = next((line.split(":", 1)[1].strip() for line in cpuinfo.splitlines() + if line.startswith("model name")), platform.processor()) + return dict(git_sha=subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip(), + git_dirty=bool(subprocess.check_output(["git", "status", "--porcelain"], cwd=root)), + binary_hashes={side: file_hash(path) + for side, path in binaries.items()}, + cpu_hca_identity=dict(cpu=cpu, logical_cpus=os.cpu_count(), kernel=platform.release(), + architecture=platform.machine(), device=device, hca_port=hca_port, + runner_affinity=sorted(os.sched_getaffinity(0)) + if hasattr(os, "sched_getaffinity") else None)) + + +def provider_counters(device, port): + base = Path("/sys/class/infiniband") / device / "ports" / str(port) + result = {} + for group in ("counters", "hw_counters"): + for path in sorted((base / group).glob("*")): + try: + result[f"{group}.{path.name}"] = int(path.read_text().strip()) + except (ValueError, OSError): + result[f"{group}.{path.name}"] = None + return result or None + + +def validate(row, case, transport): + errors = [] + if row.get("schema_version") != 1: + errors.append("missing/unsupported report schema") + if type(row.get("valid")) is not bool: + errors.append("missing/invalid validity flag") + required = ("errors_total", "report_errors", "setup_errors", "completed_in_window", "latency_count", + "submit_rejects", "timeouts", "connection_errors", "cq_errors", "payload_mismatches", + "duplicate_completions", "retries", "histogram_overflow_count", "measure_s") + for name in required: + value = row.get(name) + if type(value) is not int or value < 0: + errors.append(f"missing/invalid {name}") + if row.get("actual_path") != transport or row.get("allow_tcp_fallback") is not False: + errors.append("actual path/fallback policy mismatch") + if row.get("fallback_count") != 0: + errors.append("missing/nonzero fallback count") + if row.get("case_id") != case["case_id"] or row.get("measure_s") != case["duration_secs"]: + errors.append("case identity/duration mismatch") + for name in ("loader_count", "connection_count", "payload_bytes"): + if row.get(name) != case[name]: + errors.append(f"workload mismatch: {name}") + if row.get("latency_count") != row.get("completed_in_window"): + errors.append("latency/completion count mismatch") + measure, completed = row.get("measure_s", 0), row.get("completed_in_window", 0) + tps = row.get("rpc_tps") + if (type(tps) not in (int, float) or not math.isfinite(tps) + or type(measure) is not int or measure <= 0 or type(completed) is not int + or not math.isclose(tps, completed / measure, rel_tol=1e-9, abs_tol=1e-6)): + errors.append("TPS does not use the measured completion window") + latency = [row.get(name) for name in ("min_ns", "p50_ns", "p90_ns", "p95_ns", "p99_ns", "p999_ns", "max_ns")] + if any(type(value) is not int or value < 0 for value in latency) or latency != sorted(latency): + errors.append("invalid/missing latency bounds") + if transport == "rdma": + connections = row.get("client_connections", []) + if not isinstance(connections, list) or not all(isinstance(conn, dict) for conn in connections): + return errors + ["missing/malformed connection statuses"] + qpairs = [(conn.get("info") or {}).get("qpair_number") if isinstance(conn.get("info"), dict) + else None for conn in connections] + if (len(qpairs) != case["connection_count"] or len(set(qpairs)) != len(qpairs) + or any(type(qp) is not int or qp <= 0 for qp in qpairs) + or any(conn.get("path") != "rdma" or conn.get("open") is not True for conn in connections) + or row.get("qpair_count") != case["connection_count"]): + errors.append("actual unique QP count/path mismatch") + if row.get("valid") is True and (row.get("errors_total") != 0 or not completed): + errors.append("successful report contradicts errors/completions") + return errors + + +def flatten(value, prefix=""): + result = {} + for key, item in value.items(): + name = f"{prefix}.{key}" if prefix else key + if isinstance(item, dict): + result.update(flatten(item, name)) + else: + result[name] = ("null" if item is None else json.dumps(item, separators=(",", ":")) + if isinstance(item, (list, bool)) else item) + return result + + +def write_tables(output, rows): + flat = [flatten(row) for row in rows] + leading = ["case_id", "repeat_index", "topology", "actual_path", "loader_count", "qpair_count", + "rpc_tps", "avg_ns", "p90_ns", "p99_ns", "errors_total", "report_errors", "valid"] + fields = leading + sorted(set().union(*(row.keys() for row in flat)) - set(leading)) + for suffix, delimiter in (("csv", ","), ("tsv", "\t")): + with (output / f"results.{suffix}").open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=fields, delimiter=delimiter) + writer.writeheader() + writer.writerows(flat) + (output / "results.json").write_text(json.dumps(dict(schema_version=1, results=rows), indent=2)) + + +def print_row(row): + print(f'{row["case_id"]}: TPS={row.get("rpc_tps")} avg_ns={row.get("avg_ns")} ' + f'p90_ns={row.get("p90_ns")} p99_ns={row.get("p99_ns")} ' + f'errors={row.get("errors_total")} report_errors={row.get("report_errors")} ' + f'valid={row["valid"]}', flush=True) diff --git a/tools/benchmark/rpc_regression.py b/tools/benchmark/rpc_regression.py new file mode 100644 index 000000000..0b71615b4 --- /dev/null +++ b/tools/benchmark/rpc_regression.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +"""One manifest, CLI workload and report contract for TCP/RDMA regressions.""" + +import argparse +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import shutil +import sys + +from rpc_bench_cases import client_arguments, load_cases, server_environment +from rpc_bench_process import Runtime, ready, start, wait_client +from rpc_bench_report import identity, print_row, provider_counters, validate, write_tables + + +ROOT = Path(__file__).resolve().parents[2] + + +def arguments(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--transport", choices=("rdma", "tcp"), default="rdma") + parser.add_argument("--manifest", type=Path) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--server-bin", type=Path) + parser.add_argument("--cli-bin", type=Path) + parser.add_argument("--address") + parser.add_argument("--device", default=os.environ.get("CROWDB_RDMA_DEVICE", "rocep35s0f0")) + parser.add_argument("--hca-port", type=int, default=int(os.environ.get("CROWDB_RDMA_PORT", "1"))) + parser.add_argument("--port", type=int, default=19101 + int(os.environ.get("CROWDB_PORT_OFFSET", "0"))) + parser.add_argument("--topology", choices=("native", "containers", "containers-no-rdma"), default="native") + parser.add_argument("--image", default=os.environ.get("CROWDB_RDMA_TEST_IMAGE", "crowdb-node:dev")) + parser.add_argument("--client-cpus") + parser.add_argument("--server-cpus") + parser.add_argument("--peak-repeats", type=int, default=int(os.environ.get("RPC_BENCH_PEAK_REPEATS", "3"))) + parser.add_argument("--startup-timeout", type=float, default=10) + args = parser.parse_args() + prefix = "RDMA" if args.transport == "rdma" else "RPC" + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") + runtime = Path(os.environ.get("CROWDB_RUNTIME_ROOT", ROOT / ".crowdb-runtime")) + args.output_dir = (args.output_dir or Path(os.environ.get(f"{prefix}_BENCH_LOG_ROOT", + runtime / f"artifacts/bench/{args.transport}-rpc-{stamp}"))).resolve() + args.server_bin = (args.server_bin or Path(os.environ.get(f"{prefix}_BENCH_SERVER_BIN", + ROOT / "lib/crowdb-rpc/build/crowdb-rpc-fb-server"))).resolve() + args.cli_bin = (args.cli_bin or Path(os.environ.get(f"{prefix}_BENCH_CLI_BIN", ROOT / "target/release/crowdb-rpc-test"))).resolve() + args.address = args.address or (os.environ.get("CROWDB_RDMA_SOURCE_ADDR", "10.200.1.1") + if args.transport == "rdma" else "127.0.0.1") + if args.peak_repeats < 0 or not 0 < args.port <= 65535: + parser.error("invalid peak repeats or service port") + return args + + +def run_case(args, case, index, metadata): + directory = args.output_dir / f"case-{index:03d}" + directory.mkdir() + runtime = Runtime(ROOT, args.output_dir, shutil.which("pixi"), args.topology, args.image) + env = server_environment(case, args.transport, args.address, args.device, args.hca_port, args.port, directory / "runtime") + status_file, report_file = directory / "server-startup.json", directory / "client-result.json" + server_argv = [args.server_bin, "--host", args.address, "--port", args.port, + "--io_engines", case["io_engines"], "--io_workers", case["io_workers"], + "--send_queue_capacity", case["send_queue_capacity"], "--status_file", status_file, + "--metrics_interval", case["metrics_interval"], "--logdir", directory / "server-log"] + if args.transport == "rdma": + server_argv.append("--require_rdma") + if case["enable_nagle"]: + server_argv.append("--enable_nagle") + server_command = runtime.command(server_argv, env, "server", args.server_cpus) + client_argv = [args.cli_bin, "--log-dir", directory / "client-log", "--mode", "coroutine", + "--server-host", args.address, "--server-port", args.port, "--output", report_file, + "--device", args.device, "--source-address", args.address, "--hca-port", args.hca_port, + *client_arguments(case)] + if args.transport == "rdma": + client_argv.append("--rdma") + client_command = runtime.command(client_argv, env, "client", args.client_cpus) + (directory / "commands.json").write_text(json.dumps(dict(server=server_command, client=client_command, + parameters=case), indent=2)) + row = dict(case_id=case["case_id"], repeat_index=case.get("repeat_index", 1), schema_version=1, + valid=False, errors_total=None, report_errors=0, setup_errors=0, + client_exit=None, server_exit=None, failure=None) + server = client = None + startup, failure = None, None + before = provider_counters(args.device, args.hca_port) + try: + with (directory / "server.log").open("w") as server_log, (directory / "client.log").open("w") as client_log: + server = start(server_command, env, server_log, ROOT) + startup = ready(server, status_file, args.transport, args.startup_timeout) + runtime.server_pid = startup.get("process_id") + client = start(client_command, env, client_log, ROOT) + wait_client(client, server, case["warmup_secs"] + case["duration_secs"] + 20 + case["connection_count"] * 3) + report = json.loads(report_file.read_text()) + if not isinstance(report, dict): + raise ValueError("client result must be an object") + row |= report + problems = validate(row, case, args.transport) + if problems: + previous = row.get("report_errors") + row["report_errors"] = (previous if type(previous) is int and previous >= 0 else 0) + len(problems) + failure = "; ".join(problems) + except (OSError, ValueError, RuntimeError) as error: + failure = str(error) + category = "setup_errors" if client is None else "report_errors" + row[category] = (row.get(category) or 0) + 1 + finally: + if client is not None: + row["client_exit"] = runtime.stop(client, "client") + if server is not None: + row["server_exit"] = runtime.stop(server, "server") + row.update(metadata) + row.update(server_startup=startup, provider_counters_before=before, + provider_counters_after=provider_counters(args.device, args.hca_port), + topology=args.topology, physical_hosts=1, parameters=case) + after = row["provider_counters_after"] or {} + row["rnr_provider_counters"] = {name: dict(before=(before or {}).get(name), after=value) + for name, value in after.items() if "rnr" in name.lower()} or None + row["valid"] = (row.get("valid") is True and row.get("errors_total") == 0 + and row.get("report_errors") == 0 and row["client_exit"] == 0 + and row["server_exit"] == 0 and failure is None) + if failure is not None: + row["runner_failure"] = failure + if not row["valid"]: + print(f'FAILED CASE {case["case_id"]}: {failure}; client_exit={row["client_exit"]} ' + f'server_exit={row["server_exit"]}', file=sys.stderr) + for path in (directory / "server.log", directory / "client.log"): + if path.exists(): + print(f"{path}:\n{path.read_text()}", file=sys.stderr) + (directory / "result.json").write_text(json.dumps(row, indent=2)) + return row + + +def main(): + args = arguments() + args.output_dir.mkdir(parents=True) + cases = load_cases(args.manifest, args.transport) + (args.output_dir / "manifest.json").write_text(json.dumps(dict(schema_version=1, cases=cases), indent=2)) + metadata = identity(ROOT, dict(server=args.server_bin, client=args.cli_bin), args.device, args.hca_port) + rows = [] + for case in cases: + row = run_case(args, case, len(rows), metadata) + rows.append(row) + write_tables(args.output_dir, rows) + print_row(row) + if not row["valid"]: + return 1 + peak = max(zip(rows, cases), key=lambda pair: pair[0]["rpc_tps"])[1] + for repeat in range(args.peak_repeats): + case = peak | dict(case_id=peak["case_id"] + "-peak", repeat_index=repeat + 1) + row = run_case(args, case, len(rows), metadata) + rows.append(row) + write_tables(args.output_dir, rows) + print_row(row) + if not row["valid"]: + return 1 + print(f"Results: {args.output_dir / 'results.csv'}", flush=True) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, ValueError) as error: + print(f"error: {error}", file=sys.stderr) + sys.exit(2) diff --git a/tools/benchmark/tests/rdma_regression_test.py b/tools/benchmark/tests/rdma_regression_test.py index f4e9ad007..66204cc2a 100644 --- a/tools/benchmark/tests/rdma_regression_test.py +++ b/tools/benchmark/tests/rdma_regression_test.py @@ -1,7 +1,7 @@ # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. -"""Result and exit-status contracts, using fixture processes without an HCA.""" +"""Canonical report/readiness contracts with owned fixture processes.""" import csv import json @@ -15,82 +15,167 @@ class RdmaRegressionTest(unittest.TestCase): def setUp(self): - self.temporary = tempfile.TemporaryDirectory(prefix="rdma-runner-") - self.addCleanup(self.temporary.cleanup) - self.directory = Path(self.temporary.name) + temporary = tempfile.TemporaryDirectory(prefix="rdma-runner-") + self.addCleanup(temporary.cleanup) + self.directory = Path(temporary.name) self.root = Path(__file__).resolve().parents[3] self.server = self.directory / "server.py" - self.server.write_text( - '#!/usr/bin/env python3\nimport signal\n' - 'print("listening port=1", flush=True)\nsignal.pause()\n' - ) + self.server.write_text('''#!/usr/bin/env python3 +import json, os, signal, sys, time +from pathlib import Path +assert os.environ["CROWDB_RDMA_ALLOW_TCP_FALLBACK"] == "0" +mode = os.environ.get("RDMA_FIXTURE_SERVER_MODE", "ok") +if mode == "exit": sys.exit(3) +print("listening port=19101", flush=True) +if mode == "stdout-only": signal.pause() +path = Path(sys.argv[sys.argv.index("--status_file") + 1]) +status = dict(schema_version=1, process_id=os.getpid(), allow_tcp_fallback=False, + rdma=dict(listen_active=mode != "no-rdma", ready=True, cm_port=19101)) +path.write_text(json.dumps(status)) +if mode == "early": + time.sleep(0.2) + sys.exit(4) +signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) +signal.pause() +''') self.server.chmod(0o755) self.client = self.directory / "client.py" - self.client.write_text( - '#!/usr/bin/env python3\nimport os, sys\n' - 'assert os.environ["CROWDB_RDMA_ALLOW_TCP_FALLBACK"] == "0"\n' - 'assert sys.argv[sys.argv.index("--mode") + 1] == "coroutine"\n' - 'print(os.environ["RDMA_FIXTURE_REPORT"])\n' - 'sys.exit(int(os.environ["RDMA_FIXTURE_EXIT"]))\n' - ) + self.client.write_text('''#!/usr/bin/env python3 +import os, sys, time +from pathlib import Path +assert os.environ["CROWDB_RDMA_ALLOW_TCP_FALLBACK"] == "0" +assert sys.argv[1:3] == ["bench", os.environ["RDMA_FIXTURE_VERB"]] +for flag, expected in [("--mode", "coroutine"), ("--loader-count", "4"), + ("--connection-count", "2"), ("--cq-count", "2"), ("--wire-slot-capacity", "1024")]: + assert sys.argv[sys.argv.index(flag) + 1] == expected +assert os.environ["CROWDB_RDMA_WIRE_SLOT_CAPACITY"] == "1024" +if os.environ.get("RDMA_FIXTURE_SERVER_MODE") == "early": time.sleep(2) +Path(sys.argv[sys.argv.index("--output") + 1]).write_text(os.environ["RDMA_FIXTURE_REPORT"]) +sys.exit(int(os.environ["RDMA_FIXTURE_EXIT"])) +''') self.client.chmod(0o755) + self.manifest = self.directory / "cases.json" + self.manifest.write_text(json.dumps(dict(schema_version=1, cases=[ + dict(case_id="fixture", loader_count=4, connection_count=2, cq_count=2)]))) - def run_case(self, report, client_exit): - environment = os.environ.copy() - environment.update({ - "RDMA_BENCH_SERVER_BIN": str(self.server), - "RDMA_BENCH_CLI_BIN": str(self.client), - "RDMA_BENCH_CASES": "rdma_1w_1l_1c", - "RDMA_BENCH_LOG_ROOT": str(self.directory / "output"), - "RDMA_FIXTURE_REPORT": report, - "RDMA_FIXTURE_EXIT": str(client_exit), - }) + def run_case(self, report, client_exit=0, server_mode="ok", transport="rdma"): + prefix = "RDMA" if transport == "rdma" else "RPC" + environment = os.environ | { + f"{prefix}_BENCH_SERVER_BIN": str(self.server), + f"{prefix}_BENCH_CLI_BIN": str(self.client), + f"{prefix}_BENCH_LOG_ROOT": str(self.directory / "output"), + "RDMA_FIXTURE_REPORT": report, "RDMA_FIXTURE_EXIT": str(client_exit), + "RDMA_FIXTURE_SERVER_MODE": server_mode, + "RDMA_FIXTURE_VERB": "rdma" if transport == "rdma" else "rpc", + } + script = "bench-rdma-rpc-regression.sh" if transport == "rdma" else "bench-rpc-regression.sh" process = subprocess.run( - [shutil.which("pixi"), "run", "--", "bash", "tools/benchmark/bench-rdma-rpc-regression.sh"], - cwd=self.root, env=environment, capture_output=True, text=True, timeout=20, - ) + [shutil.which("pixi"), "run", "--", "bash", f"tools/benchmark/{script}", + "--manifest", str(self.manifest), "--peak-repeats", "0", "--startup-timeout", "1"], + cwd=self.root, env=environment, capture_output=True, text=True, timeout=20) results = self.directory / "output/results.tsv" self.assertTrue(results.exists(), process.stdout + process.stderr) - with results.open() as file: - rows = list(csv.DictReader(file, delimiter="\t")) + with results.open() as stream: + rows = list(csv.DictReader(stream, delimiter="\t")) self.assertEqual(len(rows), 1, process.stdout + process.stderr) return process, rows[0] @staticmethod - def report(errors=0): - return json.dumps({ - "total_ops": 100, - "duration_ms": 10000, - "total_errors": errors, - "by_op": {"write": {"latency_us": {"avg_us": 20.0, "p50_us": 18, "p99_us": 40}}}, - }, indent=2) + def report(errors=0, transport="rdma", duplicate_qp=False): + row = dict(schema_version=1, case_id="fixture", valid=errors == 0, errors_total=errors, + actual_path=transport, allow_tcp_fallback=False, fallback_count=0, + loader_count=4, connection_count=2, payload_bytes=128, + completed_in_window=1000, latency_count=1000, measure_s=10, rpc_tps=100.0, + qpair_count=2 if transport == "rdma" else 0, + min_ns=1, avg_ns=20.0, p50_ns=20, p90_ns=30, p95_ns=40, p99_ns=50, p999_ns=60, max_ns=100) + for name in ("report_errors", "setup_errors", "submit_rejects", "timeouts", "connection_errors", + "cq_errors", "payload_mismatches", "duplicate_completions", "retries", "histogram_overflow_count"): + row[name] = 0 + row["client_connections"] = [dict(path=transport, open=True, info=dict(qpair_number=100 + (0 if duplicate_qp else i))) + for i in range(2)] + return json.dumps(row) def test_valid_report_succeeds_and_records_zero_errors(self): - process, row = self.run_case(self.report(), 0) + process, row = self.run_case(self.report()) self.assertEqual(process.returncode, 0, process.stdout + process.stderr) - self.assertEqual(row["errors"], "0") + self.assertEqual(row["errors_total"], "0") self.assertEqual(row["report_errors"], "0") - self.assertEqual(row["valid"], "1") + self.assertEqual(row["valid"], "true") + self.assertEqual(row["parameters.cq_depth"], "2048") + self.assertEqual(row["server_exit"], "0") + + def test_tcp_uses_the_same_runner_and_canonical_schema(self): + process, row = self.run_case(self.report(transport="tcp"), transport="tcp") + self.assertEqual(process.returncode, 0, process.stdout + process.stderr) + self.assertEqual(row["actual_path"], "tcp") + self.assertEqual(row["valid"], "true") def test_failed_client_preserves_reported_error_count(self): - process, row = self.run_case(self.report(errors=7), 2) - self.assertNotEqual(process.returncode, 0, process.stdout + process.stderr) - self.assertEqual(row["errors"], "7") + process, row = self.run_case(self.report(errors=7), client_exit=2) + self.assertNotEqual(process.returncode, 0) + self.assertEqual(row["errors_total"], "7") self.assertEqual(row["client_exit"], "2") - self.assertEqual(row["valid"], "0") + self.assertEqual(row["valid"], "false") def test_missing_report_is_not_success_even_when_client_exits_zero(self): - process, row = self.run_case("human report", 0) - self.assertNotEqual(process.returncode, 0, process.stdout + process.stderr) - self.assertEqual(row["errors"], "NA") + process, row = self.run_case("human report") + self.assertNotEqual(process.returncode, 0) + self.assertEqual(row["errors_total"], "null") self.assertEqual(row["report_errors"], "1") - self.assertEqual(row["valid"], "0") + self.assertEqual(row["valid"], "false") def test_error_report_is_failure_even_when_client_exits_zero(self): - process, row = self.run_case(self.report(errors=3), 0) - self.assertNotEqual(process.returncode, 0, process.stdout + process.stderr) - self.assertEqual(row["errors"], "3") - self.assertEqual(row["valid"], "0") + process, row = self.run_case(self.report(errors=3)) + self.assertNotEqual(process.returncode, 0) + self.assertEqual(row["errors_total"], "3") + self.assertEqual(row["valid"], "false") + + def test_duplicate_qpairs_invalidate_report(self): + process, row = self.run_case(self.report(duplicate_qp=True)) + self.assertNotEqual(process.returncode, 0) + self.assertNotEqual(row["report_errors"], "0") + self.assertEqual(row["valid"], "false") + + def test_forged_success_with_errors_is_rejected(self): + report = json.loads(self.report(errors=3)) + report["valid"] = True + process, row = self.run_case(json.dumps(report)) + self.assertNotEqual(process.returncode, 0) + self.assertEqual(row["errors_total"], "3") + self.assertNotEqual(row["report_errors"], "0") + + def test_malformed_counter_and_connection_types_retain_a_failure_row(self): + report = json.loads(self.report()) + report.update(report_errors=None, client_connections=None) + process, row = self.run_case(json.dumps(report)) + self.assertNotEqual(process.returncode, 0) + self.assertNotEqual(row["report_errors"], "0") + self.assertEqual(row["valid"], "false") + + def test_failed_server_never_launches_a_successful_client(self): + process, row = self.run_case(self.report(), server_mode="exit") + self.assertNotEqual(process.returncode, 0) + self.assertEqual(row["server_exit"], "3") + self.assertEqual(row["client_exit"], "null") + self.assertEqual(row["setup_errors"], "1") + + def test_human_listening_message_cannot_replace_rdma_readiness(self): + process, row = self.run_case(self.report(), server_mode="no-rdma") + self.assertNotEqual(process.returncode, 0) + self.assertEqual(row["client_exit"], "null") + self.assertEqual(row["setup_errors"], "1") + + def test_listening_stdout_without_status_reaches_a_bounded_failure(self): + process, row = self.run_case(self.report(), server_mode="stdout-only") + self.assertNotEqual(process.returncode, 0) + self.assertEqual(row["client_exit"], "null") + self.assertEqual(row["setup_errors"], "1") + + def test_server_exit_during_workload_invalidates_and_stops_client(self): + process, row = self.run_case(self.report(), server_mode="early") + self.assertNotEqual(process.returncode, 0) + self.assertEqual(row["server_exit"], "4") + self.assertEqual(row["valid"], "false") if __name__ == "__main__": From 777580b68fc0e84b23d5a1bdc167fb30361ec8dc Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 00:23:10 +0800 Subject: [PATCH 035/258] Reconcile RDMA benchmark submissions with remote request totals --- doc/working/plan-rdma-rpc-transport.md | 22 +++++++++++++++++++ tools/benchmark/rpc_bench_report.py | 12 +++++++--- tools/benchmark/rpc_regression.py | 15 +++++++++++-- tools/benchmark/tests/rdma_regression_test.py | 10 +++++++++ 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 7a62856e9..036594779 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -776,6 +776,28 @@ Implementation checkpoint: three times and retain all rows. Require zero workload errors/fallback and report avg/P90/tails with the highest repeatable TPS. Do not claim a hardware maximum when CPU/affinity/topology still limits the result. +- Sustained checkpoint: the 24-row gradual matrix completed 203,195,362 + measured validated RPCs with zero errors/fallback. At 512 loaders, 8 QPs + and 8 CQs/pollers, peak repeats give 2,201,203 / 2,234,423 / 2,277,181 TPS. + RQ 4096 does not improve the matching four-CQ control; queue admission and + the HCA RNR-retry-error counter show no new errors. Artifact: + `.crowdb-runtime/rdma-check/artifacts/bench/rdma-rpc-20261022T160711.769136Z/`. +- The next nine-row matrix increases loaders to 4096 and QPs/CQs to 32/16. + Its 2048-loader, 32-QP, 16-CQ candidate repeats at 2,886,026 / 2,892,625 / + 2,913,750 TPS, all error-free. The middle trial has avg 707.02 us, + P90 815.10 us and P99 954.37 us; this is the throughput/latency pair for + that trial. Artifact: + `.crowdb-runtime/rdma-check/artifacts/bench/rdma-rpc-20261022T161241.180591Z/`. + All 33 rows also reconcile remote RDMA request-frame deltas exactly with + accepted client submissions. The runner now enforces this relation and its + thirteenth contract test detects a forged remote total. +- The real-HCA held receive/MR/CM lifetime, registration-budget exhaustion, + and TCP-only-peer strict/allowed fallback cases all pass separately. Workspace + `clippy --all-targets --all-features` also passes on this source. +- Next controls: vary the idle polling budget and slot size at multiple CQs, + then run TCP and matched two-container manifests. A larger pool or more QPs + alone is not evidence of a higher hardware limit; retain CPU/resource data + and the exact binary identities with every trial. - [ ] **Run matching OCI profiles and finish protocol tests**: use the same CLI workload and manifest in two host-network containers on this physical HCA; record separate process/container topology. Complete negotiated TCP diff --git a/tools/benchmark/rpc_bench_report.py b/tools/benchmark/rpc_bench_report.py index 3dfabb353..202c0fa99 100644 --- a/tools/benchmark/rpc_bench_report.py +++ b/tools/benchmark/rpc_bench_report.py @@ -50,7 +50,7 @@ def validate(row, case, transport): errors.append("missing/unsupported report schema") if type(row.get("valid")) is not bool: errors.append("missing/invalid validity flag") - required = ("errors_total", "report_errors", "setup_errors", "completed_in_window", "latency_count", + required = ("errors_total", "report_errors", "setup_errors", "submitted", "completed_in_window", "latency_count", "submit_rejects", "timeouts", "connection_errors", "cq_errors", "payload_mismatches", "duplicate_completions", "retries", "histogram_overflow_count", "measure_s") for name in required: @@ -78,13 +78,19 @@ def validate(row, case, transport): if any(type(value) is not int or value < 0 for value in latency) or latency != sorted(latency): errors.append("invalid/missing latency bounds") if transport == "rdma": + try: + received = row["server_after"]["rdma"]["rpc_frames_received"] - row["server_before"]["rdma"]["rpc_frames_received"] + if received != row["submitted"]: + errors.append("server request total does not match accepted submissions") + except (KeyError, TypeError): + errors.append("missing server request totals") connections = row.get("client_connections", []) if not isinstance(connections, list) or not all(isinstance(conn, dict) for conn in connections): return errors + ["missing/malformed connection statuses"] qpairs = [(conn.get("info") or {}).get("qpair_number") if isinstance(conn.get("info"), dict) else None for conn in connections] - if (len(qpairs) != case["connection_count"] or len(set(qpairs)) != len(qpairs) - or any(type(qp) is not int or qp <= 0 for qp in qpairs) + if (len(qpairs) != case["connection_count"] or any(type(qp) is not int or qp <= 0 for qp in qpairs) + or len(set(qpairs)) != len(qpairs) or any(conn.get("path") != "rdma" or conn.get("open") is not True for conn in connections) or row.get("qpair_count") != case["connection_count"]): errors.append("actual unique QP count/path mismatch") diff --git a/tools/benchmark/rpc_regression.py b/tools/benchmark/rpc_regression.py index 0b71615b4..2feaa745a 100644 --- a/tools/benchmark/rpc_regression.py +++ b/tools/benchmark/rpc_regression.py @@ -112,9 +112,20 @@ def run_case(args, case, index, metadata): row.update(server_startup=startup, provider_counters_before=before, provider_counters_after=provider_counters(args.device, args.hca_port), topology=args.topology, physical_hosts=1, parameters=case) + row["server_request_frames"] = None + if args.transport == "rdma": + try: + row["server_request_frames"] = row["server_after"]["rdma"]["rpc_frames_received"] - row["server_before"]["rdma"]["rpc_frames_received"] + except (KeyError, TypeError): + pass after = row["provider_counters_after"] or {} - row["rnr_provider_counters"] = {name: dict(before=(before or {}).get(name), after=value) - for name, value in after.items() if "rnr" in name.lower()} or None + row["rnr_provider_counters"] = {} + for name, value in after.items(): + if "rnr" in name.lower(): + previous = (before or {}).get(name) + delta = value - previous if type(value) is int and type(previous) is int and value >= previous else None + row["rnr_provider_counters"][name] = dict(before=previous, after=value, delta=delta) + row["rnr_provider_counters"] = row["rnr_provider_counters"] or None row["valid"] = (row.get("valid") is True and row.get("errors_total") == 0 and row.get("report_errors") == 0 and row["client_exit"] == 0 and row["server_exit"] == 0 and failure is None) diff --git a/tools/benchmark/tests/rdma_regression_test.py b/tools/benchmark/tests/rdma_regression_test.py index 66204cc2a..a14fcff23 100644 --- a/tools/benchmark/tests/rdma_regression_test.py +++ b/tools/benchmark/tests/rdma_regression_test.py @@ -85,6 +85,8 @@ def report(errors=0, transport="rdma", duplicate_qp=False): row = dict(schema_version=1, case_id="fixture", valid=errors == 0, errors_total=errors, actual_path=transport, allow_tcp_fallback=False, fallback_count=0, loader_count=4, connection_count=2, payload_bytes=128, + submitted=1100, server_before=dict(rdma=dict(rpc_frames_received=100)), + server_after=dict(rdma=dict(rpc_frames_received=1200)), completed_in_window=1000, latency_count=1000, measure_s=10, rpc_tps=100.0, qpair_count=2 if transport == "rdma" else 0, min_ns=1, avg_ns=20.0, p50_ns=20, p90_ns=30, p95_ns=40, p99_ns=50, p999_ns=60, max_ns=100) @@ -144,6 +146,14 @@ def test_forged_success_with_errors_is_rejected(self): self.assertEqual(row["errors_total"], "3") self.assertNotEqual(row["report_errors"], "0") + def test_server_request_count_must_match_all_accepted_requests(self): + report = json.loads(self.report()) + report["server_after"]["rdma"]["rpc_frames_received"] += 1 + process, row = self.run_case(json.dumps(report)) + self.assertNotEqual(process.returncode, 0) + self.assertEqual(row["server_request_frames"], "1101") + self.assertEqual(row["valid"], "false") + def test_malformed_counter_and_connection_types_retain_a_failure_row(self): report = json.loads(self.report()) report.update(report_errors=None, client_connections=None) From 44a9b2d11945fe9fca1f2f387d1f73e69449175a Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 00:30:07 +0800 Subject: [PATCH 036/258] Pin container benchmark runs to an immutable runtime image --- tools/benchmark/rpc_regression.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/benchmark/rpc_regression.py b/tools/benchmark/rpc_regression.py index 2feaa745a..37e9d3426 100644 --- a/tools/benchmark/rpc_regression.py +++ b/tools/benchmark/rpc_regression.py @@ -10,6 +10,7 @@ import os from pathlib import Path import shutil +import subprocess import sys from rpc_bench_cases import client_arguments, load_cases, server_environment @@ -147,6 +148,12 @@ def main(): cases = load_cases(args.manifest, args.transport) (args.output_dir / "manifest.json").write_text(json.dumps(dict(schema_version=1, cases=cases), indent=2)) metadata = identity(ROOT, dict(server=args.server_bin, client=args.cli_bin), args.device, args.hca_port) + metadata.update(container_image=None, container_image_id=None, application_uid=os.getuid()) + if args.topology != "native": + metadata["container_image"] = args.image + args.image = subprocess.check_output([shutil.which("pixi"), "run", "--", "docker", "image", "inspect", + args.image, "--format", "{{.Id}}"], cwd=ROOT, text=True).strip() + metadata["container_image_id"] = args.image rows = [] for case in cases: row = run_case(args, case, len(rows), metadata) From 5de5f863c436682143245782dd992b52d1a322e0 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 01:01:31 +0800 Subject: [PATCH 037/258] Propagate RDMA disconnects through the owning completion queue --- doc/design/rpc/design-crowdb-rpc-rdma.md | 9 + doc/working/plan-rdma-rpc-transport.md | 38 +++- .../include/crowdb-rpc/connection.h | 1 + .../transport/rdma/rdma_transport.h | 5 +- .../crowdb-rpc/transport/socket_transport.h | 2 + lib/crowdb-rpc/src/connection.cpp | 3 + .../src/transport/rdma/rdma_connection.cpp | 4 +- .../src/transport/rdma/rdma_poller.cpp | 8 +- lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp | 16 ++ .../src/transport/rdma/rdma_send.cpp | 2 + .../src/transport/rdma/rdma_transport.cpp | 2 + .../src/transport/socket_transport.cpp | 10 +- .../tests/common/test_rdma_transport.h | 19 +- lib/crowdb-rpc/tests/rdma_failure_test.cpp | 198 ++++++++++++++++++ 14 files changed, 306 insertions(+), 11 deletions(-) create mode 100644 lib/crowdb-rpc/tests/rdma_failure_test.cpp diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index 35f3f01fc..5f62c0575 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -247,6 +247,15 @@ completions cannot publish data into a reused buffer. QP, CM, and CQ errors move the connection to draining or failed state and quarantine uncertain buffers. +`Connection::close()` also wakes its assigned CQ through a private transport +notifier. The CQ owner requests `rdma_disconnect` once, guarded by the same +per-connection operation admission that protects posting and CM-ID lifetime. +The notifier only wakes the owner; application callbacks do not drain their +own CQ. CM retirement and transport shutdown use the same disconnect guard +before stopping admission and destroying the QP. A close removes the logical +connection from socket-transport indexes, including a close racing publication; +retained buffer leases continue to own their arena and device resources. + ## 7. Service interactions For S3/HTTP and DiskIO, HTTP or ChunkIO fills the Access Server registered send diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md index 036594779..aad98c5a9 100644 --- a/doc/working/plan-rdma-rpc-transport.md +++ b/doc/working/plan-rdma-rpc-transport.md @@ -768,7 +768,7 @@ Implementation checkpoint: execution remains pending in the task below. TCP uses writev/Nagle controls; RDMA uses bounded complete-frame SEND aggregation, so the sweep does not pretend these two transport mechanisms are the same switch. -- [~] **Run gradual native hardware profiles**: first one coroutine/QP/CQ, +- [x] **Run gradual native hardware profiles**: first one coroutine/QP/CQ, then loader sweep 1/2/4/8/16/32 and higher as needed, then QP and CQ sweeps; compare aggregation off/on, SQ/RQ 1024 and 4096, small slot classes and larger payload/slot pairs. Increase depth only when counters show it limits load. @@ -798,12 +798,43 @@ Implementation checkpoint: then run TCP and matched two-container manifests. A larger pool or more QPs alone is not evidence of a higher hardware limit; retain CPU/resource data and the exact binary identities with every trial. -- [ ] **Run matching OCI profiles and finish protocol tests**: use the same +- [~] **Run matching OCI profiles and finish protocol tests**: use the same CLI workload and manifest in two host-network containers on this physical HCA; record separate process/container topology. Complete negotiated TCP preflight, fallback-before-payload, no mixed stream, no-device TCP and non-root device/memlock acceptance. Compare matching native/container rows and explain any gap. Files: RDMA OCI runner, capability/path tests. + Add an opt-in HCA case with a deliberately expired advertised CM endpoint: + valid TCP preflight must lead to one actual CM failure, strict rejection, + or one aligned TCP fallback echo before any RDMA payload. Set the fixture's + capability before worker startup through guarded `TestSocketTransport` / + `TestRdmaTransport` helpers. Add a real HCA peer-close case after request + delivery, with fallback allowed, asserting one failed completion, retained + RDMA path, no fallback and no TCP workload frame. Files: + `tests/rdma_failure_test.cpp`, `tests/common/test_rdma_transport.h`, and + the existing guarded transport test friendship. No new hot-path lock. +- The expired-CM fixture passes strict rejection and one allowed TCP echo + after a real CM attempt, with no RDMA workload frame. The new peer-close + fixture initially failed: `Connection::close()` only published local state, + so the peer remained open and its pending request never failed. A private + transport-close notifier now wakes the assigned CQ owner; the owner calls + `rdma_disconnect` before stopping posting admission and retiring its send + queue. The existing per-QP operation gate pins the CM ID until that call + completes, and an atomic flag prevents duplicate requests from CQ/CM/shutdown. + Shutdown explicitly disconnects retained IDs before destruction. RDMA close + also unregisters socket-transport weak indexes, including a close/publication + race, while retained leases keep their arena/device ownership. +- All three real-HCA failure cases now pass in 40 ms: expired endpoint, peer + close after payload, and server shutdown after payload. The latter two prove + one failed completion with fallback allowed, retained RDMA path, no fallback, + no TCP workload frame and empty connection indexes after shutdown. +- The final close/disconnect source passes the full C++/FFI gate at + `.crowdb-runtime/artifacts/unit/cpp-93edc5b4128b456696da0095db768d20/`, + all 137 no-verbs CTest cases, all four CLI benchmark integration cases, + workspace clippy, Rust/C++ formatting and scoped tree-lint (exit zero with + style warnings). All thirteen runner contract tests pass. Six real-HCA + failure/lifetime/budget/fallback cases also pass together in 74 ms. The + release CLI rebuild exits zero; post-fix sustained benchmark checks remain. - [ ] **Gate and document each coherent change**: build with `-j32`; run all executables/tests/lints through `pixi run`, use `clean-env` before server tests, and keep complete failure output. Run affected tests, fmt, clippy, @@ -822,6 +853,9 @@ Provider references for the implementation choices: an independently opened context for a CM-owned QP. - [RDMA-CM request private-data limit](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/man/rdma_connect.3) and [accept event lifetime](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/man/rdma_accept.3). +- [RDMA disconnect and flush contract](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/man/rdma_disconnect.3), + [CM-ID destruction](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/cma.c) + and [kernel reported-event accounting](https://github.com/torvalds/linux/blob/master/drivers/infiniband/core/ucma.c). ## Historical verification record diff --git a/lib/crowdb-rpc/include/crowdb-rpc/connection.h b/lib/crowdb-rpc/include/crowdb-rpc/connection.h index 500d6b52c..6de26267c 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/connection.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/connection.h @@ -315,6 +315,7 @@ class Connection : public std::enable_shared_from_this ConnectionCallback on_frame_callback_; ConnectionCallback> on_frame_ready_; + ConnectionCallback> on_transport_close_; ConnectionCallback on_close_callback_; ConnectionCallback on_rpc_close_callback_; }; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index 687c99956..1cffa3dee 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -252,7 +252,7 @@ class RdmaTransport : public Transport void start_pollers(); void run_poller(CompletionQueue *queue); void notify_queue(uint32_t index); - void install_frame_notifier(const std::shared_ptr &connection, uint32_t index); + void install_connection_notifiers(const std::shared_ptr &connection, uint32_t index); uint32_t flush_sends(CompletionQueue &queue, uint32_t budget); uint32_t flush_connection(Connection *connection, ConnectionPools &pools, uint32_t budget); void retire_sends(CompletionQueue &queue, Connection *connection); @@ -270,6 +270,7 @@ class RdmaTransport : public Transport void destroy_connection_qp(Connection *connection, rdma_cm_id *id); static ConnectionPools *pools_for(Connection *connection); static void set_connection_pools(Connection *connection, std::shared_ptr pools); + static void request_disconnect(ConnectionPools &pools); bool publish_cm_connection(rdma_cm_id *id, const std::shared_ptr &connection, std::shared_ptr pools); @@ -330,6 +331,8 @@ class RdmaTransport : public Transport uint32_t peer_receive_slot_capacity = 0; uint32_t peer_receive_frame_limit = 1; uint32_t qp_number = 0; + rdma_cm_id *cm_id = nullptr; // immutable after publication + std::atomic disconnect_requested{false}; SendQueue send_queue; std::atomic queued_frames{0}; OutFrame *pending_frame = nullptr; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h index 21de20e87..50f641d27 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h @@ -533,6 +533,8 @@ class SocketTransport : public Transport [[nodiscard]] std::string status_json() const; #if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) + friend struct TestSocketTransport; + const RdmaTransport::Stats *rdma_stats_for_tests() const { return rdma_transport_ == nullptr ? nullptr : &rdma_transport_->stats(); diff --git a/lib/crowdb-rpc/src/connection.cpp b/lib/crowdb-rpc/src/connection.cpp index e50d6d451..506ae0264 100644 --- a/lib/crowdb-rpc/src/connection.cpp +++ b/lib/crowdb-rpc/src/connection.cpp @@ -426,6 +426,9 @@ void Connection::close() if (const auto *callback = on_close_callback_.get(); callback != nullptr && *callback) { (*callback)(this); } + if (const auto *callback = on_transport_close_.get(); callback != nullptr && *callback) { + (*callback)(); + } } void Connection::on_frame(Frame *frame) diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp index 41154930e..e53b07e47 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp @@ -161,7 +161,7 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int reservation.committed = true; connection->set_path(ConnectionPath::Rdma); connection->transport_handle = reinterpret_cast(id->qp); - install_frame_notifier(connection, pools->cq_index); + install_connection_notifiers(connection, pools->cq_index); if (!prime_receive(connection)) { destroy_connection_qp(connection.get(), id); rdma_destroy_id(id); @@ -375,7 +375,7 @@ std::shared_ptr RdmaTransport::accept() reservation.committed = true; connection->set_path(ConnectionPath::Rdma); connection->transport_handle = reinterpret_cast(id->qp); - install_frame_notifier(connection, pools->cq_index); + install_connection_notifiers(connection, pools->cq_index); if (!prime_receive(connection)) { destroy_connection_qp(connection.get(), id); rdma_destroy_id(id); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp index 1dd70e72f..2c7d214a0 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp @@ -83,13 +83,15 @@ void RdmaTransport::notify_queue(uint32_t index) (void)::write(queue.continuation_fd, &value, sizeof(value)); } -void RdmaTransport::install_frame_notifier(const std::shared_ptr &connection, uint32_t index) +void RdmaTransport::install_connection_notifiers(const std::shared_ptr &connection, uint32_t index) { - connection->set_frame_ready_notifier([state = repost_state_, index] { + auto notify = [state = repost_state_, index] { Operation operation(state); if (operation) state->transport->notify_queue(index); - }); + }; + connection->set_frame_ready_notifier(notify); + connection->on_transport_close_.set(std::move(notify)); } uint32_t RdmaTransport::process_deferred(CompletionQueue &queue, uint32_t max_completions) diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp index a8f206c8b..5f34031d8 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp @@ -2,10 +2,13 @@ // Licensed under the Apache License, Version 2.0. #include "../../status_json.h" +#include "crowdb-common/log.h" #include "rdma_cq.h" #ifdef CROWDB_RPC_HAVE_RDMA +# include + namespace crowdb::rpc { @@ -22,6 +25,7 @@ void RdmaTransport::set_connection_pools(Connection *connection, std::shared_ptr bool RdmaTransport::publish_cm_connection(rdma_cm_id *id, const std::shared_ptr &connection, std::shared_ptr pools) { + pools->cm_id = id; // The shared CM consumer must see the owner before a migrated terminal // event can retire this ID. Setup events stay private until this point. StatusJson info; @@ -48,11 +52,23 @@ bool RdmaTransport::publish_cm_connection(rdma_cm_id *id, const std::shared_ptr< return true; } +void RdmaTransport::request_disconnect(ConnectionPools &pools) +{ + // The existing posting gate pins the CM ID through disconnect. A terminal + // CM consumer stops this gate before destroying the QP and ID. + Operation posting(pools.posting_state); + if (!posting || pools.cm_id == nullptr || pools.disconnect_requested.exchange(true, std::memory_order_acq_rel)) + return; + if (rdma_disconnect(pools.cm_id) != 0) + CRB_LOG_DEBUG("rdma disconnect request completed with errno={}", errno); +} + void RdmaTransport::destroy_connection_qp(Connection *connection, rdma_cm_id *id) { connection->close(); auto *pools = pools_for(connection); if (pools != nullptr) { + request_disconnect(*pools); pools->posting_state->users.fetch_or(RepostState::STOPPED, std::memory_order_acq_rel); while ((pools->posting_state->users.load(std::memory_order_acquire) & ~RepostState::STOPPED) != 0) std::this_thread::yield(); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp index 135fd88e3..81dc395d5 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp @@ -152,6 +152,8 @@ uint32_t RdmaTransport::flush_sends(CompletionQueue &queue, uint32_t budget) // Closed producers finish publishing before their bounded queue retires. for (size_t index = 0; index < queue.connections.size();) { auto &entry = queue.connections[index]; + if (!entry.connection->is_open()) + request_disconnect(*entry.pools); if (!entry.connection->is_open() && (entry.pools->posting_state->users.fetch_or(RepostState::STOPPED, std::memory_order_acq_rel) & ~RepostState::STOPPED) == 0) { diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index f3f08738f..d780aaf49 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -291,6 +291,8 @@ void RdmaTransport::shutdown() } for (const auto &[id, connection] : cm_connections_) { (void)id; + if (auto *pools = pools_for(connection.get()); pools != nullptr) + request_disconnect(*pools); connection->close(); } { diff --git a/lib/crowdb-rpc/src/transport/socket_transport.cpp b/lib/crowdb-rpc/src/transport/socket_transport.cpp index ab5309f50..8a456b65d 100644 --- a/lib/crowdb-rpc/src/transport/socket_transport.cpp +++ b/lib/crowdb-rpc/src/transport/socket_transport.cpp @@ -889,11 +889,17 @@ size_t SocketTransport::connection_count() const void SocketTransport::attach_rdma_connection(const std::shared_ptr &conn, Connection::OnFrameCallback on_frame, Connection::OnCloseCallback on_close) { - if (conn == nullptr) + if (conn == nullptr || !conn->is_open()) return; conn->set_on_frame(std::move(on_frame)); - conn->set_on_close(std::move(on_close)); + conn->set_on_close([this, on_close = std::move(on_close)](Connection *connection) { + if (on_close) + on_close(connection); + unregister_conn(connection); + }); register_conn(conn); + if (!conn->is_open()) + unregister_conn(conn.get()); if (!workers_.empty()) { Worker *worker = get_worker(); conn->io_worker = worker; diff --git a/lib/crowdb-rpc/tests/common/test_rdma_transport.h b/lib/crowdb-rpc/tests/common/test_rdma_transport.h index 36a8a1b33..8e1f28568 100644 --- a/lib/crowdb-rpc/tests/common/test_rdma_transport.h +++ b/lib/crowdb-rpc/tests/common/test_rdma_transport.h @@ -6,6 +6,7 @@ #include "../../src/transport/rdma/rdma_cq.h" #include "../../src/transport/rdma/rdma_device.h" #include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "crowdb-rpc/transport/socket_transport.h" #if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) @@ -17,6 +18,14 @@ namespace crowdb::rpc struct TestRdmaTransport { + static void advertise_expired_endpoint_for_tests(RdmaTransport &transport, const std::string &address, + uint16_t port) + { + transport.local_capability_.cm_address = address; + transport.local_capability_.port = port; + transport.listen_active_.store(true, std::memory_order_release); + } + static void set_aggregation_for_tests(RdmaTransport &transport, bool enabled) { transport.config_.aggregation_enabled = enabled; @@ -120,7 +129,7 @@ struct TestRdmaTransport static void watch_frames(RdmaTransport &transport, const std::shared_ptr &connection, uint32_t index) { - transport.install_frame_notifier(connection, index); + transport.install_connection_notifiers(connection, index); } static bool pending(RdmaTransport &transport, uint64_t token) @@ -175,6 +184,14 @@ struct TestCompletionQueue } }; +struct TestSocketTransport +{ + static RdmaTransport &rdma_for_tests(SocketTransport &transport) + { + return *transport.rdma_transport_; + } +}; + } // namespace crowdb::rpc #endif diff --git a/lib/crowdb-rpc/tests/rdma_failure_test.cpp b/lib/crowdb-rpc/tests/rdma_failure_test.cpp new file mode 100644 index 000000000..0406fe516 --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_failure_test.cpp @@ -0,0 +1,198 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "common/test_rdma_transport.h" +#include "crowdb-rpc/client/client.h" +#include "crowdb-rpc/server/message.h" +#include "crowdb-rpc/server/server.h" + +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) + +# include + +# include +# include +# include +# include + +using namespace crowdb::rpc; + +namespace +{ +RdmaConfig test_config(const std::string &address) +{ + RdmaConfig config; + config.use_environment = false; + config.source_address = address; + if (const char *device = std::getenv("CROWDB_RDMA_DEVICE"); device != nullptr) + config.device = device; + config.sq_depth = config.rq_depth = config.posted_receive_target = 8; + config.wire_slot_capacity = 1024; + config.send_arena_bytes = config.receive_arena_bytes = 8192; + config.cq_depth = 16; + return config; +} + +bool wait_for(const std::atomic &count) +{ + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (count.load(std::memory_order_acquire) == 0 && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + return count.load(std::memory_order_acquire) != 0; +} + +struct TestCompletion +{ + std::atomic callbacks{0}; + std::atomic status{CROWDB_RPC_OK}; + std::atomic valid{false}; +}; + +void test_complete(uint64_t request_id, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, crowdb_rpc_status status, + void *context) +{ + auto &completion = *static_cast(context); + completion.status.store(status, std::memory_order_relaxed); + completion.valid.store(request_id == 91 && status == CROWDB_RPC_OK && data != nullptr && + crowdb_rpc_buffer_len(data) == 16 && + std::memcmp(crowdb_rpc_buffer_data(data), "fallback payload", 16) == 0); + if (control != nullptr) + crowdb_rpc_buffer_release(control); + if (data != nullptr) + crowdb_rpc_buffer_release(data); + completion.callbacks.fetch_add(1, std::memory_order_release); +} + +bool test_send(RpcClient &client, SocketTransport &transport, Connection *connection, TestCompletion &completion) +{ + auto *control = build_ping_request(transport.pool(), 91, 0); + auto *data = transport.pool()->alloc(16); + if (control == nullptr || data == nullptr) { + if (control != nullptr) + control->release(); + if (data != nullptr) + data->release(); + return false; + } + data->write("fallback payload", 16); + return client.send(&transport, connection, 91, control, data, 100, test_complete, &completion); +} +} // namespace + +TEST(RdmaHardwareTest, ExpiredCmEndpointFallsBackOnlyBeforePayload) +{ + const char *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) + GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; + const auto config = test_config(address); + RpcServer server; + ASSERT_TRUE(server.configure_rdma(config, true, true, false)); + ASSERT_TRUE(server.listen(address, 0)); + auto &native = TestSocketTransport::rdma_for_tests(*server.transport()); + // Publish a stale endpoint before starting workers. There is deliberately + // no CM listener on the TCP-only service port. + TestRdmaTransport::advertise_expired_endpoint_for_tests(native, address, + static_cast(server.listen_port())); + std::atomic received{0}; + server.register_handler(100, [&received](Frame *frame, Connection *connection) { + received.fetch_add(1, std::memory_order_relaxed); + return handle_echo(frame, connection); + }); + server.start(); + for (const bool allow : {false, true}) { + TestCompletion completion; + SocketTransport transport(1, 1); + ASSERT_TRUE(transport.configure_rdma(config, true, allow, false)); + transport.start(); + RpcClient client; + client.set_completion_pool_size(128); + auto connection = transport.connect(address, server.listen_port()); + const auto *stats = transport.rdma_stats_for_tests(); + ASSERT_NE(stats, nullptr); + EXPECT_EQ(stats->preflight_attempts.load(), 1U); + EXPECT_EQ(stats->cm_attempts.load(), 1U); + EXPECT_EQ(stats->fallback_count.load(), allow ? 1U : 0U); + EXPECT_EQ(stats->rpc_frames_sent.load(), 0U); + if (allow) { + ASSERT_NE(connection, nullptr); + EXPECT_EQ(connection->path(), ConnectionPath::Tcp); + client.attach(connection.get()); + ASSERT_TRUE(test_send(client, transport, connection.get(), completion)); + EXPECT_TRUE(wait_for(completion.callbacks)); + EXPECT_TRUE(completion.valid.load()); + } + else { + EXPECT_EQ(connection, nullptr); + } + transport.stop(); + EXPECT_EQ(completion.callbacks.load(), allow ? 1U : 0U); + } + server.stop(); + EXPECT_EQ(received.load(), 1U); +} + +namespace +{ +void test_peer_close_after_payload(bool stop_server) +{ + const char *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) + GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; + const auto config = test_config(address); + std::atomic received{0}; + RpcServer server; + ASSERT_TRUE(server.configure_rdma(config, true, true, true)); + ASSERT_TRUE(server.listen(address, 0)); + server.register_handler(100, [&received, stop_server](Frame *frame, Connection *connection) { + received.fetch_add(1, std::memory_order_relaxed); + delete frame; + if (!stop_server) + connection->close(); + return static_cast(nullptr); + }); + server.start(); + TestCompletion completion; + SocketTransport transport(1, 1); + ASSERT_TRUE(transport.configure_rdma(config, true, true, false)); + transport.start(); + RpcClient client; + client.set_completion_pool_size(128); + auto connection = transport.connect(address, server.listen_port()); + ASSERT_NE(connection, nullptr); + ASSERT_EQ(connection->path(), ConnectionPath::Rdma); + client.attach(connection.get()); + ASSERT_TRUE(test_send(client, transport, connection.get(), completion)); + if (stop_server) { + ASSERT_TRUE(wait_for(received)); + server.stop(); + } + EXPECT_TRUE(wait_for(completion.callbacks)); + EXPECT_EQ(completion.status.load(), CROWDB_RPC_ERR_CONN_CLOSED); + EXPECT_FALSE(connection->is_open()); + EXPECT_EQ(connection->path(), ConnectionPath::Rdma); + transport.stop(); + server.stop(); + EXPECT_EQ(completion.callbacks.load(), 1U); + EXPECT_EQ(received.load(), 1U); + const auto *stats = transport.rdma_stats_for_tests(); + ASSERT_NE(stats, nullptr); + EXPECT_EQ(stats->cm_attempts.load(), 1U); + EXPECT_EQ(stats->fallback_count.load(), 0U); + EXPECT_EQ(stats->rpc_frames_sent.load(), 1U); + EXPECT_EQ(transport.stats().rpc_frames_sent.load(), 0U); + EXPECT_EQ(transport.connection_count(), 0U); + EXPECT_EQ(server.transport()->connection_count(), 0U); +} +} // namespace + +TEST(RdmaHardwareTest, PeerCloseAfterPayloadFailsOnceWithoutTcpFallback) +{ + test_peer_close_after_payload(false); +} + +TEST(RdmaHardwareTest, ServerShutdownAfterPayloadFailsOnceWithoutTcpFallback) +{ + test_peer_close_after_payload(true); +} + +#endif From 4bf414c8084e908901d6bf93f05044a07ff4c362 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 01:07:35 +0800 Subject: [PATCH 038/258] Record verified RPC transport baselines and close completed foundation work --- .../R238-rdma-registered-buffer-foundation.md | 58 - doc/backlog/R239-rdma-rpc-transport.md | 149 --- doc/backlog/R240-rdma-datapath-stream.md | 3 +- doc/backlog/R241-rdma-chunk-kv-access.md | 2 +- doc/backlog/R242-rdma-fat-client-sdk.md | 2 +- doc/backlog/R5-rdma-alloc.md | 17 +- doc/backlog/backlog.md | 17 +- doc/design/rpc/rpc-flow-analysis.md | 127 +- doc/working/plan-rdma-chunk-kv-access.md | 4 +- doc/working/plan-rdma-datapath-stream.md | 4 +- doc/working/plan-rdma-fat-client-sdk.md | 4 +- doc/working/plan-rdma-multi-node.md | 4 +- doc/working/plan-rdma-registered-buffers.md | 123 -- doc/working/plan-rdma-rpc-transport.md | 1089 ----------------- doc/working/plan-rdma.md | 145 +-- 15 files changed, 177 insertions(+), 1571 deletions(-) delete mode 100644 doc/backlog/R238-rdma-registered-buffer-foundation.md delete mode 100644 doc/backlog/R239-rdma-rpc-transport.md delete mode 100644 doc/working/plan-rdma-registered-buffers.md delete mode 100644 doc/working/plan-rdma-rpc-transport.md diff --git a/doc/backlog/R238-rdma-registered-buffer-foundation.md b/doc/backlog/R238-rdma-registered-buffer-foundation.md deleted file mode 100644 index 77683dd50..000000000 --- a/doc/backlog/R238-rdma-registered-buffer-foundation.md +++ /dev/null @@ -1,58 +0,0 @@ - - - -### R238: RDMA — Registered host-buffer foundation - -## Problem - -R5 requires bounded registered host buffers for RDMA, but the current RPC and -access layers do not share a lifetime contract for buffers used by NIC, -DiskIO, SDK callers, and asynchronous completion. An unbounded or prematurely -reused buffer can corrupt a later stream. The foundation must be independent of -the selected RPC transport and must work when RDMA is unavailable. - -## Solution - -1. Define a process and device scoped registered-memory pool with a hard byte - budget and separate control and data capacity classes: 64 KiB, 256 KiB, and - 1 MiB. -2. Define the ownership states `available`, `posted_receive`, - `receive_in_flight`, `send_in_flight`, `disk_owned`, `caller_owned`, and - `quarantined`. A buffer has one owner and one release transition. -3. Expose an opaque buffer lease that records capacity, actual length, request - ID, stream sequence, logical offset, and release state. Release returns a - caller-owned buffer to its class pool only after all device access ends. -4. Make exhaustion bounded: producers wait or return backpressure, and no - replacement allocation bypasses the registered-memory budget. Unknown - completion state quarantines the buffer until connection quiescence. -5. Keep registration optional. A build or host without RDMA retains the TCP - pool and does not require an RDMA device at runtime. - -## Dependencies - -- R5 remains the umbrella registered-memory goal. -- R4 bounded memory pool supplies the budget and admission principles. -- The output is consumed by R239 RDMA RPC transport and R240/R241 data paths. - -## Acceptance - -- Given a configured byte budget, when all classes are allocated, the next - allocation returns bounded backpressure and never exceeds the budget. - Invariant: registered bytes are bounded. Unit test. -- Given a buffer in each ownership state, when the valid owner releases it, - the next state is deterministic and no second owner can release it. - Invariant: single ownership. Unit test. -- Given an uncertain completion, when the connection closes, the buffer is - quarantined and is not returned to a new stream. Invariant: no reuse before - quiescence. Integration test. -- Given RDMA is unavailable, when the pool is initialized, TCP allocation - remains usable without an RDMA runtime. Invariant: optional capability. - Integration test. - -Run: - -```bash -pixi run -- cargo fmt --all -- --check -pixi run -- cargo test -p crowdb-rpc --all-targets -pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings -``` diff --git a/doc/backlog/R239-rdma-rpc-transport.md b/doc/backlog/R239-rdma-rpc-transport.md deleted file mode 100644 index 923f778b3..000000000 --- a/doc/backlog/R239-rdma-rpc-transport.md +++ /dev/null @@ -1,149 +0,0 @@ - - - -### R239: RPC — RC SEND/RECV transport with TCP fallback - -## Problem - -CROWDB RPC has a common TCP transport but no usable RDMA transport. RDMA must -reuse RPC correlation and framing while combining epoll wakeups with bounded CQ -polling. It must be optional: unsupported endpoints must use TCP without -repeated RDMA attempts, and failures before payload submission must fall back -without changing request semantics. - -## Solution - -1. Add an RC SEND/RECV transport behind the common RPC `Transport` interface. - One or more complete RPC frames are copied into one fixed-size registered - wire slot and sent with one SGE. Optional bounded transport aggregation - preserves each frame's identity and ordering, drains available frames - without a batching timer, and never exceeds the local send slot or the - peer's advertised receive slot, including all headers and metadata. - Receive slots are pre-posted from a registered arena and are - reused only after Frame ownership is released. The complete wire message - must fit the slot capacity. -2. Use service discovery for coarse RDMA capability and TCP handshake for the - current CM endpoint, device/GID identity, capability generation, protocol - version, and supported data classes. Local configuration and probing can - disable RDMA before CM setup. -3. Assign each QP to one shared CQ and each CQ to one fixed polling/event - worker. Busy workers poll and dispatch bounded batches continuously; idle - workers arm notifications, poll again to close the race, then sleep in - epoll. Acknowledge completion events before resuming polling. Generic - socket workers do not compete for CQ ownership. CM remains a separately - serialized lifecycle event source. -4. Select RDMA as the preferred path for one logical connection and TCP as its - one-way fallback. A TCP connection is never upgraded to RDMA. The shared - request ID is used for idempotent or duplicate-suppressed retry. -5. Define CM, QP, receive posting, CQ error, drain, and shutdown states. A - partial or uncertain RDMA stream fails as one operation and is never joined - with TCP bytes. -6. Provide OCI image test profiles. The image contains `libibverbs`, - `librdmacm`, and the provider libraries; the host supplies the kernel RDMA - driver and HCA devices. The current hardware profile runs multiple - containers on one physical host with `--network=host`, `--privileged`, and - `--ulimit memlock=-1`, sharing one configured HCA and opening independent - QPs. A later restricted profile maps only the required `/dev/infiniband` - devices and uses `IPC_LOCK` plus a bounded memlock limit. -7. Keep two current E2E topology profiles. The TCP-only profile exposes no - RDMA devices and verifies that capability discovery avoids all RDMA - attempts. The shared-HCA profile runs multiple logical containers on one - physical host and verifies multiple QPs and fallback. Physical multi-node - topology is tracked by a separate follow-up plan and is outside R239 scope. -8. Use host networking for the current container profile. Every container - receives one `CROWDB_PORT_OFFSET`; each service applies that same offset to - its documented default port. The RDMA HCA device and physical HCA port are - shared and are not offset. The default RDMA CM service port is `19101`, so - the effective port is `19101 + CROWDB_PORT_OFFSET`. -9. Provide `crowdb-cli bench rdma` sharing the coroutine echo workload with - `bench rpc`, driven by the existing RPC regression scripts. Explicitly - configure loaders, QPs, CQs, queue depths, slot capacity and aggregation; - prohibit TCP fallback in RDMA tests and verify the actual path. Measure - validated completions for ten seconds after warmup, with separate drain - accounting, nanosecond latency histograms, categorized errors and canonical - machine-readable results. Report all workload/resource settings, RPC TPS, - SEND rate, avg/P90/tail latency and validity together. Application logical - operations remain a separate metric from correlated RPC completions. - -## Dependencies - -- R238 registered host-buffer foundation. -- `design-crowdb-rpc.md` common frame and correlation contract. -- Existing TCP epoll worker and transport error semantics. -- R240 and R241 consume this transport. - -## Acceptance - -- Given both peers advertise RDMA, when the TCP handshake completes, the - client establishes the negotiated RC path and sends a valid one-SGE wire - slot. - Invariant: shared RPC framing. Integration test. -- Given no RDMA capability or local RDMA configuration, when a connection is - created, TCP is selected without an RDMA attempt. Invariant: optional RDMA. - Integration test. -- Given RDMA setup fails before payload submission, when an idempotent request - is retried, TCP returns one correlated response. Invariant: one-way fallback. - Integration test. -- Given an RDMA payload partially completes, when the connection fails, no TCP - bytes are appended and uncertain buffers remain quarantined. Invariant: - partial transfer isolation. Integration test. -- Given QPs assigned to multiple CQs and busy/idle traffic transitions, when - completions and CM events arrive, each CQ's fixed owner drains completions, - arms/polls before idle sleep, acknowledges wakeups and stops cleanly without - competing socket workers or lost wakeups. - Invariant: event/poll composition. Integration test. -- Given aggregation is enabled and frames approach the peer slot limit, when - frames are submitted, only complete fitting frames share a SEND; every RPC - retains identity, ordering and completion ownership. A malformed suffix - publishes no frames, and a held frame view prevents receive-slot repost. - Invariant: bounded aggregate ownership. Integration test. -- Given an OCI image with no RDMA devices, when the TCP-only profile starts, - service discovery selects TCP without attempting RDMA. Invariant: optional - capability. E2E test. -- Given multiple containers sharing one HCA, when the shared-HCA profile opens - multiple logical connections, each QP remains correlated and bounded, while - the test reports a single physical host rather than physical node isolation. - Invariant: topology declaration matches test claim. E2E test. -- Given a prepared OCI runtime with the required RDMA devices and memlock - limit, when the RPC flow runs as a non-root application user, the container - completes CM/QP/send/receive/repost without invoking `sudo`, loading modules, - moving links, or entering host namespaces. Invariant: privileged host setup - is separated from unprivileged application execution. E2E test. -- Given the RDMA benchmark runner uses the same echo workload as the TCP - regression benchmark, when it runs on a prepared HCA, it reports throughput, - latency including avg/P90/tails, categorized errors, all queue/QP/CQ/slot - settings and actual RDMA path for matching ten-second coroutine cases. - Missing reports, nonzero errors, fallback and failed processes fail the run. - Invariant: - transport regressions and large flow bugs are visible before downstream use. - Benchmark test. -- Given an application linked against `crowdb-rpc`, when deployment provides - verbs/rdmacm libraries, HCA device access, memlock, and peer reachability, - the application uses the ordinary RPC API and the library selects RDMA or - TCP without setup-script calls or namespace operations in application code. - Invariant: RDMA deployment concerns stay outside the library consumer. - Integration test and documented runtime contract. - -Run: - -```bash -pixi run -- cargo fmt --all -- --check -pixi run -- cargo test -p crowdb-rpc --all-targets -pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings -``` - -The initial local command shape is: - -```bash -docker run --rm --network=host --privileged --ulimit memlock=-1 \ - -e CROWDB_RDMA_DEVICE=mlx5_0 \ - -e CROWDB_RDMA_PORT=1 \ - crowdb:test -``` - -The second container uses the same physical host HCA and its own logical -connection. The restricted container profile is a follow-up hardening step -after the shared-HCA profile passes. Physical multi-node execution is specified -in the separate follow-up plan. With host networking, each container must use -a distinct port offset; the offset applies to TCP/RPC and RDMA CM service -ports, while `CROWDB_RDMA_PORT` remains the HCA physical port number. diff --git a/doc/backlog/R240-rdma-datapath-stream.md b/doc/backlog/R240-rdma-datapath-stream.md index 776339afd..4b01d33b5 100644 --- a/doc/backlog/R240-rdma-datapath-stream.md +++ b/doc/backlog/R240-rdma-datapath-stream.md @@ -28,8 +28,7 @@ I/O still uses it. ## Dependencies -- R238 registered buffers. -- R239 RDMA RPC transport. +- Accepted [registered buffers and RDMA RPC](../design/rpc/design-crowdb-rpc-rdma.md). - Existing ChunkIO aggregation, Access Server S3, DiskIO asynchronous I/O, and integrity contracts. diff --git a/doc/backlog/R241-rdma-chunk-kv-access.md b/doc/backlog/R241-rdma-chunk-kv-access.md index 46740045d..68c98e06e 100644 --- a/doc/backlog/R241-rdma-chunk-kv-access.md +++ b/doc/backlog/R241-rdma-chunk-kv-access.md @@ -26,7 +26,7 @@ without changing access semantics. ## Dependencies -- R238 and R239. +- Accepted [registered buffers and RDMA RPC](../design/rpc/design-crowdb-rpc-rdma.md). - Existing Chunk-KV client/server and Access Server access semantics. - Existing ChunkIO aggregation and request-number system. diff --git a/doc/backlog/R242-rdma-fat-client-sdk.md b/doc/backlog/R242-rdma-fat-client-sdk.md index 64ded00cc..0c84551fd 100644 --- a/doc/backlog/R242-rdma-fat-client-sdk.md +++ b/doc/backlog/R242-rdma-fat-client-sdk.md @@ -31,7 +31,7 @@ recycling memory while the caller still reads it. ## Dependencies -- R238, R239, and R241. +- Accepted registered buffers/RDMA RPC and R241. - Existing Access Server access-core and FFI conventions. - R220 remains a separate Dataset SDK plan; this requirement supplies the transport and lease contract it may reuse. diff --git a/doc/backlog/R5-rdma-alloc.md b/doc/backlog/R5-rdma-alloc.md index 7d517487b..c1b2bd097 100644 --- a/doc/backlog/R5-rdma-alloc.md +++ b/doc/backlog/R5-rdma-alloc.md @@ -5,9 +5,9 @@ ## Status -This umbrella requirement is decomposed into R238 through R242. The child -requirements are independently implementable and retain this requirement's -registered-host-buffer, RDMA RPC, datapath, Chunk-KV, and fat-client goals. +The registered host-buffer foundation and common RDMA RPC transport are +accepted. Remaining child requirements R240 through R242 retain the datapath, +Chunk-KV, and fat-client goals. ## Problem @@ -20,13 +20,10 @@ defined in [`design-crowdb-rpc-rdma.md`](../design/rpc/design-crowdb-rpc-rdma.md ## Solution -1. Implement the registered host-buffer foundation in - [R238](R238-rdma-registered-buffer-foundation.md), including 64 KiB, - 256 KiB, and 1 MiB classes, bounded ownership, and quarantine. -2. Implement the common RC SEND/RECV RPC transport in - [R239](R239-rdma-rpc-transport.md), using service discovery plus TCP - handshake endpoint exchange, epoll wakeups, bounded CQ polling, and - RDMA-to-TCP fallback. +1. Reuse the accepted [registered-buffer and RDMA RPC contract](../design/rpc/design-crowdb-rpc-rdma.md): + bounded 64 KiB/256 KiB/1 MiB classes, explicit ownership and quarantine, + TCP preflight, per-CQ polling/event workers and one-way fallback. +2. Retain the verified transport and benchmark invariants when adding services. 3. Implement the S3/HTTP/DiskIO stream in [R240](R240-rdma-datapath-stream.md), reusing existing ChunkIO aggregation and 1 MiB datapath buffers. diff --git a/doc/backlog/backlog.md b/doc/backlog/backlog.md index e74742514..585befbb1 100644 --- a/doc/backlog/backlog.md +++ b/doc/backlog/backlog.md @@ -34,14 +34,15 @@ live in the linked document; implementation follows **Status:** Deferred until the basic TCP S3 path is stable and measured. - **[R240](R240-rdma-datapath-stream.md)** — RDMA S3/HTTP/DiskIO stream using existing ChunkIO aggregation. **Area:** access server / S3 / DiskIO / RDMA. - **Complexity:** High. **Dependencies:** R238 and R239. + **Complexity:** High. **Dependencies:** accepted registered buffers and RDMA + RPC. - **[R241](R241-rdma-chunk-kv-access.md)** — RDMA Access Server to Chunk-KV transport with bounded metadata and aggregate messages. **Area:** access - server / Chunk-KV / RDMA. **Complexity:** High. **Dependencies:** R238 and - R239. + server / Chunk-KV / RDMA. **Complexity:** High. **Dependencies:** accepted + registered buffers and RDMA RPC. - **[R242](R242-rdma-fat-client-sdk.md)** — native fat-client access SDK and FFI buffer leases. **Area:** access server / SDK / RDMA. **Complexity:** High. - **Dependencies:** R238, R239, and R241. + **Dependencies:** accepted RDMA RPC and R241. - **[R185](R185-access-iceberg-cache-invalidation.md)** — bounded Iceberg cache and invalidation. **Area:** access server / Iceberg / Group 0 / Chunk-KV. **Complexity:** Medium. **Status:** Deferred pending focused cache measurements. @@ -108,13 +109,7 @@ live in the linked document; implementation follows **Area:** crowdb-tree engine. **Complexity:** Medium. - **[R5](R5-rdma-alloc.md)** — RDMA host-buffer RPC and data transport umbrella. **Area:** RPC / access server / Chunk-KV / DiskIO / SDK / RDMA. - **Complexity:** High. **Status:** Decomposed into R238–R242. -- **[R238](R238-rdma-registered-buffer-foundation.md)** — bounded registered - host-buffer classes, ownership, leases, and quarantine. **Area:** RPC / RDMA. - **Complexity:** High. **Dependencies:** R4 and R5. -- **[R239](R239-rdma-rpc-transport.md)** — RC SEND/RECV RPC transport with - epoll/CQ integration and TCP fallback. **Area:** RPC / RDMA. **Complexity:** - High. **Dependencies:** R238. + **Complexity:** High. **Status:** Foundation/transport accepted; R240–R242 remain. - **[R60](R60-tree-scan-sibling-leaf-readahead.md)** — sibling-leaf readahead on cold scans. **Area:** crowdb-tree / scan / DiskIO. **Complexity:** Medium. **Status:** Deferred pending cold file/block-backed measurements. diff --git a/doc/design/rpc/rpc-flow-analysis.md b/doc/design/rpc/rpc-flow-analysis.md index d2b57fff6..1426ece05 100644 --- a/doc/design/rpc/rpc-flow-analysis.md +++ b/doc/design/rpc/rpc-flow-analysis.md @@ -30,18 +30,135 @@ The callback mode avoids a tokio scheduler round trip. The tokio mode uses `RpcClient::call`, an oneshot channel, and a task wake for each response. Both models share `RpcClient`, `Connection`, `BufferPool`, and `RpcServer`. -The hot path uses connection pointers in epoll/kqueue user data, per-worker +The TCP hot path uses connection pointers in epoll/kqueue user data, per-worker receive buffers, caller-thread `writev`, batched server responses, and a slab completion pool with a map fallback. The timeout reaper handles lost responses outside the hot path. Reference-count operations are O(1); frame construction, the echo memcpy, flatbuffer serialization, and kernel socket -copies are unavoidable. +copies contribute to this workload's cost. + +RDMA uses the same native coroutine loaders and echo handler. Submission enters +a bounded per-QP send queue; its assigned CQ owner packs complete RPC frames +into a registered slot and posts one SGE. Each CQ has one fixed polling/event +thread. Receive frame views retain their registered slot until the final +consumer releases it; socket workers handle CM rather than competing for CQ +polling. The echo workload still copies its response payload and does not +measure a storage or zero-copy application path. ## 2. Latest Benchmark Results -Both runs use 128B values, a standalone server, loopback, and a 20s duration. -The Linux run is the current regression reference. The macOS run is the latest -retained standalone baseline. `nagle` means TCP coalescing is enabled. +### Linux TCP/RDMA — 2026-10-23 + +The common runners are `bench-rdma-rpc-regression.sh` and +`bench-rpc-regression.sh`. They launch fb-server and `crowdb-cli bench rdma` +or `bench rpc` from the same manifest and retain canonical JSON/CSV/TSV, +complete logs, binary hashes, source identity, settings and raw peer status. +Missing reports, process failures, nonzero errors, fallback or path mismatches +fail the runner. RDMA remote request totals must equal accepted submissions. + +Hardware: Ryzen 9 5950X, 16 physical/32 logical CPUs, approximately 62 GiB RAM, +`rocep35s0f0` port 1, 100 Gb/s link, PCIe 8 GT/s x4. Both peers share this +physical host/HCA. Native means two processes; OCI means two host-network +containers using independent QPs, UID 1000, required Infiniband device mappings +and unlimited memlock. CPU affinity/frequency are not pinned. This topology +measures local HCA traversal rather than physical-node isolation. + +Every trial warms up for 1 s, counts validated completions in a 10 s window, +then drains separately. Coroutine loaders have one request in flight each; +there is no producer OS thread per loader. Latency is native submit-to-response +callback RTT: payload construction and Rust response validation are outside +the latency sample but remain inside throughput work. Histograms use bounded +nanosecond buckets, exact count/sum/extrema, nearest-rank upper bounds and +explicit zero/overflow counts. Resource and transport counter deltas include +warmup and drain, as declared in each row. + +The following rows select the median TPS of three peak repeats and retain +latency from that same trial. All use 128-byte payloads, zero errors and valid +reports. RDMA rows use clean source `55ffee38`; the Nagle-enabled TCP control +is retained from the preceding runner revision with its own binary identity. + +| Path / topology | Loaders | Connections / QPs | CQ / pollers per peer | Socket workers per peer | Aggregation / Nagle | RPC TPS | Avg us | P90 us | P99 us | Errors | Valid | +|--------------------------|---------|-------------------|-----------------------|-------------------------|---------------------|-----------|--------|----------|----------|--------|-------| +| RDMA native | 2048 | 16 | 16 | 1 | aggregation on | 2,903,006 | 704.47 | 819.20 | 987.14 | 0 | true | +| RDMA two OCI containers | 2048 | 16 | 16 | 1 | aggregation on | 2,954,105 | 692.26 | 851.97 | 1015.81 | 0 | true | +| TCP native | 1000 | 32 | n/a | 16 | Nagle off | 1,196,115 | 834.50 | 1679.36 | 6815.74 | 0 | true | +| TCP native | 1000 | 32 | n/a | 16 | Nagle on | 2,114,935 | 471.48 | 872.45 | 1597.44 | 0 | true | + +RDMA queue settings for both topology rows: + +| Code variable | Requested value | Observed / scope | +|------------------------------------|------------------------------|------------------------------------------| +| `io_engines`, `io_workers` | 1, 1 | per peer | +| `cq_count`, `poller_count` | 16, 16 | one fixed thread per CQ per peer | +| `sq_depth`, `rq_depth` | 1024, 1024 | per QP; provider values in canonical row | +| `posted_receive_target` | 1024 | per QP | +| `cq_depth` | 2048 | per CQ; provider returns 4095 | +| `wire_slot_capacity` | 1024 bytes | all headers/control/data included | +| `send_arena_bytes` | 1 MiB | per QP | +| `receive_arena_bytes` | 1 MiB | per QP | +| `send_queue_capacity` | 4096 | per QP | +| `poll_batch`, `verbs_poll_batch` | 256, 32 | per owner | +| `poll_empty_budget` | 256 | per owner | +| `aggregation_max_bytes` | 1024 | also bounded by peer receive slot | +| `aggregation_max_frames` | 256 | complete frames only, no flush timer | +| `registered_bytes` | derived budget | 34 MiB per peer at 16 QPs | +| `rnr_nak_retry_err` | cumulative provider counter | 53 before/after; delta 0 | +| CPU migrations/quarantined bytes | unavailable aggregate | null, never fabricated zero | + +CPU user/system time, context switches, affinity, SEND counts/rate, +frames per SEND, effective capacities and all error categories are present +in the wide canonical tables alongside the raw before/after peer snapshots. +The primary artifacts under `.crowdb-runtime/rdma-check/artifacts/bench/` are +`rdma-rpc-20261022T170138.574260Z` (native), +`rdma-rpc-20261022T170225.046025Z` (OCI), +`tcp-rpc-20261022T170313.500461Z` (TCP Nagle off) and +`tcp-rpc-20261022T162548.233330Z` (TCP Nagle on). +RDMA repeat TPS values are 2,892,325 / 2,907,907 / 2,903,006 native and +2,896,746 / 2,954,105 / 3,002,761 OCI. The OCI median is 1.76% higher; +the observed variation does not establish a container overhead or benefit. + +The staged sweep retained 24 rows and 203,195,362 measured completions at +`rdma-rpc-20261022T160711.769136Z`, then nine higher-load rows at +`rdma-rpc-20261022T161241.180591Z`. With one QP/CQ, increasing coroutine +loaders from 1 to 512 raises throughput from about 60k to 566k TPS. +Aggregation raises the 512-loader case to about 701k. With eight QPs, CQ +counts 2/4/8 produce about 880k/1.48M/2.21M TPS. At higher load, 16 CQs +reach approximately 2.9M; 32 CQs per peer fall to 2.02M and P99 4.00 ms +because both peers together have 64 polling threads on 32 logical CPUs. +RQ 4096 and larger 64/256 KiB slots do not improve matching controls. +Polling-budget and slot controls are retained at +`rdma-rpc-20261022T162311.140989Z`; this is the best repeatable tested +configuration, not proof of the HCA's hardware maximum. + +Payload controls at `rdma-rpc-20261022T163008.055655Z` cover 512 bytes through +1,048,490 bytes with matching 1 KiB/64 KiB/256 KiB/1 MiB wire slots, +64 loaders, two QPs/CQs and SQ/RQ/post target 64. Large-payload echo throughput +plateaus around 1.5 GB/s of validated payload. The full-payload validation, +echo copies and PCIe x4 link all warrant profiling; the 100 Gb/s link rate +alone does not predict this round-trip workload's TPS. The largest payload +has average/P90 latency about 44 ms, so small-message latency must not be +presented as the latency of every size/load case. The no-device TCP OCI +corpus at `tcp-rpc-20261022T163555.799432Z` also passes with zero RDMA attempts. + +Historical audit: the 1,186,876 TPS value once labeled RDMA came from TCP +socket reads/writev in +`.crowdb-runtime/artifacts/bench/rdma-rpc-20261018-220617/server/metrics.log`. +Its old TSV records error rows despite a successful script exit. The new +matching TCP Nagle-off median is 1,196,115 TPS (+0.78%), explaining that +baseline without attributing it to RDMA. The older 2,213,182 Nagle-on TCP +reference is 4.4% above the new 2,114,935 control, but timing, validation and +histogram contracts changed; it is not a controlled transport regression. + +The accepted transport source passes full C++/FFI verification at +`.crowdb-runtime/artifacts/unit/cpp-93edc5b4128b456696da0095db768d20/`, all +137 no-verbs CTest cases, four CLI benchmark cases, thirteen runner contract +cases, formatting and workspace lint. Separate real-HCA cases cover held +receive/MR lifetime, registration exhaustion, TCP-only peers, expired CM +endpoints and peer/server close after payload. Close tests allow fallback but +require exactly one failed completion, retained RDMA path and no TCP payload. + +The sections below retain earlier 20-second TCP references. `Nagle` means TCP +coalescing is enabled; old histogram precision and measurement scopes differ. ### Linux — 2026-08-27 diff --git a/doc/working/plan-rdma-chunk-kv-access.md b/doc/working/plan-rdma-chunk-kv-access.md index 1e90d00fb..5279bd772 100644 --- a/doc/working/plan-rdma-chunk-kv-access.md +++ b/doc/working/plan-rdma-chunk-kv-access.md @@ -4,8 +4,8 @@ # RDMA Chunk-KV access Plan Upstream: [R241](../backlog/R241-rdma-chunk-kv-access.md), -[root RDMA plan](plan-rdma.md), [R238](../backlog/R238-rdma-registered-buffer-foundation.md), -and [R239](../backlog/R239-rdma-rpc-transport.md). +[root RDMA plan](plan-rdma.md), [registered buffers](../design/rpc/design-crowdb-rpc-rdma.md#4-registered-arenas-and-slot-ownership), +and [RDMA RPC transport](../design/rpc/design-crowdb-rpc-rdma.md). Goal: use the common RDMA stream for Access Server to Chunk-KV operations while preserving existing aggregation, request numbering, generation checks, and TCP diff --git a/doc/working/plan-rdma-datapath-stream.md b/doc/working/plan-rdma-datapath-stream.md index 0cdfe602b..a3ddaaee2 100644 --- a/doc/working/plan-rdma-datapath-stream.md +++ b/doc/working/plan-rdma-datapath-stream.md @@ -4,8 +4,8 @@ # RDMA datapath stream Plan Upstream: [R240](../backlog/R240-rdma-datapath-stream.md), -[root RDMA plan](plan-rdma.md), [R238](../backlog/R238-rdma-registered-buffer-foundation.md), -and [R239](../backlog/R239-rdma-rpc-transport.md). +[root RDMA plan](plan-rdma.md), [registered buffers](../design/rpc/design-crowdb-rpc-rdma.md#4-registered-arenas-and-slot-ownership), +and [RDMA RPC transport](../design/rpc/design-crowdb-rpc-rdma.md). Goal: connect existing HTTP/ChunkIO aggregation to the RDMA RPC stream and DiskIO receive loop without introducing another aggregation or CPU payload diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index e4b0301bf..1b7ff4f9b 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -4,8 +4,8 @@ # RDMA fat client SDK and FFI lease Plan Upstream: [R242](../backlog/R242-rdma-fat-client-sdk.md), -[root RDMA plan](plan-rdma.md), [R238](../backlog/R238-rdma-registered-buffer-foundation.md), -[R239](../backlog/R239-rdma-rpc-transport.md), and +[root RDMA plan](plan-rdma.md), [registered buffers](../design/rpc/design-crowdb-rpc-rdma.md#4-registered-arenas-and-slot-ownership), +[RDMA RPC transport](../design/rpc/design-crowdb-rpc-rdma.md), and [R241](../backlog/R241-rdma-chunk-kv-access.md). Goal: expose Access Server semantics through a native fat-client SDK that can diff --git a/doc/working/plan-rdma-multi-node.md b/doc/working/plan-rdma-multi-node.md index 81230e323..0a2aa6981 100644 --- a/doc/working/plan-rdma-multi-node.md +++ b/doc/working/plan-rdma-multi-node.md @@ -4,7 +4,7 @@ # RDMA physical multi-node validation Plan Upstream: [R5](../backlog/R5-rdma-alloc.md), with transport behavior supplied -by [R239](../backlog/R239-rdma-rpc-transport.md) and the current +by [RDMA RPC transport](../design/rpc/design-crowdb-rpc-rdma.md) and the current [RDMA implementation plan](plan-rdma.md). Goal: validate the already implemented RDMA RPC and data paths across physical @@ -33,7 +33,7 @@ HCA physical port is not part of the offset. - [ ] **Bind endpoint devices**: configure one local device/address mapping per container and verify that the negotiated peer identity matches the intended node. -- [ ] **Run transport corpus**: exercise CM/QP setup, two-SGE frames, +- [ ] **Run transport corpus**: exercise CM/QP setup, bounded wire-slot frames, receive repost, request correlation, disconnect/error handling, and pre-payload TCP fallback across the fabric. - [ ] **Run service corpus**: compare TCP and RDMA results for Access Server, diff --git a/doc/working/plan-rdma-registered-buffers.md b/doc/working/plan-rdma-registered-buffers.md deleted file mode 100644 index a8826797c..000000000 --- a/doc/working/plan-rdma-registered-buffers.md +++ /dev/null @@ -1,123 +0,0 @@ - - - -# RDMA registered host-buffer foundation Plan - -Upstream: [R238](../backlog/R238-rdma-registered-buffer-foundation.md), -[R5](../backlog/R5-rdma-alloc.md), and -[RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md). - -Goal: provide bounded registered host buffers and an explicit lease/ownership -contract for every later RDMA RPC and data-path component. - -## Progress - -Capacity classes, ownership states, lease metadata, bounded atomic admission, -lock-free freelists, quarantine exclusion, and C/Rust metadata accessors are -implemented in `buffer.h`, `buffer.cpp`, `c_api.h`, and the FFI buffer wrapper. -The pool is device-independent so TCP-only builds retain the bounded contract. -The verbs MR arena and basic send/receive handoff are complete. Its allocation -and recycle path uses bounded atomic class freelists and an atomic arena bump -pointer; owner transitions, premature-release quarantine, inflight shutdown -quarantine, and FFI ownership accessors now use the common contract. - -The verbs pool now pre-registers one region and carves aligned capacity-class -slices from it. Each lease records the region MR handle and lkey, and -quarantined leases are retained out of the reusable free lists until pool -shutdown. - -## Phase 1: inventory and contract - -- [x] **Map existing buffer ownership**: document the current `Buffer`, - `BufferPool`, `OutFrame`, callback, and FFI ownership transitions before - adding registration. Files: `lib/crowdb-rpc/include/crowdb-rpc/buffer.h`, - `lib/crowdb-rpc/include/crowdb-rpc/transport.h`, - `lib/crowdb-rpc/include/crowdb-rpc/connection.h`, - `lib/crowdb-rpc/ffi/src/sys.rs`. -- [x] **Define capacity classes**: add named constants/configuration for 64 KiB, - 256 KiB, and 1 MiB data capacities plus the 1 KiB control capacity. Reject - zero, overflowed, and unsupported capacities before pool admission. Files: - `lib/crowdb-rpc/include/crowdb-rpc/buffer.h`, configuration headers, and - corresponding Rust FFI types. -- [x] **Define lease metadata**: specify request ID, stream ID, sequence, - logical offset, actual length, capacity, registration identity, and release - state. Keep the handle opaque across FFI. Files: buffer and FFI headers, - `lib/crowdb-rpc/ffi/src/sys.rs`. - -## Phase 2: registered pool - -- [x] **Unify device registration owner**: make the verbs-backed arena use the - common registration identity and failure contract while retaining one MR per - arena. Registration failure must leave TCP allocation usable. Files: - `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h` and - `lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp`. -- [x] **Unify class freelists**: carve all four classes from bounded registered - regions, preserve the byte and slot budgets, and avoid a separate pool path - with different recycle semantics. Files: same pool files and pool tests. -- [x] **Complete ownership transitions**: make acquire, post, completion, disk - handoff, caller release, and quarantine explicit at every async boundary; - reject a second release or recycle before completion. Files: pool, - transport completion paths, and ownership tests. -- [x] **Complete shutdown and quarantine**: drain known operations, keep - uncertain buffers out of reuse until connection quiescence, and deregister - only after device access ends. Files: pool, connection shutdown, and tests. -- [x] **Review hot-path synchronization**: keep allocation/recycle lock-free; - lifecycle destruction is separate from the atomic freelists. Files: pool - implementation. - -## Phase 3: FFI lease surface - -- [x] **Expose lease accessors**: return opaque handle, data pointer, actual - length, capacity, sequence, and logical offset without exposing verbs structs. - Files: `lib/crowdb-rpc/ffi/src/sys.rs`, `client.rs`, `server.rs`, and C ABI - declarations. -- [x] **Enforce release ordering**: caller release returns a buffer only after - the C++ owner confirms completion; Rust must not retain a dereferenceable - view after release. Files: FFI wrapper and ownership tests. - -## Tests - -Unit tests: - -- class rounding/selection and overflow rejection; -- byte-budget admission and per-class counts; -- every valid ownership transition and invalid double release; -- quarantine exclusion and shutdown accounting; -- lease metadata and opaque-handle accessors. - -Integration tests: - -- registration failure falls back to the ordinary TCP pool; -- a completed receive lease is not reusable until explicit release; -- a disk-owned lease is not reusable until disk completion is reported; -- concurrent acquire/release does not exceed the configured byte budget. - -E2E tests: - -- run the TCP-only OCI profile without RDMA devices and confirm the service - remains usable; -- run the HCA local-loop profile and confirm registered buffers survive a complete - send, receive, and release cycle. - -## Consolidated files - -- `lib/crowdb-rpc/include/crowdb-rpc/buffer.h` -- `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h` -- `lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp` -- `lib/crowdb-rpc/ffi/src/sys.rs` -- `lib/crowdb-rpc/ffi/src/client.rs` -- `lib/crowdb-rpc/ffi/src/server.rs` -- `lib/crowdb-rpc/tests/` buffer and lease tests - -## Verification gates - -```bash -pixi run -- cargo fmt --all -- --check -pixi run -- cargo test -p crowdb-rpc --all-targets -pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings -pixi run tree-lint -``` - -Completion requires all ownership, budget, shutdown, and TCP-only tests to -pass. R238 is complete once the recorded gates below pass; R239 transport work -may consume the stable lease contract. diff --git a/doc/working/plan-rdma-rpc-transport.md b/doc/working/plan-rdma-rpc-transport.md deleted file mode 100644 index aad98c5a9..000000000 --- a/doc/working/plan-rdma-rpc-transport.md +++ /dev/null @@ -1,1089 +0,0 @@ - - - -# RDMA RPC transport Plan - -Upstream: [R239](../backlog/R239-rdma-rpc-transport.md), -[root RDMA plan](plan-rdma.md), and -[RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md). - -Goal: complete RC SEND/RECV ownership and fallback acceptance, together with -CLI coroutine benchmarks, bounded RPC aggregation, and one polling/event -worker per CQ, using the existing RPC regression runners. - -## Progress - -The registered-buffer foundation and initial RC SEND/RECV flow are committed. -SEND/RECV tokens retain stable owners, receive leases wait for final C++/FFI -release, and strict RDMA tests reject TCP fallback. Nanosecond histogram and -coroutine handoff/joining fixes are also committed and verified below. - -The current CQ implementation assigns each QP to one bounded CQ, with one -fixed polling/event thread per CQ and separate CM event handling. Early -receives wait for application handler publication; QP teardown gates concurrent -posting. Per-connection arena ownership no longer occupies application -`Connection::user_data`. CQ and provider-returned QP capacity values are -retained, and poll-call counters use per-CQ shards. The CQ diff passed focused -tests, live HCA diagnostics, the full C++/FFI gate, formatting and workspace -clippy and scoped tree-lint. Tree-lint exits zero with style warnings. - -Application reply allocation now uses a separate pool, and real HCA tests -prove held registered/receive leases survive pool/transport/CM-ID destruction. -The C ABI now returns independent handles for retained buffer references. - -Portable and verbs-backed pools now share the same arena/lifetime machinery. -Registration admission remains charged while retired connections retain MRs. -TCP control preflight now precedes CM setup; the compact CM offer binds the -preflight identity/generation and peer receive admission. Strict policy has -passed a two-QP real-HCA RPC round trip and a TCP-only peer rejection case. -Bounded transport aggregation is implemented and verified below. -The CLI workload and shared regression orchestration are now implemented. -Still pending: expanded sustained comparisons, remaining failure injection, -and the two-container acceptance profiles. Do not treat the -historical benchmark comments or older subtask ledger as completed acceptance. -Physical multi-node validation remains outside this scope in -[the follow-up plan](plan-rdma-multi-node.md). - -## Current continuation checkpoint - -- [x] **Use standard queue terminology**: `sq_depth`, `rq_depth`, - `posted_receive_target`, `cq_depth`, `wire_slot_capacity`, `send_arena_bytes`, - and `receive_arena_bytes` are the configuration variables. SQ/RQ map to - verbs `max_send_wr`/`max_recv_wr`; byte budgets use 64-bit values so a 4096 - entry RQ with 1 MiB slots can represent its full 4 GiB receive arena. -- [x] **Verify queue/buffer regressions**: the 32-way C++ build and 26 focused - configuration, buffer, work ownership, CQ/CM, and epoll tests passed. -- [x] **Verify the small-message functional baseline**: 1000 requests, - 128-byte payload, one worker, one QP, one in-flight request, 1 KiB wire - slots, `sq_depth = rq_depth = posted_receive_target = 1024`, and 1 MiB send - and receive arenas passed at 71,682 TPS, P50 13.08 us, P95 17.99 us, - P99 20.69 us, and zero RPC errors. Posted receives flushed at normal peer - shutdown are reported separately by the transport counters. These short - fixed-count results are functional evidence only, not sustained load evidence. -- [x] **Complete staged small-message load acceptance**: discuss these actual code variables - with the user, then run each profile for one second of warmup and ten - seconds of measured traffic. Count successful completions inside that - interval; exclude warmup and post-window draining. Increase - `producer_count`, `io_workers`, and `connection_count` through 1, 2, 4, 8, - 16, and 32, with a separate one-in-flight serial baseline and 32 in flight - per producer for pipelined profiles. Validate 1024 and 4096 RQ depths; - use small wire slots for small-message tests and larger slots/arenas for - later realistic payloads. `RdmaHardwareTest.RpcServerClientTimedLoad` and - `tools/rdma/run-timed-load.py` retain complete stage logs and CSV results - below `.crowdb-runtime/artifacts/rdma/`. Completion correlation, payload - contents, duplicate completion, accepted-versus-completed counts, and - server request totals must agree at every stage. -- [x] **Rerun code gates**: `pixi run test-cpp`, the 26 Rust RPC FFI tests, - `cargo fmt --all -- --check`, `rs-lint`, changed C++ formatting, and - `tree-lint` passed on this diff. Tree-lint exits zero with style warnings. - The first full C++ run hit the existing 128-byte temporary-path limit; - rerunning under `CROWDB_RUNTIME_ROOT=.crowdb-runtime/rdma-check` passed. - Full gate artifacts: - `.crowdb-runtime/artifacts/unit/cpp-3ab318e0c5a04f27a8328be78ca1cab6/`. -- [ ] **Complete remaining transport and OCI acceptance**: negotiated TCP - capability preflight, real registered-lease PD/MR lifetime verification, - and multi-container profiles remain open. Do not treat the historical - gate record below as verification of these unfinished contracts. - -### Timed load evidence - -- `tools/rdma/run-timed-load.py` now runs one-second warmup followed by a - ten-second completion window for every profile. It validates unique QPs, - RDMA path selection, request correlation, 128-byte payload contents, drain - counts, and server totals. Metrics retain bounded per-producer histograms - and in-flight slots rather than one entry per request. -- The 16-worker run exposed competing CM consumers taking passive setup - events. Passive setup now uses a private CM channel until ESTABLISHED, - and shared CM event handling has one atomic consumer. -- The 32-worker run exposed a lost continuation: a competing worker could - clear `continuation_fd_` before acquiring the CQ. Only the CQ consumer now - clears it. The bounded epoll continuation test reproduces that contention - deterministically; all 26 focused tests pass after the fix. -- Every profile in the corrected matrix completed its full ten seconds with - zero RPC, correlation, or payload errors. The single-in-flight baseline - completed 626,983 requests (62,698 TPS, P99 26.24 us). -- All rows below use `producer_count = connection_count`, - `in_flight_per_producer = 32`, `wire_slot_capacity = 1024`, - `sq_depth = rq_depth = posted_receive_target = 1024`. Each I/O worker - count is **per peer**, with both peers on one physical host/HCA. - -| Producers / QPs | TPS: I/O workers = producers | TPS: I/O workers = 1 | -|-----------------|------------------------------|---------------------| -| 1 | 182,431 | 183,340 | -| 2 | 292,921 | 322,559 | -| 4 | 327,623 | 356,282 | -| 8 | 354,189 | 396,683 | -| 16 | 206,622 | 392,360 | -| 32 | 83,218 | 292,424 | - -- Complete corrected matrix: - `.crowdb-runtime/artifacts/rdma/20261022T100944Z/results.csv`. -- Fixed-I/O-worker control: - `.crowdb-runtime/artifacts/rdma/20261022T101119Z/results.csv`. -- The `rq_depth = posted_receive_target = 4096` control, with - `receive_arena_bytes = 4 MiB` per QP and `io_workers = 1`, passed at - 369,719 TPS for 8 producers/QPs (P99 761.86 us) and 278,266 TPS for 32 - (P99 4653.06 us), both with zero errors. Complete logs: - `.crowdb-runtime/artifacts/rdma/20261022T101244Z/results.csv`. -- The control supports investigating shared-CQ worker contention and CPU - scheduling before increasing receive depth. It does not establish the HCA - hardware throughput limit. CPU affinity and frequency are not fixed. -- Remaining contract work is separate from this sustained small-message - acceptance: complete TCP capability preflight before CM probing and prove - fallback, retain real PD/MR lifetimes through held leases, and run the OCI - profiles. Existing capability serialization and state predicates alone do - not prove that every connection follows the negotiated setup contract. - -### Benchmark audit and unified measurement - -User decisions: use coroutine loaders for the primary throughput benchmark; -keep producer-thread stress as diagnostic coverage; include errors in every -performance result; prohibit TCP fallback in RDMA benchmarks; redesign the -latency histogram to resolve sub-65-us RTTs. - -- [x] **Audit historical baseline**: the retained 128-byte run at - `.crowdb-runtime/artifacts/bench/rdma-rpc-20261018-220617/server/metrics.log` - records TCP socket read/writev data for all cases. Its 32-connection count - is 3,560,660 payloads over approximately three seconds, matching the - script comment's 1,186,876 TPS. Treat that number as TCP-path evidence, - not a verified RDMA baseline. The old `results.tsv` files contain error - rows despite successful shell exit; the script also expects JSON while - the current CLI emits a human report. -- [x] **Add explicit fallback policy**: `CROWDB_RDMA_ALLOW_TCP_FALLBACK=0` - sets `SocketTransport::allow_tcp_fallback_` false. Unavailable, disabled, - or uncompiled RDMA then returns connection failure before opening a TCP - socket. The default remains enabled; invalid policy values disable TCP - fallback. Count only allowed fallback attempts. Five path-policy tests - and five existing capability tests pass. - The full 126-test native RPC suite passes (hardware-only cases skip there), - and the actual hardware worker round trip also passes with fallback disabled. -- [x] **Reject false benchmark success**: both RDMA load runners set the - strict policy. The regression script preserves complete per-case client - output, records `client_exit`, `report_errors`, and `valid`, and returns - nonzero for a failed client or invalid report. Missing RPC error counts - are `NA`, not zero. A rebuilt-client invalid-device reproduction returns - client exit 2 and script exit 1. Retained evidence: - `.crowdb-runtime/artifacts/rdma/strict-policy-script-fixed/`. - Changed C++ formatting/tree-lint and workspace clippy pass. The historical - table is now annotated as unverified RDMA data instead of a regression gate. - Four runner contract tests cover a successful report, a failed client with - preserved error count, a missing report despite exit zero, and reported RPC - errors despite exit zero. These use fixture processes without an HCA: - `pixi run -- python -m unittest discover -s tools/benchmark/tests -p rdma_regression_test.py -v`. -- [ ] **Unify the workload and report contract**: use one - coroutine-driven RPC echo workload, one case manifest, - and one JSON/CSV schema for native processes and two host-network - containers. Require actual RDMA path and unique QP counts. Verify request - correlation and response length/content; include submission rejection and - native coroutine retry counters, which are not fully represented by the - current Rust `EchoHandler` response-error counter. - Audit the built-in echo allocator before using it as the common workload: - it allocates replies from `conn->pool()`, which is currently the RDMA - receive pool, while the diagnostic echo uses `server.pool()`. A completely - preposted receive pool leaves no allocation admission for those replies. -- [x] **Unify histogram semantics**: use nanosecond input, fixed bounded - memory, mergeable per-polling-owner shards, explicit zero/overflow counts, - exact count/sum/min/max, and avg/P50/P90/P95/P99/P99.9 output. Remove the - common C++/Rust histograms' 65.5-us underflow floor; do not retain a separate - benchmark-only precision contract. Specify a shared bucket layout and - golden vectors. Record timing origin and exclude warmup/drain from TPS. -- [x] **Replace competing CQ consumers before expanding the performance matrix**: - `RdmaTransport` now assigns QPs to a bounded CQ pool and starts one fixed - polling/event thread per CQ. Generic epoll workers handle CM events only. - Busy polling continues under traffic; idle workers arm, poll again, then - sleep on their private epoll engine. `poller_count` derives from `cq_count`. - Native and OCI comparison with the same manifest remains in phase E; - neither single-host topology establishes physical-node isolation. - -Required result schema (client/server resource values are separate): - -| Group | Required columns | -|-------------|------------------------------------------------------------------------------------------------------------------------------| -| Identity | case_id, run_id, git_sha, binary hashes, topology, CPU/HCA identity, CPU affinity, actual path, fallback policy/count | -| Workload | client_mode, loader_count, producer_threads, in_flight_per_loader, connection_count, qpair_count, logical_ops_per_rpc, payload_bytes, wire_bytes, aggregation_enabled, aggregation_max_bytes, aggregation_max_frames, frames_per_send | -| Scheduling | io_engines, io_workers, cq_count, poller_count, poll_batch, verbs_poll_batch, poll_empty_budget, epoll_wakeups, CQ event/poll counts | -| Capacity | requested/effective sq_depth, rq_depth, posted_receive_target, cq_depth, wire_slot_capacity, arena bytes, application_buffer_capacity, send_queue_capacity, queue/pool high-water marks | -| Measurement | warmup_s, measure_s, repeat_index, latency_scope, submitted, completed_in_window, drain_completed, rpc_tps, logical_ops_s, payload_Bps | -| Latency | latency_count, avg_ns, p50_ns, p90_ns, p95_ns, p99_ns, p999_ns, min_ns, max_ns, histogram_zero_count, histogram_overflow_count | -| Errors | errors_total, setup_errors, submit_rejects, timeouts, connection_errors, cq_errors, payload_mismatches, duplicate_completions, retries, report_errors, client_exit, server_exit, valid | -| Cost | client/server CPU time, context switches, CPU migrations, registered bytes, quarantined bytes, available RNR/provider counters | - -All fields belong to one machine-readable row; unavailable values are null. -Human tables can show projections joined by case_id but always include errors -and validity. Normal shutdown WR flushes are separate from workload failures. -Each measured operation is one validated RPC round trip. Transport aggregation -may put multiple complete RPC frames in one SEND slot; count every correlated -RPC once and report SEND count and frames per SEND separately. Application -aggregation, if benchmarked, must report logical_ops_per_rpc and logical_ops_s -separately from rpc_tps. - -Recommended sweep: single-loader latency baseline; then sweep coroutine count -with one QP/CQ/poller; then QP count; then CQ/poller count; then queue depths -and payload/slot size. Use one-second warmup and ten-second measurement, repeat -candidate peaks, and retain all trials and errors. Choose the highest repeatable -zero-error TPS and show its latency rather than selecting unrelated best -throughput and latency fields from different trials. Wire header/control bytes -count toward slot capacity, including at the 1-MiB class boundary. - -## Agreed implementation sequence - -The user selected joint implementation of the benchmark and CQ architecture -on 2026-10-22, then requested this plan before code changes. The primary runner -is `tools/benchmark/bench-rdma-rpc-regression.sh`: start the RPC server and use -`crowdb-cli bench rdma` as its client, sharing the workload implementation with -`bench rpc`. Thread-driven gtests remain transport diagnostics. Transport -aggregation is permitted for max TPS; `qpair_count` and `wire_slot_capacity` -are independent, reported parameters. These decisions replace earlier notes -deferring multiple CQs or forbidding transport coalescing. - -### A. Contract and reliable timing - -- [x] **Record the agreed scope**: amend this plan, the root decisions, and - R239's high-level contract; preserve historical measurements as historical. - Files: this plan, `plan-rdma.md`, `../backlog/R239-rdma-rpc-transport.md`. -- [x] **Replace the histogram floor**: use a shared C++/Rust logarithmic - nanosecond bucket layout with bounded memory, explicit zero/overflow, exact - count/sum/min/max, and P50/P90/P95/P99/P99.9. Keep existing metric callers - compatible, expose mergeable snapshots, and verify shared golden vectors, - sub-microsecond values, bucket boundaries, empty/reset and overflow cases. - Files: `lib/crowdb-common/{cpp,rust}/` histogram implementations and tests; - `app/crowdb-cli/src/commands/bench/{loader,result}.rs`. -- [x] **Make coroutine completion safe before parallel CQ benchmarks**: - `co_on_complete` currently fails a CAS against `CO_RUNNING`, then stores - `CO_RESPONSE_READY`; `await_suspend` can change to `CO_SUSPENDED` between - those operations and lose the resume. Replace that handoff with one atomic - publication. Replace concurrent `coroutine_handle::done()` inspection and - forced timeout resume/destruction with explicit final-suspend publication - and callback quiescence. Keep timeout handling in the client's request - completion/reaper path. Add deterministic completion-before/after-suspend - and concurrent handoff tests, plus real coroutine echo/error coverage. - Files: `client/co_client.cpp`, a private coroutine state header, RPC tests - and FFI coroutine API. This is prerequisite to attributing multi-CQ errors - or performance to the transport. - -Verified coroutine checkpoint: - -- `co_on_complete` publishes readiness with one atomic exchange, and final - suspension publishes completion once. The joining thread sleeps while - waiting and checks callback quiescence before reclaiming state or Rust - context; it never inspects a concurrently running coroutine handle or - force-resumes a pending await. Closed-connection submission rejection is - counted and stops that coroutine. Configure the existing request reaper for - stalled requests; its ordinary completion path handles timeouts. -- Rust `co_spawn` returns native `CoStats`, including submission rejection - errors; `CoBenchHandler::on_response_data` borrows response bytes for payload - validation. CLI report integration remains pending in phase D. -- Three C++ handoff cases pass, including 2,000 concurrent publication trials. - Three Rust integration cases pass: 2,048 validated echoes across 64 - coroutines/four connections/four I/O workers, timeout completion, and a - rejected closed connection. The full 29-test RPC FFI suite, 133-case native - RPC CTest suite (four hardware cases skipped), fmt, workspace clippy and - scoped tree-lint pass. No sanitizer or multi-CQ claim is made by these tests. -- The FFI build now tracks private source headers as well as public headers; - changes to the coroutine state or private transport headers cannot silently - leave the native client stale. - -Verified histogram checkpoint: - -- C++/Rust share 18 golden vectors, inclusive bucket bounds, nearest-rank - quantiles, exact 0..255 ns resolution, 128 sub-buckets per magnitude below - `2^40 ns`, explicit zero/overflow counts and a 128-bit exact sum. Owned - shards merge without atomics; final cumulative captures are exact after - producers join. Concurrent live/window captures remain approximate. -- Both language suites cover sub-microsecond samples, empty/reset, nearest - rank, exact extrema, overflow/carry, shard merge and concurrent final counts. - All 79 Rust common tests, workspace clippy, formatting and scoped tree-lint - pass. Latest full C++/FFI gate: - `.crowdb-runtime/artifacts/unit/cpp-0837d1ed9b1e4171a8c7afdd54f955c8/`. -- The diagnostic thread workload now uses the common histogram. One 10 s - hardware run with 1 s warmup, one producer/QP/worker, 32 in flight and 1 KiB - slots completed 1,767,704 measured RPCs with zero errors or histogram overflow - (176,770 TPS; avg 174.61 us; P90 195.58 us; P99 215.04 us). Submitted count - equals measured + drained. This verifies the migrated diagnostic histogram, - not the future coroutine max-TPS claim. Artifact: - `.crowdb-runtime/artifacts/rdma/20261022T111938Z/results.csv`. - -### B. CQ owners and resource ownership - -- [x] **Create the bounded CQ pool**: add `RdmaConfig::cq_count`, - `verbs_poll_batch`, and strict environment validation. Each CQ owns its - completion channel, wake fd, fixed poller and bounded WR table. Assign each - QP to one CQ at creation; SEND/RECV and repost retain that assignment. - Distinguish requested from provider-effective depths and reject insufficient - CQ capacity for the configured QPs and posted receives before traffic. - Files: `rdma_transport.h`, `rdma_config.cpp`, `rdma_connection.cpp`, - `rdma_work_request.cpp`, new private CQ implementation under `transport/rdma/`. -- [x] **Run one polling/event worker per CQ**: continuously poll while busy, - dispatch callbacks on the owner, arm notification after the empty budget, - poll again before sleeping, acknowledge events, and wake promptly on submit - and stop. Keep CM management separate from CQ completion ownership; generic - epoll workers cannot compete for that CQ. Stop producers, wake/join pollers, - drain/quarantine WRs, then destroy QPs/CQs/channels. Test idle/busy transitions, - arm races, contention, multi-CQ mapping, partial setup and repeated teardown. - Files: RDMA transport/completion sources, `socket_transport.cpp`, RPC tests. - Retain arenas in the connection's private transport context; leave - `Connection::user_data` to its caller. Gate per-QP posting before destruction, - and publish migrated CM IDs together with their owner map under the existing - lifecycle mutex. Reclaim a completed WR entry before invoking an application - callback or releasing source-buffer owners, so exact-depth handoffs can - admit a reply/repost. Poll-call counters count verbs invocations and use - per-CQ atomics with aggregation on status reads. - A CQ worker may receive data before CM returns and the caller installs its - frame handler. Retain those completed receives in a bounded per-CQ deferred - list, with their WR/connection/pool owners, until handler publication wakes - the CQ owner. Connection-level capability frames are consumed at the common - connection seam. Cancellation takes exclusive CQ ownership to retire deferred - work; shutdown joins pollers before destroying QPs/CQs. No hot-path lock is - introduced. Test early receive/late handler, deferred cancellation, fixed-owner - busy/idle wakeups and late buffer release. -- [x] **Separate application and receive allocators**: built-in echo and - default handlers allocate from the common application pool, not a fully - preposted receive arena. Receive posting and repost explicitly use the - connection's retained RDMA receive pool. Test `posted_receive_target == - rq_depth` with request and response payload validation. Files: `connection.h`, - RDMA setup/completion, `c_api.cpp`, `server/handler.cpp`, transport tests. -- [x] **Finish real registered-lease shutdown**: retain PD/context/MR owners - until the last held receive lease is released, including after transport - shutdown; no deregister-before-release, double repost, premature recycle or - silent PD leak. Files: RDMA buffer pool and lifecycle sources, lease tests. - `RdmaBufferPool` becomes a facade over a shared arena; each outstanding - buffer retains that arena, including buffers returned by `register_buffer`. - The arena retains a shared `RdmaDevice` PD owner and the `rdma_get_devices` - context list. Deregister MR before releasing PD, then release the list. - Replace raw-pointer freelist heads with tagged entry indices and atomic - links to prevent concurrent allocation/recycle ABA and link data races. - Enforce a hard total buffer-entry bound as well as the arena byte bound. - Verify the C ABI handle layer too: `crowdb_rpc_buffer_ref` currently - increments the inner buffer reference but returns the same wrapper, while - `crowdb_rpc_buffer_release` deletes that wrapper. Return an independent - opaque handle per retained reference and test both release orders, an - external owner released exactly once, and a cloned real registered lease - surviving pool/PD facade destruction. This is part of the lease contract. -- [x] **Finish portable pool parity**: the no-verbs `RegisteredBufferPool` - still uses raw-pointer freelist heads and a facade pointer in outstanding - buffers. Reuse the bounded shared-arena/index freelist machinery for that - pool too, retaining detach semantics and optional registration. Add no-device - concurrent recycle, held lease after facade destruction, byte/descriptor - bounds, quarantine and detach tests. Make `SystemBufferPool` outstanding - admission one atomic reservation so parallel reply allocation cannot exceed - its declared count. Files: `buffer.h`, `buffer.cpp`, private common arena - sources, RDMA pool wrapper and buffer tests. This closes the foundation - contract without leaving concurrency safety dependent on HCA availability. - The shared private `BufferArena` owns descriptors; `BufferRegion` owns its - allocation and optional MR/PD owner. Detached portable descriptors keep arena - ownership until their final callback, returning index admission only then; - the detached byte range stays committed. Add `CROWDB_RPC_ENABLE_RDMA=OFF` - as a supported CMake build switch so the no-verbs gate is reproducible even - on this host with rdma-core installed. -- [x] **Bound registration across connection retirement**: add a byte - reservation to the existing `RdmaDevice` resource owner. A held lease must - retain that reservation after its QP releases CQ admission; a new QP cannot - bypass the registered-memory bound by replacing an arena that is still held. - Add `registered_memory_budget_bytes` (zero derives a checked bound from the - CQ-admitted QP count, per-QP arenas and compatibility copy pools), current and - high-water reserved bytes, and registration rejection/failure counters. Keep - reservations atomic and retained by the MR owner until deregistration. - Reject setup when either send or receive MR creation fails: a non-null pool - facade alone does not prove registration succeeded. Test retained-lease - exhaustion/release and concurrent reservations, plus invalid/overflowed - budgets. Files: `rdma_device.{h,cpp}`, RDMA config/setup/pool sources and - resource lifetime tests. Expose these values in phase D's status/report API. - Include final detached-buffer quarantine accounting in the shared arena and - verify it with a no-device case. - -Registration budget checkpoint: - -- `RdmaDevice` retains atomic byte reservations through MR deregistration. - `RdmaConfig::registration_budget()` checks the derived bound and explicit - minimum; malformed 64-bit environment values and arithmetic overflow fail - configuration. Pool registration success is checked before connection - publication. Registration rejection and provider failure have separate - counters; failed construction returns the temporary byte charge. -- Forty focused buffer/configuration/completion cases pass (one pre-CM copy - compatibility case skips). New cases cover 160,000 concurrent reservations, - failed-MR rollback, explicit/derived/overflowed budgets, and a real retired - MR that blocks replacement until its final buffer release. -- The real HCA setup case admits one QP, rejects a second QP when the explicit - byte budget is exhausted despite spare CQ capacity, and preserves the first - connection and exact current/high-water charge. The held-receive/registered - copy test also passes ten repeated transport/CM teardown cycles with this - byte reservation owner in place. -- Full enabled C++/FFI gate, all 29 Rust RPC FFI tests, formatting, scoped - tree-lint and workspace clippy pass. Complete gate artifact: - `.crowdb-runtime/artifacts/unit/cpp-cecb4460f864486aa296056ea8209115/`. - The updated no-verbs build also passes all 124 CTest cases. - -Portable pool verification: - -- `BufferArena`/`BufferRegion` back both registered pool facades; detach keeps - its arena owner, and final detach release returns descriptor admission. Six - portable tests cover held/detached leases, the hard descriptor and byte - bounds, quarantine, 160,000 concurrent recycling iterations and concurrent - `SystemBufferPool` count admission. -- `CROWDB_RPC_ENABLE_RDMA=OFF` builds successfully; all 123 CTest cases pass - in that build. The enabled build passes all 42 focused buffer/completion - cases (one compatibility-copy case skips before CM adoption). The real HCA - held receive/registered-copy test passes ten repeated CM setup/teardown cycles - after the shared-arena extraction. -- Full enabled C++/FFI gate and all 29 Rust RPC FFI tests pass: - `.crowdb-runtime/artifacts/unit/cpp-26a123727c20408cbc7ef7f73be952f1/`. - Formatting, scoped tree-lint and workspace clippy pass. Tree-lint retains its - non-blocking style warning set. - -Application allocator verification: - -- `SocketTransport` passes its application pool into RDMA connections; - `prime_receive` allocates from private `ConnectionPools::recv` explicitly. - The C API and native test use the same built-in `handle_echo` implementation. -- A real HCA worker round trip with RQ/post target 1024 and exactly 1 MiB of - 1 KiB receive slots completed 1000 payload-validated requests with zero - request errors. The server asserts the connection application pool equals - `RpcServer::pool()`. This fixed-count case is functional evidence only. - -Registered lifetime verification: - -- A real registered lease, cloned through the C ABI, remains readable after - its pool facade is destroyed. Final release destroys the arena/MR before - the retained PD owner; `ibv_dealloc_pd` returns zero. Two separate C ABI cases - reproduce the original shared-wrapper bug before the fix and pass afterward, - covering both release orders and exactly one external-owner callback. -- Eight producer threads complete 160,000 allocate/clone/recycle iterations - without duplicate live slot ownership or payload corruption. Oversized - allocation failures preserve smaller-class descriptor admission. -- The live local-HCA CM case holds a receive lease and a `register_buffer` - copy across destruction of both transports, both CM IDs, and caller-held - connection handles. Both payloads remain readable; releasing their final - references expires the respective device owners. Ten repeated setup/teardown - cycles pass. This covers real MRs rather than only the emulated pool. -- Closed-connection flushes count in `shutdown_flushes`; open-connection flush - and other CQ errors still fail the stream. The focused regression verifies - these counters independently. -- Full C++/FFI gate before the final C ABI delta passes at - `.crowdb-runtime/artifacts/unit/cpp-df0a8b16e65c44dfb32c065e6a50cd1e/`. - After that delta, all 149 native RPC CTest cases pass (five opt-in cases skip - there), all 29 Rust RPC FFI tests pass, changed formatting and scoped - tree-lint and workspace clippy pass. -- The same 2-QP/2-CQ/2-poller, 2-producer/window-32 diagnostic profile now - completes 7,828,933 measured RPCs in 10 s: 782,893 TPS, avg 78.96 us, - P90 88.58 us, P99 100.35 us, zero request errors/overflow, 57 drained. - The earlier CQ-only revision measured 827,807 TPS; this single run is about - 5.4% lower and is not sufficient to attribute a stable regression or identify - max TPS. Keep both source/binary identities; do not sacrifice lease ownership - to recover throughput. Latest artifact: - `.crowdb-runtime/artifacts/rdma/20261022T131359Z/`. - -CQ verification: - -- 27 focused capability/configuration/completion tests pass, including CQ - reservation rollback, partitioned tokens, independent polling gates, deferred - receive ownership, wake-on-handler-publication, cancellation and exact-depth - callback admission. A fixture allocation failure was diagnosed in GDB and - fixed by separating application-buffer admission from the WR exhaustion - being tested; it was not a transport CQ crash. -- The final diagnostic uses two native processes on one HCA, two QPs/two CQs - per peer, one polling thread per CQ, one generic I/O worker per peer, two - producer threads, 32 in flight each, 128-byte payload, 1 KiB slots, - SQ/RQ/post target 1024, CQ depth 4096 per CQ, and 1 MiB send/receive arenas - per QP. After 1 s warmup, 10 s measured traffic completed 8,278,071 RPCs: - 827,807 TPS, avg 74.92 us, P90 86.02 us, P99 97.28 us, zero RPC errors and - zero histogram overflow; 62 requests drained afterward. Actual QP/CQ/poller - counts are asserted. This is a transport diagnostic, not the primary CLI - coroutine max-TPS result. Artifact and binary/source identity: - `.crowdb-runtime/artifacts/rdma/20261022T124101Z/`. -- A prior successful diagnostic at - `.crowdb-runtime/artifacts/rdma/20261022T122640Z/` measured 829,227 TPS with - its retained binary hash. The subsequent source revision corrected/sharded - poll-call statistics; do not combine measurements from these two binaries. -- `.crowdb-runtime/artifacts/rdma/20261022T122423Z/` is invalid: the runner - started with the previous binary after a failed incremental build. Its - `INVALID.md` explicitly excludes it from acceptance/comparisons. Later runs - require a successful build before launch and retain binary/source hashes. -- The full C++ gate and all 29 Rust RPC FFI tests pass on the final CQ source. - Complete logs and results: - `.crowdb-runtime/artifacts/unit/cpp-c333ccb1d15145bfadc725c037c7be19/`. - Workspace clippy and changed C++ formatting also pass. - -### C. Bounded aggregation - -- [x] **Negotiate receive admission before coalescing**: finish TCP capability - preflight before the CM attempt; reuse its TCP connection only for allowed - pre-payload fallback. Publish actual CM address/port, per-instance generation, - device/GID identity, receive slot capacity and supported aggregate frame count. - Bind the CM private-data exchange to the preflight generation and capacities; - reject stale/mismatched peers before publishing an RDMA connection. Direct - low-level CM diagnostics also negotiate receive slot capacities. Strict RDMA - permits the TCP control preflight but never TCP workload bytes. Keep no-device - strict rejection before opening a socket. Use an unregistered RAII TCP - control socket: close it on RDMA success and transfer its aligned stream to - the TCP worker only for allowed fallback. This avoids registering a short - lived probe in a data worker and racing descriptor reuse. This is a prerequisite - for the peer-slot bound below; the remaining OCI/failure matrix stays in E. - Keep the compact CM offer within the RC request's 56-byte private-data limit; - use a fixed 48-byte offer with explicit encoding rather than a native struct - dump. It carries protocol, receive capacity/frame limit, local generation, - expected peer generation and endpoint identity digests. The full TCP - capability carries the CM endpoint and identities. Copy event private data - before acknowledging the CM event. Direct CM diagnostics omit only the - preflight expectation, while still negotiating receive admission. - Files: RDMA capability/CM sources, `socket_transport.cpp`, server readiness, - capability/path and hardware tests. -- [x] **Coalesce complete frames on the CQ owner**: use bounded lock-free - admission and per-QP queues; drain available frames without adding a waiting - timer. Keep per-QP ordering and common frame/request IDs. Limit a SEND to - `min(aggregation_max_bytes, local send slot capacity, peer receive slot - capacity)` including every header/control byte and to - `aggregation_max_frames`. Disabled mode sends one frame. Reject oversized - single frames before admission; do not fragment or silently fall back. - Files: RDMA submit/owner loop, `transport.h`, config/capability validation. - Reuse the common `SendQueue` MPSC primitive in `ConnectionPools` for RDMA - admission. Keep it separate from `Connection::send_queue`: TCP close drains - that queue under `in_send_`, which cannot also act as the RDMA CQ consumer. - Keep a bounded - per-CQ connection directory, changed only under its existing CQ ownership - gate during cold setup/retirement. The CQ owner drains per-QP queues with a - fixed SEND budget and retains one overflow frame for the next slot; do not - read a lifecycle-mutated owner map in the polling loop or introduce an - atomic-shared-pointer load with hidden locking on every poll. Wake a sleeping - owner with a pending-publication/idle handshake rather than an eventfd write - for every RPC. Retire admitted/pending frames on every failure path. -- [x] **Retain aggregate owners through completion**: one SEND WR retains all - constituent frames until CQE; failure closes the uncertain stream and - quarantines its slot once. Validate the entire received aggregate before - publishing any frame. Give each frame an independent immutable buffer view - sharing the receive-slot lease; repost only after all C++/FFI views release. - Test exact slot fit, one byte over, multi-frame correlation/order, malformed - suffix, held middle frame, send errors, bounded exhaustion and shutdown. - Files: RDMA completion/WR code, buffer views and focused RPC tests. - Link constituent `OutFrame` owners in the SEND WR without a per-frame vector - allocation. Parse the full receive slot into bounded header/offset records - before the first callback. Each published payload view has its own descriptor - and view offset, sharing only the parent slot lease; `ref_clone` on the parent - alone cannot represent different immutable offsets safely. Keep a parent - reference until every frame has been dispatched. Defer a mixed capability/data - aggregate if the application handler is not yet published. -- [x] **Expose aggregation observations**: export actual frames/SEND, wire - bytes, queue high-water and rejects. Keep RPC TPS distinct from transport - SEND rate. Export TCP frames/writev and Nagle settings alongside RDMA SEND - aggregation; phase E owns the sustained comparisons. - Files: transport statistics, C API/Rust FFI and benchmark reports. - -Aggregation checkpoint: - -- `rdma_aggregate.{h,cpp}` validates all header/control/payload ranges and a - maximum of 256 frames before dispatch. `Buffer::slice` creates independent - descriptors retaining one parent receive lease. `rdma_receive.cpp` holds the - parent across all callbacks and postpones mixed early aggregates as a whole. -- `rdma_send.cpp` owns per-QP bounded MPSC admission and one pending next-slot - frame. The assigned CQ rotates QPs through its SEND budget, bounds posts by - effective SQ capacity, and retains the linked source chain until completion. - Queue retirement waits for producer publication. CQ cancellation uses an - atomic connection identity before reading retained owners from a WR that - may concurrently be reused by another QP. -- Producers wake an idle CQ using one atomic IDLE/PENDING handshake; active - traffic performs no eventfd write per RPC, and busy polling no longer reads - the continuation fd on every iteration. Config exposes - `aggregation_enabled`, `aggregation_max_bytes`, `aggregation_max_frames`, - and `send_queue_capacity`; receive capabilities advertise parser support. -- All 37 focused aggregate/completion/configuration/send cases pass. These - include exact fit and one byte over, malformed suffix, independent held - middle frame, capability/data deferral, chain release after CQ error, - frame/byte limits, disabled aggregation, queue exhaustion and failed post. - Send-post fixtures use real registered slots and a controlled QP post hook. -- Real two-process strict RDMA, two QPs, window 32 completes 1000 validated - requests with zero RPC/CQ errors; aggregation uses 225 client SENDs and - 248 server SENDs (including control traffic). Held receive/MR lifetime and - registration-budget HCA cases also pass. This fixed-count result is - functional evidence only; phase E owns sustained comparisons. -- Full C++/FFI gate, all 29 Rust RPC FFI tests, formatting, scoped tree-lint - and workspace clippy pass. Full gate artifact: - `.crowdb-runtime/artifacts/unit/cpp-00a90cd3faa7480f894c80dee7ab03bc/`. - The no-verbs build passes all 133 CTest cases. Native statistics are present; - status/FFI/CLI exposure is verified in phase D; sustained comparisons are - the active continuation in phase E. - -Preflight verification: - -- Version 2 TCP capability carries CM address/port, generation/device/GID, - receive slot and frame limit. The 48-byte CM offer binds both generations - and explicit identity digests; the digest is a consistency check, not peer - authentication. Event private data is copied before acknowledgement. -- The raw nonblocking control socket uses one absolute deadline for connect, - write, header and capability reads. Truncated and stalled replies fail - closed. An allowed TCP-only peer reuses the aligned socket for a validated - echo; strict policy records no CM attempt or fallback. -- All 169 enabled native CTest cases pass (opt-in hardware cases skip there). - The no-verbs build passes all 130 cases. Full C++/FFI gate, including all - 29 Rust RPC FFI tests, passes at - `.crowdb-runtime/artifacts/unit/cpp-8022af25b2b643cea3d99b3bc35b35d2/`. - A separate real-HCA two-process, two-QP worker case passes 1000 validated - requests with zero request/CQ errors and fallback disabled. This is - functional evidence, not a sustained benchmark. - -Build parallelism checkpoint: - -- Native CMake builds use `-j32`. The RPC FFI build dependency enables `cc`'s - `parallel` feature so standalone Cargo builds also share Cargo's jobserver - budget instead of compiling native sources serially. `cargo check -j32` - passes; the FFI suite validates the resulting library separately. - -Status/configuration checkpoint: - -- Portable `RdmaConfig` now supports explicit cold configuration, with a - versioned C ABI and safe Rust wrapper. Configure before listen; explicit - values bypass process environment. Client configuration disables its CM - listener, while the fb-server starts its listener before readiness output. -- Versioned owned JSON snapshots expose actual path, cached QP/CQ/provider - capacities, requested queue/slot/arena/aggregation settings, native errors, - memory budgets, CPU time, context switches and affinity. Unsupported fields - remain null; `quarantined_bytes` has no global retained-arena snapshot yet - and must not be represented by the old unused zero-valued transport counter. -- Benchmark status control RPC uses message type 101. The CLI will query it - through an explicit short-lived TCP management connection before/after - traffic, since the complete status JSON is larger than small 1 KiB wire - slots. These are control exchanges outside the measurement interval, not - workload fallback; report `control_path=tcp` separately from verified RDMA - workload QPs and strict fallback count. Keep no extra management worker - active during measured traffic. Reuse the same status schema as local FFI. -- Native client error reads now include flushed + current metric windows. - Stop periodic flush and join callbacks before exact final totals; a flush - must not erase evidence of earlier submit/retry/timeout/missed responses. -- All 186 enabled native CTest cases and all 137 no-verbs cases pass; opt-in - HCA cases skip in the ordinary gate. All 33 Rust FFI tests pass, including - four new status/configuration cases. The opt-in real-HCA typed configuration - test separately proves immutable provider/QP status survives shutdown. - Formatting, scoped tree-lint and FFI clippy pass. The legacy two-process - strict two-QP worker round trip failed at its first client connection - (`rdma_worker_test.cpp:165`, null connection); diagnose this separately from - the passing typed-configuration HCA test before proceeding. The preceding - checkpoint prematurely recorded that last run as passing. - Readiness/runner acceptance and CLI integration remain pending; - this checkpoint makes no new performance claim. -- The first divergence was an accidental environment-key rename while moving - portable configuration: the runner's `CROWDB_RDMA_WIRE_SLOT_CAPACITY=1024` - was ignored, leaving 1 MiB slots and rejecting its 1 MiB receive arena. - Restore the established key and add an exact environment-contract regression. - The corrected two-process/two-QP aggregate diagnostic passes 1000 requests - with zero RPC/CQ errors, and all 187 enabled native tests pass. Scoped - tree-lint passes; this is functional verification only. - -### D. CLI coroutine workload and canonical results - -Implementation checkpoint: - -- `bench rdma` and `bench rpc` now share the timed coroutine echo workload, - bounded loader correlation slots and per-completion-owner histogram shards. - `LatencyHistogram::capture_total` merges those shards after callbacks join. - Input uses the agreed queue/slot variables; CLI aliases preserve existing - `--loader-num`, `--connections` and `--value-size` invocations. -- Canonical JSON goes directly to `--output`; it records warmup/window/drain, - native submission failures and correlation counters, verified payloads, - unique QPs and native client/server status. Missing required status counters - invalidate a run. Resource/transport deltas explicitly cover warmup/drain; - only validated response callbacks inside the duration contribute to TPS. -- Four CLI integration cases pass for TCP coroutine/Tokio timing, corrupted - payload rejection, timeout reporting, unavailable strict RDMA and invalid - input. The initial integration run exposed explicit TCP being rejected by - the strict fallback guard. `rdma_enabled_` now distinguishes a deliberately - selected TCP connection from a failed RDMA attempt; legacy strict RDMA still - fails closed. The native regression tests this combination with environment - RDMA enabled and fallback disabled. All 187 enabled CTest cases and scoped - tree-lint pass. Common histogram merge tests and CLI clippy also pass. -- All four CLI integration cases pass on the final source; the four shared - histogram tests, workspace clippy, formatting and scoped tree-lint pass. - Full C++/FFI gate passes at - `.crowdb-runtime/artifacts/unit/cpp-b17dc30f9af94d00ad083d4062a3c79e/`. - The no-verbs build passes all 137 CTest cases and all 33 FFI tests pass. -- The primary regression script now launches the fb-server and release CLI, - consumes startup/status/result JSON files, and verifies errors, actual path - and unique QPs. Its first strict native HCA run uses one CQ/poller per peer, - one generic I/O worker per peer, 128-byte payload, 1024-byte wire slots, - SQ/RQ/post target 1024 and 1 MiB send/receive arenas per QP. Each row has - 1 s warmup and 10 s measured traffic, with zero errors/fallback: - one loader gives 58,570 TPS (avg 16.54 us, P90 16.13 us); 32 loaders give - 337,553 TPS with aggregation disabled and 436,374 TPS enabled. These are - initial single trials, not a maximum or repeatability claim. Wide canonical - tables, complete settings/counters and binary identities are retained at - `.crowdb-runtime/rdma-check/artifacts/bench/rdma-rpc-20261022T155651.230011Z/`. - -- [x] **Add `bench rdma` with shared echo workload**: reuse `bench rpc` - coroutine machinery; expose `qpair_count`, loader count, warmup/duration, - slot/aggregation and CQ/queue knobs using the same code variable names as - the report. RDMA command requires RDMA and disables TCP fallback. Preserve - existing TCP command behavior; no OS thread per loader. Reject inconsistent - settings before starting. Files: CLI `bench.rs`, `bench/verb.rs`, - `bench/rpc.rs`, shared workload/config modules. - Add cold, typed C ABI configuration before listen/start, rather than changing - process environment from a multithreaded Rust runtime. Explicit configuration - overrides environment defaults. Capture QP number/CQ/effective queue/peer-slot - information before publishing each connection; status must read immutable - snapshots rather than dereferencing a QP during teardown. Return versioned - transport status JSON in an owned C ABI buffer for FFI, and expose the same - status through an fb-server benchmark control RPC while workload QPs remain - live. Server readiness follows actual CM listen/start. -- [x] **Validate and measure each completion**: expose response data to - `CoBenchHandler`, verify request ID, length and payload, count submission - rejects/retries/timeouts/connection/payload/duplicate errors, and reconcile - C++ coroutine statistics with Rust results. Warm up for one second, measure - completions for ten seconds using a monotonic deadline, then drain separately; - exclude setup/warmup/drain from TPS. Record native monotonic submission-to- - response-callback RTT in nanoseconds, only for validated responses. State - explicitly that request construction and Rust validation are outside the - latency sample; validation remains inside throughput work. Do not combine - C++ and Rust clock values to approximate a validation-inclusive interval. - Never destroy a coroutine while a late - completion can resume it (the lifecycle prerequisite is in phase A). - Files: `co_client.cpp`, FFI `co_bench.rs`, CLI echo - handler and lifecycle tests. -- [x] **Write the canonical report**: emit a versioned JSON result directly - to a requested file, plus human output; serialize all schema columns above. - Use explicit null for unavailable counters. Record measured errors and mark - invalid on setup/client/server/report failure or RDMA path mismatch. Include - client/server resource settings and actual QP/CQ/poller/path/fallback stats. - Files: CLI benchmark result/transport FFI modules and report tests. - -### E. One runner, staged evidence and requirement completion - -- [x] **Unify regression orchestration**: keep the named RDMA regression - script as the entry point and share common orchestration with TCP regression. - Launch the RPC server then the CLI client from one case manifest; wait for - actual RDMA readiness, not just the TCP listen message. Save complete logs, - canonical JSON and a wide CSV/TSV with errors/validity; propagate failures. - Record git/binary hashes, topology, CPU/HCA, affinity, CPU time and supported - provider counters. Tests cover invalid/missing reports, reported errors, - failed/early-exit server, disabled fallback and actual argument propagation. - Files: `tools/benchmark/bench-{rdma-,}rpc-regression.sh`, common runner and - `tools/benchmark/tests/rdma_regression_test.py`. -- Runner checkpoint: both shell entry points dispatch to `rpc_regression.py`; - `rpc_bench_cases.py` resolves one JSON manifest and derives queue/arena - capacities, `rpc_bench_process.py` owns readiness and process lifetimes, and - `rpc_bench_report.py` validates/exports canonical JSON plus wide CSV/TSV. - Every child executable runs through Pixi. A native server PID from its owned - startup status permits graceful shutdown while preserving the actual exit - code. Binary hashes, source revision/dirty state, CPU/HCA/affinity and readable - provider counters accompany every row. The runner stops at an invalid case - and prints its full server/client logs. Twelve fixture contract tests pass, - including both entry points, malformed/forged reports, missing readiness, - duplicate QPs and early server exit. Shell syntax and Python compilation pass. - Native/container command construction shares this manifest; actual OCI - execution remains pending in the task below. TCP uses writev/Nagle controls; - RDMA uses bounded complete-frame SEND aggregation, so the sweep does not - pretend these two transport mechanisms are the same switch. -- [x] **Run gradual native hardware profiles**: first one coroutine/QP/CQ, - then loader sweep 1/2/4/8/16/32 and higher as needed, then QP and CQ sweeps; - compare aggregation off/on, SQ/RQ 1024 and 4096, small slot classes and larger - payload/slot pairs. Increase depth only when counters show it limits load. - Each case uses 1 s warmup + 10 s measured traffic. Repeat candidate peaks - three times and retain all rows. Require zero workload errors/fallback and - report avg/P90/tails with the highest repeatable TPS. Do not claim a hardware - maximum when CPU/affinity/topology still limits the result. -- Sustained checkpoint: the 24-row gradual matrix completed 203,195,362 - measured validated RPCs with zero errors/fallback. At 512 loaders, 8 QPs - and 8 CQs/pollers, peak repeats give 2,201,203 / 2,234,423 / 2,277,181 TPS. - RQ 4096 does not improve the matching four-CQ control; queue admission and - the HCA RNR-retry-error counter show no new errors. Artifact: - `.crowdb-runtime/rdma-check/artifacts/bench/rdma-rpc-20261022T160711.769136Z/`. -- The next nine-row matrix increases loaders to 4096 and QPs/CQs to 32/16. - Its 2048-loader, 32-QP, 16-CQ candidate repeats at 2,886,026 / 2,892,625 / - 2,913,750 TPS, all error-free. The middle trial has avg 707.02 us, - P90 815.10 us and P99 954.37 us; this is the throughput/latency pair for - that trial. Artifact: - `.crowdb-runtime/rdma-check/artifacts/bench/rdma-rpc-20261022T161241.180591Z/`. - All 33 rows also reconcile remote RDMA request-frame deltas exactly with - accepted client submissions. The runner now enforces this relation and its - thirteenth contract test detects a forged remote total. -- The real-HCA held receive/MR/CM lifetime, registration-budget exhaustion, - and TCP-only-peer strict/allowed fallback cases all pass separately. Workspace - `clippy --all-targets --all-features` also passes on this source. -- Next controls: vary the idle polling budget and slot size at multiple CQs, - then run TCP and matched two-container manifests. A larger pool or more QPs - alone is not evidence of a higher hardware limit; retain CPU/resource data - and the exact binary identities with every trial. -- [~] **Run matching OCI profiles and finish protocol tests**: use the same - CLI workload and manifest in two host-network containers on this physical - HCA; record separate process/container topology. Complete negotiated TCP - preflight, fallback-before-payload, no mixed stream, no-device TCP and - non-root device/memlock acceptance. Compare matching native/container rows - and explain any gap. Files: RDMA OCI runner, capability/path tests. - Add an opt-in HCA case with a deliberately expired advertised CM endpoint: - valid TCP preflight must lead to one actual CM failure, strict rejection, - or one aligned TCP fallback echo before any RDMA payload. Set the fixture's - capability before worker startup through guarded `TestSocketTransport` / - `TestRdmaTransport` helpers. Add a real HCA peer-close case after request - delivery, with fallback allowed, asserting one failed completion, retained - RDMA path, no fallback and no TCP workload frame. Files: - `tests/rdma_failure_test.cpp`, `tests/common/test_rdma_transport.h`, and - the existing guarded transport test friendship. No new hot-path lock. -- The expired-CM fixture passes strict rejection and one allowed TCP echo - after a real CM attempt, with no RDMA workload frame. The new peer-close - fixture initially failed: `Connection::close()` only published local state, - so the peer remained open and its pending request never failed. A private - transport-close notifier now wakes the assigned CQ owner; the owner calls - `rdma_disconnect` before stopping posting admission and retiring its send - queue. The existing per-QP operation gate pins the CM ID until that call - completes, and an atomic flag prevents duplicate requests from CQ/CM/shutdown. - Shutdown explicitly disconnects retained IDs before destruction. RDMA close - also unregisters socket-transport weak indexes, including a close/publication - race, while retained leases keep their arena/device ownership. -- All three real-HCA failure cases now pass in 40 ms: expired endpoint, peer - close after payload, and server shutdown after payload. The latter two prove - one failed completion with fallback allowed, retained RDMA path, no fallback, - no TCP workload frame and empty connection indexes after shutdown. -- The final close/disconnect source passes the full C++/FFI gate at - `.crowdb-runtime/artifacts/unit/cpp-93edc5b4128b456696da0095db768d20/`, - all 137 no-verbs CTest cases, all four CLI benchmark integration cases, - workspace clippy, Rust/C++ formatting and scoped tree-lint (exit zero with - style warnings). All thirteen runner contract tests pass. Six real-HCA - failure/lifetime/budget/fallback cases also pass together in 74 ms. The - release CLI rebuild exits zero; post-fix sustained benchmark checks remain. -- [ ] **Gate and document each coherent change**: build with `-j32`; run all - executables/tests/lints through `pixi run`, use `clean-env` before server - tests, and keep complete failure output. Run affected tests, fmt, clippy, - changed C++ format/tree-lint, then full C++/FFI gates. Update permanent design - to implemented behavior; remove requirements/plans only after all acceptance - passes. Keep an explicit verified/pending checkpoint after each commit. - -Provider references for the implementation choices: - -- [libibverbs QP capacity terminology](https://github.com/linux-rdma/rdma-core/blob/master/libibverbs/man/ibv_create_qp.3). -- [RDMA-CM receive posting before connection readiness](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/man/rdma_create_qp.3). -- [RDMA-CM RNR behavior and default delay](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/man/rdma_accept.3). -- [RDMA-CM device context retention](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/cma.c) - (`rdma_get_devices` increments device references; `rdma_free_devices` - releases them). Retaining the matching CM context list avoids substituting - an independently opened context for a CM-owned QP. -- [RDMA-CM request private-data limit](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/man/rdma_connect.3) - and [accept event lifetime](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/man/rdma_accept.3). -- [RDMA disconnect and flush contract](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/man/rdma_disconnect.3), - [CM-ID destruction](https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/cma.c) - and [kernel reported-event accounting](https://github.com/torvalds/linux/blob/master/drivers/infiniband/core/ucma.c). - -## Historical verification record - -The C++ unit/integration gate and Rust FFI tests pass after the teardown and -completion-error changes; `tree-lint` completes with the repository's existing -clang-tidy warning set. The OCI runner now mounts a named data volume and uses -the configured RDMA local interface by default. TCP-only and unprivileged -shared-HCA startup smoke tests both reached the container readiness message; -the unprivileged smoke correctly fell back to TCP because this container host -did not expose a usable HCA device. A privileged multi-container HCA run still -requires the prepared local-loop topology. - -## R239 status checkpoint - -Completed: - -- active and passive RDMA-CM address, route, connect, and accept flow; -- connection-level PD and registered send/receive slot pools; -- initial receive posting, CQ completion dispatch, owner-release repost, and - RNR retry configuration; -- capability serialization and explicit pre-payload fallback policy; TCP preflight remains pending; -- epoll registration for CM and CQ wakeup events; -- live HCA local-loop CM/QP/send/receive E2E with all frames received and clean CQ - status, and clean client/server exit; -- actual `RpcServer`/`RpcClient` worker-path request/response round trip with - sequential frames and clean teardown; -- contiguous one-SGE wire-slot send/receive flow with 128B, 1KiB, and 4KiB - payload benchmark smoke tests reporting zero errors; -- non-root runner selection and the library consumer/deployment contract. - -Remaining implementation and acceptance work: - -- add mock/integration coverage for CM disconnect, QP error, CQ flush/error, - epoll wakeup, receive exhaustion, and pre-payload TCP fallback; -- make each completion's `wr_id` resolve a stable connection/request/slot - owner and test success, flush, protection, retry, and remote-access errors; -- run the single-node HCA local-loop multi-container OCI profile against the - built image; -- remove temporary hardware diagnostics, normalize failure logging, and run - the full C++ format/tree-lint gates; -- implement multi-CQ and a fixed polling/event worker per CQ together with - the CLI coroutine benchmark, as agreed in the implementation sequence above; -- rerun the RDMA benchmark concurrency and payload cases after the acceptance - changes. - Earlier script results claiming 442k/414k/376k ops/s did not establish the - RDMA path; replace them with validated canonical results from the CLI runner. - -## Phase 1: common transport seam - -- [x] **Trace TCP submission**: map `Transport::submit`, `Connection`, - `OutFrame`, parser dispatch, completion callbacks, and connection shutdown. - Files: `lib/crowdb-rpc/include/crowdb-rpc/transport.h`, `connection.h`, - `out_frame.h`, TCP transport and worker sources. -- [x] **Add RDMA capability state**: define unavailable, probing, connecting, - ready, draining, failed, and TCP-fallback states. Keep path state per logical - connection and prevent TCP-to-RDMA upgrade. Files: transport/connection - headers and state tests. -- [x] **Define endpoint handshake data**: represent coarse discovery result, - CM address/port, GID/device identity, capability generation, protocol - version, and supported data classes. Validate stale generation and peer - identity before CM setup. Files: RPC control schema and endpoint config. - -## Phase 2: CM and QP lifecycle - -- [x] **Implement device selection**: select configured HCA, port, local - address, and GID; fail closed to TCP when local probing finds no usable - device. Files: `rdma_transport.h/.cpp` and configuration adapter. -- [x] **Implement active/passive CM flow**: create IDs, bind/listen or resolve - address/route, create QP, connect/accept, and publish ready only after initial - receive buffers are posted. Files: RDMA transport sources. -- [x] **Implement connection teardown**: map CM disconnect and CQ completion - errors to one close/fail transition. Retire the connection pools before QP/ID - destruction, quarantine uncertain buffers, and wake the owning engine on stop. - -## Phase 3: event/poll loop - -- [x] **Extend epoll registration**: add external event-fd registration and - removal to the existing `EpollEngine` without changing TCP readiness rules. - Files: `lib/crowdb-rpc/include/crowdb-rpc/transport/epoll/epoll_engine.h`, - `lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp`. -- [ ] **Drain CM events**: acknowledge all received CM events, update the - connection state, and re-arm the CM channel where required. -- [x] **Create shared CQ**: all QPs use the transport-wide CQ and completion - channel, so multiple connections share one completion stream. -- [x] **Add bounded CQ polling**: drain the shared CQ with an empty-poll - budget, then arm CQ notification for the next epoll wakeup. -- [x] **Make epoll a CQ wakeup**: acknowledge the CQ event and resume polling; - epoll does not scan or dispatch per-connection CQs. -- [x] **Make stop wakeable**: stop wakes the owning epoll engine, then shutdown - drains known CQ work before destroying IDs and pools. - -## Phase 4: frame submission and completion - -- [x] **Build one-SGE wire slots**: allocate one fixed-size registered slot - from the send ring, copy header/control/payload contiguously, and post one - SEND SGE with the actual wire length. Reject messages larger than the slot. -- [x] **Post one-slot receives**: prepost one full registered receive slot per - receive ring entry, associate each with a stable `wr_id`, and replenish only - after the owner releases it. -- [x] **Resolve WR identity**: `wr_id` identifies connection, request ID, - stream ID, sequence, logical offset, buffers, and ownership state. Success, - flush, local protection, retry, and remote access errors are accounted for; - injected provider-error coverage remains in the acceptance checklist. -- [x] **Dispatch common callbacks**: validate lengths and class before parser - publication, then invoke the existing C++ completion path and FFI callback. - -## Phase 5: fallback and observability - -- [ ] **Implement pre-payload fallback**: if CM/QP setup or first submission - fails before payload, select TCP with the same request identity. -- [x] **Reject mixed completion**: after any payload submission, fail the stream - on uncertainty and never enqueue a TCP suffix. -- [x] **Add metrics**: posted receives, fallback count, quarantined bytes, CQ - poll/empty-poll counts, CM disconnects, flush, local-protection, retry, and - remote-access errors are exposed; QP state, CQ depth, and RNR counters remain. - -## Tests - -Unit tests: - -- endpoint capability and generation validation; -- connection-state transitions and illegal TCP-to-RDMA upgrade; -- frame class/length validation and one-SGE wire-slot construction; -- `wr_id` mapping and completion state transitions; -- fallback boundary and request-ID reuse rules. - -Integration tests: - -- RDMA-disabled build and host selects TCP without CM calls; -- mocked CM/CQ setup, reconnect, CQ error, flush completion, and shutdown; -- multiple connections sharing one CQ pool preserve completion ownership and - request correlation; -- epoll external event wakes worker and drains all current CQ completions; -- send/receive sequence preserves exact control/data lengths inside one slot; -- pre-payload failure retries idempotent request on TCP; -- partial payload failure never appends TCP data. - -E2E tests: - -- TCP-only OCI profile; -- single-node HCA local-loop profile with multiple containers sharing one HCA. -- non-root OCI profile: host prepares RDMA and network topology, the container - receives only the required `/dev/infiniband` devices and memlock allowance, - and the RPC flow runs as the application user without `sudo`, module loads, - link moves, or host namespace entry. - -## Consolidated files - -- `lib/crowdb-common/cpp/include/crowdb-common/metrics/latency_histogram.h` -- `lib/crowdb-common/cpp/src/metrics/latency_histogram.cpp` -- `lib/crowdb-common/rust/src/metrics/histogram.rs` and `histogram/data.rs` -- `lib/crowdb-common/testdata/latency_histogram.csv` and both language tests -- `lib/crowdb-rpc/src/client/co_client.cpp` and private coroutine state header -- `lib/crowdb-rpc/src/buffer_arena.{h,cpp}`, `registered_buffer_pool.cpp`, - shared buffer facade and registered/real-MR lifetime tests -- `lib/crowdb-rpc/ffi/src/co_bench.rs` and C/Rust statistics bindings -- `app/crowdb-cli/src/commands/bench/{verb,rpc,loader,result}.rs` -- `tools/benchmark/bench-rdma-rpc-regression.sh`, shared RPC runner and tests -- `lib/crowdb-rpc/include/crowdb-rpc/transport.h` -- `lib/crowdb-rpc/include/crowdb-rpc/connection.h` -- `lib/crowdb-rpc/include/crowdb-rpc/transport/epoll/epoll_engine.h` -- `lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp` -- `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h` -- `lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp` -- `lib/crowdb-rpc/tests/` transport, fallback, and event-loop tests -- OCI image and single-node multi-container E2E runner configuration - -## Verification gates - -```bash -pixi run -- cargo fmt --all -- --check -pixi run -- cargo test -p crowdb-rpc --all-targets -pixi run -- cargo test -p crowdb-rpc-ffi --all-targets -pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings -pixi run rs-lint -pixi run tree-lint -``` - -Do not hand off to R240/R241 until a framed RPC stream works over RDMA and TCP -fallback has passed the partial-transfer boundary tests. - -## Container launch contract - -The host performs privileged RDMA setup with `setup-rdma.sh local`. Each -container receives the required `/dev/infiniband` devices, device-access group, -the selected HCA mapping, and bounded memlock limit. The RPC process runs as -the application user and does not use `sudo`, load kernel modules, move links, -or enter host namespaces. - -The local hardware smoke test is: - -```bash -sudo tools/rdma/setup-rdma.sh local -tools/rdma/run-local-loop.sh local -``` - -Multiple containers on one node use the same HCA local-loop model and open -independent QPs. Physical multi-node OCI runs are outside this plan and are -specified in the separate follow-up plan. No namespace operation is part of -the test or application path. - -## Library consumer contract - -An application linking `crowdb-rpc` does not run the RDMA setup script and does -not need to enter a network namespace. Deployment prepares the host or -container once; the application then uses the normal RPC server/client API. - -The deployment layer must provide: - -- a loaded RDMA kernel provider and an `ACTIVE` HCA port with a routable local - address/GID; -- `libibverbs`, `librdmacm`, and the matching provider library in the runtime - image; -- access for the application user to the required `/dev/infiniband` devices; -- a sufficient memlock limit for registered buffers; -- peer reachability on the selected RDMA address and port. - -The application layer only supplies the normal RPC endpoint configuration. The -library discovers usable devices and selects RDMA before TCP when the peer -advertises compatible capability. `CROWDB_RDMA_DEVICE`, `CROWDB_RDMA_PORT`, and -`CROWDB_RDMA_SOURCE_ADDR` are deployment overrides, not required API calls. -For host-network multi-container runs, deployment also supplies one -`CROWDB_PORT_OFFSET` per container. The library and service launchers apply the -same offset to every default TCP/RPC and RDMA CM service port; each colocated -RPC service adds its stable `CROWDB_RDMA_SERVICE_SLOT` to the CM port so -services do not contend for one listener. The HCA physical port selected by -`CROWDB_RDMA_PORT` is unchanged. The default RDMA CM service port is `19101`. -When any local capability, CM, QP, or pre-payload setup step is unavailable, -the library reports the reason and uses TCP without requiring application code -changes. After payload submission, the connection fails as one stream and does -not append a TCP suffix. diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index 979be972c..fc9b0d950 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -5,9 +5,7 @@ Upstream: [R5](../backlog/R5-rdma-alloc.md), [RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md), and child -backlogs [R238](../backlog/R238-rdma-registered-buffer-foundation.md), -[R239](../backlog/R239-rdma-rpc-transport.md), -[R240](../backlog/R240-rdma-datapath-stream.md), +backlogs [R240](../backlog/R240-rdma-datapath-stream.md), [R241](../backlog/R241-rdma-chunk-kv-access.md), and [R242](../backlog/R242-rdma-fat-client-sdk.md). @@ -16,40 +14,29 @@ the preferred connection path, TCP as the one-way fallback, shared aggregation semantics, bounded buffer leases, and automated OCI verification using a single-node HCA local-loop with multiple containers. +This root plan persists until the umbrella goal is complete. Completed child +plans are deleted; remaining child plans retain their implementation detail. + ## Current implementation status -R238's foundation implementation is committed. The common RPC buffer layer and verbs-backed per-connection -arenas share the four capacity classes, bounded admission, explicit owner -transitions, safe release ordering, quarantine handling, shutdown quarantine, -and C/Rust lease metadata accessors. R239's transport path is implemented and -passes the prepared HCA -local-loop plus the real `RpcServer`/`RpcClient` worker round trip. Its -requirement is still open because negotiated capability preflight, -pre-payload fallback, aggregation/CLI benchmarking and -multi-container acceptance have not all passed. Live HCA tests now verify -held registered/receive leases after pool, transport and CM-ID destruction; -the final release deregisters MR before PD release. Application reply allocation -is separate from receive admission, and every CQ has a fixed polling/event -owner. The transport plan records the corresponding gates and diagnostics. - -The latest transport handoff closes the completion ownership gap: SEND and -RECV work requests now carry stable connection/request/stream metadata, receive -leases remain caller-owned until their final C++/FFI reference is released -before repost, and CQ send failures close the uncertain RDMA stream after quarantining -its buffers. The worker test checks a non-empty response payload as well as -request correlation. The current local-loop runtime is prepared. The single-worker baseline with -1024 receive WRs and small wire slots passes. The staged 1/2/4/8/16/32 -producer/QP profiles now each run for ten measured seconds after warmup, -with zero errors after fixing CM event ownership and CQ continuation races. -The 1024 and 4096 RQ profiles, full C++ gate, Rust RPC FFI suite, formatting, -clippy, and tree-lint pass. The transport plan records exact counts, latency, -and the fixed-I/O-worker control; protocol and OCI acceptance remain open. - -This is the root plan. The five child plans contain implementation details; -only this plan controls phase order, cross-backlog decisions, and final -acceptance. The agreed joint CQ/benchmark/aggregation work is detailed in -[the transport plan](plan-rdma-rpc-transport.md#agreed-implementation-sequence). -Update the formal design with each implemented architecture change. +Registered host buffers and the common RC SEND/RECV transport are accepted. +The shared arena has bounded lock-free admission, retained MR/PD/device +ownership, explicit leases and quarantine. TCP preflight binds CM setup to the +current endpoint and receive admission. QPs have one fixed polling/event owner +per CQ; bounded aggregation preserves complete frames. Strict RDMA CLI tests +share the coroutine workload and canonical results with TCP. + +The final transport source `55ffee38` passes full C++/FFI gates, 137 no-verbs +cases, CLI/runner contract tests, real HCA lifetime/budget/fallback/disconnect +cases, formatting and workspace lint. Native and two non-root containers on +one shared HCA pass repeated 1 s warmup + 10 s measurement profiles. Median +RDMA throughput is 2.903M TPS native and 2.954M TPS in OCI, with zero errors or +fallback. The old 1.187M value labeled RDMA was TCP; matching current TCP +measures 1.196M TPS. Exact identities, settings and artifact names are retained +in [RPC flow analysis](../design/rpc/rpc-flow-analysis.md). + +Current work is R240, then R241, using their existing plans. R242 and physical +multi-node validation remain outside the user's requested continuation. ## Fixed design decisions @@ -109,85 +96,15 @@ before advancing; implement R240, then R241, using their existing [Chunk-KV/access plan](plan-rdma-chunk-kv-access.md), with their own gates. Do not create duplicate plan files. R242 remains outside this requested continuation. -1. Finish the R238 foundation contract and its gates. Do not add new R239 - transport behavior until the common and verbs-backed pools agree on class, - ownership, quarantine, and shutdown semantics. -2. Close R239 acceptance gaps: injectable CM/CQ failure coverage, stable - `wr_id` ownership mapping, pre-payload retry on TCP, partial-transfer - isolation, and the single-node multi-container OCI profile. -3. Run the R238/R239 handoff gates, complete R240, then complete R241. - R242 remains downstream of R241 and the stable FFI lease surface. - -R240 and R241 start after R239 passes its transport acceptance. -R242 starts only after R241 establishes the access semantics and after the FFI -lease contract from R238 is stable. - -## Phase 0: baseline and decision lock - -- [x] **Lock hardware test model**: use `setup-rdma.sh local` and - `run-local-loop.sh local` for single-node validation; do not use namespace - setup in product or acceptance tests. Multiple containers on one node share - the local HCA loop and open independent QPs. -- [ ] **Record baseline**: run the existing TCP RPC, Access Server, DiskIO, and - Chunk-KV gates before modifying production code. Root files: affected crate - manifests and CI configuration. -- [ ] **Confirm host capabilities**: record `ibv_devices`, `rdma link`, the - selected HCA port, GID/IP mapping, and whether the local HCA loop is usable. - Record the container-to-QP mapping for the single-node shared-HCA run. - Do not treat an unavailable host device as an implementation failure. -- [ ] **Lock configuration names**: agree on RDMA enablement, device/port/local - address, control/data class defaults, registered bytes, receive depth, and - fallback policy. Root files: configuration schema and formal design. -- [ ] **Prepare OCI image**: include `libibverbs`, `librdmacm`, and provider - libraries; keep a TCP-only image invocation that exposes no RDMA device. - Root files: Dockerfile/OCI test scripts and CI documentation. - -Exit condition: TCP baseline passes, the host topology is recorded, and the -configuration contract is stable. If the host has no RDMA, continue with the -TCP-only profile and build/mocked tests. - -## Phase 1: R238 registered buffers - -- [x] **Close R238 pool contract**: align `RegisteredBufferPool` and the - verbs-backed `RdmaBufferPool` on the four classes, lease metadata, owner - transitions, quarantine accounting, and bounded shutdown. Keep registration - optional and do not introduce a new lock on the allocation/recycle path. - Files: `lib/crowdb-rpc/include/crowdb-rpc/buffer.h`, - `lib/crowdb-rpc/src/buffer.cpp`, and - `lib/crowdb-rpc/src/transport/rdma/rdma_transport.*`. -- [x] **Run R238 gates**: unit and integration tests pass with RDMA disabled - and enabled build configurations, including registration failure and - uncertain completion cases. -- [x] **Review handoff**: verify that R239 can obtain control/data buffers and - that every asynchronous owner has a release transition. -- [x] **Close portable pool parity**: share the verified arena lifetime and - tagged-index freelist machinery with the no-verbs registered pool, then - verify concurrent bounds, held leases, quarantine and detach without an HCA. - The detailed transport plan owns this continuation before aggregation. -- [x] **Bound retired registration admission**: keep registered bytes charged - to the existing device owner while old caller leases retain their MRs, reject - failed MR setup before publishing a connection, and verify the current/high - water budget and rejection counters. The transport plan owns this final - resource admission continuation. - -Exit condition: no buffer can be recycled while transport, disk, or caller -ownership remains unresolved. - -## Phase 2: R239 common RDMA RPC - -- [ ] **Close R239 acceptance**: add mock/integration coverage for CQ - success/error/flush, CM disconnect/QP error, receive exhaustion, stable - `wr_id` ownership, pre-payload fallback, and no mixed TCP/RDMA stream. -- [x] **Local-loop verification**: run `setup-rdma.sh local` followed by - `run-local-loop.sh local`; both CM roles, framed SEND/RECV, receive repost, - and exact payload lengths have passed on the prepared HCA. -- [ ] **Container local-loop verification**: start multiple containers on one - node after the local setup; verify each container opens independent QPs over - the same HCA port and that request correlation and bounded buffers remain - correct. - -Exit condition: R239 can carry a framed stream, report completion to the common - RPC layer, and leave TCP fully usable when RDMA is absent or fails. +1. Complete R240 using its existing datapath stream plan and service gates. +2. Complete R241 using its existing Chunk-KV/access plan and parity gates. +3. Keep R242 downstream until the user includes that scope. + +Before modifying each service path, run its focused baseline corpus and record +it in that child's plan. Transport baseline, hardware topology, queue names and +OCI runtime preparation are already verified. Service acceptance must prove +actual RDMA, ownership through disk/caller completion, and bounded pressure; +transport echo acceptance alone does not prove those properties. ## Phase 3: R240 and R241 service integration From b42f0e026485d2c6a99892a202aca2295a406c3b Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 01:16:56 +0800 Subject: [PATCH 039/258] Specify registered producer handoff and direct datapath buffer layout --- doc/backlog/R240-rdma-datapath-stream.md | 20 +++-- doc/working/plan-rdma-datapath-stream.md | 98 +++++++++++++++++++++++- doc/working/plan-rdma.md | 5 +- 3 files changed, 116 insertions(+), 7 deletions(-) diff --git a/doc/backlog/R240-rdma-datapath-stream.md b/doc/backlog/R240-rdma-datapath-stream.md index 4b01d33b5..5873496ad 100644 --- a/doc/backlog/R240-rdma-datapath-stream.md +++ b/doc/backlog/R240-rdma-datapath-stream.md @@ -13,16 +13,26 @@ I/O still uses it. ## Solution -1. Feed already aggregated ChunkIO buffers to R239. RDMA does not aggregate or - flush; it preserves actual payload length and stream sequence. -2. Use a 1 KiB control block plus a 1 MiB data capacity for the datapath. A - logical object is an ordered sequence of messages when larger than one - message. +1. Feed already aggregated ChunkIO buffers to the common transport without + another application aggregation queue or flush timer. Ordinary RPC transport + coalescing remains bounded by both peers' wire-slot admission. +2. Use a 1 KiB control region plus a 1 MiB data region for the direct datapath. + `wire_slot_capacity` bounds their total, including the RPC header. Negotiate + this layout before payload submission; reject incompatible RDMA layouts + before selecting the path. Direct messages use separate registered SGEs for + control and immutable payload, so mirror fanout can retain the same payload + without modifying its bytes or copying it behind different request headers. + A logical object is an ordered sequence of messages when larger than one + message. This layout sends one application message per SEND; the existing + contiguous layout retains bounded complete-frame aggregation. 3. Let Access Server fill its registered send buffer from the HTTP path. Let DiskIO own its registered receive buffer after CQ completion and submit that buffer directly to asynchronous disk I/O. 4. Repost a DiskIO receive buffer only after disk completion. Apply bounded receive-window backpressure when DiskIO or the SDK consumer is slow. + Receive data starts at the registered data-region base, preserving disk + alignment. Control storage and data storage share the configured registered + byte budget. A held or quarantined lease cannot be replaced outside it. 5. Preserve R239 fallback: before payload submission the operation may use TCP; after partial RDMA transfer the stream fails without TCP byte splicing. diff --git a/doc/working/plan-rdma-datapath-stream.md b/doc/working/plan-rdma-datapath-stream.md index a3ddaaee2..2347aea6c 100644 --- a/doc/working/plan-rdma-datapath-stream.md +++ b/doc/working/plan-rdma-datapath-stream.md @@ -11,6 +11,101 @@ Goal: connect existing HTTP/ChunkIO aggregation to the RDMA RPC stream and DiskIO receive loop without introducing another aggregation or CPU payload copy. +## Execution checkpoint + +The transport/foundation acceptance is complete at `55ffee38`; cleanup is +committed at `d6bedf94`. Preserve the verified contiguous-slot benchmark path. +The six existing DiskIO-client integration cases pass before service changes. +The full C++ gate already covers DiskIO engines and AlignedWriter. + +Observed flow and the first missing seam: + +- `NativeBodyAllocator` / `NativeOwner` in `crowdb-access-s3` supplies Hyper's + body receive regions. `NativeFramedOwner` finalizes physical frame regions + in place and supplies immutable `Bytes` views. S3 `write_object_body` passes + those owners through `UploadBuffer::Framed` to + `LargeAsyncObjectWriter::on_framed_data`. Do not route this path through + `frame_tail` or introduce another HTTP assembly copy. +- `RoutedDiskWriter` delegates views to `DiskioClient::write_views`, then + `WireClient::write_segment_views` and `RpcClient::call_chain`. Existing + grouping preserves order and `ordering_zone_offset`. The current RDMA + `flush_connection` copies every view into a new contiguous wire slot. +- `DiskioServer::handle_write` clones `request->data_buf`, deletes the Frame, + and retains the clone through `AlignedWriter::submit_ordered` completion. + Its current `data_buf->data` access must honor `payload_data()` for views. + AlignedWriter passes aligned full blocks directly, with explicit edge-block + staging only when required by device alignment. +- Reads currently allocate from `conn->pool()` and may stage O_DIRECT data. + Allocate eligible direct reads from registered producer storage. Rust + `WireClient::await_read_response` already returns `Bytes::from_owner(Buffer)`; + the final caller release can therefore own receive repost admission. + +### Direct layout and implementation order + +The existing one-SGE path copies complete frames and remains the ordinary RPC +benchmark path. A zero-copy service producer needs an immutable registered +payload independent of each destination's control header: mirror targets have +different disk addresses and may send concurrently. Add an explicitly +negotiated control/data layout for this service path, using the QP's existing +two-SGE capability. This avoids rewriting payload prefixes during fanout. +The layout is an implementation decision for the required zero-copy path; +it does not add application aggregation or a flush timer. + +- [~] **Add negotiated direct layout**: add `RdmaWireLayout::Contiguous` and + `ControlData`, defaulting to `Contiguous`. For `ControlData`, the control + region is `RDMA_CONTROL_CAPACITY` including the 14-byte RPC header; derive + data capacity from `wire_slot_capacity - RDMA_CONTROL_CAPACITY` and validate + it against the existing classes. The 1 MiB data default therefore has total + `wire_slot_capacity = 1049600`, rather than claiming 1 MiB of payload fits + in a 1 MiB total wire slot. Advertise layout in the versioned TCP capability + and bind it in the compact CM offer. A mismatched layout fails before payload + and follows the existing explicit fallback policy. Reject direct layout with + transport aggregation enabled or provider SGE capacity below two. Extend + typed C/Rust cold config, environment parsing and status with the same names. + Files: `rdma_config.h`, capability/offer/preflight modules, config ABI/FFI, + `rdma_connection.cpp`, config/status tests. +- [ ] **Allocate and send immutable registered data**: expose bounded producer + allocation from the transport's device-owned arena. Provide a unique Rust + `WritableBuffer` with a `MaybeUninit` capacity view and consuming commit; + freeze into the existing owner-backed `Bytes` contract. Validate address + ranges against the owning arena before using an MR key; retain the exact + underlying allocation through SEND completion and propagate quarantine to + that allocation. Build the small per-request header/control in its own + registered region, pad that region to 1 KiB, and post control plus payload + as two SGEs. Keep one WR per complete RPC and count direct/copied payload + bytes in status. No range registry or new hot-path lock. Files: buffer arena, + RDMA pool/send/work-request modules, C ABI/FFI buffer producer API. +- [ ] **Receive into aligned data storage**: prepost two registered SGEs in + direct layout, with 1 KiB control and a separately aligned data allocation. + Charge both against `receive_arena_bytes`; descriptor count covers both and + arithmetic is checked. Validate the complete control/header/length before + dispatch; retain data ownership through disk/caller completion. Release or + quarantine both regions exactly once on post/CQE/cancel/shutdown failure. + Files: `rdma_receive.cpp`, receive posting, WorkRequest and repost context, + real HCA ownership tests. Keep generic receive credit implicit in RQ slots. +- [ ] **Connect the actual HTTP producer**: allow `NativeBodyAllocator` to + obtain its owner storage from the retained DiskIO producer allocator after + route setup. Preserve its existing byte admission, AtomicWaker pressure and + framed-owner API. Wire it through `ChunkIoClient` / `RoutedDiskWriter` / + S3 storage setup. A bounded native allocator failure pauses intake; it must + not silently allocate an unregistered replacement. Retain TCP allocation + when the service selects TCP. Cover owner views, slices and mirror fanout. +- [ ] **Close DiskIO handoff and read lifetime**: use exact view pointer/length, + validate request bounds before disk submission, transition data to DiskOwned + until the callback returns, and retain connection state across delayed I/O. + Registered read storage must cover engine completion and SEND completion; + client receive storage remains held through returned Bytes. Preserve existing + alignment handling and short-I/O/error semantics. Test delayed I/O, queue + pressure, held caller buffers, error, cancellation and connection shutdown. +- [ ] **Verify real service flows**: run focused baseline and changed tests, + then real DiskIO plus HTTP/ChunkIO upload/download of objects larger than + 1 MiB on the prepared HCA. Assert direct-payload counters and same data + addresses at producer/SEND and receive/disk handoff, exact bytes and existing + frame integrity/order. Repeat TCP-only and two non-root OCI profiles. Keep + per-case manifests, settings, errors and complete logs. Run required fmt, + clippy/tree-lint and full affected gates before cleanup; then use R241's own + plan for Chunk-KV integration. + ## Phase 1: map existing data flow - [ ] **Trace upload flow**: identify where HTTP body bytes enter ChunkIO, @@ -27,7 +122,8 @@ copy. ## Phase 2: upload to DiskIO - [ ] **Allocate 1 MiB datapath class**: select the configured 1 MiB data - capacity plus 1 KiB control buffer for S3/HTTP/DiskIO streams. + capacity plus 1 KiB control buffer for S3/HTTP/DiskIO streams; the configured + total wire slot includes both regions and the header. - [ ] **Fill registered send buffer**: make the HTTP/ChunkIO producer write directly into the send-owned registered buffer and submit its actual length. The producer must stop when the send/registered budget is exhausted. diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index fc9b0d950..c0a29c174 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -41,7 +41,10 @@ multi-node validation remain outside the user's requested continuation. ## Fixed design decisions - Use one common RPC abstraction for TCP and RDMA. -- Use RC SEND/RECV, one contiguous wire slot and one SGE per message. +- Use RC SEND/RECV. The accepted contiguous RPC layout uses one wire slot + and one SGE per message. R240 adds a negotiated control/data layout with + separate SGEs for immutable producer payload and per-target control; its + total wire admission includes both regions. - Use service discovery for coarse RDMA capability and TCP handshake for the current RDMA endpoint, capability generation, and data classes. - Select RDMA before the first payload; fallback is RDMA to TCP only. From 390addb497739bc5c87963b085a6b3786fd6e900 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 01:48:35 +0800 Subject: [PATCH 040/258] Retain producer buffer storage and admission across server teardown --- doc/design/rpc/design-crowdb-rpc-rdma.md | 16 +++ doc/working/plan-rdma-datapath-stream.md | 52 ++++++++ lib/crowdb-rpc/ffi/src/buffer.rs | 4 + lib/crowdb-rpc/ffi/src/lib.rs | 2 + lib/crowdb-rpc/ffi/src/producer_buffer.rs | 100 ++++++++++++++ lib/crowdb-rpc/ffi/src/sys.rs | 7 + .../ffi/tests/producer_buffer_test.rs | 38 ++++++ lib/crowdb-rpc/include/crowdb-rpc/buffer.h | 19 +-- lib/crowdb-rpc/include/crowdb-rpc/c_api.h | 22 +++- .../transport/rdma/rdma_transport.h | 10 +- .../crowdb-rpc/transport/socket_transport.h | 23 ++-- lib/crowdb-rpc/src/buffer.cpp | 59 --------- lib/crowdb-rpc/src/c_api/producer.cpp | 52 ++++++++ lib/crowdb-rpc/src/system_buffer_pool.cpp | 122 ++++++++++++++++++ .../src/transport/producer_buffer.cpp | 21 +++ .../src/transport/rdma/rdma_producer.cpp | 18 +++ .../src/transport/rdma/rdma_transport.cpp | 1 + .../src/transport/socket_transport.cpp | 3 +- lib/crowdb-rpc/tests/producer_buffer_test.cpp | 100 ++++++++++++++ .../tests/system_buffer_lifetime_test.cpp | 44 +++++++ 20 files changed, 624 insertions(+), 89 deletions(-) create mode 100644 lib/crowdb-rpc/ffi/src/producer_buffer.rs create mode 100644 lib/crowdb-rpc/ffi/tests/producer_buffer_test.rs create mode 100644 lib/crowdb-rpc/src/c_api/producer.cpp create mode 100644 lib/crowdb-rpc/src/system_buffer_pool.cpp create mode 100644 lib/crowdb-rpc/src/transport/producer_buffer.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_producer.cpp create mode 100644 lib/crowdb-rpc/tests/producer_buffer_test.cpp create mode 100644 lib/crowdb-rpc/tests/system_buffer_lifetime_test.cpp diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index 5f62c0575..aa7fe36ed 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -173,6 +173,22 @@ release. Their detached ranges remain committed and are not reused by another stream. Cached and quarantined descriptors both count toward the hard total. Application-pool count admission also uses one atomic reservation. +`SystemBufferPool` retains its admission counter and recycler in a shared +state owned by every outstanding allocation. Destroying the facade therefore +does not invalidate a TCP/application buffer still held by an asynchronous +consumer. Clones share one allocation charge; its final release returns that +charge. The transport owns any default application-pool facade it creates. + +`allocate_producer_buffer` lends storage from the device-owned send arena after +CM context adoption publishes that arena. Its registration charge and arena +owner survive the transport. `require_registered` rejects unavailable +registration; an exhausted active arena never substitutes an ordinary heap +allocation. Without an active registered arena, optional allocation uses the +ordinary bounded application pool. `WritableBuffer` exposes only unique +uninitialized capacity until its producer publishes an initialized prefix; +immutable views then retain the lease through their final release. Allocation +alone does not bypass the contiguous transport's payload copy. + The device owner charges MR creation against `registered_memory_budget_bytes`. Zero derives a checked bound from all CQ-admitted QPs, their send/receive arenas and the two compatibility copy diff --git a/doc/working/plan-rdma-datapath-stream.md b/doc/working/plan-rdma-datapath-stream.md index 2347aea6c..a3b63486e 100644 --- a/doc/working/plan-rdma-datapath-stream.md +++ b/doc/working/plan-rdma-datapath-stream.md @@ -17,6 +17,8 @@ The transport/foundation acceptance is complete at `55ffee38`; cleanup is committed at `d6bedf94`. Preserve the verified contiguous-slot benchmark path. The six existing DiskIO-client integration cases pass before service changes. The full C++ gate already covers DiskIO engines and AlignedWriter. +The 31 focused ChunkIO configuration/mirror/stream cases and 11 Access Server +configuration/service cases also pass before production changes. Observed flow and the first missing seam: @@ -51,6 +53,43 @@ two-SGE capability. This avoids rewriting payload prefixes during fanout. The layout is an implementation decision for the required zero-copy path; it does not add application aggregation or a flush timer. +- [x] **Expose bounded producer allocation first**: add native transport and + C ABI allocation from the already device-owned send arena, with explicit + `require_registered` policy and no replacement allocation on exhaustion. + Publish the arena pointer only after CM/device adoption using release/acquire; + the existing operation gate pins shutdown. Add unique writable Rust storage + with an uninitialized-capacity view and an explicit initialization contract. + Test no-device allocation, hard bounds, pointer-preserving commit and real + registered storage retained across server destruction. This API alone does + not establish zero-copy SEND; direct layout and producer wiring follow. + The new ordinary-buffer lifetime test first reproduces a SIGSEGV in + `Buffer::release()` after the `SystemBufferPool` facade is destroyed (GDB + confirms the retained raw recycler pointer). Move ordinary pool admission + and recycling into a retained shared State, keeping its CAS admission and + avoiding a new lock. Outstanding buffers must remain releasable after their + facade is destroyed, as registered leases already do. Verify existing detach + and concurrent outstanding-admission tests alongside the new regression. + Use the admission reservation itself as the allocation's State lifetime + reference, plus one facade bit. This retains the prior one-CAS acquire and + one-counter-release hot path; final release and facade destruction atomically + choose the sole State destructor. Add their concurrent race test. + The first HCA producer case reached its readiness assertion before calling + `RpcServer::start`; the listener is created during start. Add that missing + setup action and rerun the case without changing deadlines or assertions. + The next run confirms listen readiness but rejects allocation: PD/arenas are + adopted on the first real CM connection, not on listen. Establish a real + client QP before the allocation assertions; also assert required allocation + is unavailable before that CM adoption. This matches route-before-producer + initialization in the planned service wiring. + The corrected real-HCA test passes in 14 ms: exact arena exhaustion, + no optional system replacement, release/reallocation, and a cloned lease + surviving server/CM teardown. Held receive and peer-close cases also pass. + The final production source passes full C++/FFI verification at + `.crowdb-runtime/artifacts/unit/cpp-9266930ffd4142d18fd3d5c5318082de/`, + all 141 no-verbs CTest cases (one HCA case skips there), both new Rust + producer tests, workspace clippy/rs-lint, formatting and scoped tree-lint. + Tree-lint exits zero with style warnings. The test's corrected startup/CM + setup is rebuilt and verified separately after that full gate. - [~] **Add negotiated direct layout**: add `RdmaWireLayout::Contiguous` and `ControlData`, defaulting to `Contiguous`. For `ControlData`, the control region is `RDMA_CONTROL_CAPACITY` including the 14-byte RPC header; derive @@ -62,6 +101,9 @@ it does not add application aggregation or a flush timer. and follows the existing explicit fallback policy. Reject direct layout with transport aggregation enabled or provider SGE capacity below two. Extend typed C/Rust cold config, environment parsing and status with the same names. + Preserve the existing C ABI configuration initializer/struct size. Add a + versioned configure entry accepting the existing config plus `wire_layout`, + rather than writing new fields past a version-one caller's storage. Files: `rdma_config.h`, capability/offer/preflight modules, config ABI/FFI, `rdma_connection.cpp`, config/status tests. - [ ] **Allocate and send immutable registered data**: expose bounded producer @@ -207,3 +249,13 @@ pixi run tree-lint Do not hand off to R242 until both upload and download have explicit lease ownership tests and the DiskIO receive loop proves repost-after-disk-completion. + +## Provider references + +- [libibverbs receive work requests](https://github.com/linux-rdma/rdma-core/blob/master/libibverbs/man/ibv_post_recv.3) + defines the SGE address/length/lkey array and completion-before-buffer-reuse + requirement. CrowDB additionally waits for the disk/caller lease to end. +- [libibverbs send work requests](https://github.com/linux-rdma/rdma-core/blob/master/libibverbs/man/ibv_post_send.3) + defines scatter/gather submission and its completion ownership. The fixed + 1 KiB control region, total wire admission and layout negotiation are + CrowDB's own protocol contract. diff --git a/lib/crowdb-rpc/ffi/src/buffer.rs b/lib/crowdb-rpc/ffi/src/buffer.rs index 966658ed2..7f856055b 100644 --- a/lib/crowdb-rpc/ffi/src/buffer.rs +++ b/lib/crowdb-rpc/ffi/src/buffer.rs @@ -167,6 +167,10 @@ impl std::fmt::Debug for Buffer { } impl Buffer { + pub(crate) fn handle(&self) -> sys::crowdb_rpc_buffer_t { + self.handle + } + /// Transfer a receive buffer out of its pool, preserving its allocation. /// Returns false for pool types that cannot detach receive memory. pub fn detach(&mut self) -> bool { diff --git a/lib/crowdb-rpc/ffi/src/lib.rs b/lib/crowdb-rpc/ffi/src/lib.rs index 5821bdb95..7bd07a854 100644 --- a/lib/crowdb-rpc/ffi/src/lib.rs +++ b/lib/crowdb-rpc/ffi/src/lib.rs @@ -18,6 +18,7 @@ mod connection_pool; mod cpp_global_metrics; pub mod logging; mod owned_route; +mod producer_buffer; mod server; pub mod sys; #[cfg(feature = "test-util")] @@ -33,6 +34,7 @@ pub use logging::{ shutdown_logging, }; pub use owned_route::{OwnedClientRoute, RpcClientHandle}; +pub use producer_buffer::WritableBuffer; pub use server::{Connection, RpcError, RpcServer, ServerRequest}; pub use transport::RdmaConfig; diff --git a/lib/crowdb-rpc/ffi/src/producer_buffer.rs b/lib/crowdb-rpc/ffi/src/producer_buffer.rs new file mode 100644 index 000000000..009c2d868 --- /dev/null +++ b/lib/crowdb-rpc/ffi/src/producer_buffer.rs @@ -0,0 +1,100 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Unique producer storage before immutable buffer publication. + +use std::mem::MaybeUninit; + +use crate::{sys, Buffer, RpcError, RpcServer}; + +/// A unique allocation that cannot be submitted or cloned while writable. +/// Its storage retains the native recycler independently of the server. +#[derive(Debug)] +pub struct WritableBuffer { + buffer: Buffer, +} + +impl WritableBuffer { + #[must_use] + pub fn capacity(&self) -> usize { + self.buffer.lease_metadata().capacity as usize + } + + #[must_use] + pub fn registration_id(&self) -> u32 { + self.buffer.lease_metadata().registration_id + } + + /// Spare storage for an I/O producer. No initialized bytes are published. + /// + /// # Panics + /// Panics if foreign code has violated this allocation's unique ownership. + pub fn uninit_bytes_mut(&mut self) -> &mut [MaybeUninit] { + let capacity = self.capacity(); + // SAFETY: this type alone owns the uncommitted native allocation. + let pointer = unsafe { sys::crowdb_rpc_buffer_mutable_data(self.buffer.handle()) }; + assert!( + !pointer.is_null(), + "producer buffer is no longer uniquely writable" + ); + // SAFETY: native capacity covers this allocation and MaybeUninit does + // not require its bytes to have been initialized. The mutable borrow + // prevents publication or release until the returned view expires. + unsafe { std::slice::from_raw_parts_mut(pointer.cast::>(), capacity) } + } + + /// Initialize from caller bytes and publish an immutable buffer. + /// + /// # Errors + /// Rejects input larger than the allocation. + pub fn write(mut self, bytes: &[u8]) -> Result { + if bytes.len() > self.capacity() { + return Err(RpcError::InvalidArg); + } + self.buffer.write(bytes); + Ok(self.buffer) + } + + /// Publish an initialized prefix without copying it. + /// + /// # Safety + /// The caller must have initialized every byte in `0..initialized_length` + /// and must have ended all writes and external mutable aliases to it. + /// + /// # Errors + /// Rejects an empty, oversized or no-longer-unique allocation. + pub unsafe fn assume_init(self, initialized_length: usize) -> Result { + let length = u32::try_from(initialized_length).map_err(|_| RpcError::InvalidArg)?; + // SAFETY: the caller guarantees initialized storage; native code also + // checks capacity, unique ownership and the uncommitted state. + if unsafe { sys::crowdb_rpc_buffer_commit(self.buffer.handle(), length) } == 0 { + Err(RpcError::InvalidArg) + } else { + Ok(self.buffer) + } + } +} + +impl RpcServer { + /// Allocate bounded producer storage. Required registration never falls + /// back to system allocation; an exhausted active arena also returns None. + /// Capacity is limited to the largest host-buffer class (1 MiB). + #[must_use] + pub fn allocate_producer_buffer( + &self, + capacity: u32, + require_registered: bool, + ) -> Option { + // SAFETY: the server is borrowed; the returned lease owns its storage. + let handle = unsafe { + sys::crowdb_rpc_server_alloc_producer_buffer( + self.handle(), + capacity, + i32::from(require_registered), + ) + }; + (!handle.is_null()).then(|| WritableBuffer { + buffer: Buffer::from_raw(handle), + }) + } +} diff --git a/lib/crowdb-rpc/ffi/src/sys.rs b/lib/crowdb-rpc/ffi/src/sys.rs index 6a92e8cec..e91da432e 100644 --- a/lib/crowdb-rpc/ffi/src/sys.rs +++ b/lib/crowdb-rpc/ffi/src/sys.rs @@ -162,6 +162,13 @@ extern "C" { pub fn crowdb_rpc_buffer_write(buf: crowdb_rpc_buffer_t, data: *const u8, len: u32); pub fn crowdb_rpc_buffer_data(buf: crowdb_rpc_buffer_t) -> *const u8; pub fn crowdb_rpc_buffer_len(buf: crowdb_rpc_buffer_t) -> u32; + pub fn crowdb_rpc_buffer_mutable_data(buf: crowdb_rpc_buffer_t) -> *mut u8; + pub fn crowdb_rpc_buffer_commit(buf: crowdb_rpc_buffer_t, initialized_length: u32) -> c_int; + pub fn crowdb_rpc_server_alloc_producer_buffer( + server: crowdb_rpc_server_t, + capacity: u32, + require_registered: c_int, + ) -> crowdb_rpc_buffer_t; pub fn crowdb_rpc_buffer_capacity(buf: crowdb_rpc_buffer_t) -> u32; pub fn crowdb_rpc_buffer_request_id(buf: crowdb_rpc_buffer_t) -> u64; pub fn crowdb_rpc_buffer_stream_id(buf: crowdb_rpc_buffer_t) -> u64; diff --git a/lib/crowdb-rpc/ffi/tests/producer_buffer_test.rs b/lib/crowdb-rpc/ffi/tests/producer_buffer_test.rs new file mode 100644 index 000000000..2f9b4f7ff --- /dev/null +++ b/lib/crowdb-rpc/ffi/tests/producer_buffer_test.rs @@ -0,0 +1,38 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_rpc_ffi::{BufferPool, RdmaConfig, RpcServer}; + +#[test] +fn producer_admission_stays_bounded_and_bytes_keep_the_original_allocation() { + let pool = BufferPool::new(1); + let server = RpcServer::new(Some(&pool)); + server.configure_rdma(&RdmaConfig::default()).unwrap(); + assert!(server.allocate_producer_buffer(512, true).is_none()); + assert!(server.allocate_producer_buffer(0, false).is_none()); + assert!(server.allocate_producer_buffer((1 << 20) + 1, false).is_none()); + let mut writable = server.allocate_producer_buffer(512, false).unwrap(); + assert_eq!(writable.registration_id(), 0); + assert!(server.allocate_producer_buffer(1, false).is_none()); + let pointer = writable.uninit_bytes_mut().as_ptr().cast::(); + let buffer = writable.write(b"initialized producer data").unwrap(); + assert_eq!(buffer.bytes().as_ptr(), pointer); + let bytes = bytes::Bytes::from_owner(buffer); + let tail = bytes.slice(12..); + drop(bytes); + assert!(server.allocate_producer_buffer(1, false).is_none()); + drop(server); + drop(pool); + assert_eq!(tail, b"producer data"[..]); + drop(tail); +} + +#[test] +fn oversize_initialization_releases_producer_admission() { + let pool = BufferPool::new(1); + let server = RpcServer::new(Some(&pool)); + server.configure_rdma(&RdmaConfig::default()).unwrap(); + let buffer = server.allocate_producer_buffer(1, false).unwrap(); + assert!(buffer.write(b"too large").is_err()); + assert!(server.allocate_producer_buffer(1, false).is_some()); +} diff --git a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h index 59be719c8..842aa2569 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h @@ -161,25 +161,20 @@ class SystemBufferPool : public BufferPool public: // max_buffers: outstanding allocation bound (safety valve). SystemBufferPool(uint32_t max_buffers = 8192); - ~SystemBufferPool() override = default; + ~SystemBufferPool() override; + SystemBufferPool(const SystemBufferPool &) = delete; + SystemBufferPool &operator=(const SystemBufferPool &) = delete; Buffer *alloc(uint32_t capacity) override; void recycle(Buffer *buf) override; bool detach(Buffer *buf) override; - [[nodiscard]] uint32_t max_buffers() const - { - return max_buffers_; - } - - [[nodiscard]] uint32_t outstanding() const - { - return outstanding_.load(std::memory_order_relaxed); - } + [[nodiscard]] uint32_t max_buffers() const; + [[nodiscard]] uint32_t outstanding() const; private: - uint32_t max_buffers_; - std::atomic outstanding_{0}; // in-use buffers (allocated, not yet freed) + struct State; + State *state_; }; // Bounded registered-memory abstraction. The pool remains usable without an diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h index e90455ec2..468d29512 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h @@ -37,13 +37,17 @@ crowdb_rpc_buffer_t crowdb_rpc_buffer_alloc(crowdb_rpc_pool_t pool, uint32_t cap void crowdb_rpc_buffer_write(crowdb_rpc_buffer_t buf, const uint8_t *data, uint32_t len); const uint8_t *crowdb_rpc_buffer_data(crowdb_rpc_buffer_t buf); uint32_t crowdb_rpc_buffer_len(crowdb_rpc_buffer_t buf); -uint32_t crowdb_rpc_buffer_capacity(crowdb_rpc_buffer_t buf); -uint64_t crowdb_rpc_buffer_request_id(crowdb_rpc_buffer_t buf); -uint64_t crowdb_rpc_buffer_stream_id(crowdb_rpc_buffer_t buf); -uint64_t crowdb_rpc_buffer_sequence(crowdb_rpc_buffer_t buf); -uint64_t crowdb_rpc_buffer_logical_offset(crowdb_rpc_buffer_t buf); -uint32_t crowdb_rpc_buffer_registration_id(crowdb_rpc_buffer_t buf); -int crowdb_rpc_buffer_ownership(crowdb_rpc_buffer_t buf); +// Writable access requires the unique, uncommitted allocation owner. Commit +// only an initialized prefix; success permanently ends writable access. +uint8_t *crowdb_rpc_buffer_mutable_data(crowdb_rpc_buffer_t handle); +int crowdb_rpc_buffer_commit(crowdb_rpc_buffer_t handle, uint32_t initialized_length); +uint32_t crowdb_rpc_buffer_capacity(crowdb_rpc_buffer_t buf); +uint64_t crowdb_rpc_buffer_request_id(crowdb_rpc_buffer_t buf); +uint64_t crowdb_rpc_buffer_stream_id(crowdb_rpc_buffer_t buf); +uint64_t crowdb_rpc_buffer_sequence(crowdb_rpc_buffer_t buf); +uint64_t crowdb_rpc_buffer_logical_offset(crowdb_rpc_buffer_t buf); +uint32_t crowdb_rpc_buffer_registration_id(crowdb_rpc_buffer_t buf); +int crowdb_rpc_buffer_ownership(crowdb_rpc_buffer_t buf); void crowdb_rpc_buffer_set_metadata(crowdb_rpc_buffer_t buf, uint64_t request_id, uint64_t stream_id, uint64_t sequence, uint64_t logical_offset, uint32_t registration_id); int crowdb_rpc_buffer_transition(crowdb_rpc_buffer_t buf, int from, int to); @@ -107,6 +111,10 @@ crowdb_rpc_buffer_t crowdb_rpc_server_status_json(crowdb_rpc_server_t server); crowdb_rpc_buffer_t crowdb_rpc_conn_status_json(crowdb_rpc_conn_t connection); void crowdb_rpc_server_register_status_handler(crowdb_rpc_server_t server, uint16_t msg_type); int crowdb_rpc_server_rdma_listening(crowdb_rpc_server_t server); +// Null means bounded exhaustion/unavailable registration. No replacement +// system allocation is made when require_registered is nonzero. +crowdb_rpc_buffer_t crowdb_rpc_server_alloc_producer_buffer(crowdb_rpc_server_t server, uint32_t capacity, + int require_registered); crowdb_rpc_server_t crowdb_rpc_server_create(crowdb_rpc_pool_t pool); crowdb_rpc_server_t crowdb_rpc_server_create_with_workers(crowdb_rpc_pool_t pool, uint32_t num_workers); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index 1cffa3dee..ecc350d6e 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -110,7 +110,14 @@ class RdmaTransport : public Transport bool submit(Connection *conn, OutFrame *frame) override; bool post_receive(Connection *conn, Buffer *slot); Buffer *register_buffer(Buffer *buf) override; - void shutdown() override; + Buffer *allocate_producer_buffer(uint32_t capacity); + + [[nodiscard]] bool has_producer_pool() const + { + return producer_pool_.load(std::memory_order_acquire) != nullptr; + } + + void shutdown() override; // RDMA-specific: connect to a peer endpoint. // Returns a new Connection on success, nullptr on failure. @@ -193,6 +200,7 @@ class RdmaTransport : public Transport SystemBufferPool default_application_pool_; BufferPool *application_pool_; std::unique_ptr send_pool_; + std::atomic producer_pool_{nullptr}; std::unique_ptr recv_pool_; ConnectionHandler connection_handler_; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h index 50f641d27..9f51f9e53 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h @@ -349,6 +349,10 @@ class SocketTransport : public Transport return buf; } // noop for TCP + // Producer storage is bounded by the selected transport's arena/pool. + // A required registration never substitutes an ordinary allocation. + Buffer *allocate_producer_buffer(uint32_t capacity, bool require_registered); + void shutdown() override; // Shared I/O logic (called by Worker::run_loop). @@ -543,15 +547,16 @@ class SocketTransport : public Transport #endif private: - std::shared_ptr connect_tcp(const std::string &addr, int port); - void prepare_tcp_socket(int fd) const; - BufferPool *pool_; - bool allow_tcp_fallback_{true}; - RdmaConfig rdma_config_; - bool rdma_explicit_configuration_ = false; - bool rdma_enabled_ = false; - bool rdma_listen_enabled_ = false; - std::atomic started_{false}; + std::shared_ptr connect_tcp(const std::string &addr, int port); + void prepare_tcp_socket(int fd) const; + BufferPool *pool_; + std::unique_ptr owned_pool_; + bool allow_tcp_fallback_{true}; + RdmaConfig rdma_config_; + bool rdma_explicit_configuration_ = false; + bool rdma_enabled_ = false; + bool rdma_listen_enabled_ = false; + std::atomic started_{false}; #ifdef CROWDB_RPC_HAVE_RDMA std::unique_ptr rdma_transport_; #endif diff --git a/lib/crowdb-rpc/src/buffer.cpp b/lib/crowdb-rpc/src/buffer.cpp index 0ddabade9..2ced1097f 100644 --- a/lib/crowdb-rpc/src/buffer.cpp +++ b/lib/crowdb-rpc/src/buffer.cpp @@ -151,63 +151,4 @@ void Buffer::set_metadata(uint64_t request, uint64_t stream, uint64_t seq, uint6 registration_id = registration; } -// ── SystemBufferPool ────────────────────────────────────────────── -// -// Direct heap allocation — no free-list, no mutex. Each alloc does -// posix_memalign + new (Buffer + refcount); each recycle does free + -// delete. glibc per-thread arenas handle small-allocation recycling -// efficiently without a userspace pool. - -SystemBufferPool::SystemBufferPool(uint32_t max_buffers) : max_buffers_(max_buffers) -{ -} - -Buffer *SystemBufferPool::alloc(uint32_t capacity) -{ - uint32_t count = outstanding_.load(std::memory_order_relaxed); - do { - if (count >= max_buffers_) - return nullptr; - } while (!outstanding_.compare_exchange_weak(count, count + 1, std::memory_order_relaxed)); - - auto *buf = new Buffer; - buf->capacity = capacity; - buf->type = BufferType::System; - buf->pool = this; - buf->ref = new std::atomic(1); - - // posix_memalign for cache-line alignment (64 bytes). - void *ptr = nullptr; - if (posix_memalign(&ptr, 64, capacity) != 0) { - delete buf->ref; - delete buf; - outstanding_.fetch_sub(1, std::memory_order_relaxed); - return nullptr; - } - buf->data = static_cast(ptr); - buf->len = 0; - return buf; -} - -void SystemBufferPool::recycle(Buffer *buf) -{ - std::free(buf->data); - delete buf->ref; - delete buf; - outstanding_.fetch_sub(1, std::memory_order_relaxed); -} - -bool SystemBufferPool::detach(Buffer *buf) -{ - const BufferOwnership state = - buf == nullptr ? BufferOwnership::Quarantined : buf->ownership.load(std::memory_order_acquire); - if (buf == nullptr || buf->pool != this || buf->ref->load(std::memory_order_acquire) != 1 || - (state != BufferOwnership::Available && state != BufferOwnership::CallerOwned)) { - return false; - } - buf->pool = nullptr; - outstanding_.fetch_sub(1, std::memory_order_relaxed); - return true; -} - } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/c_api/producer.cpp b/lib/crowdb-rpc/src/c_api/producer.cpp new file mode 100644 index 000000000..74406a2b8 --- /dev/null +++ b/lib/crowdb-rpc/src/c_api/producer.cpp @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/c_api_internal.h" + +namespace +{ +crowdb::rpc::Buffer *writable_buffer(crowdb_rpc_buffer_t handle) +{ + auto *buffer = handle == nullptr ? nullptr : handle->buf; + return buffer != nullptr && buffer->len == 0 && buffer->ref != nullptr && + buffer->ref->load(std::memory_order_acquire) == 1 && + buffer->ownership.load(std::memory_order_acquire) == crowdb::rpc::BufferOwnership::Available + ? buffer + : nullptr; +} +} // namespace + +extern "C" uint8_t *crowdb_rpc_buffer_mutable_data(crowdb_rpc_buffer_t handle) +{ + auto *buffer = writable_buffer(handle); + return buffer == nullptr ? nullptr : buffer->data; +} + +extern "C" int crowdb_rpc_buffer_commit(crowdb_rpc_buffer_t handle, uint32_t initialized_length) +{ + auto *buffer = writable_buffer(handle); + if (buffer == nullptr || initialized_length == 0 || initialized_length > buffer->capacity) + return 0; + buffer->len = initialized_length; + return 1; +} + +extern "C" crowdb_rpc_buffer_t crowdb_rpc_server_alloc_producer_buffer(crowdb_rpc_server_t server, uint32_t capacity, + int require_registered) +{ + if (server == nullptr || server->server == nullptr || (require_registered != 0 && require_registered != 1)) + return nullptr; + try { + auto release = [](crowdb::rpc::Buffer *buffer) { buffer->release(); }; + std::unique_ptr buffer( + server->server->transport()->allocate_producer_buffer(capacity, require_registered != 0), release); + if (buffer == nullptr) + return nullptr; + auto *handle = new crowdb_rpc_buffer_s{buffer.get()}; + (void)buffer.release(); + return handle; + } + catch (...) { + return nullptr; + } +} diff --git a/lib/crowdb-rpc/src/system_buffer_pool.cpp b/lib/crowdb-rpc/src/system_buffer_pool.cpp new file mode 100644 index 000000000..4e07d5aed --- /dev/null +++ b/lib/crowdb-rpc/src/system_buffer_pool.cpp @@ -0,0 +1,122 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/buffer.h" + +#include + +namespace crowdb::rpc +{ +// Outstanding buffers retain allocation admission and the recycler itself. +// The facade may disappear while an asynchronous caller still owns a lease. +struct SystemBufferPool::State final : BufferPool +{ + explicit State(uint32_t limit) : maximum(limit) + { + } + + static constexpr uint64_t FACADE_REFERENCE = uint64_t{1} << 63; + const uint32_t maximum; + // Low bits are both bounded admission and allocation lifetime references. + std::atomic references{FACADE_REFERENCE}; + static_assert(std::atomic::is_always_lock_free); + + void release_allocation() + { + if (references.fetch_sub(1, std::memory_order_acq_rel) == 1) { + delete this; + } + } + + void release_facade() + { + if (references.fetch_and(~FACADE_REFERENCE, std::memory_order_acq_rel) == FACADE_REFERENCE) { + delete this; + } + } + + Buffer *alloc(uint32_t capacity) override + { + uint64_t count = references.load(std::memory_order_relaxed); + do { + if ((count & FACADE_REFERENCE) == 0 || (count & ~FACADE_REFERENCE) >= maximum) { + return nullptr; + } + } while (!references.compare_exchange_weak(count, count + 1, std::memory_order_relaxed)); + try { + auto buffer = std::make_unique(); + auto ref = std::make_unique>(1); + void *memory = nullptr; + if (posix_memalign(&memory, 64, capacity) != 0) { + release_allocation(); + return nullptr; + } + buffer->capacity = capacity; + buffer->type = BufferType::System; + buffer->pool = this; + buffer->ref = ref.release(); + buffer->data = static_cast(memory); + return buffer.release(); + } + catch (...) { + release_allocation(); + throw; + } + } + + void recycle(Buffer *buffer) override + { + std::free(buffer->data); + delete buffer->ref; + delete buffer; + release_allocation(); + } + + bool detach(Buffer *buffer) override + { + const auto state = + buffer == nullptr ? BufferOwnership::Quarantined : buffer->ownership.load(std::memory_order_acquire); + if (buffer == nullptr || buffer->pool != this || buffer->ref->load(std::memory_order_acquire) != 1 || + (state != BufferOwnership::Available && state != BufferOwnership::CallerOwned)) { + return false; + } + buffer->pool = nullptr; + release_allocation(); + return true; + } +}; + +SystemBufferPool::SystemBufferPool(uint32_t max_buffers) : state_(new State(max_buffers)) +{ +} + +SystemBufferPool::~SystemBufferPool() +{ + state_->release_facade(); +} + +Buffer *SystemBufferPool::alloc(uint32_t capacity) +{ + return state_->alloc(capacity); +} + +void SystemBufferPool::recycle(Buffer *buffer) +{ + state_->recycle(buffer); +} + +bool SystemBufferPool::detach(Buffer *buffer) +{ + return state_->detach(buffer); +} + +uint32_t SystemBufferPool::max_buffers() const +{ + return state_->maximum; +} + +uint32_t SystemBufferPool::outstanding() const +{ + return static_cast(state_->references.load(std::memory_order_relaxed) & ~State::FACADE_REFERENCE); +} +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/producer_buffer.cpp b/lib/crowdb-rpc/src/transport/producer_buffer.cpp new file mode 100644 index 000000000..347c53981 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/producer_buffer.cpp @@ -0,0 +1,21 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/transport/socket_transport.h" + +namespace crowdb::rpc +{ +Buffer *SocketTransport::allocate_producer_buffer(uint32_t capacity, bool require_registered) +{ + if (capacity == 0 || capacity > RDMA_DATAPATH_CAPACITY) + return nullptr; +#ifdef CROWDB_RPC_HAVE_RDMA + if (rdma_transport_ != nullptr) { + auto *buffer = rdma_transport_->allocate_producer_buffer(capacity); + if (buffer != nullptr || rdma_transport_->has_producer_pool() || require_registered) + return buffer; + } +#endif + return require_registered ? nullptr : pool_->alloc(capacity); +} +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_producer.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_producer.cpp new file mode 100644 index 000000000..3f0e0a66c --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_producer.cpp @@ -0,0 +1,18 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/transport/rdma/rdma_transport.h" + +#ifdef CROWDB_RPC_HAVE_RDMA +namespace crowdb::rpc +{ +Buffer *RdmaTransport::allocate_producer_buffer(uint32_t capacity) +{ + Operation operation(repost_state_); + if (!operation) + return nullptr; + auto *pool = producer_pool_.load(std::memory_order_acquire); + return pool == nullptr ? nullptr : pool->alloc(capacity); +} +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index d780aaf49..bd224bdc9 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -169,6 +169,7 @@ bool RdmaTransport::adopt_cm_context(struct ibv_context *context) capability_available_.store(false, std::memory_order_release); return false; } + producer_pool_.store(send_pool_.get(), std::memory_order_release); register_events(event_engine_); return ready(); } diff --git a/lib/crowdb-rpc/src/transport/socket_transport.cpp b/lib/crowdb-rpc/src/transport/socket_transport.cpp index 8a456b65d..c3da9a734 100644 --- a/lib/crowdb-rpc/src/transport/socket_transport.cpp +++ b/lib/crowdb-rpc/src/transport/socket_transport.cpp @@ -556,7 +556,8 @@ SocketTransport::SocketTransport(uint32_t io_engines, uint32_t io_workers, Buffe } } if (pool_ == nullptr) { - pool_ = new SystemBufferPool(); // own a default pool + owned_pool_ = std::make_unique(); + pool_ = owned_pool_.get(); } #ifdef CROWDB_RPC_HAVE_RDMA rdma_config_ = std::move(rdma_config); diff --git a/lib/crowdb-rpc/tests/producer_buffer_test.cpp b/lib/crowdb-rpc/tests/producer_buffer_test.cpp new file mode 100644 index 000000000..52dcfa3f3 --- /dev/null +++ b/lib/crowdb-rpc/tests/producer_buffer_test.cpp @@ -0,0 +1,100 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/c_api.h" + +#include + +#include +#include +#include +#include + +namespace +{ +using TestBuffer = std::unique_ptr; +using TestServer = std::unique_ptr; +using TestConnection = std::unique_ptr; +} // namespace + +TEST(ProducerBufferTest, CommitRequiresUniqueInitializedPrefixAndEndsMutableAccess) +{ + auto pool = crowdb_rpc_pool_create(1); + ASSERT_NE(pool, nullptr); + TestBuffer buffer(crowdb_rpc_buffer_alloc(pool, 16), crowdb_rpc_buffer_release); + ASSERT_NE(buffer, nullptr); + auto *pointer = crowdb_rpc_buffer_mutable_data(buffer.get()); + ASSERT_NE(pointer, nullptr); + std::memcpy(pointer, "producer payload", 16); + TestBuffer clone(crowdb_rpc_buffer_ref(buffer.get()), crowdb_rpc_buffer_release); + ASSERT_NE(clone, nullptr); + EXPECT_EQ(crowdb_rpc_buffer_mutable_data(buffer.get()), nullptr); + EXPECT_EQ(crowdb_rpc_buffer_commit(buffer.get(), 16), 0); + clone.reset(); + EXPECT_EQ(crowdb_rpc_buffer_commit(buffer.get(), 0), 0); + EXPECT_EQ(crowdb_rpc_buffer_commit(buffer.get(), 17), 0); + ASSERT_EQ(crowdb_rpc_buffer_commit(buffer.get(), 16), 1); + EXPECT_EQ(crowdb_rpc_buffer_commit(buffer.get(), 16), 0); + EXPECT_EQ(crowdb_rpc_buffer_mutable_data(buffer.get()), nullptr); + EXPECT_EQ(crowdb_rpc_buffer_data(buffer.get()), pointer); + crowdb_rpc_pool_destroy(pool); + EXPECT_EQ(std::memcmp(crowdb_rpc_buffer_data(buffer.get()), "producer payload", 16), 0); +} + +TEST(RdmaHardwareTest, ProducerArenaStaysBoundedAndLeaseOutlivesServer) +{ + const auto *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) + GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; + crowdb_rpc_rdma_config_t config; + crowdb_rpc_rdma_config_init(&config); + config.enabled = config.listen_enabled = 1; + config.allow_tcp_fallback = 0; + config.sq_depth = config.rq_depth = config.posted_receive_target = 8; + config.cq_depth = 16; + config.wire_slot_capacity = 1024; + config.send_arena_bytes = config.receive_arena_bytes = 8192; + config.source_address = address; + config.device = std::getenv("CROWDB_RDMA_DEVICE"); + TestServer server(crowdb_rpc_server_create(nullptr), crowdb_rpc_server_destroy); + ASSERT_NE(server, nullptr); + ASSERT_EQ(crowdb_rpc_server_configure_rdma(server.get(), &config), CROWDB_RPC_OK); + ASSERT_EQ(crowdb_rpc_server_listen(server.get(), address, 0), CROWDB_RPC_OK); + crowdb_rpc_server_start(server.get()); + ASSERT_EQ(crowdb_rpc_server_rdma_listening(server.get()), 1); + EXPECT_EQ(crowdb_rpc_server_alloc_producer_buffer(server.get(), 1024, 1), nullptr); + // Listening advertises the endpoint; the first established QP adopts + // its CM device context and publishes the registered producer arena. + TestServer client(crowdb_rpc_server_create(nullptr), crowdb_rpc_server_destroy); + config.listen_enabled = 0; + ASSERT_EQ(crowdb_rpc_server_configure_rdma(client.get(), &config), CROWDB_RPC_OK); + crowdb_rpc_server_start(client.get()); + TestConnection connection(crowdb_rpc_connect(client.get(), address, crowdb_rpc_server_port(server.get())), + crowdb_rpc_conn_destroy); + ASSERT_NE(connection, nullptr); + std::vector held; + for (uint32_t i = 0; i < 8; ++i) { + TestBuffer buffer(crowdb_rpc_server_alloc_producer_buffer(server.get(), 1024, 1), crowdb_rpc_buffer_release); + ASSERT_NE(buffer, nullptr); + EXPECT_NE(crowdb_rpc_buffer_registration_id(buffer.get()), 0U); + auto *pointer = crowdb_rpc_buffer_mutable_data(buffer.get()); + ASSERT_NE(pointer, nullptr); + std::memcpy(pointer, "registered producer", 19); + ASSERT_EQ(crowdb_rpc_buffer_commit(buffer.get(), 19), 1); + held.push_back(std::move(buffer)); + } + EXPECT_EQ(crowdb_rpc_server_alloc_producer_buffer(server.get(), 1024, 1), nullptr); + EXPECT_EQ(crowdb_rpc_server_alloc_producer_buffer(server.get(), 1024, 0), nullptr); + held.pop_back(); + TestBuffer replacement(crowdb_rpc_server_alloc_producer_buffer(server.get(), 1024, 1), crowdb_rpc_buffer_release); + ASSERT_NE(replacement, nullptr); + EXPECT_NE(crowdb_rpc_buffer_registration_id(replacement.get()), 0U); + const auto *pointer = crowdb_rpc_buffer_data(held.front().get()); + TestBuffer clone(crowdb_rpc_buffer_ref(held.front().get()), crowdb_rpc_buffer_release); + ASSERT_NE(clone, nullptr); + server.reset(); + held.clear(); + replacement.reset(); + EXPECT_EQ(crowdb_rpc_buffer_data(clone.get()), pointer); + EXPECT_EQ(std::memcmp(pointer, "registered producer", 19), 0); +} diff --git a/lib/crowdb-rpc/tests/system_buffer_lifetime_test.cpp b/lib/crowdb-rpc/tests/system_buffer_lifetime_test.cpp new file mode 100644 index 000000000..24ffb20c3 --- /dev/null +++ b/lib/crowdb-rpc/tests/system_buffer_lifetime_test.cpp @@ -0,0 +1,44 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/buffer.h" + +#include + +#include +#include +#include +#include + +TEST(SystemBufferPoolTest, HeldLeaseReleasesAfterFacadeDestruction) +{ + auto pool = std::make_unique(1); + auto *buffer = pool->alloc(16); + ASSERT_NE(buffer, nullptr); + buffer->write("producer payload", 16); + auto *second_owner = buffer->ref_clone(); + ASSERT_NE(second_owner, nullptr); + EXPECT_EQ(pool->outstanding(), 1U); + EXPECT_EQ(pool->alloc(1), nullptr); + pool.reset(); + EXPECT_EQ(std::memcmp(buffer->payload_data(), "producer payload", 16), 0); + buffer->release(); + EXPECT_EQ(std::memcmp(second_owner->payload_data(), "producer payload", 16), 0); + second_owner->release(); +} + +TEST(SystemBufferPoolTest, FacadeDestructionRacesFinalLeaseReleaseSafely) +{ + for (uint32_t iteration = 0; iteration < 1000; ++iteration) { + auto pool = std::make_unique(1); + auto *buffer = pool->alloc(16); + ASSERT_NE(buffer, nullptr); + std::barrier ready(2); + std::jthread release([buffer, &ready] { + ready.arrive_and_wait(); + buffer->release(); + }); + ready.arrive_and_wait(); + pool.reset(); + } +} From 3aa96c23907afca054a7858e013b13ee3661467a Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 02:28:11 +0800 Subject: [PATCH 041/258] Negotiate aligned control and data regions for direct RDMA payloads --- doc/design/rpc/design-crowdb-rpc-rdma.md | 47 ++++- doc/working/plan-rdma-datapath-stream.md | 41 +++- lib/crowdb-rpc/ffi/src/lib.rs | 2 +- lib/crowdb-rpc/ffi/src/sys.rs | 5 + lib/crowdb-rpc/ffi/src/transport.rs | 15 +- .../ffi/tests/transport_config_test.rs | 19 +- lib/crowdb-rpc/include/crowdb-rpc/buffer.h | 1 + lib/crowdb-rpc/include/crowdb-rpc/c_api.h | 4 + .../transport/rdma/rdma_capability.h | 8 +- .../crowdb-rpc/transport/rdma/rdma_config.h | 71 ++++--- .../transport/rdma/rdma_transport.h | 15 +- lib/crowdb-rpc/src/buffer.cpp | 3 + lib/crowdb-rpc/src/buffer_arena.cpp | 47 ++++- lib/crowdb-rpc/src/buffer_arena.h | 7 +- lib/crowdb-rpc/src/c_api/transport.cpp | 25 ++- .../src/transport/rdma/rdma_buffer_pool.cpp | 10 +- .../src/transport/rdma/rdma_capability.cpp | 52 ++++- .../src/transport/rdma/rdma_completion.cpp | 13 +- .../src/transport/rdma/rdma_config.cpp | 19 +- .../src/transport/rdma/rdma_connection.cpp | 16 +- lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp | 13 +- lib/crowdb-rpc/src/transport/rdma/rdma_cq.h | 3 + .../src/transport/rdma/rdma_direct_frame.cpp | 47 +++++ .../src/transport/rdma/rdma_direct_frame.h | 16 ++ .../transport/rdma/rdma_direct_receive.cpp | 111 +++++++++++ .../src/transport/rdma/rdma_direct_send.cpp | 147 ++++++++++++++ .../src/transport/rdma/rdma_offer.cpp | 17 +- .../src/transport/rdma/rdma_offer.h | 13 +- lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp | 15 ++ .../src/transport/rdma/rdma_receive.cpp | 12 +- .../src/transport/rdma/rdma_send.cpp | 15 +- .../src/transport/rdma/rdma_status.cpp | 8 + .../src/transport/rdma/rdma_transport.cpp | 8 +- .../src/transport/rdma/rdma_work_request.cpp | 27 ++- .../src/transport/socket_connect.cpp | 3 +- lib/crowdb-rpc/tests/rdma_config_test.cpp | 35 ++++ .../tests/rdma_direct_frame_test.cpp | 60 ++++++ .../tests/rdma_direct_hardware_test.cpp | 179 ++++++++++++++++++ lib/crowdb-rpc/tests/rdma_offer_test.cpp | 31 +++ .../tests/registered_buffer_test.cpp | 35 ++++ .../tests/transport_config_test.cpp | 7 + tools/rpc-test/src/args.rs | 1 + tools/rpc-test/src/workload/report.rs | 13 +- tools/rpc-test/src/workload/status.rs | 1 + 44 files changed, 1114 insertions(+), 123 deletions(-) create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_direct_frame.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_direct_frame.h create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_direct_receive.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_direct_send.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_direct_frame_test.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_direct_hardware_test.cpp diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index aa7fe36ed..bb90a28f4 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -49,10 +49,13 @@ version, and supported data classes. The client verifies that the generation and peer identity are current before creating the RDMA connection. `SocketTransport` performs this control exchange on a temporary nonblocking -TCP socket with one deadline. Version 2 also advertises receive slot capacity -and the maximum number of frames accepted in one SEND. A 48-byte CM offer +TCP socket with one deadline. The 160-byte version 3 capability advertises +receive slot capacity, the maximum number of frames accepted in one SEND and +`wire_layout`. Older capability versions can be decoded for pre-payload TCP +fallback; an RDMA connection requires the current protocol and matching layout. +A 48-byte CM offer echoes the expected peer generation and identity digest, and declares the -sender's generation, digest and receive admission. Both CM sides validate it +sender's generation, digest, layout and receive admission. Both CM sides validate it before publishing the logical connection. The digest detects inconsistent or stale discovery; it does not authenticate a peer. Low-level CM diagnostics may omit the TCP expectation but still negotiate receive admission. @@ -93,7 +96,7 @@ admission without fragmentation or fallback. A logical result larger than the selected capacity is an ordered sequence identified by request ID, stream ID, sequence, logical offset, and START/DATA/END/ERROR flags. -Each SEND uses one SGE pointing at the slot and the actual wire length. Each +In the default `Contiguous` layout, each SEND uses one SGE pointing at the slot and the actual wire length. Each RECV is posted with one SGE pointing at a complete receive slot and its full capacity. The sender and receiver do not share addresses and do not need equal ring counts; the receive slot is a bounded window for incoming messages. One @@ -104,6 +107,26 @@ before posting SEND. This copy is the transport boundary needed to make the wire message contiguous. RPC correlation and logical operation counts remain per frame, independently of SEND count. +`ControlData` is the negotiated direct datapath layout. It posts a fixed +1024-byte control region, including the 14-byte RPC header, followed by the +actual payload length in a separate SGE. The control FlatBuffer is zero padded +inside that region. Receive uses two SGEs with matching region capacities; +data starts at its own registered allocation base. Data classes of 64 KiB and +above have 4096-byte alignment. Control and data consume the configured receive +arena budget; a 1 MiB data capacity requires `wire_slot_capacity = 1049600`. +Header and CQE lengths must match exactly before dispatch. This layout sends +one RPC per SEND and rejects transport aggregation or fewer than two provider +SGEs during setup. It leaves the HTTP/ChunkIO producer's aggregation intact. + +Immutable contiguous data already owned by the device's producer arena or +the connection's receive arena is borrowed by exact allocation range. SEND +retains that allocation without rewriting it, allowing different mirror +control headers to share the same payload. Unregistered or noncontiguous data +uses a bounded copy into the device arena. Per-CQ `direct_payload_bytes` and +`copied_payload_bytes` distinguish these paths; `direct_sends` counts all SENDs +using this layout. A failed SEND quarantines the canonical source allocation, +including when the WR retains a nested view. + Each QP has a bounded `SendQueue` in `ConnectionPools`. Producers validate and transfer immutable frame ownership through its MPSC admission; the CQ owner alone drains it, carries one next-slot frame and posts SENDs within effective @@ -172,6 +195,10 @@ portable buffers keep the region alive and reserve their descriptor until final release. Their detached ranges remain committed and are not reused by another stream. Cached and quarantined descriptors both count toward the hard total. Application-pool count admission also uses one atomic reservation. +When a requested class cannot obtain a fresh allocation, it may reuse an +available larger class without extending the arena. The returned buffer keeps +its actual capacity class. This preserves admission after differently sized +HTTP owners have carved the arena. `SystemBufferPool` retains its admission counter and recycler in a shared state owned by every outstanding allocation. Destroying the facade therefore @@ -311,7 +338,11 @@ one 8192-entry CQ admits four QPs with the default send and posted-receive limits. `poller_count` follows the number of initialized CQs. Small-message benchmarks select 1024-byte slots and matching arena budgets explicitly. -The versioned C ABI and Rust `RdmaConfig` expose cold configuration. Owned +The versioned C ABI and Rust `RdmaConfig` expose cold configuration. +`crowdb_rpc_server_configure_rdma_v2` accepts `wire_layout` separately from the +unchanged version-one config structure; the original entry selects contiguous +layout. `CROWDB_RDMA_WIRE_LAYOUT` uses 0 for contiguous and 1 for control/data. +Owned status JSON contains requested settings, actual QP/CQ/provider capacities, transport errors, registration budgets and process resource counters. Connection status caches immutable provider values before publication and retains them @@ -347,8 +378,10 @@ slot class only when the complete wire message fits that class. being mistaken for a distinct operation. - **I6:** FFI completion never transfers a buffer whose ownership or length is unresolved. -- **I7:** A SEND uses one contiguous registered slot and a RECV uses one - complete posted slot; sender SGE boundaries never define receiver parsing. +- **I7:** Contiguous layout parses complete frames inside one registered slot. + Control/data layout uses its negotiated fixed control region and independent + data capacity. Both enforce total wire admission and retain every region + until transport and consumer access complete. - **I8:** Every QP is assigned to exactly one shared CQ, and every shared CQ is polled by exactly one worker at a time. - **I9:** Epoll notification only wakes an idle CQ polling worker; completion diff --git a/doc/working/plan-rdma-datapath-stream.md b/doc/working/plan-rdma-datapath-stream.md index a3b63486e..b73b32037 100644 --- a/doc/working/plan-rdma-datapath-stream.md +++ b/doc/working/plan-rdma-datapath-stream.md @@ -90,7 +90,7 @@ it does not add application aggregation or a flush timer. producer tests, workspace clippy/rs-lint, formatting and scoped tree-lint. Tree-lint exits zero with style warnings. The test's corrected startup/CM setup is rebuilt and verified separately after that full gate. -- [~] **Add negotiated direct layout**: add `RdmaWireLayout::Contiguous` and +- [x] **Add negotiated direct layout**: add `RdmaWireLayout::Contiguous` and `ControlData`, defaulting to `Contiguous`. For `ControlData`, the control region is `RDMA_CONTROL_CAPACITY` including the 14-byte RPC header; derive data capacity from `wire_slot_capacity - RDMA_CONTROL_CAPACITY` and validate @@ -106,7 +106,25 @@ it does not add application aggregation or a flush timer. rather than writing new fields past a version-one caller's storage. Files: `rdma_config.h`, capability/offer/preflight modules, config ABI/FFI, `rdma_connection.cpp`, config/status tests. -- [ ] **Allocate and send immutable registered data**: expose bounded producer + Implementation and the first 38 focused cases pass, including real HCA + full 1 MiB direct echo, 4096-byte receive alignment, exact payload/copy + counters, all four receive leases held without repost, release-triggered + repost, and source/receive leases retained after both CM transports die. + A second HCA case proves layout mismatch does not attempt CM and only uses + TCP with explicit fallback enabled. Portable cases cover exact total wire + length, malformed/truncated control, unknown layout, two-SGE requirements, + version-two decode and CM identity binding. Full C++/FFI verification passes + at `.crowdb-runtime/artifacts/unit/cpp-90779b2686624240909737a33e7c2a99/`: + 543 Tree, 168 RPC and 64 DiskIO native cases; all FFI cases including the + versioned direct-layout configuration pass. TCP-only build has 146 CTest + items, zero failures, with the producer HCA case skipped as expected. + Workspace all-feature clippy, rs-lint, Rust/C++ formatting, scoped tree-lint + (exit zero with style warnings) and all four CLI benchmark report tests pass. + The CLI test command initially named `bench_rpc_test`; Cargo lists the actual + target as `rpc_bench_test`, which passes after correcting only the command. + Final real-HCA lease/producer/layout cases pass at + `.crowdb-runtime/rdma-check/artifacts/r240-direct/native-hca.{log,xml}`. +- [x] **Allocate and send immutable registered data**: expose bounded producer allocation from the transport's device-owned arena. Provide a unique Rust `WritableBuffer` with a `MaybeUninit` capacity view and consuming commit; freeze into the existing owner-backed `Bytes` contract. Validate address @@ -117,7 +135,7 @@ it does not add application aggregation or a flush timer. as two SGEs. Keep one WR per complete RPC and count direct/copied payload bytes in status. No range registry or new hot-path lock. Files: buffer arena, RDMA pool/send/work-request modules, C ABI/FFI buffer producer API. -- [ ] **Receive into aligned data storage**: prepost two registered SGEs in +- [x] **Receive into aligned data storage**: prepost two registered SGEs in direct layout, with 1 KiB control and a separately aligned data allocation. Charge both against `receive_arena_bytes`; descriptor count covers both and arithmetic is checked. Validate the complete control/header/length before @@ -125,13 +143,28 @@ it does not add application aggregation or a flush timer. quarantine both regions exactly once on post/CQE/cancel/shutdown failure. Files: `rdma_receive.cpp`, receive posting, WorkRequest and repost context, real HCA ownership tests. Keep generic receive credit implicit in RQ slots. -- [ ] **Connect the actual HTTP producer**: allow `NativeBodyAllocator` to +- [~] **Connect the actual HTTP producer**: allow `NativeBodyAllocator` to obtain its owner storage from the retained DiskIO producer allocator after route setup. Preserve its existing byte admission, AtomicWaker pressure and framed-owner API. Wire it through `ChunkIoClient` / `RoutedDiskWriter` / S3 storage setup. A bounded native allocator failure pauses intake; it must not silently allocate an unregistered replacement. Retain TCP allocation when the service selects TCP. Cover owner views, slices and mirror fanout. + Add a cold `RpcServer` construction path receiving layout/data-capacity + defaults before native environment overrides. DiskIO server and its semantic + client select control/data with 1 MiB data capacity; ordinary RPC construction + retains contiguous defaults. Existing typed explicit configuration remains + authoritative. Do not set a process-wide direct-layout environment variable: + the Access Server's metadata transports must retain their own layout. + Expose a clonable producer allocator retaining the client's `Arc` + through `RoutedDiskWriter` and `ChunkIoClient`. `NativeOwner` may hold its + writable allocation as backing storage while its established disjoint-region + rules govern initialization and immutable framed views. Drop that backing + before waking allocator credit waiters, so a resumed receiver observes the + returned native admission. Native pool exhaustion is `WouldBlock`, not a + malloc replacement or a terminal upload error. Keep the existing AtomicWaker + release wakeups; add a bounded retry wake for native credits held by other + producers or asynchronous transport completion. - [ ] **Close DiskIO handoff and read lifetime**: use exact view pointer/length, validate request bounds before disk submission, transition data to DiskOwned until the callback returns, and retain connection state across delayed I/O. diff --git a/lib/crowdb-rpc/ffi/src/lib.rs b/lib/crowdb-rpc/ffi/src/lib.rs index 7bd07a854..54c252811 100644 --- a/lib/crowdb-rpc/ffi/src/lib.rs +++ b/lib/crowdb-rpc/ffi/src/lib.rs @@ -36,7 +36,7 @@ pub use logging::{ pub use owned_route::{OwnedClientRoute, RpcClientHandle}; pub use producer_buffer::WritableBuffer; pub use server::{Connection, RpcError, RpcServer, ServerRequest}; -pub use transport::RdmaConfig; +pub use transport::{RdmaConfig, RdmaWireLayout}; pub use sys::{CrowdbRpcLatencyStats, CrowdbRpcTransportStats}; diff --git a/lib/crowdb-rpc/ffi/src/sys.rs b/lib/crowdb-rpc/ffi/src/sys.rs index e91da432e..f8f438918 100644 --- a/lib/crowdb-rpc/ffi/src/sys.rs +++ b/lib/crowdb-rpc/ffi/src/sys.rs @@ -155,6 +155,11 @@ extern "C" { server: crowdb_rpc_server_t, config: *const CrowdbRpcRdmaConfig, ) -> crowdb_rpc_status; + pub fn crowdb_rpc_server_configure_rdma_v2( + server: crowdb_rpc_server_t, + config: *const CrowdbRpcRdmaConfig, + wire_layout: u32, + ) -> crowdb_rpc_status; pub fn crowdb_rpc_server_status_json(server: crowdb_rpc_server_t) -> crowdb_rpc_buffer_t; pub fn crowdb_rpc_conn_status_json(connection: crowdb_rpc_conn_t) -> crowdb_rpc_buffer_t; pub fn crowdb_rpc_server_register_status_handler(server: crowdb_rpc_server_t, msg_type: u16); diff --git a/lib/crowdb-rpc/ffi/src/transport.rs b/lib/crowdb-rpc/ffi/src/transport.rs index ab4c71021..70efdb601 100644 --- a/lib/crowdb-rpc/ffi/src/transport.rs +++ b/lib/crowdb-rpc/ffi/src/transport.rs @@ -8,6 +8,15 @@ use std::mem::MaybeUninit; use crate::{sys, Buffer, Connection, RpcError, RpcServer}; +/// Layout selected during TCP preflight and bound into the CM offer. +#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)] +#[repr(u32)] +pub enum RdmaWireLayout { + #[default] + Contiguous = 0, + ControlData = 1, +} + /// Explicit configuration overrides native environment defaults before listen. #[derive(Debug, Clone)] #[allow(clippy::struct_excessive_bools)] @@ -22,6 +31,7 @@ pub struct RdmaConfig { pub rq_depth: u32, pub posted_receive_target: u32, pub wire_slot_capacity: u32, + pub wire_layout: RdmaWireLayout, pub poll_batch: u32, pub verbs_poll_batch: u32, pub poll_empty_budget: u32, @@ -60,6 +70,7 @@ impl Default for RdmaConfig { rq_depth: raw.rq_depth, posted_receive_target: raw.posted_receive_target, wire_slot_capacity: raw.wire_slot_capacity, + wire_layout: RdmaWireLayout::Contiguous, poll_batch: raw.poll_batch, verbs_poll_batch: raw.verbs_poll_batch, poll_empty_budget: raw.poll_empty_budget, @@ -111,7 +122,9 @@ impl RpcServer { raw.source_address = source.as_ptr(); raw.gid = gid.as_ptr(); // SAFETY: handle is borrowed, struct and CStrings live through the call. - let status = unsafe { sys::crowdb_rpc_server_configure_rdma(self.handle(), &raw) }; + let status = unsafe { + sys::crowdb_rpc_server_configure_rdma_v2(self.handle(), &raw, config.wire_layout as u32) + }; if status == sys::CROWDB_RPC_OK { Ok(()) } else { diff --git a/lib/crowdb-rpc/ffi/tests/transport_config_test.rs b/lib/crowdb-rpc/ffi/tests/transport_config_test.rs index 9caa52c80..93955e5b9 100644 --- a/lib/crowdb-rpc/ffi/tests/transport_config_test.rs +++ b/lib/crowdb-rpc/ffi/tests/transport_config_test.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use crowdb_protocol::fb::{ConnectionPingRequest, ConnectionPingRequestArgs}; -use crowdb_rpc_ffi::{Buffer, RdmaConfig, RpcClient, RpcServer}; +use crowdb_rpc_ffi::{Buffer, RdmaConfig, RdmaWireLayout, RpcClient, RpcServer}; use flatbuffers::FlatBufferBuilder; use serde_json::Value; @@ -39,6 +39,23 @@ fn invalid_queue_bounds_fail_before_start() { assert!(server.configure_rdma(&config).is_err()); } +#[test] +fn typed_control_data_layout_accounts_for_control_and_forbids_aggregation() { + let server = RpcServer::new(None); + let mut config = RdmaConfig { + wire_layout: RdmaWireLayout::ControlData, + wire_slot_capacity: (1 << 20) + 1024, + receive_arena_bytes: 1024 * ((1 << 20) + 1024), + ..RdmaConfig::default() + }; + server.configure_rdma(&config).unwrap(); + config.aggregation_enabled = true; + assert!(server.configure_rdma(&config).is_err()); + config.aggregation_enabled = false; + config.receive_arena_bytes -= 1; + assert!(server.configure_rdma(&config).is_err()); +} + #[tokio::test] async fn status_control_rpc_returns_owned_versioned_json() { let server = RpcServer::new(None); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h index 842aa2569..b20f4ce0a 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h @@ -73,6 +73,7 @@ struct Buffer uintptr_t registration_handle = 0; uint8_t *allocation_data = nullptr; uint32_t allocation_capacity = 0; + Buffer *allocation_owner = nullptr; uint32_t view_offset = 0; Buffer *next_pool = nullptr; Buffer *next_all = nullptr; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h index 468d29512..7c5b03bcb 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h @@ -106,6 +106,10 @@ typedef struct crowdb_rpc_rdma_config void crowdb_rpc_rdma_config_init(crowdb_rpc_rdma_config_t *config); // Explicit cold configuration overrides environment; rejected after listen. crowdb_rpc_status crowdb_rpc_server_configure_rdma(crowdb_rpc_server_t server, const crowdb_rpc_rdma_config_t *config); +// Version-two configuration adds the negotiated wire layout without changing +// the version-one struct or initializer. wire_layout: 0 contiguous, 1 control/data. +crowdb_rpc_status crowdb_rpc_server_configure_rdma_v2(crowdb_rpc_server_t server, + const crowdb_rpc_rdma_config_t *config, uint32_t wire_layout); // Owned, versioned UTF-8 JSON snapshots. Release using buffer_release. crowdb_rpc_buffer_t crowdb_rpc_server_status_json(crowdb_rpc_server_t server); crowdb_rpc_buffer_t crowdb_rpc_conn_status_json(crowdb_rpc_conn_t connection); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h index 49f8f8d3b..cee92ccfd 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h @@ -23,6 +23,11 @@ enum class RdmaPathState : uint8_t { TcpFallback, }; +enum class RdmaWireLayout : uint32_t { Contiguous = 0, ControlData = 1 }; + +[[nodiscard]] uint32_t rdma_data_slot_capacity(uint32_t wire_capacity, RdmaWireLayout layout); +[[nodiscard]] bool valid_rdma_wire_capacity(uint32_t wire_capacity, RdmaWireLayout layout); + struct RdmaEndpointCapability { bool rdma_available = false; @@ -35,11 +40,12 @@ struct RdmaEndpointCapability uint32_t receive_slot_capacity = 0; uint32_t receive_frame_limit = 1; std::string cm_address{}; + RdmaWireLayout wire_layout = RdmaWireLayout::Contiguous; }; constexpr uint16_t RDMA_HANDSHAKE_MAGIC = 0x5244; constexpr uint16_t RDMA_HANDSHAKE_MSG_TYPE = 0xFFFE; -constexpr uint16_t RDMA_PROTOCOL_VERSION = 2; +constexpr uint16_t RDMA_PROTOCOL_VERSION = 3; constexpr size_t RDMA_LEGACY_HANDSHAKE_SIZE = 128; constexpr size_t RDMA_HANDSHAKE_SIZE = 160; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h index d73b27a90..239535025 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h @@ -3,7 +3,7 @@ #pragma once -#include "crowdb-rpc/buffer.h" +#include "crowdb-rpc/transport/rdma/rdma_capability.h" #include @@ -11,38 +11,51 @@ namespace crowdb::rpc { struct RdmaConfig { - bool use_environment = true; - std::string device; - std::string source_address{}; - std::string gid{}; - bool require_tcp_preflight = false; - uint8_t port = 1; - uint32_t cq_depth = 8192; - uint32_t cq_count = 1; - uint32_t wire_slot_capacity = RDMA_DATAPATH_CAPACITY; - uint64_t send_arena_bytes = 64ULL << 20; - uint64_t receive_arena_bytes = 1ULL << 30; - uint64_t registered_memory_budget_bytes = 0; - uint32_t sq_depth = 1024; - uint32_t rq_depth = 1024; - uint32_t qp_max_send_sge = 2; - uint32_t qp_max_recv_sge = 2; - uint32_t poll_batch = 256; - uint32_t verbs_poll_batch = 32; - uint32_t poll_empty_budget = 256; - uint32_t shutdown_drain_ms = 3000; - uint32_t posted_receive_target = 1024; - uint8_t min_rnr_timer = 1; - uint8_t retry_count = 7; - uint8_t rnr_retry_count = 7; - bool aggregation_enabled = false; - uint32_t aggregation_max_bytes = RDMA_DATAPATH_CAPACITY; - uint32_t aggregation_max_frames = 256; - uint32_t send_queue_capacity = 4096; + bool use_environment = true; + std::string device; + std::string source_address{}; + std::string gid{}; + bool require_tcp_preflight = false; + uint8_t port = 1; + uint32_t cq_depth = 8192; + uint32_t cq_count = 1; + uint32_t wire_slot_capacity = RDMA_DATAPATH_CAPACITY; + RdmaWireLayout wire_layout = RdmaWireLayout::Contiguous; + uint64_t send_arena_bytes = 64ULL << 20; + uint64_t receive_arena_bytes = 1ULL << 30; + uint64_t registered_memory_budget_bytes = 0; + uint32_t sq_depth = 1024; + uint32_t rq_depth = 1024; + uint32_t qp_max_send_sge = 2; + uint32_t qp_max_recv_sge = 2; + uint32_t poll_batch = 256; + uint32_t verbs_poll_batch = 32; + uint32_t poll_empty_budget = 256; + uint32_t shutdown_drain_ms = 3000; + uint32_t posted_receive_target = 1024; + uint8_t min_rnr_timer = 1; + uint8_t retry_count = 7; + uint8_t rnr_retry_count = 7; + bool aggregation_enabled = false; + uint32_t aggregation_max_bytes = RDMA_DATAPATH_CAPACITY; + uint32_t aggregation_max_frames = 256; + uint32_t send_queue_capacity = 4096; [[nodiscard]] bool apply_environment(); [[nodiscard]] bool valid() const; [[nodiscard]] uint64_t registration_budget() const; + + [[nodiscard]] uint32_t data_slot_capacity() const + { + return rdma_data_slot_capacity(wire_slot_capacity, wire_layout); + } + + [[nodiscard]] uint64_t receive_control_bytes() const + { + return wire_layout == RdmaWireLayout::ControlData + ? static_cast(posted_receive_target) * RDMA_CONTROL_CAPACITY + : 0; + } }; } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index ecc350d6e..87eae5306 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -47,6 +47,7 @@ class RdmaBufferPool : public BufferPool ~RdmaBufferPool() override; Buffer *alloc(uint32_t capacity) override; + Buffer *borrow_range(const uint8_t *data, uint32_t size); void recycle(Buffer *buf) override; [[nodiscard]] uint64_t quarantined_bytes() const; @@ -102,6 +103,10 @@ class RdmaTransport : public Transport std::atomic submit_rejects{0}; std::atomic send_queue_high_water{0}; std::atomic send_errors{0}; + std::atomic receive_post_errors{0}; + std::atomic direct_sends{0}; + std::atomic direct_payload_bytes{0}; + std::atomic copied_payload_bytes{0}; }; explicit RdmaTransport(RdmaConfig config = {}, BufferPool *application_pool = nullptr); @@ -214,6 +219,7 @@ class RdmaTransport : public Transport enum class WorkKind : uint8_t { Send, Receive }; struct ConnectionPools; + bool create_connection_pools(ConnectionPools &pools); struct WorkRequest { @@ -223,6 +229,7 @@ class RdmaTransport : public Transport std::atomic connection_identity{nullptr}; std::shared_ptr pools; Buffer *slot = nullptr; + Buffer *prefix = nullptr; OutFrame *frame = nullptr; uint64_t request_id = 0; uint64_t stream_id = 0; @@ -263,13 +270,17 @@ class RdmaTransport : public Transport void install_connection_notifiers(const std::shared_ptr &connection, uint32_t index); uint32_t flush_sends(CompletionQueue &queue, uint32_t budget); uint32_t flush_connection(Connection *connection, ConnectionPools &pools, uint32_t budget); + uint32_t flush_direct_connection(Connection *connection, ConnectionPools &pools, uint32_t budget); + bool post_direct(Connection *connection, ConnectionPools &pools, Buffer *slot, Buffer *prefix, OutFrame *frame); + bool post_direct_receive(Connection *connection, Buffer *slot); void retire_sends(CompletionQueue &queue, Connection *connection); bool post_aggregate(Connection *connection, ConnectionPools &pools, Buffer *slot, OutFrame *frames, uint32_t frame_count); static constexpr uint64_t CM_EVENT_TAG = 1; static constexpr uint64_t CQ_EVENT_BASE = 2; - WorkRequest *acquire_work(WorkKind kind, Connection *conn, Buffer *slot, OutFrame *frame = nullptr); + WorkRequest *acquire_work(WorkKind kind, Connection *conn, Buffer *slot, OutFrame *frame = nullptr, + Buffer *prefix = nullptr); WorkRequest *claim_work(uint64_t token); void release_work(WorkRequest *work); void dispatch_completion(const struct ibv_wc &wc); @@ -334,6 +345,7 @@ class RdmaTransport : public Transport std::shared_ptr posting_state = std::make_shared(); std::unique_ptr send; std::unique_ptr recv; + std::unique_ptr recv_control; uint32_t cq_index = 0; uint32_t reserved_capacity = 0; uint32_t peer_receive_slot_capacity = 0; @@ -358,6 +370,7 @@ class RdmaTransport : public Transport void release_completed_frame(WorkRequest *work, const struct ibv_wc &wc); void release_completed_receive(WorkRequest *work, const struct ibv_wc &wc); + void release_completed_direct_receive(WorkRequest *work, const struct ibv_wc &wc); void record_completion_error(const struct ibv_wc &wc, bool expected_flush = false); static void repost_frame_release(void *context, Buffer *slot); bool adopt_cm_context(struct ibv_context *context); diff --git a/lib/crowdb-rpc/src/buffer.cpp b/lib/crowdb-rpc/src/buffer.cpp index 2ced1097f..23774ef0a 100644 --- a/lib/crowdb-rpc/src/buffer.cpp +++ b/lib/crowdb-rpc/src/buffer.cpp @@ -120,6 +120,7 @@ Buffer *Buffer::slice(uint32_t offset, uint32_t size) view->registration_id = registration_id; view->registration_key = registration_key; view->registration_handle = registration_handle; + view->allocation_owner = allocation_owner == nullptr ? this : allocation_owner; view->set_metadata(request_id, stream_id, sequence, logical_offset, registration_id); view->free_cb = [](void *parent) { static_cast(parent)->release(); }; view->free_ctx = this; @@ -133,6 +134,8 @@ bool Buffer::transition(BufferOwnership from, BufferOwnership to) bool Buffer::quarantine() { + if (allocation_owner != nullptr) + (void)allocation_owner->quarantine(); BufferOwnership state = ownership.load(std::memory_order_acquire); while (state != BufferOwnership::Quarantined) { if (ownership.compare_exchange_weak(state, BufferOwnership::Quarantined, std::memory_order_acq_rel, diff --git a/lib/crowdb-rpc/src/buffer_arena.cpp b/lib/crowdb-rpc/src/buffer_arena.cpp index 1007ba1ef..a8a595789 100644 --- a/lib/crowdb-rpc/src/buffer_arena.cpp +++ b/lib/crowdb-rpc/src/buffer_arena.cpp @@ -8,23 +8,24 @@ namespace crowdb::rpc { -std::shared_ptr BufferRegion::allocate(uint64_t bytes, uint32_t registration_id) +std::shared_ptr BufferRegion::allocate(uint64_t bytes, uint32_t registration_id, uint32_t alignment) { auto region = std::make_shared(); region->bytes = bytes; region->registration_id = registration_id; void *memory = nullptr; - if (bytes != 0 && posix_memalign(&memory, 64, bytes) == 0) { + if (bytes != 0 && posix_memalign(&memory, alignment, bytes) == 0) { region->data = static_cast(memory); region->owner = std::shared_ptr(memory, std::free); } return region; } -BufferArena::BufferArena(std::shared_ptr allocation, uint32_t count) +BufferArena::BufferArena(std::shared_ptr allocation, uint32_t count, uint32_t large_alignment) : region(std::move(allocation)), bytes(region == nullptr ? 0 : region->bytes), - capacity(count) + capacity(count), + large_alignment_(large_alignment) { if (region != nullptr && region->data != nullptr && count != 0 && count != UINT32_MAX) entries_ = std::make_unique(count); @@ -55,14 +56,15 @@ Buffer *BufferArena::fresh(uint32_t size) return nullptr; } while (!next_entry_.compare_exchange_weak(index, index + 1, std::memory_order_relaxed)); } - uint64_t offset = next_offset.load(std::memory_order_relaxed); - uint64_t aligned; + uint64_t offset = next_offset.load(std::memory_order_relaxed); + uint64_t aligned; + const uint64_t mask = (size >= RDMA_METADATA_CAPACITY ? large_alignment_ : 64U) - 1U; do { - if (offset > bytes || offset > UINT64_MAX - 63U) { + if (offset > bytes || offset > UINT64_MAX - mask) { push_entry(unused_entries_, index); return nullptr; } - aligned = (offset + 63U) & ~uint64_t{63}; + aligned = (offset + mask) & ~mask; if (aligned > bytes || size > bytes - aligned) { push_entry(unused_entries_, index); return nullptr; @@ -128,8 +130,13 @@ Buffer *BufferArena::alloc(uint32_t requested) auto *buffer = pop(static_cast(cls)); if (buffer == nullptr) buffer = fresh(size); + if (buffer == nullptr) { + for (size_t bucket = static_cast(cls) + 1; bucket < free_heads_.size() && buffer == nullptr; ++bucket) + buffer = pop(bucket); + } if (buffer == nullptr) return nullptr; + (void)classify_rdma_capacity(buffer->capacity, &cls, &size); buffer->reset_view(); buffer->len = 0; buffer->ref->store(1, std::memory_order_relaxed); @@ -140,6 +147,30 @@ Buffer *BufferArena::alloc(uint32_t requested) return buffer; } +Buffer *BufferArena::borrow_range(const uint8_t *data, uint32_t size) +{ + if (region == nullptr || region->registration_handle == 0 || entries_ == nullptr || size == 0) + return nullptr; + const auto address = reinterpret_cast(data); + const auto begin = reinterpret_cast(region->data); + if (address < begin || size > bytes || address - begin > bytes - size) + return nullptr; + const auto count = next_entry_.load(std::memory_order_acquire); + for (uint32_t index = 0; index < count; ++index) { + auto *buffer = entries_[index].buffer.load(std::memory_order_acquire); + if (buffer == nullptr) + continue; + const auto start = reinterpret_cast(buffer->allocation_data); + if (address >= start && address - start <= buffer->allocation_capacity && + size <= buffer->allocation_capacity - (address - start)) { + // The immutable source owner still pins this allocation. Retain + // that same allocation, not merely any MR containing the address. + return buffer->slice(static_cast(address - start), size); + } + } + return nullptr; +} + void BufferArena::recycle(Buffer *buffer) { // Retain this receiver until after publication; another allocator may diff --git a/lib/crowdb-rpc/src/buffer_arena.h b/lib/crowdb-rpc/src/buffer_arena.h index 249fcd9c7..c18cd4afb 100644 --- a/lib/crowdb-rpc/src/buffer_arena.h +++ b/lib/crowdb-rpc/src/buffer_arena.h @@ -16,7 +16,8 @@ struct BufferRegion uint32_t registration_id = 0; std::shared_ptr owner; - static std::shared_ptr allocate(uint64_t bytes, uint32_t registration_id = 0); + static std::shared_ptr allocate(uint64_t bytes, uint32_t registration_id = 0, + uint32_t alignment = 64); }; class BufferArena : public BufferPool, public std::enable_shared_from_this @@ -35,14 +36,16 @@ class BufferArena : public BufferPool, public std::enable_shared_from_this next_offset{0}; std::atomic quarantined{0}; - BufferArena(std::shared_ptr region, uint32_t count); + BufferArena(std::shared_ptr region, uint32_t count, uint32_t large_alignment = 64); ~BufferArena() override; Buffer *alloc(uint32_t requested) override; + Buffer *borrow_range(const uint8_t *data, uint32_t size); void recycle(Buffer *buffer) override; bool detach(Buffer *buffer) override; void quarantine_inflight(); private: + const uint32_t large_alignment_; std::unique_ptr entries_; std::array, 4> free_heads_{}; std::atomic unused_entries_{0}; diff --git a/lib/crowdb-rpc/src/c_api/transport.cpp b/lib/crowdb-rpc/src/c_api/transport.cpp index 0ad2db010..faad622b9 100644 --- a/lib/crowdb-rpc/src/c_api/transport.cpp +++ b/lib/crowdb-rpc/src/c_api/transport.cpp @@ -6,14 +6,15 @@ using namespace crowdb::rpc; -#define CONFIG_NUMBERS(X) \ - X(cq_count) \ - X(cq_depth) \ - X(sq_depth) \ - X(rq_depth) \ - X(posted_receive_target) X(wire_slot_capacity) X(poll_batch) X(verbs_poll_batch) X(poll_empty_budget) \ - X(send_queue_capacity) X(aggregation_max_bytes) X(aggregation_max_frames) X(send_arena_bytes) \ - X(receive_arena_bytes) X(registered_memory_budget_bytes) +#define CONFIG_NUMBERS(X) \ + X(cq_count) \ + X(cq_depth) \ + X(sq_depth) \ + X(rq_depth) \ + X(posted_receive_target) \ + X(wire_slot_capacity) X(poll_batch) X(verbs_poll_batch) X(poll_empty_budget) X(send_queue_capacity) \ + X(aggregation_max_bytes) X(aggregation_max_frames) X(send_arena_bytes) X(receive_arena_bytes) \ + X(registered_memory_budget_bytes) extern "C" void crowdb_rpc_rdma_config_init(crowdb_rpc_rdma_config_t *config) { @@ -33,6 +34,13 @@ extern "C" void crowdb_rpc_rdma_config_init(crowdb_rpc_rdma_config_t *config) extern "C" crowdb_rpc_status crowdb_rpc_server_configure_rdma(crowdb_rpc_server_t server, const crowdb_rpc_rdma_config_t *config) +{ + return crowdb_rpc_server_configure_rdma_v2(server, config, 0); +} + +extern "C" crowdb_rpc_status crowdb_rpc_server_configure_rdma_v2(crowdb_rpc_server_t server, + const crowdb_rpc_rdma_config_t *config, + uint32_t wire_layout) { if (server == nullptr || config == nullptr || config->struct_size != sizeof(*config) || config->version != 1 || config->enabled > 1 || config->allow_tcp_fallback > 1 || config->listen_enabled > 1 || @@ -40,6 +48,7 @@ extern "C" crowdb_rpc_status crowdb_rpc_server_configure_rdma(crowdb_rpc_server_ return CROWDB_RPC_ERR_INVALID_ARG; try { RdmaConfig native; + native.wire_layout = static_cast(wire_layout); native.use_environment = false; native.port = static_cast(config->port); native.aggregation_enabled = config->aggregation_enabled != 0; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp index 56f0a6b95..c914a24ca 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp @@ -32,7 +32,7 @@ std::shared_ptr register_region(ibv_pd *pd, uint64_t size, std::sh { if (pd == nullptr) return nullptr; - auto region = BufferRegion::allocate(size); + auto region = BufferRegion::allocate(size, 0, 4096); if (region->data == nullptr) return nullptr; auto registration = std::make_shared(); @@ -52,7 +52,8 @@ std::shared_ptr register_region(ibv_pd *pd, uint64_t size, std::sh RdmaBufferPool::RdmaBufferPool(ibv_pd *pd, uint64_t mr_size, uint32_t max_buffers, std::shared_ptr registration_owner) - : arena_(std::make_shared(register_region(pd, mr_size, std::move(registration_owner)), max_buffers)) + : arena_( + std::make_shared(register_region(pd, mr_size, std::move(registration_owner)), max_buffers, 4096)) { } @@ -63,6 +64,11 @@ Buffer *RdmaBufferPool::alloc(uint32_t capacity) return arena_->alloc(capacity); } +Buffer *RdmaBufferPool::borrow_range(const uint8_t *data, uint32_t size) +{ + return arena_->borrow_range(data, size); +} + void RdmaBufferPool::recycle(Buffer *buffer) { arena_->recycle(buffer); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp index 96e2f5f2d..f11509282 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp @@ -11,6 +11,23 @@ namespace crowdb::rpc { +uint32_t rdma_data_slot_capacity(uint32_t wire_capacity, RdmaWireLayout layout) +{ + if (layout == RdmaWireLayout::Contiguous) + return wire_capacity; + return layout == RdmaWireLayout::ControlData && wire_capacity > RDMA_CONTROL_CAPACITY + ? wire_capacity - RDMA_CONTROL_CAPACITY + : 0; +} + +bool valid_rdma_wire_capacity(uint32_t wire_capacity, RdmaWireLayout layout) +{ + BufferCapacityClass cls; + uint32_t capacity = 0; + const auto data_capacity = rdma_data_slot_capacity(wire_capacity, layout); + return classify_rdma_capacity(data_capacity, &cls, &capacity) && capacity == data_capacity; +} + std::array serialize_rdma_handshake(const RdmaEndpointCapability &capability) { std::array bytes{}; @@ -42,6 +59,11 @@ std::array serialize_rdma_handshake(const RdmaEndp bytes[132 + i] = static_cast(capability.receive_frame_limit >> (i * 8)); } std::memcpy(bytes.data() + 136, capability.cm_address.data(), std::min(16, capability.cm_address.size())); + if (capability.protocol_version >= 3) { + const auto layout = static_cast(capability.wire_layout); + for (size_t i = 0; i < 4; ++i) + bytes[152 + i] = static_cast(layout >> (i * 8)); + } return bytes; } @@ -60,7 +82,7 @@ bool parse_rdma_handshake(std::span bytes, RdmaEndpointCapability }; if (get16(0) != RDMA_HANDSHAKE_MAGIC) return false; - if (get16(2) >= RDMA_PROTOCOL_VERSION && bytes.size() < RDMA_HANDSHAKE_SIZE) + if (get16(2) >= 2 && bytes.size() < RDMA_HANDSHAKE_SIZE) return false; *out = RdmaEndpointCapability{}; out->protocol_version = get16(2); @@ -77,7 +99,7 @@ bool parse_rdma_handshake(std::span bytes, RdmaEndpointCapability (static_cast(bytes[offset + 2]) << 16) | (static_cast(bytes[offset + 3]) << 24); } - if (out->protocol_version >= RDMA_PROTOCOL_VERSION) { + if (out->protocol_version >= 2) { for (size_t i = 0; i < 4; ++i) { out->receive_slot_capacity |= static_cast(bytes[128 + i]) << (i * 8); if (i == 0) @@ -88,6 +110,17 @@ bool parse_rdma_handshake(std::span bytes, RdmaEndpointCapability out->cm_address.resize(out->cm_address.find('\0') == std::string::npos ? out->cm_address.size() : out->cm_address.find('\0')); } + if (out->protocol_version >= 3) { + uint32_t layout = 0; + for (size_t i = 0; i < 4; ++i) { + layout |= static_cast(bytes[152 + i]) << (i * 8); + if (bytes[156 + i] != 0) + return false; + } + if (layout > static_cast(RdmaWireLayout::ControlData)) + return false; + out->wire_layout = static_cast(layout); + } return true; } @@ -99,16 +132,15 @@ bool validate_rdma_capability(const RdmaEndpointCapability &capability, uint16_t capability.gid.empty() || (!expected_device.empty() && capability.device != expected_device)) { return false; } - if (expected_protocol >= RDMA_PROTOCOL_VERSION) { - BufferCapacityClass cls; - uint32_t size = 0; - in_addr address{}; - in6_addr gid{}; + if (expected_protocol >= 2) { + in_addr address{}; + in6_addr gid{}; if (::inet_pton(AF_INET, capability.cm_address.c_str(), &address) != 1 || ::inet_pton(AF_INET6, capability.gid.c_str(), &gid) != 1 || capability.generation == 0 || - !classify_rdma_capacity(capability.receive_slot_capacity, &cls, &size) || - size != capability.receive_slot_capacity || capability.receive_frame_limit == 0 || - capability.receive_frame_limit > 256) + !valid_rdma_wire_capacity(capability.receive_slot_capacity, capability.wire_layout) || + capability.receive_frame_limit == 0 || capability.receive_frame_limit > 256 || + (capability.wire_layout == RdmaWireLayout::ControlData && + (expected_protocol < 3 || capability.receive_frame_limit != 1))) return false; } return capability.data_classes[0] == RDMA_CONTROL_CAPACITY && diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp index 2b9964efb..8b4b2618b 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp @@ -163,7 +163,7 @@ bool RdmaTransport::prime_receive(const std::shared_ptr &connection) return false; BufferPool *pool = pools->recv.get(); for (uint32_t i = 0; i < depth; ++i) { - Buffer *slot = pool->alloc(config_.wire_slot_capacity); + Buffer *slot = pool->alloc(config_.data_slot_capacity()); if (slot == nullptr) { return false; } @@ -209,6 +209,7 @@ void RdmaTransport::release_completed_frame(WorkRequest *work, const ibv_wc &wc) auto connection = std::move(work->conn); const auto pools = std::move(work->pools); Buffer *slot = work->slot; + Buffer *prefix = work->prefix; if (pools != nullptr && pools->sends_in_flight != 0) --pools->sends_in_flight; @@ -218,6 +219,8 @@ void RdmaTransport::release_completed_frame(WorkRequest *work, const ibv_wc &wc) // suffix to be appended to this RDMA connection. if (work->slot != nullptr) (void)work->slot->quarantine(); + if (prefix != nullptr) + (void)prefix->quarantine(); if (connection != nullptr) connection->close(); } @@ -227,9 +230,17 @@ void RdmaTransport::release_completed_frame(WorkRequest *work, const ibv_wc &wc) if (connection != nullptr) connection->close(); } + if (wc.status == IBV_WC_SUCCESS && prefix != nullptr && + !prefix->transition(BufferOwnership::SendInFlight, BufferOwnership::Available)) { + (void)prefix->quarantine(); + if (connection != nullptr) + connection->close(); + } release_work(work); if (slot != nullptr) slot->release(); + if (prefix != nullptr) + prefix->release(); release_rdma_frames(frame, wc.status != IBV_WC_SUCCESS); } diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp index e41054195..0f208dfdf 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp @@ -31,6 +31,10 @@ bool RdmaConfig::apply_environment() destination = static_cast(parsed); return true; }; + auto layout = static_cast(wire_layout); + if (!read("CROWDB_RDMA_WIRE_LAYOUT", layout) || layout > static_cast(RdmaWireLayout::ControlData)) + return false; + wire_layout = static_cast(layout); return (port != 1 || read("CROWDB_RDMA_PORT", port)) && read("CROWDB_RDMA_SQ_DEPTH", sq_depth) && read("CROWDB_RDMA_RQ_DEPTH", rq_depth) && read("CROWDB_RDMA_POSTED_RECEIVE_TARGET", posted_receive_target) && read("CROWDB_RDMA_WIRE_SLOT_CAPACITY", wire_slot_capacity) && @@ -48,15 +52,14 @@ bool RdmaConfig::apply_environment() bool RdmaConfig::valid() const { - BufferCapacityClass capacity_class; - uint32_t capacity = 0; - return port > 0 && classify_rdma_capacity(wire_slot_capacity, &capacity_class, &capacity) && - capacity == wire_slot_capacity && sq_depth > 0 && rq_depth > 0 && posted_receive_target > 0 && - posted_receive_target <= rq_depth && cq_depth > posted_receive_target && - cq_depth <= static_cast(INT32_MAX) && cq_count > 0 && cq_count <= 256 && - static_cast(cq_count) * cq_depth < UINT32_MAX && poll_batch > 0 && + const bool direct = wire_layout == RdmaWireLayout::ControlData; + return port > 0 && valid_rdma_wire_capacity(wire_slot_capacity, wire_layout) && + (!direct || (!aggregation_enabled && qp_max_send_sge >= 2 && qp_max_recv_sge >= 2)) && sq_depth > 0 && + rq_depth > 0 && posted_receive_target > 0 && posted_receive_target <= rq_depth && + cq_depth > posted_receive_target && cq_depth <= static_cast(INT32_MAX) && cq_count > 0 && + cq_count <= 256 && static_cast(cq_count) * cq_depth < UINT32_MAX && poll_batch > 0 && poll_batch <= static_cast(INT32_MAX) && verbs_poll_batch > 0 && verbs_poll_batch <= 256 && - poll_empty_budget > 0 && send_arena_bytes >= wire_slot_capacity && + poll_empty_budget > 0 && send_arena_bytes >= data_slot_capacity() && receive_arena_bytes >= static_cast(posted_receive_target) * wire_slot_capacity && min_rnr_timer <= 31 && retry_count <= 7 && rnr_retry_count <= 7 && registration_budget() != 0 && aggregation_max_bytes >= HEADER_SIZE && aggregation_max_bytes <= RDMA_DATAPATH_CAPACITY && diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp index e53b07e47..189e08c06 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp @@ -145,13 +145,17 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int rdma_destroy_id(id); return nullptr; } + if (config_.wire_layout == RdmaWireLayout::ControlData && + (qp_attr.cap.max_send_sge < 2 || qp_attr.cap.max_recv_sge < 2)) { + rdma_destroy_qp(id); + rdma_destroy_id(id); + return nullptr; + } auto pools = std::make_shared(config_.send_queue_capacity); pools->effective_capacity = qp_attr.cap; pools->cq_index = reservation.queue->index; pools->reserved_capacity = reservation.capacity; - pools->send = make_pool(config_.send_arena_bytes, config_.sq_depth); - pools->recv = make_pool(config_.receive_arena_bytes, config_.rq_depth); - if (pools->send == nullptr || pools->recv == nullptr) { + if (!create_connection_pools(*pools)) { rdma_destroy_qp(id); rdma_destroy_id(id); return nullptr; @@ -363,9 +367,9 @@ std::shared_ptr RdmaTransport::accept() pools->effective_capacity = qp_attr.cap; pools->cq_index = reservation.queue->index; pools->reserved_capacity = reservation.capacity; - pools->send = make_pool(config_.send_arena_bytes, config_.sq_depth); - pools->recv = make_pool(config_.receive_arena_bytes, config_.rq_depth); - if (pools->send == nullptr || pools->recv == nullptr) { + if ((config_.wire_layout == RdmaWireLayout::ControlData && + (qp_attr.cap.max_send_sge < 2 || qp_attr.cap.max_recv_sge < 2)) || + !create_connection_pools(*pools)) { rdma_destroy_qp(id); rdma_destroy_id(id); return nullptr; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp index 38e897025..3d20ada86 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp @@ -65,14 +65,23 @@ bool RdmaTransport::create_completion_queues(ibv_context *context) void RdmaTransport::refresh_poll_stats() const { - uint64_t polls = 0; - uint64_t empty = 0; + uint64_t polls = 0; + uint64_t empty = 0; + uint64_t direct_sends = 0; + uint64_t direct_bytes = 0; + uint64_t copied_bytes = 0; for (const auto &queue : completion_queues_) { polls += queue->poll_calls.load(std::memory_order_relaxed); empty += queue->empty_polls.load(std::memory_order_relaxed); + direct_sends += queue->direct_sends.load(std::memory_order_relaxed); + direct_bytes += queue->direct_payload_bytes.load(std::memory_order_relaxed); + copied_bytes += queue->copied_payload_bytes.load(std::memory_order_relaxed); } stats_.cq_polls.store(polls, std::memory_order_relaxed); stats_.cq_empty_polls.store(empty, std::memory_order_relaxed); + stats_.direct_sends.store(direct_sends, std::memory_order_relaxed); + stats_.direct_payload_bytes.store(direct_bytes, std::memory_order_relaxed); + stats_.copied_payload_bytes.store(copied_bytes, std::memory_order_relaxed); stats_.registered_memory_budget_bytes.store(memory_stats_->budget, std::memory_order_relaxed); stats_.registered_bytes.store(memory_stats_->current.load(std::memory_order_acquire), std::memory_order_relaxed); stats_.registered_bytes_high_water.store(memory_stats_->high_water.load(std::memory_order_relaxed), diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h index 713e63b98..e93a07bef 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h @@ -50,6 +50,9 @@ struct RdmaTransport::CompletionQueue std::atomic reserved_capacity{0}; std::atomic poll_calls{0}; std::atomic empty_polls{0}; + std::atomic direct_sends{0}; + std::atomic direct_payload_bytes{0}; + std::atomic copied_payload_bytes{0}; }; } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_direct_frame.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_frame.cpp new file mode 100644 index 000000000..d067d9b4c --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_frame.cpp @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_direct_frame.h" + +#include "rdma_aggregate.h" + +#include +#include + +namespace crowdb::rpc +{ +bool rdma_direct_frame_size(const OutFrame &frame, uint32_t wire_limit, uint32_t &size) +{ + uint32_t plain = 0; + if (!rdma_frame_size(frame, UINT32_MAX, plain)) + return false; + const uint64_t total = static_cast(RDMA_CONTROL_CAPACITY) + plain - HEADER_SIZE - frame.control->len; + if (total > wire_limit) + return false; + size = static_cast(total); + return true; +} + +void write_rdma_direct_control(uint8_t *destination, const OutFrame &frame) +{ + std::fill_n(destination, RDMA_CONTROL_CAPACITY, uint8_t{0}); + auto header = frame.header; + header.msg_size = RDMA_CONTROL_CAPACITY - HEADER_SIZE; + header.data_size = 0; + for (uint8_t index = 0; index < frame.data_views(); ++index) + header.data_size += frame.data_view(index)->len; + serialize_header(destination, header); + std::memcpy(destination + HEADER_SIZE, frame.control->data, frame.control->len); +} + +bool parse_rdma_direct_control(std::span control, uint32_t data_capacity, uint32_t wire_length, + Header &header) +{ + if (control.size() != RDMA_CONTROL_CAPACITY || wire_length < RDMA_CONTROL_CAPACITY) + return false; + header = parse_header(control.data()); + return header.magic == MAGIC && header.msg_offset == HEADER_SIZE && + header.msg_size == RDMA_CONTROL_CAPACITY - HEADER_SIZE && header.data_size <= data_capacity && + static_cast(RDMA_CONTROL_CAPACITY) + header.data_size == wire_length; +} +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_direct_frame.h b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_frame.h new file mode 100644 index 000000000..de77fe6f3 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_frame.h @@ -0,0 +1,16 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/transport.h" + +#include + +namespace crowdb::rpc +{ +bool rdma_direct_frame_size(const OutFrame &frame, uint32_t wire_limit, uint32_t &size); +void write_rdma_direct_control(uint8_t *destination, const OutFrame &frame); +bool parse_rdma_direct_control(std::span control, uint32_t data_capacity, uint32_t wire_length, + Header &header); +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_direct_receive.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_receive.cpp new file mode 100644 index 000000000..215c90f28 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_receive.cpp @@ -0,0 +1,111 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "rdma_direct_frame.h" + +#ifdef CROWDB_RPC_HAVE_RDMA +namespace crowdb::rpc +{ +bool RdmaTransport::post_direct_receive(Connection *connection, Buffer *slot) +{ + Operation operation(repost_state_); + if (!operation || connection == nullptr || slot == nullptr || slot->data == nullptr || + slot->capacity < config_.data_slot_capacity()) + return false; + auto *pools = pools_for(connection); + if (pools == nullptr || pools->recv_control == nullptr) + return false; + Operation posting(pools->posting_state); + if (!posting || !connection->is_open() || connection->transport_handle == 0) + return false; + auto release = [](Buffer *buffer) { buffer->release(); }; + std::unique_ptr prefix(pools->recv_control->alloc(RDMA_CONTROL_CAPACITY), release); + if (prefix == nullptr || !prefix->transition(BufferOwnership::Available, BufferOwnership::PostedReceive)) + return false; + if (!slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive) && + !slot->transition(BufferOwnership::CallerOwned, BufferOwnership::PostedReceive)) { + (void)prefix->transition(BufferOwnership::PostedReceive, BufferOwnership::Available); + return false; + } + auto rollback = [&] { + (void)slot->transition(BufferOwnership::PostedReceive, BufferOwnership::Available); + (void)prefix->transition(BufferOwnership::PostedReceive, BufferOwnership::Available); + }; + if (slot->ref_clone() == nullptr) { + rollback(); + return false; + } + auto *work = acquire_work(WorkKind::Receive, connection, slot, nullptr, prefix.get()); + if (work == nullptr) { + rollback(); + slot->release(); + return false; + } + std::array regions{}; + regions[0] = {reinterpret_cast(prefix->data), RDMA_CONTROL_CAPACITY, prefix->registration_key}; + regions[1] = {reinterpret_cast(slot->data), config_.data_slot_capacity(), slot->registration_key}; + ibv_recv_wr request{}; + request.wr_id = work->token.load(std::memory_order_acquire); + request.sg_list = regions.data(); + request.num_sge = 2; + ibv_recv_wr *bad = nullptr; + if (ibv_post_recv(reinterpret_cast(connection->transport_handle), &request, &bad) != 0) { + release_work(work); + rollback(); + slot->release(); + return false; + } + (void)prefix.release(); + stats_.posted_receives.fetch_add(1, std::memory_order_relaxed); + return true; +} + +void RdmaTransport::release_completed_direct_receive(WorkRequest *work, const ibv_wc &completion) +{ + auto *slot = work->slot; + auto *prefix = work->prefix; + Header header; + const bool valid = completion.status == IBV_WC_SUCCESS && work->conn->is_open() && + slot->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight) && + prefix->transition(BufferOwnership::PostedReceive, BufferOwnership::Available) && + parse_rdma_direct_control(std::span(prefix->data, RDMA_CONTROL_CAPACITY), + config_.data_slot_capacity(), completion.byte_len, header) && + slot->transition(BufferOwnership::ReceiveInFlight, BufferOwnership::CallerOwned); + if (!valid) { + if (completion.status != IBV_WC_SUCCESS) + record_completion_error(completion, !work->conn->is_open()); + else if (work->conn->is_open()) + stats_.protocol_errors.fetch_add(1, std::memory_order_relaxed); + auto connection = work->conn; + (void)slot->quarantine(); + (void)prefix->quarantine(); + release_work(work); + connection->close(); + slot->release(); + prefix->release(); + return; + } + auto connection = std::move(work->conn); + auto pools = std::move(work->pools); + slot->last_release_cb = &RdmaTransport::repost_frame_release; + slot->last_release_ctx = new RepostContext{repost_state_, connection, std::move(pools)}; + auto *frame = new Frame; + frame->header = header; + frame->control.assign(prefix->data + HEADER_SIZE, prefix->data + RDMA_CONTROL_CAPACITY); + extract_control_fields(frame->control.data(), header.msg_size, frame->request_id, frame->rpc_create_nano, + frame->response_create_nano); + frame->data_buf = slot->slice(0, header.data_size); + release_work(work); + prefix->release(); + stats_.receives.fetch_add(1, std::memory_order_relaxed); + stats_.rpc_frames_received.fetch_add(1, std::memory_order_relaxed); + stats_.wire_bytes_received.fetch_add(completion.byte_len, std::memory_order_relaxed); + if (connection->is_open()) + connection->on_frame(frame); + else + delete frame; + slot->release(); +} +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_direct_send.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_send.cpp new file mode 100644 index 000000000..53f119eee --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_send.cpp @@ -0,0 +1,147 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_aggregate.h" +#include "rdma_cq.h" +#include "rdma_direct_frame.h" + +#ifdef CROWDB_RPC_HAVE_RDMA +# include +# include + +namespace crowdb::rpc +{ +namespace +{ +const uint8_t *contiguous_data(const OutFrame &frame) +{ + const uint8_t *first = nullptr; + uintptr_t next = 0; + for (uint8_t index = 0; index < frame.data_views(); ++index) { + const auto *view = frame.data_view(index); + if (view->len == 0) + continue; + const auto address = reinterpret_cast(view->payload_data()); + if (first == nullptr) { + first = view->payload_data(); + next = address; + } + if (address != next || view->len > UINTPTR_MAX - next) + return nullptr; + next += view->len; + } + return first; +} +} // namespace + +uint32_t RdmaTransport::flush_direct_connection(Connection *connection, ConnectionPools &pools, uint32_t budget) +{ + const auto limit = std::min(config_.wire_slot_capacity, pools.peer_receive_slot_capacity); + uint32_t sent = 0; + auto &queue = *completion_queues_[pools.cq_index]; + while (sent < budget && connection->is_open() && + pools.sends_in_flight < std::min(config_.sq_depth, pools.effective_capacity.max_send_wr)) { + if (pools.pending_frame == nullptr && pools.send_queue.drain(&pools.pending_frame, 1) == 0) + break; + auto *frame = pools.pending_frame; + uint32_t bytes = 0; + if (!rdma_direct_frame_size(*frame, limit, bytes)) { + connection->close(); + break; + } + auto *prefix = pools.send->alloc(RDMA_CONTROL_CAPACITY); + if (prefix == nullptr) + break; + write_rdma_direct_control(prefix->data, *frame); + prefix->len = RDMA_CONTROL_CAPACITY; + const auto length = bytes - RDMA_CONTROL_CAPACITY; + Buffer *data = nullptr; + bool borrowed = false; + if (length != 0) { + const auto *source = contiguous_data(*frame); + if (source != nullptr) { + data = send_pool_->borrow_range(source, length); + if (data == nullptr) + data = pools.recv->borrow_range(source, length); + } + borrowed = data != nullptr; + if (data == nullptr) { + data = send_pool_->alloc(length); + if (data != nullptr) { + uint32_t offset = 0; + for (uint8_t index = 0; index < frame->data_views(); ++index) { + const auto *view = frame->data_view(index); + if (view->len != 0) + std::memcpy(data->data + offset, view->payload_data(), view->len); + offset += view->len; + } + data->len = length; + queue.copied_payload_bytes.fetch_add(length, std::memory_order_relaxed); + } + } + if (data == nullptr) { + prefix->release(); + break; + } + if (borrowed) + (void)data->transition(BufferOwnership::CallerOwned, BufferOwnership::Available); + data->request_id = frame->request_id; + } + pools.pending_frame = nullptr; + pools.queued_frames.fetch_sub(1, std::memory_order_release); + frame->next_aggregate = nullptr; + if (!post_direct(connection, pools, data == nullptr ? prefix : data, data == nullptr ? nullptr : prefix, frame)) + break; + queue.direct_sends.fetch_add(1, std::memory_order_relaxed); + if (borrowed) + queue.direct_payload_bytes.fetch_add(length, std::memory_order_relaxed); + ++sent; + } + return sent; +} + +bool RdmaTransport::post_direct(Connection *connection, ConnectionPools &pools, Buffer *slot, Buffer *prefix, + OutFrame *frame) +{ + Operation posting(pools.posting_state); + WorkRequest *work = nullptr; + if (posting && connection->is_open() && connection->transport_handle != 0 && + slot->transition(BufferOwnership::Available, BufferOwnership::SendInFlight) && + (prefix == nullptr || prefix->transition(BufferOwnership::Available, BufferOwnership::SendInFlight))) + work = acquire_work(WorkKind::Send, connection, slot, frame, prefix); + if (work != nullptr) { + auto *control = prefix == nullptr ? slot : prefix; + std::array regions{}; + regions[0] = {reinterpret_cast(control->data), RDMA_CONTROL_CAPACITY, control->registration_key}; + if (prefix != nullptr) + regions[1] = {reinterpret_cast(slot->payload_data()), slot->len, slot->registration_key}; + ibv_send_wr request{}; + request.wr_id = work->token.load(std::memory_order_acquire); + request.sg_list = regions.data(); + request.num_sge = prefix == nullptr ? 1 : 2; + request.opcode = IBV_WR_SEND; + request.send_flags = IBV_SEND_SIGNALED; + ibv_send_wr *bad = nullptr; + if (ibv_post_send(reinterpret_cast(connection->transport_handle), &request, &bad) == 0) { + ++pools.sends_in_flight; + stats_.sends.fetch_add(1, std::memory_order_relaxed); + stats_.rpc_frames_sent.fetch_add(1, std::memory_order_relaxed); + stats_.wire_bytes_sent.fetch_add(RDMA_CONTROL_CAPACITY + (prefix == nullptr ? 0 : slot->len), + std::memory_order_relaxed); + return true; + } + release_work(work); + } + stats_.send_errors.fetch_add(1, std::memory_order_relaxed); + connection->close(); + (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); + slot->release(); + if (prefix != nullptr) { + (void)prefix->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); + prefix->release(); + } + release_rdma_frames(frame); + return false; +} +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_offer.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_offer.cpp index f8ae11608..ff4c4630d 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_offer.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_offer.cpp @@ -32,6 +32,7 @@ std::array serialize_rdma_offer(const RdmaEndpointC put(bytes, 4, RDMA_PROTOCOL_VERSION, 2); put(bytes, 6, local.receive_frame_limit, 2); put(bytes, 8, local.receive_slot_capacity, 4); + put(bytes, 12, static_cast(local.wire_layout), 4); put(bytes, 16, local.generation, 8); put(bytes, 24, expected_peer == nullptr ? 0 : expected_peer->generation, 8); put(bytes, 32, rdma_capability_identity(local), 8); @@ -42,20 +43,21 @@ std::array serialize_rdma_offer(const RdmaEndpointC bool parse_rdma_offer(std::span bytes, RdmaCmOffer *out) { if (out == nullptr || bytes.size() < RDMA_CM_OFFER_SIZE || get(bytes, 0, 4) != OFFER_MAGIC || - get(bytes, 4, 2) != RDMA_PROTOCOL_VERSION || get(bytes, 12, 4) != 0) + get(bytes, 4, 2) != RDMA_PROTOCOL_VERSION || + get(bytes, 12, 4) > static_cast(RdmaWireLayout::ControlData)) return false; RdmaCmOffer offer; offer.receive_frame_limit = static_cast(get(bytes, 6, 2)); offer.receive_slot_capacity = static_cast(get(bytes, 8, 4)); + offer.wire_layout = static_cast(get(bytes, 12, 4)); offer.generation = get(bytes, 16, 8); offer.expected_generation = get(bytes, 24, 8); offer.identity = get(bytes, 32, 8); offer.expected_identity = get(bytes, 40, 8); - BufferCapacityClass cls; - uint32_t size = 0; - if (!classify_rdma_capacity(offer.receive_slot_capacity, &cls, &size) || size != offer.receive_slot_capacity || - offer.receive_frame_limit == 0 || offer.receive_frame_limit > 256 || offer.generation == 0 || - offer.identity == 0 || (offer.expected_generation == 0) != (offer.expected_identity == 0)) + if (!valid_rdma_wire_capacity(offer.receive_slot_capacity, offer.wire_layout) || offer.receive_frame_limit == 0 || + offer.receive_frame_limit > 256 || offer.generation == 0 || + (offer.wire_layout == RdmaWireLayout::ControlData && offer.receive_frame_limit != 1) || offer.identity == 0 || + (offer.expected_generation == 0) != (offer.expected_identity == 0)) return false; *out = offer; return true; @@ -73,6 +75,8 @@ std::array serialize_rdma_reply(const RdmaEndpointC bool validate_rdma_offer(const RdmaCmOffer &offer, const RdmaEndpointCapability &local, const RdmaEndpointCapability *expected_peer, bool require_preflight) { + if (offer.wire_layout != local.wire_layout) + return false; if (require_preflight && offer.expected_generation == 0) return false; if (offer.expected_generation != 0 && @@ -81,6 +85,7 @@ bool validate_rdma_offer(const RdmaCmOffer &offer, const RdmaEndpointCapability return expected_peer == nullptr || (offer.generation == expected_peer->generation && offer.identity == rdma_capability_identity(*expected_peer) && offer.receive_slot_capacity == expected_peer->receive_slot_capacity && + offer.wire_layout == expected_peer->wire_layout && offer.receive_frame_limit == expected_peer->receive_frame_limit); } } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_offer.h b/lib/crowdb-rpc/src/transport/rdma/rdma_offer.h index 21263421f..4389efe9a 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_offer.h +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_offer.h @@ -11,12 +11,13 @@ constexpr size_t RDMA_CM_OFFER_SIZE = 48; struct RdmaCmOffer { - uint16_t receive_frame_limit = 0; - uint32_t receive_slot_capacity = 0; - uint64_t generation = 0; - uint64_t expected_generation = 0; - uint64_t identity = 0; - uint64_t expected_identity = 0; + uint16_t receive_frame_limit = 0; + uint32_t receive_slot_capacity = 0; + RdmaWireLayout wire_layout = RdmaWireLayout::Contiguous; + uint64_t generation = 0; + uint64_t expected_generation = 0; + uint64_t identity = 0; + uint64_t expected_identity = 0; }; std::array serialize_rdma_offer(const RdmaEndpointCapability &local, diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp index 5f34031d8..5ad585809 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp @@ -12,6 +12,16 @@ namespace crowdb::rpc { +bool RdmaTransport::create_connection_pools(ConnectionPools &pools) +{ + const auto control_bytes = config_.receive_control_bytes(); + pools.send = make_pool(config_.send_arena_bytes, config_.sq_depth); + pools.recv = make_pool(config_.receive_arena_bytes - control_bytes, config_.rq_depth); + if (control_bytes != 0) + pools.recv_control = make_pool(control_bytes, config_.posted_receive_target); + return pools.send != nullptr && pools.recv != nullptr && (control_bytes == 0 || pools.recv_control != nullptr); +} + RdmaTransport::ConnectionPools *RdmaTransport::pools_for(Connection *connection) { return static_cast(connection->transport_context_.get()); @@ -35,6 +45,11 @@ bool RdmaTransport::publish_cm_connection(rdma_cm_id *id, const std::shared_ptr< info.number("effective_rq_depth", pools->effective_capacity.max_recv_wr); info.number("effective_cq_depth", completion_queues_[pools->cq_index]->effective_depth); info.number("peer_receive_slot_capacity", pools->peer_receive_slot_capacity); + info.number("peer_receive_data_capacity", + rdma_data_slot_capacity(pools->peer_receive_slot_capacity, config_.wire_layout)); + info.number("effective_max_send_sge", pools->effective_capacity.max_send_sge); + info.number("effective_max_recv_sge", pools->effective_capacity.max_recv_sge); + info.string("wire_layout", config_.wire_layout == RdmaWireLayout::Contiguous ? "contiguous" : "control_data"); info.number("peer_receive_frame_limit", pools->peer_receive_frame_limit); connection->transport_info_json_ = info.finish(); std::scoped_lock lock(cm_mu_); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp index ca26bb2e7..fac93e3d7 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp @@ -10,6 +10,10 @@ namespace crowdb::rpc { void RdmaTransport::release_completed_receive(WorkRequest *work, const ibv_wc &wc) { + if (work->prefix != nullptr) { + release_completed_direct_receive(work, wc); + return; + } auto fail = [&] { if (wc.status != IBV_WC_SUCCESS) record_completion_error(wc, !work->conn->is_open()); @@ -73,8 +77,12 @@ void RdmaTransport::repost_frame_release(void *context, Buffer *slot) (void)slot->quarantine(); { Operation operation(repost->state); - if (operation && slot->ownership.load(std::memory_order_acquire) == BufferOwnership::Available) - (void)repost->state->transport->post_receive(repost->connection.get(), slot); + if (operation && repost->connection->is_open() && + slot->ownership.load(std::memory_order_acquire) == BufferOwnership::Available && + !repost->state->transport->post_receive(repost->connection.get(), slot)) { + repost->state->transport->stats_.receive_post_errors.fetch_add(1, std::memory_order_relaxed); + repost->connection->close(); + } } slot->release(); } diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp index 81dc395d5..2380f8478 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp @@ -3,6 +3,7 @@ #include "rdma_aggregate.h" #include "rdma_cq.h" +#include "rdma_direct_frame.h" #ifdef CROWDB_RPC_HAVE_RDMA # include @@ -30,10 +31,14 @@ bool RdmaTransport::submit(Connection *connection, OutFrame *frame) if (pools == nullptr || pools->send == nullptr || pools->cq_index >= completion_queues_.size()) return reject(); Operation posting(pools->posting_state); - uint32_t bytes = 0; + uint32_t bytes = 0; + const bool direct = config_.wire_layout == RdmaWireLayout::ControlData; const auto limit = - std::min({config_.wire_slot_capacity, pools->peer_receive_slot_capacity, config_.aggregation_max_bytes}); - if (!posting || !connection->is_open() || !rdma_frame_size(*frame, limit, bytes)) + direct + ? std::min(config_.wire_slot_capacity, pools->peer_receive_slot_capacity) + : std::min({config_.wire_slot_capacity, pools->peer_receive_slot_capacity, config_.aggregation_max_bytes}); + if (!posting || !connection->is_open() || + !(direct ? rdma_direct_frame_size(*frame, limit, bytes) : rdma_frame_size(*frame, limit, bytes))) return reject(); uint32_t queued = pools->queued_frames.load(std::memory_order_relaxed); do { @@ -54,6 +59,8 @@ bool RdmaTransport::submit(Connection *connection, OutFrame *frame) uint32_t RdmaTransport::flush_connection(Connection *connection, ConnectionPools &pools, uint32_t budget) { + if (config_.wire_layout == RdmaWireLayout::ControlData) + return flush_direct_connection(connection, pools, budget); const auto byte_limit = std::min({config_.wire_slot_capacity, pools.peer_receive_slot_capacity, config_.aggregation_max_bytes}); const auto frame_limit = @@ -81,6 +88,8 @@ uint32_t RdmaTransport::flush_connection(Connection *connection, ConnectionPools if (bytes > byte_limit - slot->len) break; copy_rdma_frame(slot->data + slot->len, *frame); + completion_queues_[pools.cq_index]->copied_payload_bytes.fetch_add( + bytes - HEADER_SIZE - frame->control->len, std::memory_order_relaxed); slot->len += bytes; frame->next_aggregate = nullptr; *tail = frame; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp index b46a59ef4..40a3971f1 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp @@ -21,6 +21,10 @@ std::string RdmaTransport::status_json() const status.number("cm_port", capability.port); status.number("generation", capability.generation); status.boolean("aggregation_enabled", config_.aggregation_enabled); + status.string("wire_layout", config_.wire_layout == RdmaWireLayout::Contiguous ? "contiguous" : "control_data"); + status.number("data_slot_capacity", config_.data_slot_capacity()); + status.number("control_slot_capacity", + config_.wire_layout == RdmaWireLayout::ControlData ? RDMA_CONTROL_CAPACITY : 0); # define CONFIG_FIELD(name) status.number(#name, config_.name) CONFIG_FIELD(port); CONFIG_FIELD(cq_count); @@ -48,6 +52,10 @@ std::string RdmaTransport::status_json() const COUNTER_FIELD(cq_errors); COUNTER_FIELD(protocol_errors); COUNTER_FIELD(send_errors); + COUNTER_FIELD(receive_post_errors); + COUNTER_FIELD(direct_sends); + COUNTER_FIELD(direct_payload_bytes); + COUNTER_FIELD(copied_payload_bytes); COUNTER_FIELD(submit_rejects); COUNTER_FIELD(send_queue_high_water); COUNTER_FIELD(fallback_count); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index bd224bdc9..27bbacd14 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -79,8 +79,10 @@ RdmaTransport::RdmaTransport(RdmaConfig config, BufferPool *application_pool) local_capability_.data_classes = {RDMA_CONTROL_CAPACITY, RDMA_METADATA_CAPACITY, RDMA_AGGREGATE_CAPACITY, RDMA_DATAPATH_CAPACITY}; local_capability_.receive_slot_capacity = config_.wire_slot_capacity; - local_capability_.receive_frame_limit = RDMA_MAX_AGGREGATE_FRAMES; - local_capability_.cm_address = config_.source_address; + local_capability_.wire_layout = config_.wire_layout; + local_capability_.receive_frame_limit = + config_.wire_layout == RdmaWireLayout::ControlData ? 1 : RDMA_MAX_AGGREGATE_FRAMES; + local_capability_.cm_address = config_.source_address; int device_count = 0; ibv_device **devices = ibv_get_device_list(&device_count); @@ -189,6 +191,8 @@ std::unique_ptr RdmaTransport::make_pool(uint64_t bytes, uint32_ bool RdmaTransport::post_receive(Connection *conn, Buffer *slot) { + if (config_.wire_layout == RdmaWireLayout::ControlData) + return post_direct_receive(conn, slot); Operation operation(repost_state_); if (!operation) return false; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp index 00f18e711..74d329133 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp @@ -4,13 +4,15 @@ #include "crowdb-rpc/transport/rdma/rdma_transport.h" #include "rdma_aggregate.h" #include "rdma_cq.h" +#include "rdma_direct_frame.h" #ifdef CROWDB_RPC_HAVE_RDMA namespace crowdb::rpc { -RdmaTransport::WorkRequest *RdmaTransport::acquire_work(WorkKind kind, Connection *conn, Buffer *slot, OutFrame *frame) +RdmaTransport::WorkRequest *RdmaTransport::acquire_work(WorkKind kind, Connection *conn, Buffer *slot, OutFrame *frame, + Buffer *prefix) { auto owner = conn->weak_from_this().lock(); if (owner == nullptr) @@ -31,6 +33,7 @@ RdmaTransport::WorkRequest *RdmaTransport::acquire_work(WorkKind kind, Connectio work.connection_identity.store(conn, std::memory_order_relaxed); work.pools = pools == nullptr ? nullptr : pools->shared_from_this(); work.slot = slot; + work.prefix = prefix; work.frame = frame; work.request_id = frame == nullptr ? slot->request_id : frame->request_id; work.stream_id = slot->stream_id; @@ -60,8 +63,9 @@ void RdmaTransport::release_work(WorkRequest *work) { work->conn.reset(); work->pools.reset(); - work->slot = nullptr; - work->frame = nullptr; + work->slot = nullptr; + work->prefix = nullptr; + work->frame = nullptr; work->token.store(0, std::memory_order_release); } @@ -74,11 +78,20 @@ void RdmaTransport::dispatch_completion(const ibv_wc &wc) // work record owns dispatch, even after disconnect or QP destruction. if (work->kind == WorkKind::Receive && wc.status == IBV_WC_SUCCESS && !work->conn->has_frame_handler()) { RdmaAggregate aggregate; - const bool valid = wc.byte_len <= work->slot->capacity && - parse_rdma_aggregate(std::span(work->slot->data, wc.byte_len), aggregate); + bool valid = false; bool application = false; - for (uint32_t index = 0; valid && index < aggregate.count; ++index) - application |= aggregate.frames[index].header.msg_type != RDMA_HANDSHAKE_MSG_TYPE; + if (work->prefix != nullptr) { + Header header; + valid = parse_rdma_direct_control(std::span(work->prefix->data, RDMA_CONTROL_CAPACITY), + config_.data_slot_capacity(), wc.byte_len, header); + application = valid && header.msg_type != RDMA_HANDSHAKE_MSG_TYPE; + } + else { + valid = wc.byte_len <= work->slot->capacity && + parse_rdma_aggregate(std::span(work->slot->data, wc.byte_len), aggregate); + for (uint32_t index = 0; valid && index < aggregate.count; ++index) + application |= aggregate.frames[index].header.msg_type != RDMA_HANDSHAKE_MSG_TYPE; + } if (application) { const uint32_t index = static_cast(wc.wr_id) / config_.cq_depth; completion_queues_[index]->deferred.push_back({work, wc}); diff --git a/lib/crowdb-rpc/src/transport/socket_connect.cpp b/lib/crowdb-rpc/src/transport/socket_connect.cpp index fa6cd13d5..22f354448 100644 --- a/lib/crowdb-rpc/src/transport/socket_connect.cpp +++ b/lib/crowdb-rpc/src/transport/socket_connect.cpp @@ -26,7 +26,8 @@ std::shared_ptr SocketTransport::connect(const std::string &addr, in stats.preflight_attempts.fetch_add(1, std::memory_order_relaxed); auto probe = RdmaPreflight::connect(addr, port, local_rdma_capability(), rdma_transport_->source_address()); if (probe != nullptr && - validate_rdma_capability(probe->peer, RDMA_PROTOCOL_VERSION, probe->peer.generation, {})) { + validate_rdma_capability(probe->peer, RDMA_PROTOCOL_VERSION, probe->peer.generation, {}) && + probe->peer.wire_layout == rdma_transport_->config().wire_layout) { const auto &cm_address = probe->peer.cm_address; const auto destination = cm_address == "0.0.0.0" ? addr : cm_address; auto connection = rdma_transport_->connect(destination, probe->peer.port, &probe->peer); diff --git a/lib/crowdb-rpc/tests/rdma_config_test.cpp b/lib/crowdb-rpc/tests/rdma_config_test.cpp index 4fb9b7711..b4af1139f 100644 --- a/lib/crowdb-rpc/tests/rdma_config_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_config_test.cpp @@ -10,6 +10,41 @@ using crowdb::rpc::RdmaConfig; +TEST(RdmaConfigTest, ControlDataChargesBothRegionsAndRequiresTwoSges) +{ + RdmaConfig config; + config.wire_layout = crowdb::rpc::RdmaWireLayout::ControlData; + config.wire_slot_capacity = crowdb::rpc::RDMA_DATAPATH_CAPACITY + crowdb::rpc::RDMA_CONTROL_CAPACITY; + config.receive_arena_bytes = uint64_t{config.posted_receive_target} * config.wire_slot_capacity; + EXPECT_TRUE(config.valid()); + EXPECT_EQ(config.data_slot_capacity(), crowdb::rpc::RDMA_DATAPATH_CAPACITY); + EXPECT_EQ(config.receive_control_bytes(), uint64_t{config.posted_receive_target} * 1024); + --config.receive_arena_bytes; + EXPECT_FALSE(config.valid()); + ++config.receive_arena_bytes; + config.aggregation_enabled = true; + EXPECT_FALSE(config.valid()); + config.aggregation_enabled = false; + config.qp_max_send_sge = 1; + EXPECT_FALSE(config.valid()); + config.qp_max_send_sge = 2; + config.qp_max_recv_sge = 1; + EXPECT_FALSE(config.valid()); + config.qp_max_recv_sge = 2; + config.wire_layout = static_cast(2); + EXPECT_FALSE(config.valid()); +} + +TEST(RdmaConfigTest, EnvironmentRejectsUnknownWireLayout) +{ + const TestEnvironment valid("CROWDB_RDMA_WIRE_LAYOUT", "1"); + RdmaConfig config; + ASSERT_TRUE(config.apply_environment()); + EXPECT_EQ(config.wire_layout, crowdb::rpc::RdmaWireLayout::ControlData); + const TestEnvironment invalid("CROWDB_RDMA_WIRE_LAYOUT", "2"); + EXPECT_FALSE(config.apply_environment()); +} + TEST(RdmaConfigTest, PostedReceiveTargetMustFitRqAndRegisteredBudget) { RdmaConfig config; diff --git a/lib/crowdb-rpc/tests/rdma_direct_frame_test.cpp b/lib/crowdb-rpc/tests/rdma_direct_frame_test.cpp new file mode 100644 index 000000000..9201e4f08 --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_direct_frame_test.cpp @@ -0,0 +1,60 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "../src/transport/rdma/rdma_aggregate.h" +#include "../src/transport/rdma/rdma_direct_frame.h" +#include "crowdb-rpc/server/message.h" + +#include + +using namespace crowdb::rpc; + +TEST(RdmaDirectFrameTest, FullDatapathLengthIncludesFixedControlRegion) +{ + SystemBufferPool pool; + auto *data = pool.alloc(RDMA_DATAPATH_CAPACITY); + ASSERT_NE(data, nullptr); + data->len = RDMA_DATAPATH_CAPACITY; + auto *frame = build_out_frame(17, 100, build_ping_request(&pool, 17, 0), data); + uint32_t size = 0; + const auto limit = RDMA_DATAPATH_CAPACITY + RDMA_CONTROL_CAPACITY; + EXPECT_FALSE(rdma_direct_frame_size(*frame, limit - 1, size)); + ASSERT_TRUE(rdma_direct_frame_size(*frame, limit, size)); + EXPECT_EQ(size, limit); + std::array control; + write_rdma_direct_control(control.data(), *frame); + Header decoded; + ASSERT_TRUE(parse_rdma_direct_control(control, RDMA_DATAPATH_CAPACITY, size, decoded)); + EXPECT_EQ(decoded.data_size, RDMA_DATAPATH_CAPACITY); + EXPECT_EQ(decoded.msg_size, RDMA_CONTROL_CAPACITY - HEADER_SIZE); + uint64_t id = 0, created = 0, responded = 0; + extract_control_fields(control.data() + HEADER_SIZE, decoded.msg_size, id, created, responded); + EXPECT_EQ(id, 17U); + EXPECT_FALSE(parse_rdma_direct_control(control, RDMA_DATAPATH_CAPACITY - 1, size, decoded)); + EXPECT_FALSE(parse_rdma_direct_control(control, RDMA_DATAPATH_CAPACITY, size - 1, decoded)); + EXPECT_FALSE(parse_rdma_direct_control(control, RDMA_DATAPATH_CAPACITY, size + 1, decoded)); + EXPECT_FALSE(parse_rdma_direct_control(std::span(control).first(1023), RDMA_DATAPATH_CAPACITY, size, decoded)); + control[0] ^= 1; + EXPECT_FALSE(parse_rdma_direct_control(control, RDMA_DATAPATH_CAPACITY, size, decoded)); + release_rdma_frames(frame); +} + +TEST(RdmaDirectFrameTest, ZeroDataAndControlBoundRejectMalformedFrames) +{ + SystemBufferPool pool; + auto *frame = build_out_frame(91, 100, build_ping_request(&pool, 91, 0), nullptr); + uint32_t size = 0; + EXPECT_TRUE(rdma_direct_frame_size(*frame, RDMA_CONTROL_CAPACITY, size)); + EXPECT_EQ(size, RDMA_CONTROL_CAPACITY); + frame->control->len = RDMA_CONTROL_CAPACITY - HEADER_SIZE + 1; + EXPECT_FALSE(rdma_direct_frame_size(*frame, UINT32_MAX, size)); + frame->control->len = 0; + std::array control; + write_rdma_direct_control(control.data(), *frame); + Header header; + EXPECT_TRUE(parse_rdma_direct_control(control, 1024, 1024, header)); + header.msg_size = 0; + serialize_header(control.data(), header); + EXPECT_FALSE(parse_rdma_direct_control(control, 1024, 1024, header)); + release_rdma_frames(frame); +} diff --git a/lib/crowdb-rpc/tests/rdma_direct_hardware_test.cpp b/lib/crowdb-rpc/tests/rdma_direct_hardware_test.cpp new file mode 100644 index 000000000..4bcd85afd --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_direct_hardware_test.cpp @@ -0,0 +1,179 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "../src/transport/rdma/rdma_aggregate.h" +#include "common/test_rdma_transport.h" +#include "crowdb-rpc/server/message.h" +#include "crowdb-rpc/server/server.h" + +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) +# include + +# include +# include + +using namespace crowdb::rpc; + +namespace +{ +RdmaConfig direct_config() +{ + RdmaConfig config; + config.use_environment = false; + config.wire_layout = RdmaWireLayout::ControlData; + config.wire_slot_capacity = RDMA_DATAPATH_CAPACITY + RDMA_CONTROL_CAPACITY; + config.sq_depth = config.rq_depth = config.posted_receive_target = 4; + config.cq_depth = 8; + config.send_arena_bytes = 4U * RDMA_DATAPATH_CAPACITY; + config.receive_arena_bytes = uint64_t{4} * config.wire_slot_capacity; + config.source_address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + if (const auto *device = std::getenv("CROWDB_RDMA_DEVICE")) + config.device = device; + return config; +} + +template bool drain_until(RdmaTransport &client, RdmaTransport &server, Ready ready) +{ + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!ready() && std::chrono::steady_clock::now() < deadline) { + client.drain_cq(); + server.drain_cq(); + } + return ready(); +} +} // namespace + +TEST(RdmaHardwareTest, DirectDataPreservesBytesAlignmentAndLeaseAdmission) +{ + const auto *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) + GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; + const auto config = direct_config(); + SystemBufferPool application; + auto server = std::make_unique(config, &application); + auto client = std::make_unique(config, &application); + ASSERT_TRUE(server->listen(address, 19391)); + std::shared_ptr incoming; + std::thread acceptor([&] { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (incoming == nullptr && std::chrono::steady_clock::now() < deadline) { + incoming = server->accept(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + }); + auto outgoing = client->connect(address, 19391); + acceptor.join(); + ASSERT_NE(incoming, nullptr); + ASSERT_NE(outgoing, nullptr); + std::vector server_held, client_held; + const std::array lengths{0, 13, RDMA_DATAPATH_CAPACITY, 65536}; + incoming->set_on_frame([&](Frame *frame, Connection *connection) { + EXPECT_GE(frame->request_id, 1U); + EXPECT_LE(frame->request_id, lengths.size()); + const auto expected = lengths[frame->request_id - 1]; + EXPECT_EQ(frame->data_buf->len, expected); + EXPECT_EQ(reinterpret_cast(frame->data_buf->payload_data()) % 4096, 0U); + EXPECT_TRUE(std::all_of(frame->data_buf->payload_data(), frame->data_buf->payload_data() + expected, + [id = frame->request_id](uint8_t byte) { return byte == id; })); + server_held.push_back(frame->data_buf->ref_clone()); + auto *reply = + build_out_frame(frame->request_id, 100, build_ping_response(&application, frame->request_id, 0, 0), + frame->data_buf->ref_clone()); + delete frame; + const bool accepted = server->submit(connection, reply); + EXPECT_TRUE(accepted); + if (!accepted) + release_rdma_frames(reply); + }); + outgoing->set_on_frame([&](Frame *frame, Connection *) { + const auto expected = lengths[frame->request_id - 1]; + EXPECT_EQ(frame->data_buf->len, expected); + EXPECT_TRUE(std::all_of(frame->data_buf->payload_data(), frame->data_buf->payload_data() + expected, + [id = frame->request_id](uint8_t byte) { return byte == id; })); + client_held.push_back(frame->data_buf->ref_clone()); + delete frame; + }); + Buffer *source_held = nullptr; + uint64_t total = 0; + for (size_t index = 0; index < lengths.size(); ++index) { + Buffer *data = nullptr; + if (lengths[index] != 0) { + data = index == 3 ? application.alloc(lengths[index]) : client->allocate_producer_buffer(lengths[index]); + ASSERT_NE(data, nullptr); + std::memset(data->data, static_cast(index + 1), lengths[index]); + data->len = lengths[index]; + if (index == 2) + source_held = data->ref_clone(); + total += lengths[index]; + } + auto *frame = build_out_frame(index + 1, 100, build_ping_request(&application, index + 1, 0), data); + ASSERT_TRUE(client->submit(outgoing.get(), frame)); + ASSERT_TRUE(drain_until(*client, *server, [&] { return client_held.size() == index + 1; })); + } + ASSERT_EQ(server_held.size(), lengths.size()); + EXPECT_EQ(server->stats().posted_receives.load(), 4U); + EXPECT_EQ(client->stats().posted_receives.load(), 4U); + EXPECT_EQ(server->stats().copied_payload_bytes.load(), 0U); + EXPECT_EQ(client->stats().copied_payload_bytes.load(), lengths.back()); + EXPECT_EQ(server->stats().direct_payload_bytes.load(), total); + EXPECT_EQ(client->stats().direct_payload_bytes.load(), total - lengths.back()); + server_held.front()->release(); + server_held.erase(server_held.begin()); + EXPECT_EQ(server->stats().posted_receives.load(), 5U); + EXPECT_EQ(client->stats().posted_receives.load(), 4U); + // Exhaust receive admission with caller leases, then destroy both peers. + // The held receive and registered source must still retain their own MR/PD. + const auto server_device = TestRdmaTransport::device_owner(*server); + const auto client_device = TestRdmaTransport::device_owner(*client); + EXPECT_EQ(server->stats().receive_post_errors.load(), 0U); + EXPECT_EQ(client->stats().receive_post_errors.load(), 0U); + EXPECT_EQ(server->stats().cq_errors.load(), 0U); + EXPECT_EQ(client->stats().cq_errors.load(), 0U); + server.reset(); + client.reset(); + incoming.reset(); + outgoing.reset(); + EXPECT_FALSE(client_device.expired()); + EXPECT_FALSE(server_device.expired()); + EXPECT_EQ(source_held->payload_data()[RDMA_DATAPATH_CAPACITY - 1], 3U); + for (auto *buffer : server_held) + buffer->release(); + for (auto *buffer : client_held) + buffer->release(); + source_held->release(); + EXPECT_TRUE(client_device.expired()); + EXPECT_TRUE(server_device.expired()); +} + +TEST(RdmaHardwareTest, LayoutMismatchFailsBeforeCmOrUsesExplicitTcpFallback) +{ + const auto *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) + GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; + auto config = direct_config(); + auto contiguous = config; + contiguous.wire_layout = RdmaWireLayout::Contiguous; + contiguous.wire_slot_capacity = RDMA_DATAPATH_CAPACITY; + RpcServer server; + ASSERT_TRUE(server.configure_rdma(contiguous, true, false, true)); + ASSERT_TRUE(server.listen(address, 0)); + server.start(); + for (bool fallback : {false, true}) { + SocketTransport client(1, 1); + ASSERT_TRUE(client.configure_rdma(config, true, fallback, false)); + client.start(); + auto connection = client.connect(address, server.listen_port()); + if (fallback) { + ASSERT_NE(connection, nullptr); + EXPECT_EQ(connection->path(), ConnectionPath::Tcp); + } + else { + EXPECT_EQ(connection, nullptr); + } + const auto *stats = client.rdma_stats_for_tests(); + ASSERT_NE(stats, nullptr); + EXPECT_EQ(stats->cm_attempts.load(), 0U); + EXPECT_EQ(stats->fallback_count.load(), static_cast(fallback)); + } +} +#endif diff --git a/lib/crowdb-rpc/tests/rdma_offer_test.cpp b/lib/crowdb-rpc/tests/rdma_offer_test.cpp index 83d0a648d..fa0f92666 100644 --- a/lib/crowdb-rpc/tests/rdma_offer_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_offer_test.cpp @@ -97,6 +97,37 @@ TEST(RdmaOfferTest, PrivateDataBindsPreflightIdentityAndReceiveAdmission) EXPECT_FALSE(validate_rdma_offer(reply, client, &server, true)); } +TEST(RdmaOfferTest, ControlDataLayoutIsBoundByBothCapabilityAndPrivateOffer) +{ + auto client = test_capability(11); + auto server = test_capability(17); + client.wire_layout = server.wire_layout = RdmaWireLayout::ControlData; + client.receive_slot_capacity = server.receive_slot_capacity = RDMA_DATAPATH_CAPACITY + RDMA_CONTROL_CAPACITY; + const auto wire = serialize_rdma_handshake(client); + RdmaEndpointCapability decoded; + ASSERT_TRUE(parse_rdma_handshake(wire, &decoded)); + EXPECT_EQ(decoded.wire_layout, RdmaWireLayout::ControlData); + EXPECT_TRUE(validate_rdma_capability(decoded, RDMA_PROTOCOL_VERSION, 11, "test-device")); + RdmaCmOffer offer; + ASSERT_TRUE(parse_rdma_offer(serialize_rdma_offer(client, &server), &offer)); + EXPECT_TRUE(validate_rdma_offer(offer, server, &client, true)); + server.wire_layout = RdmaWireLayout::Contiguous; + server.receive_slot_capacity = RDMA_DATAPATH_CAPACITY; + EXPECT_FALSE(validate_rdma_offer(offer, server, &client, true)); + auto invalid = wire; + invalid[152] = 2; + EXPECT_FALSE(parse_rdma_handshake(invalid, &decoded)); + invalid = wire; + invalid[156] = 1; + EXPECT_FALSE(parse_rdma_handshake(invalid, &decoded)); + client.protocol_version = 2; + client.wire_layout = RdmaWireLayout::Contiguous; + client.receive_slot_capacity = RDMA_DATAPATH_CAPACITY; + ASSERT_TRUE(parse_rdma_handshake(serialize_rdma_handshake(client), &decoded)); + EXPECT_EQ(decoded.wire_layout, RdmaWireLayout::Contiguous); + EXPECT_TRUE(validate_rdma_capability(decoded, 2, 11, "test-device")); +} + TEST(RdmaOfferTest, MissingPreflightAndMalformedAdmissionFailClosed) { const auto client = test_capability(1); diff --git a/lib/crowdb-rpc/tests/registered_buffer_test.cpp b/lib/crowdb-rpc/tests/registered_buffer_test.cpp index b7bc1b9b5..38edfe0d3 100644 --- a/lib/crowdb-rpc/tests/registered_buffer_test.cpp +++ b/lib/crowdb-rpc/tests/registered_buffer_test.cpp @@ -32,6 +32,41 @@ TEST(RegisteredBufferTest, HeldLeaseRetainsArenaAfterFacadeDestruction) EXPECT_TRUE(arena.expired()); } +TEST(RegisteredBufferTest, SmallerProducerReusesLargerFreeAllocationWithinHardBudget) +{ + RegisteredBufferPool pool(RDMA_DATAPATH_CAPACITY, 1); + auto *large = pool.alloc(RDMA_DATAPATH_CAPACITY); + ASSERT_NE(large, nullptr); + const auto *pointer = large->data; + large->release(); + auto *small = pool.alloc(32); + ASSERT_NE(small, nullptr); + EXPECT_EQ(small->data, pointer); + EXPECT_EQ(small->capacity, RDMA_DATAPATH_CAPACITY); + EXPECT_EQ(small->capacity_class, BufferCapacityClass::Datapath); + EXPECT_EQ(pool.committed_bytes(), RDMA_DATAPATH_CAPACITY); + EXPECT_EQ(pool.alloc(32), nullptr); + small->release(); +} + +TEST(RegisteredBufferTest, QuarantinedNestedViewRetainsAndRetiresCanonicalAllocation) +{ + RegisteredBufferPool pool(1024, 1); + auto *root = pool.alloc(32); + ASSERT_NE(root, nullptr); + auto *view = root->slice(4, 16); + ASSERT_NE(view, nullptr); + auto *nested = view->slice(2, 8); + ASSERT_NE(nested, nullptr); + root->release(); + view->release(); + EXPECT_TRUE(nested->quarantine()); + EXPECT_EQ(nested->allocation_owner->ownership.load(), BufferOwnership::Quarantined); + nested->release(); + EXPECT_EQ(pool.alloc(32), nullptr); + EXPECT_EQ(pool.quarantined_bytes(), 1024U); +} + TEST(RegisteredBufferTest, DetachedLeaseRetainsRegionAndCanReleaseAfterFacade) { auto pool = std::make_unique(4096, 4); diff --git a/lib/crowdb-rpc/tests/transport_config_test.cpp b/lib/crowdb-rpc/tests/transport_config_test.cpp index d78ee08ac..92d959abd 100644 --- a/lib/crowdb-rpc/tests/transport_config_test.cpp +++ b/lib/crowdb-rpc/tests/transport_config_test.cpp @@ -47,6 +47,13 @@ TEST(TransportConfigTest, VersionedCAbiRejectsMalformedBoundsBeforePublication) EXPECT_EQ(crowdb_rpc_server_configure_rdma(server, &config), CROWDB_RPC_ERR_INVALID_ARG); config.cq_count = 1; EXPECT_EQ(crowdb_rpc_server_configure_rdma(server, &config), CROWDB_RPC_OK); + EXPECT_EQ(crowdb_rpc_server_configure_rdma_v2(server, &config, 2), CROWDB_RPC_ERR_INVALID_ARG); + config.wire_slot_capacity = RDMA_DATAPATH_CAPACITY + RDMA_CONTROL_CAPACITY; + config.receive_arena_bytes = uint64_t{config.posted_receive_target} * config.wire_slot_capacity; + EXPECT_EQ(crowdb_rpc_server_configure_rdma(server, &config), CROWDB_RPC_ERR_INVALID_ARG); + EXPECT_EQ(crowdb_rpc_server_configure_rdma_v2(server, &config, 1), CROWDB_RPC_OK); + EXPECT_EQ(config.struct_size, sizeof(config)); + EXPECT_EQ(config.version, 1U); crowdb_rpc_server_destroy(server); } diff --git a/tools/rpc-test/src/args.rs b/tools/rpc-test/src/args.rs index 35ec177e9..8c5aa9c0e 100644 --- a/tools/rpc-test/src/args.rs +++ b/tools/rpc-test/src/args.rs @@ -165,6 +165,7 @@ impl RpcArgs { rq_depth: q.rq_depth, posted_receive_target: q.posted_receive_target, wire_slot_capacity: q.wire_slot_capacity, + wire_layout: crowdb_rpc_ffi::RdmaWireLayout::Contiguous, send_arena_bytes: q.send_arena_bytes, receive_arena_bytes: q.receive_arena_bytes, registered_memory_budget_bytes: q.registered_memory_budget_bytes, diff --git a/tools/rpc-test/src/workload/report.rs b/tools/rpc-test/src/workload/report.rs index 67c1c66d3..d23faaaf4 100644 --- a/tools/rpc-test/src/workload/report.rs +++ b/tools/rpc-test/src/workload/report.rs @@ -139,10 +139,15 @@ pub(super) fn finish( || histogram.overflow_count != 0; let cq_errors = sum_rdma_delta(report, "cq_errors"); let fallback = sum_rdma_delta(report, "fallback_count"); - let transport_errors: u64 = ["protocol_errors", "send_errors", "registration_failures"] - .into_iter() - .map(|field| sum_rdma_delta(report, field)) - .sum(); + let transport_errors: u64 = [ + "protocol_errors", + "send_errors", + "receive_post_errors", + "registration_failures", + ] + .into_iter() + .map(|field| sum_rdma_delta(report, field)) + .sum(); let errors = native.total_errors + get(Count::PayloadMismatches) + get(Count::Duplicates) diff --git a/tools/rpc-test/src/workload/status.rs b/tools/rpc-test/src/workload/status.rs index 1409c28b6..36814c767 100644 --- a/tools/rpc-test/src/workload/status.rs +++ b/tools/rpc-test/src/workload/status.rs @@ -33,6 +33,7 @@ pub(super) fn validate(status: &Value, rdma: bool) -> Result<(), String> { "cq_errors", "protocol_errors", "send_errors", + "receive_post_errors", "submit_rejects", "fallback_count", "registration_failures", From 95ff6eef5ab9b2f014301c1faedfbeb3f601204a Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 02:48:31 +0800 Subject: [PATCH 042/258] Fill HTTP framed owners from the selected DiskIO producer arena --- Cargo.lock | 1 + app/crowdb-access-server/src/main.rs | 6 +- app/crowdb-diskio/src/dio_main.cpp | 4 +- doc/design/rpc/design-crowdb-rpc-rdma.md | 15 ++ doc/working/plan-rdma-datapath-stream.md | 32 ++- lib/crowdb-access-s3/Cargo.toml | 2 + lib/crowdb-access-s3/src/native_buffer.rs | 84 ++++--- .../src/native_buffer/allocator.rs | 47 +++- .../tests/native_producer_test.rs | 196 +++++++++++++++++ lib/crowdb-chunk-client/src/client.rs | 7 + .../src/client/disk_metrics.rs | 3 + .../src/disk_io/disk_writer.rs | 5 + .../src/disk_io/routing.rs | 4 + lib/crowdb-diskio-client/src/semantic.rs | 19 +- .../src/semantic/connection.rs | 7 +- lib/crowdb-rpc/ffi/src/lib.rs | 2 +- lib/crowdb-rpc/ffi/src/producer_buffer.rs | 24 ++ lib/crowdb-rpc/ffi/src/server.rs | 38 +++- lib/crowdb-rpc/ffi/src/sys.rs | 7 + .../ffi/tests/transport_config_test.rs | 9 + lib/crowdb-rpc/include/crowdb-rpc/c_api.h | 4 + .../include/crowdb-rpc/c_api_internal.h | 6 + .../include/crowdb-rpc/server/server.h | 3 +- .../crowdb-rpc/transport/rdma/rdma_config.h | 60 ++--- .../crowdb-rpc/transport/socket_transport.h | 7 +- lib/crowdb-rpc/src/c_api.cpp | 194 ----------------- lib/crowdb-rpc/src/c_api/server.cpp | 205 ++++++++++++++++++ lib/crowdb-rpc/src/server/server.cpp | 4 +- .../src/transport/rdma/rdma_config.cpp | 11 + .../src/transport/socket_transport.cpp | 9 +- 30 files changed, 734 insertions(+), 281 deletions(-) create mode 100644 lib/crowdb-access-s3/tests/native_producer_test.rs create mode 100644 lib/crowdb-rpc/src/c_api/server.cpp diff --git a/Cargo.lock b/Cargo.lock index 76c1fc35c..d2f722c24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -693,6 +693,7 @@ dependencies = [ "crowdb-common", "crowdb-kv-client", "crowdb-protocol", + "crowdb-rpc-ffi", "flatbuffers", "futures", "hmac", diff --git a/app/crowdb-access-server/src/main.rs b/app/crowdb-access-server/src/main.rs index 523b43849..16d7a735b 100644 --- a/app/crowdb-access-server/src/main.rs +++ b/app/crowdb-access-server/src/main.rs @@ -216,7 +216,11 @@ async fn run_s3( "CROWDB_S3_NATIVE_BUDGET_BYTES", )? .unwrap_or(256 * 1024 * 1024); - let body_allocator = Arc::new(NativeBodyAllocator::new(native_budget, 1024 * 1024)?); + let mut body_allocator = NativeBodyAllocator::new(native_budget, 1024 * 1024)?; + if let Some(producer) = chunks.producer_allocator() { + body_allocator = body_allocator.with_producer_allocator(producer)?; + } + let body_allocator = Arc::new(body_allocator); let handler = Arc::new( S3Dispatcher::new( authenticator, diff --git a/app/crowdb-diskio/src/dio_main.cpp b/app/crowdb-diskio/src/dio_main.cpp index ed383e38a..6fbdff432 100644 --- a/app/crowdb-diskio/src/dio_main.cpp +++ b/app/crowdb-diskio/src/dio_main.cpp @@ -205,7 +205,9 @@ int main(int argc, char *argv[]) } // Create + start the RPC server. - crowdb::rpc::RpcServer server(nullptr, 1, cfg.rpc_workers); + const auto rdma_defaults = crowdb::rpc::RdmaConfig::for_layout(crowdb::rpc::RdmaWireLayout::ControlData, + crowdb::rpc::RDMA_DATAPATH_CAPACITY); + crowdb::rpc::RpcServer server(nullptr, 1, cfg.rpc_workers, rdma_defaults); if (!server.listen(cfg.bind_address, cfg.listen_port)) { std::fprintf(stderr, "error: failed to listen on %s:%d\n", cfg.bind_address.c_str(), cfg.listen_port); return 1; diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index bb90a28f4..99d797439 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -216,6 +216,21 @@ uninitialized capacity until its producer publishes an initialized prefix; immutable views then retain the lease through their final release. Allocation alone does not bypass the contiguous transport's payload copy. +The DiskIO client exposes a retained `ProducerBufferAllocator` through the +ChunkIO disk-writer interface. S3 attaches it to `NativeBodyAllocator` before +sharing that allocator. Hyper fills disjoint payload regions in this native +backing; `NativeFramedOwner` finalizes frame regions in place and publishes +immutable views. The backing lease returns before HTTP credit waiters wake. +An exhausted active producer arena pauses body intake. HTTP owner release +wakes existing AtomicWakers; a 1 ms retry wake also observes credits returned +by other producers or native SEND completion. This retry does not flush or +reassemble application messages. + +Service constructors can supply layout and data-capacity defaults before +environment overrides. DiskIO server/client select control/data with 1 MiB +data capacity. Ordinary RPC constructors retain contiguous defaults; this +selection alone does not enable RDMA or change the explicit fallback policy. + The device owner charges MR creation against `registered_memory_budget_bytes`. Zero derives a checked bound from all CQ-admitted QPs, their send/receive arenas and the two compatibility copy diff --git a/doc/working/plan-rdma-datapath-stream.md b/doc/working/plan-rdma-datapath-stream.md index b73b32037..4a89f3d2d 100644 --- a/doc/working/plan-rdma-datapath-stream.md +++ b/doc/working/plan-rdma-datapath-stream.md @@ -143,7 +143,7 @@ it does not add application aggregation or a flush timer. quarantine both regions exactly once on post/CQE/cancel/shutdown failure. Files: `rdma_receive.cpp`, receive posting, WorkRequest and repost context, real HCA ownership tests. Keep generic receive credit implicit in RQ slots. -- [~] **Connect the actual HTTP producer**: allow `NativeBodyAllocator` to +- [x] **Connect the actual HTTP producer**: allow `NativeBodyAllocator` to obtain its owner storage from the retained DiskIO producer allocator after route setup. Preserve its existing byte admission, AtomicWaker pressure and framed-owner API. Wire it through `ChunkIoClient` / `RoutedDiskWriter` / @@ -165,13 +165,41 @@ it does not add application aggregation or a flush timer. malloc replacement or a terminal upload error. Keep the existing AtomicWaker release wakeups; add a bounded retry wake for native credits held by other producers or asynchronous transport completion. -- [ ] **Close DiskIO handoff and read lifetime**: use exact view pointer/length, + The producer seam and 12 existing native-owner cases plus two new native + admission/view cases pass. The new opt-in real HCA case also passes: Hyper's + provider regions fill one complete 1 MiB framed owner, `BufferChain` retains + those views, and native status reports exactly 1 MiB direct payload and zero + client transport copy/fallback. This is provider-to-HCA integration, not yet + the full HTTP/DiskIO end-to-end acceptance. The first native build after + splitting server C ABI functions fails because the pool handle definition + was local to the old translation unit; move it into the existing internal + opaque-handle header. The corrected native build exits zero before tests. + All DiskIO-client tests and all RPC FFI tests pass with the new constructor. + Workspace clippy catches a missing doc-code backtick around `DiskIO`; fix + only that documentation warning and rerun the gate. + Final full C++/FFI gate passes at + `.crowdb-runtime/artifacts/unit/cpp-cba652da52b248cca3721baebc228927/`. + All 31 focused ChunkIO and 11 Access Server config/service cases pass; + workspace all-feature clippy, rs-lint, Rust/C++ format, scoped tree-lint + (zero exit with style warnings), and 146 TCP-only CTest items also pass + (one HCA-only item skipped). This closes producer wiring, while disk lifetime + and complete HTTP/service/OCI acceptance remain below. +- [~] **Close DiskIO handoff and read lifetime**: use exact view pointer/length, validate request bounds before disk submission, transition data to DiskOwned until the callback returns, and retain connection state across delayed I/O. Registered read storage must cover engine completion and SEND completion; client receive storage remains held through returned Bytes. Preserve existing alignment handling and short-I/O/error semantics. Test delayed I/O, queue pressure, held caller buffers, error, cancellation and connection shutdown. + Retain `SocketTransport` and its default application pool through asynchronous + DiskIO callbacks: `RpcServer` owns a shared transport, and that transport owns + its default pool. Expose the shared transport for the DiskIO constructor; + completion closures retain it and their connection, with global metric + pointers and static response builders rather than raw `DiskioServer *`. + A stopped connection skips reply submission. The existing AlignedWriter + destructor drains its own queued operations; its block-size-one fast path, + reads and fsync callbacks must remain independently safe after the service + facade is destroyed. No new completion-path mutex is needed. - [ ] **Verify real service flows**: run focused baseline and changed tests, then real DiskIO plus HTTP/ChunkIO upload/download of objects larger than 1 MiB on the prepared HCA. Assert direct-payload counters and same data diff --git a/lib/crowdb-access-s3/Cargo.toml b/lib/crowdb-access-s3/Cargo.toml index 1630cf175..6213ac116 100644 --- a/lib/crowdb-access-s3/Cargo.toml +++ b/lib/crowdb-access-s3/Cargo.toml @@ -25,6 +25,7 @@ crowdb-access-multipart = { path = "../crowdb-access-multipart" } crowdb-chunk-kv-client = { path = "../crowdb-chunk-kv-client" } crowdb-kv-client = { path = "../crowdb-kv-client" } crowdb-protocol = { path = "../crowdb-protocol" } +crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi" } flatbuffers = { workspace = true } futures = "0.3" hyper = { workspace = true, features = ["http1", "server"] } @@ -41,6 +42,7 @@ sha2 = "0.10" subtle = "2" thiserror = { workspace = true } tracing = { workspace = true } +tokio = { workspace = true, features = ["time"] } uuid = { version = "1", features = ["v4"] } zeroize = { version = "1", features = ["derive"] } diff --git a/lib/crowdb-access-s3/src/native_buffer.rs b/lib/crowdb-access-s3/src/native_buffer.rs index 8268091e3..d5a44ce27 100644 --- a/lib/crowdb-access-s3/src/native_buffer.rs +++ b/lib/crowdb-access-s3/src/native_buffer.rs @@ -6,10 +6,11 @@ #![allow(unsafe_code)] use std::cell::UnsafeCell; -use std::future::poll_fn; +use std::future::{poll_fn, Future}; use std::io; use std::mem::MaybeUninit; use std::ops::Range; +use std::pin::Pin; use std::ptr::NonNull; use std::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; use std::sync::{Arc, Weak}; @@ -25,6 +26,7 @@ use crowdb_protocol::frame::{ encode_frame_regions, prepare_frame_regions, set_frame_chunk_id, FrameError, FrameMagic, FRAME_FOOTER_BYTES, FRAME_V2_HEADER_BYTES, MAX_FRAME_BYTES, MAX_FRAME_V2_PAYLOAD_BYTES, }; +use crowdb_rpc_ffi::{ProducerBufferAllocator, WritableBuffer}; use hyper::body::{Bytes, Http1BodyReceiveBuffer, Http1BodyReceiveProvider}; #[derive(Clone)] @@ -66,6 +68,7 @@ struct ReceiverState { payload_lengths: Vec, append_slot: Option, credit_wait_started: Option, + native_retry: Option>>, magic: FrameMagic, owner_bytes: usize, } @@ -88,6 +91,7 @@ struct AllocatorState { backpressure_events: AtomicUsize, backpressure_wait_ns: AtomicUsize, credit_waiters: ArcSwap>>, + producer: Option, } #[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Serialize)] @@ -168,7 +172,8 @@ impl NativeBodyReceiver { return Poll::Ready(Ok(())); } let owner_bytes = state.owner_bytes; - if !self.allocator.try_reserve(owner_bytes) { + let mut owner = self.allocator.try_owner(owner_bytes)?; + if owner.is_none() { if state.credit_wait_started.is_none() { state.credit_wait_started = Some(Instant::now()); self.allocator @@ -177,28 +182,20 @@ impl NativeBodyReceiver { .fetch_add(1, Ordering::Relaxed); } self.credit_waker.register(cx.waker()); - if !self.allocator.try_reserve(owner_bytes) { + owner = self.allocator.try_owner(owner_bytes)?; + if owner.is_none() { + self.poll_native_credit(state, cx); return Poll::Pending; } } + state.native_retry = None; if let Some(started) = state.credit_wait_started.take() { self.allocator.state.backpressure_wait_ns.fetch_add( usize::try_from(started.elapsed().as_nanos()).unwrap_or(usize::MAX), Ordering::Relaxed, ); } - let owner = match NativeOwner::new(owner_bytes, Arc::clone(&self.allocator.state)) { - Ok(owner) => Arc::new(owner), - Err(error) => { - self.allocator - .state - .retained_bytes - .fetch_sub(owner_bytes, Ordering::AcqRel); - self.allocator.wake_credit_waiters(); - return Poll::Ready(Err(error)); - } - }; - self.allocator.state.allocations.fetch_add(1, Ordering::Relaxed); + let owner = owner.expect("owner was admitted above"); if let Some(prefix) = state.pending_prefetched.take() { if let Err(error) = self.populate_prefetched(state, owner, &prefix) { return Poll::Ready(Err(error)); @@ -209,6 +206,22 @@ impl NativeBodyReceiver { Poll::Ready(Ok(())) } + fn poll_native_credit(&self, state: &mut ReceiverState, cx: &mut Context<'_>) { + if self.allocator.state.producer.is_none() { + return; + } + // Other producers and native SEND completion can return arena credit + // without dropping this HTTP allocator's owner. Keep their retry wake + // bounded; ordinary owner release still wakes through AtomicWaker. + let retry = state + .native_retry + .get_or_insert_with(|| Box::pin(tokio::time::sleep(std::time::Duration::from_millis(1)))); + if retry.as_mut().poll(cx).is_ready() { + state.native_retry = None; + cx.waker().wake_by_ref(); + } + } + fn populate_prefetched( &self, state: &mut ReceiverState, @@ -733,22 +746,10 @@ impl Http1BodyReceiveProvider for NativeBodyReceiver { "HTTP read-ahead exceeds one native owner", )); } - if !self.allocator.try_reserve(owner_bytes) { + let Some(owner) = self.allocator.try_owner(owner_bytes)? else { state.pending_prefetched = Some(data.clone()); return Ok(data); - } - let owner = match NativeOwner::new(owner_bytes, Arc::clone(&self.allocator.state)) { - Ok(owner) => Arc::new(owner), - Err(error) => { - self.allocator - .state - .retained_bytes - .fetch_sub(owner_bytes, Ordering::AcqRel); - self.allocator.wake_credit_waiters(); - return Err(error); - } }; - self.allocator.state.allocations.fetch_add(1, Ordering::Relaxed); self.populate_prefetched(state, owner, &data)?; Ok(data) } @@ -758,6 +759,7 @@ struct NativeOwner { pointer: NonNull, capacity: usize, allocator: Arc, + native: Option, } // SAFETY: provider-issued regions are disjoint physical-frame payload slots. @@ -767,6 +769,21 @@ unsafe impl Sync for NativeOwner {} impl NativeOwner { fn new(capacity: usize, allocator: Arc) -> io::Result { + if let Some(producer) = &allocator.producer { + let capacity32 = u32::try_from(capacity) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "native owner exceeds u32"))?; + let mut native = producer.allocate(capacity32).ok_or_else(|| { + io::Error::new(io::ErrorKind::WouldBlock, "native producer arena exhausted") + })?; + let pointer = NonNull::new(native.uninit_bytes_mut().as_mut_ptr().cast::()) + .expect("native allocation is non-null"); + return Ok(Self { + pointer, + capacity, + allocator, + native: Some(native), + }); + } // SAFETY: malloc returns either a suitably aligned allocation of at // least `capacity` bytes or null; ownership is immediately wrapped. let pointer = NonNull::new(unsafe { libc::malloc(capacity).cast::() }) @@ -775,14 +792,21 @@ impl NativeOwner { pointer, capacity, allocator, + native: None, }) } } impl Drop for NativeOwner { fn drop(&mut self) { - // SAFETY: `pointer` came from malloc and this owner frees it once. - unsafe { libc::free(self.pointer.as_ptr().cast()) }; + let native = self.native.take(); + if native.is_none() { + // SAFETY: absent native backing, pointer came from malloc and + // this owner frees it once, after all immutable views are gone. + unsafe { libc::free(self.pointer.as_ptr().cast()) }; + } + // Return native admission before waking credit waiters. + drop(native); self.allocator .retained_bytes .fetch_sub(self.capacity, Ordering::AcqRel); diff --git a/lib/crowdb-access-s3/src/native_buffer/allocator.rs b/lib/crowdb-access-s3/src/native_buffer/allocator.rs index e758a72d0..6bf614233 100644 --- a/lib/crowdb-access-s3/src/native_buffer/allocator.rs +++ b/lib/crowdb-access-s3/src/native_buffer/allocator.rs @@ -34,6 +34,7 @@ impl NativeBodyAllocator { backpressure_events: AtomicUsize::new(0), backpressure_wait_ns: AtomicUsize::new(0), credit_waiters: ArcSwap::from_pointee(Vec::new()), + producer: None, }), }) } @@ -91,7 +92,9 @@ impl NativeBodyAllocator { .physical_length(payload_bytes as u64) .ok() .and_then(|bytes| usize::try_from(bytes).ok()) - .filter(|bytes| *bytes <= self.state.budget_bytes) + .filter(|bytes| { + *bytes <= self.state.budget_bytes && (self.state.producer.is_none() || *bytes <= 1 << 20) + }) .ok_or(NativeBufferConfigError::InvalidOwnerSize)?; Ok(self.receiver_with_owner_bytes( physical.max( @@ -112,6 +115,26 @@ impl NativeBodyAllocator { self } + /// Reuse the selected `DiskIO` transport's native producer arena. + /// + /// # Errors + /// Rejects owner sizes above the largest registered class (1 MiB). + /// + /// # Panics + /// Panics if this allocator has already been shared or used. + pub fn with_producer_allocator( + mut self, + producer: super::ProducerBufferAllocator, + ) -> Result { + if self.state.owner_bytes > 1 << 20 { + return Err(NativeBufferConfigError::InvalidOwnerSize); + } + Arc::get_mut(&mut self.state) + .expect("attach producer before sharing native allocator") + .producer = Some(producer); + Ok(self) + } + fn receiver_with_owner_bytes(&self, owner_bytes: usize) -> NativeBodyReceiver { let credit_waker = Arc::new(AtomicWaker::new()); self.state.credit_waiters.rcu(|current| { @@ -136,6 +159,7 @@ impl NativeBodyAllocator { payload_lengths: vec![0; owner_bytes.div_ceil(MAX_FRAME_BYTES)], append_slot: None, credit_wait_started: None, + native_retry: None, magic: FrameMagic::RepoLargeV2, owner_bytes, }), @@ -164,6 +188,27 @@ impl NativeBodyAllocator { } } + pub(super) fn try_owner(&self, bytes: usize) -> std::io::Result>> { + if !self.try_reserve(bytes) { + return Ok(None); + } + match super::NativeOwner::new(bytes, Arc::clone(&self.state)) { + Ok(owner) => { + self.state.allocations.fetch_add(1, Ordering::Relaxed); + Ok(Some(Arc::new(owner))) + } + Err(error) => { + self.state.retained_bytes.fetch_sub(bytes, Ordering::AcqRel); + if error.kind() == std::io::ErrorKind::WouldBlock { + Ok(None) + } else { + self.wake_credit_waiters(); + Err(error) + } + } + } + } + pub(super) fn wake_credit_waiters(&self) { self.state.wake_credit_waiters(); } diff --git a/lib/crowdb-access-s3/tests/native_producer_test.rs b/lib/crowdb-access-s3/tests/native_producer_test.rs new file mode 100644 index 000000000..d35302d83 --- /dev/null +++ b/lib/crowdb-access-s3/tests/native_producer_test.rs @@ -0,0 +1,196 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::future::poll_fn; +use std::sync::Arc; +use std::time::Duration; + +use crowdb_access_s3::native_buffer::NativeBodyAllocator; +use crowdb_chunk_client::FramedWriteBuffer; +use crowdb_protocol::common::ChunkId; +use crowdb_protocol::fb::{ConnectionPingRequest, ConnectionPingRequestArgs}; +use crowdb_protocol::frame::{parse_frame, FrameMagic, MAX_FRAME_BYTES, MAX_FRAME_PAYLOAD_BYTES}; +use crowdb_rpc_ffi::{ + Buffer, BufferChain, BufferPool, ProducerBufferAllocator, RdmaConfig, RpcClient, RpcServer, +}; +use flatbuffers::FlatBufferBuilder; +use hyper::body::Http1BodyReceiveProvider; + +#[tokio::test] +async fn native_credit_returned_by_another_producer_wakes_http_intake() { + let pool = BufferPool::new(1); + let server = Arc::new(RpcServer::new(Some(&pool))); + server.configure_rdma(&RdmaConfig::default()).unwrap(); + let source = ProducerBufferAllocator::new(server, false); + let held = source.allocate(64 * 1024).unwrap(); + let allocator = NativeBodyAllocator::new(2 * MAX_FRAME_BYTES, MAX_FRAME_BYTES) + .unwrap() + .with_producer_allocator(source) + .unwrap(); + let receiver = allocator.object_receiver(); + let mut waiting = Box::pin(poll_fn(|cx| receiver.poll_next_buffer(cx, 13))); + assert!(tokio::time::timeout(Duration::from_millis(20), &mut waiting) + .await + .is_err()); + assert_eq!(allocator.retained_bytes(), 0); + assert_eq!(allocator.allocation_count(), 0); + drop(held); + let mut region = tokio::time::timeout(Duration::from_secs(1), waiting) + .await + .unwrap() + .unwrap(); + for byte in &mut region.spare_capacity_mut()[..13] { + byte.write(7); + } + region.advance(13).unwrap(); + let payload = receiver.on_data_ready(region).unwrap(); + assert!(payload.iter().all(|byte| *byte == 7)); + assert!(allocator.metrics_snapshot().backpressure_events > 0); + drop(payload); + drop(receiver); + assert_eq!(allocator.retained_bytes(), 0); +} + +#[tokio::test] +async fn framed_native_views_retain_storage_and_keep_original_payload_addresses() { + let pool = BufferPool::new(1); + let server = Arc::new(RpcServer::new(Some(&pool))); + server.configure_rdma(&RdmaConfig::default()).unwrap(); + let source = ProducerBufferAllocator::new(server, false); + let allocator = NativeBodyAllocator::new(MAX_FRAME_BYTES, MAX_FRAME_BYTES) + .unwrap() + .with_producer_allocator(source.clone()) + .unwrap(); + let receiver = allocator.object_receiver(); + receiver.enable_owner_handoff(); + let mut region = poll_fn(|cx| receiver.poll_next_buffer(cx, MAX_FRAME_PAYLOAD_BYTES)) + .await + .unwrap(); + let pointer = region.spare_capacity_mut().as_ptr().cast::(); + for byte in &mut region.spare_capacity_mut()[..MAX_FRAME_PAYLOAD_BYTES] { + byte.write(0x5a); + } + region.advance(MAX_FRAME_PAYLOAD_BYTES).unwrap(); + let payload = receiver.on_data_ready(region).unwrap(); + let mut owner = receiver.take_ready_owner().unwrap(); + owner.prepare_frames(FrameMagic::RepoLargeV1, 123).unwrap(); + let chunk = ChunkId { high: 17, low: 31 }; + let range = owner + .finalize_frame(0, FrameMagic::RepoLargeV1, chunk, 123) + .unwrap(); + let views = owner.views(range).unwrap(); + assert_eq!(views.len(), 1); + assert_eq!(views[0].as_ptr().wrapping_add(14), pointer); + assert!(parse_frame(&views[0], chunk) + .unwrap() + .payload + .iter() + .all(|byte| *byte == 0x5a)); + assert!(source.allocate(32).is_none()); + let mirror = views[0].clone(); + drop(views); + drop(owner); + drop(receiver); + drop(payload); + assert!(source.allocate(32).is_none()); + assert_eq!(mirror.as_ptr().wrapping_add(14), pointer); + drop(mirror); + assert_eq!(allocator.retained_bytes(), 0); + assert!(source.allocate(32).is_some()); +} + +#[tokio::test] +async fn hardware_http_framed_owner_uses_direct_payload_send() { + if std::env::var_os("CROWDB_RDMA_E2E").is_none() { + return; + } + let address = std::env::var("CROWDB_RDMA_E2E_LISTEN_ADDR").unwrap(); + let config = RdmaConfig { + enabled: true, + listen_enabled: true, + allow_tcp_fallback: false, + wire_layout: crowdb_rpc_ffi::RdmaWireLayout::ControlData, + wire_slot_capacity: (1 << 20) + 1024, + sq_depth: 4, + rq_depth: 4, + posted_receive_target: 4, + cq_depth: 8, + send_arena_bytes: 4 << 20, + receive_arena_bytes: 4 * ((1 << 20) + 1024), + device: std::env::var("CROWDB_RDMA_DEVICE").unwrap_or_default(), + source_address: address.clone(), + ..RdmaConfig::default() + }; + let server = RpcServer::new(None); + server.configure_rdma(&config).unwrap(); + server.listen(&address, 0).unwrap(); + server.register_echo_handler(100); + server.start(); + let client = Arc::new(RpcServer::new(None)); + client + .configure_rdma(&RdmaConfig { + listen_enabled: false, + ..config + }) + .unwrap(); + client.start(); + let connection = client.connect(&address, server.port()).unwrap(); + let rpc = RpcClient::new(); + rpc.attach(&connection); + let allocator = NativeBodyAllocator::new(1 << 20, 1 << 20) + .unwrap() + .with_producer_allocator(ProducerBufferAllocator::new(Arc::clone(&client), true)) + .unwrap(); + let receiver = allocator.object_receiver(); + receiver.enable_owner_handoff(); + for index in 0..16 { + let mut region = poll_fn(|cx| receiver.poll_next_buffer(cx, MAX_FRAME_PAYLOAD_BYTES)) + .await + .unwrap(); + for byte in &mut region.spare_capacity_mut()[..MAX_FRAME_PAYLOAD_BYTES] { + byte.write(index); + } + region.advance(MAX_FRAME_PAYLOAD_BYTES).unwrap(); + drop(receiver.on_data_ready(region).unwrap()); + } + let mut owner = receiver.take_ready_owner().unwrap(); + owner.prepare_frames(FrameMagic::RepoLargeV1, 123).unwrap(); + let chunk = ChunkId { high: 11, low: 19 }; + for index in 0..16 { + owner + .finalize_frame(index, FrameMagic::RepoLargeV1, chunk, 123) + .unwrap(); + } + let views = owner.views(0..(1 << 20)).unwrap(); + let expected = views[0].clone(); + let mut builder = FlatBufferBuilder::new(); + let request = ConnectionPingRequest::create( + &mut builder, + &ConnectionPingRequestArgs { + id: 91, + rpc_create_nano: 0, + }, + ); + builder.finish(request, None); + let response = rpc + .call_chain( + &client, + &connection, + 91, + Buffer::from_bytes(builder.finished_data()), + BufferChain::from_owned_bytes(views).unwrap(), + 100, + ) + .unwrap() + .await + .unwrap(); + assert_eq!(response.data.unwrap().as_slice(), expected.as_ref()); + let status: serde_json::Value = serde_json::from_slice(&client.transport_status_json().unwrap()).unwrap(); + assert_eq!(status["rdma"]["direct_payload_bytes"], 1 << 20); + assert_eq!(status["rdma"]["copied_payload_bytes"], 0); + assert_eq!(status["rdma"]["fallback_count"], 0); + assert_eq!(status["rdma"]["protocol_errors"], 0); + assert_eq!(allocator.prefix_copy_bytes(), 0); + client.stop(); + server.stop(); +} diff --git a/lib/crowdb-chunk-client/src/client.rs b/lib/crowdb-chunk-client/src/client.rs index 78552bf8f..37d670144 100644 --- a/lib/crowdb-chunk-client/src/client.rs +++ b/lib/crowdb-chunk-client/src/client.rs @@ -103,6 +103,12 @@ struct ClientTopology { } impl ChunkIoClient { + /// Reuse the DiskIO transport's bounded native producer allocation source. + #[must_use] + pub fn producer_allocator(&self) -> Option { + self.disk_writer.producer_allocator() + } + /// Discover services and build lock-free DiskIO routing. pub async fn connect(config: ChunkIoClientConfig) -> Result { let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( @@ -645,6 +651,7 @@ impl ChunkAllocator for MetricsChunkAllocator { } } + async fn discover_current_range_bindings( service: &ServiceRegistryClient, kv: Arc, diff --git a/lib/crowdb-chunk-client/src/client/disk_metrics.rs b/lib/crowdb-chunk-client/src/client/disk_metrics.rs index 03814fb71..e210e9dad 100644 --- a/lib/crowdb-chunk-client/src/client/disk_metrics.rs +++ b/lib/crowdb-chunk-client/src/client/disk_metrics.rs @@ -12,6 +12,9 @@ pub(super) struct MetricsDiskWriter { #[async_trait] impl DiskWriter for MetricsDiskWriter { + fn producer_allocator(&self) -> Option { + self.inner.producer_allocator() + } async fn write(&self, seg: &Segment, unit_bytes: u64, data: Bytes) -> Result<()> { let bytes = u64::try_from(data.len()).unwrap_or(u64::MAX); let mut operation = self.metrics.diskio_write.start(); diff --git a/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs b/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs index d5dacf4a3..69303036e 100644 --- a/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs +++ b/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs @@ -12,6 +12,11 @@ use crate::{IoError, Result}; /// Block-IO seam. Production routing is owned by `crowdb-diskio-client`. #[async_trait] pub trait DiskWriter: Send + Sync { + /// Optional native storage for an owner-backed upstream producer. + fn producer_allocator(&self) -> Option { + None + } + /// Write `data` to the disk/zone/offset described by `seg`. async fn write(&self, seg: &Segment, unit_bytes: u64, data: Bytes) -> Result<()>; diff --git a/lib/crowdb-chunk-client/src/disk_io/routing.rs b/lib/crowdb-chunk-client/src/disk_io/routing.rs index c16615dc2..0985a71e3 100644 --- a/lib/crowdb-chunk-client/src/disk_io/routing.rs +++ b/lib/crowdb-chunk-client/src/disk_io/routing.rs @@ -104,6 +104,10 @@ impl RoutedDiskWriter { #[async_trait] impl DiskWriter for RoutedDiskWriter { + fn producer_allocator(&self) -> Option { + Some(self.client.producer_allocator()) + } + async fn write(&self, seg: &Segment, unit_bytes: u64, data: Bytes) -> Result<()> { let target = Self::target(seg, unit_bytes)?; self.client diff --git a/lib/crowdb-diskio-client/src/semantic.rs b/lib/crowdb-diskio-client/src/semantic.rs index fcb207d96..4cba99f9e 100644 --- a/lib/crowdb-diskio-client/src/semantic.rs +++ b/lib/crowdb-diskio-client/src/semantic.rs @@ -314,6 +314,14 @@ impl std::fmt::Debug for DiskioClient { } impl DiskioClient { + /// Producer storage uses the same device arena as the discovered routes. + /// An exhausted active arena returns no allocation, including in mixed + /// TCP/RDMA routing; it never expands the native registered budget. + #[must_use] + pub fn producer_allocator(&self) -> crowdb_rpc_ffi::ProducerBufferAllocator { + crowdb_rpc_ffi::ProducerBufferAllocator::new(Arc::clone(&self.server), false) + } + /// Classify an actual wire return code without broadening ordinary read retries. #[cfg(feature = "test-util")] #[must_use] @@ -330,7 +338,16 @@ impl DiskioClient { #[cfg(feature = "test-util")] pub fn connect_for_tests(routes: Vec, config: DiskioClientConfig) -> DiskioResult { config.validate(false)?; - let server = Arc::new(RpcServer::with_engines(None, 1, config.rpc_workers)); + let server = Arc::new( + RpcServer::with_rdma_layout( + None, + 1, + config.rpc_workers, + crowdb_rpc_ffi::RdmaWireLayout::ControlData, + 1 << 20, + ) + .map_err(|error| DiskioError::TransportUnavailable(format!("create RPC client: {error}")))?, + ); server.set_send_queue_capacity(config.send_queue_capacity); server .listen("127.0.0.1", 0) diff --git a/lib/crowdb-diskio-client/src/semantic/connection.rs b/lib/crowdb-diskio-client/src/semantic/connection.rs index 9324fd843..b4092d71c 100644 --- a/lib/crowdb-diskio-client/src/semantic/connection.rs +++ b/lib/crowdb-diskio-client/src/semantic/connection.rs @@ -8,7 +8,7 @@ use super::{ }; use arc_swap::ArcSwap; use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient, ServiceRegistryClient}; -use crowdb_rpc_ffi::{ConnectionPoolIndex, RpcServer}; +use crowdb_rpc_ffi::{ConnectionPoolIndex, RdmaWireLayout, RpcServer}; use std::sync::{atomic::AtomicU64, Arc}; impl DiskioClient { @@ -83,7 +83,10 @@ impl DiskioClient { scope: Option, ) -> DiskioResult { config.validate(false)?; - let server = Arc::new(RpcServer::with_engines(None, 1, config.rpc_workers)); + let server = Arc::new( + RpcServer::with_rdma_layout(None, 1, config.rpc_workers, RdmaWireLayout::ControlData, 1 << 20) + .map_err(|error| DiskioError::TransportUnavailable(format!("create RPC client: {error}")))?, + ); server.set_send_queue_capacity(config.send_queue_capacity); server .listen("127.0.0.1", 0) diff --git a/lib/crowdb-rpc/ffi/src/lib.rs b/lib/crowdb-rpc/ffi/src/lib.rs index 54c252811..ac8f0e059 100644 --- a/lib/crowdb-rpc/ffi/src/lib.rs +++ b/lib/crowdb-rpc/ffi/src/lib.rs @@ -34,7 +34,7 @@ pub use logging::{ shutdown_logging, }; pub use owned_route::{OwnedClientRoute, RpcClientHandle}; -pub use producer_buffer::WritableBuffer; +pub use producer_buffer::{ProducerBufferAllocator, WritableBuffer}; pub use server::{Connection, RpcError, RpcServer, ServerRequest}; pub use transport::{RdmaConfig, RdmaWireLayout}; diff --git a/lib/crowdb-rpc/ffi/src/producer_buffer.rs b/lib/crowdb-rpc/ffi/src/producer_buffer.rs index 009c2d868..72be6c29f 100644 --- a/lib/crowdb-rpc/ffi/src/producer_buffer.rs +++ b/lib/crowdb-rpc/ffi/src/producer_buffer.rs @@ -4,9 +4,33 @@ //! Unique producer storage before immutable buffer publication. use std::mem::MaybeUninit; +use std::sync::Arc; use crate::{sys, Buffer, RpcError, RpcServer}; +/// Retained allocation source shared with a service's existing RPC transport. +#[derive(Clone, Debug)] +pub struct ProducerBufferAllocator { + server: Arc, + require_registered: bool, +} + +impl ProducerBufferAllocator { + #[must_use] + pub fn new(server: Arc, require_registered: bool) -> Self { + Self { + server, + require_registered, + } + } + + #[must_use] + pub fn allocate(&self, capacity: u32) -> Option { + self.server + .allocate_producer_buffer(capacity, self.require_registered) + } +} + /// A unique allocation that cannot be submitted or cloned while writable. /// Its storage retains the native recycler independently of the server. #[derive(Debug)] diff --git a/lib/crowdb-rpc/ffi/src/server.rs b/lib/crowdb-rpc/ffi/src/server.rs index fdfe10972..dc8607404 100644 --- a/lib/crowdb-rpc/ffi/src/server.rs +++ b/lib/crowdb-rpc/ffi/src/server.rs @@ -55,6 +55,28 @@ impl RpcServer { /// differences. Linux (epoll) is unaffected. A warning is logged /// once per process when the clamp activates. pub fn with_engines(pool: Option<&crate::BufferPool>, io_engines: u32, io_workers: u32) -> Self { + Self::with_rdma_layout( + pool, + io_engines, + io_workers, + crate::RdmaWireLayout::Contiguous, + 1 << 20, + ) + .expect("valid RPC engine and layout defaults") + } + + /// Set service layout/capacity defaults before environment configuration. + /// This preserves the process's existing RDMA enablement/fallback policy. + /// + /// # Errors + /// Rejects invalid engine counts or a data capacity outside the RDMA classes. + pub fn with_rdma_layout( + pool: Option<&crate::BufferPool>, + io_engines: u32, + io_workers: u32, + layout: crate::RdmaWireLayout, + data_capacity: u32, + ) -> Result { #[cfg(feature = "test-util")] crate::logging::init_test_logging(); @@ -71,9 +93,19 @@ impl RpcServer { io_workers }; let pool_handle = pool.map(|p| p.handle()).unwrap_or(ptr::null_mut()); - let handle = - unsafe { sys::crowdb_rpc_server_create_with_engines(pool_handle, io_engines, effective_workers) }; - RpcServer { handle } + let handle = unsafe { + sys::crowdb_rpc_server_create_with_layout( + pool_handle, + io_engines, + effective_workers, + layout as u32, + data_capacity, + ) + }; + if handle.is_null() { + return Err(RpcError::InvalidArg); + } + Ok(RpcServer { handle }) } /// Listen on the given address and port. If port is 0, the OS assigns diff --git a/lib/crowdb-rpc/ffi/src/sys.rs b/lib/crowdb-rpc/ffi/src/sys.rs index f8f438918..1eb758b14 100644 --- a/lib/crowdb-rpc/ffi/src/sys.rs +++ b/lib/crowdb-rpc/ffi/src/sys.rs @@ -215,6 +215,13 @@ extern "C" { io_engines: u32, io_workers: u32, ) -> crowdb_rpc_server_t; + pub fn crowdb_rpc_server_create_with_layout( + pool: crowdb_rpc_pool_t, + io_engines: u32, + io_workers: u32, + wire_layout: u32, + data_capacity: u32, + ) -> crowdb_rpc_server_t; pub fn crowdb_rpc_server_destroy(server: crowdb_rpc_server_t); pub fn crowdb_rpc_server_clear_handlers(server: crowdb_rpc_server_t); pub fn crowdb_rpc_server_listen( diff --git a/lib/crowdb-rpc/ffi/tests/transport_config_test.rs b/lib/crowdb-rpc/ffi/tests/transport_config_test.rs index 93955e5b9..82cc3415a 100644 --- a/lib/crowdb-rpc/ffi/tests/transport_config_test.rs +++ b/lib/crowdb-rpc/ffi/tests/transport_config_test.rs @@ -39,6 +39,15 @@ fn invalid_queue_bounds_fail_before_start() { assert!(server.configure_rdma(&config).is_err()); } +#[test] +fn service_layout_defaults_validate_capacity_and_preserve_explicit_tcp() { + assert!(RpcServer::with_rdma_layout(None, 1, 1, RdmaWireLayout::ControlData, 4096).is_err()); + let server = RpcServer::with_rdma_layout(None, 1, 1, RdmaWireLayout::ControlData, 1 << 20).unwrap(); + server.configure_rdma(&RdmaConfig::default()).unwrap(); + let status: Value = serde_json::from_slice(&server.transport_status_json().unwrap()).unwrap(); + assert!(status["rdma"].is_null()); +} + #[test] fn typed_control_data_layout_accounts_for_control_and_forbids_aggregation() { let server = RpcServer::new(None); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h index 7c5b03bcb..381d21180 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h @@ -124,6 +124,10 @@ crowdb_rpc_server_t crowdb_rpc_server_create(crowdb_rpc_pool_t pool); crowdb_rpc_server_t crowdb_rpc_server_create_with_workers(crowdb_rpc_pool_t pool, uint32_t num_workers); crowdb_rpc_server_t crowdb_rpc_server_create_with_engines(crowdb_rpc_pool_t pool, uint32_t io_engines, uint32_t io_workers); +// Service defaults precede environment overrides; this does not enable RDMA. +crowdb_rpc_server_t crowdb_rpc_server_create_with_layout(crowdb_rpc_pool_t pool, uint32_t io_engines, + uint32_t io_workers, uint32_t wire_layout, + uint32_t data_capacity); // Set per-connection send queue capacity (backpressure bound). Must be // called before listen/connect creates connections. Default 1024. // Rounded up to next power of two internally. diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api_internal.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api_internal.h index 06fcb0265..dc6d1b8c0 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api_internal.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api_internal.h @@ -11,6 +11,12 @@ // Concrete struct definitions for the C API opaque handles. // Defined here so both c_api.cpp and co_client.cpp can access them. +struct crowdb_rpc_pool_s +{ + crowdb::rpc::BufferPool *pool; + bool owns; +}; + struct crowdb_rpc_buffer_s { crowdb::rpc::Buffer *buf; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/server/server.h b/lib/crowdb-rpc/include/crowdb-rpc/server/server.h index 4df3f8f23..356f3645e 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/server/server.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/server/server.h @@ -26,7 +26,8 @@ class RpcServer public: // Multi-engine ctor: io_engines independent epoll/kqueue instances, // with io_workers total workers (per-engine = io_workers / io_engines). - RpcServer(BufferPool *pool = nullptr, uint32_t io_engines = 1, uint32_t io_workers = 1); + RpcServer(BufferPool *pool = nullptr, uint32_t io_engines = 1, uint32_t io_workers = 1, + RdmaConfig rdma_config = {}); ~RpcServer(); // Listen on the given address + port. Must be called before start(). diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h index 239535025..069da96b8 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h @@ -11,35 +11,37 @@ namespace crowdb::rpc { struct RdmaConfig { - bool use_environment = true; - std::string device; - std::string source_address{}; - std::string gid{}; - bool require_tcp_preflight = false; - uint8_t port = 1; - uint32_t cq_depth = 8192; - uint32_t cq_count = 1; - uint32_t wire_slot_capacity = RDMA_DATAPATH_CAPACITY; - RdmaWireLayout wire_layout = RdmaWireLayout::Contiguous; - uint64_t send_arena_bytes = 64ULL << 20; - uint64_t receive_arena_bytes = 1ULL << 30; - uint64_t registered_memory_budget_bytes = 0; - uint32_t sq_depth = 1024; - uint32_t rq_depth = 1024; - uint32_t qp_max_send_sge = 2; - uint32_t qp_max_recv_sge = 2; - uint32_t poll_batch = 256; - uint32_t verbs_poll_batch = 32; - uint32_t poll_empty_budget = 256; - uint32_t shutdown_drain_ms = 3000; - uint32_t posted_receive_target = 1024; - uint8_t min_rnr_timer = 1; - uint8_t retry_count = 7; - uint8_t rnr_retry_count = 7; - bool aggregation_enabled = false; - uint32_t aggregation_max_bytes = RDMA_DATAPATH_CAPACITY; - uint32_t aggregation_max_frames = 256; - uint32_t send_queue_capacity = 4096; + // Service defaults are applied before environment overrides and probing. + static RdmaConfig for_layout(RdmaWireLayout layout, uint32_t data_capacity); + bool use_environment = true; + std::string device; + std::string source_address{}; + std::string gid{}; + bool require_tcp_preflight = false; + uint8_t port = 1; + uint32_t cq_depth = 8192; + uint32_t cq_count = 1; + uint32_t wire_slot_capacity = RDMA_DATAPATH_CAPACITY; + RdmaWireLayout wire_layout = RdmaWireLayout::Contiguous; + uint64_t send_arena_bytes = 64ULL << 20; + uint64_t receive_arena_bytes = 1ULL << 30; + uint64_t registered_memory_budget_bytes = 0; + uint32_t sq_depth = 1024; + uint32_t rq_depth = 1024; + uint32_t qp_max_send_sge = 2; + uint32_t qp_max_recv_sge = 2; + uint32_t poll_batch = 256; + uint32_t verbs_poll_batch = 32; + uint32_t poll_empty_budget = 256; + uint32_t shutdown_drain_ms = 3000; + uint32_t posted_receive_target = 1024; + uint8_t min_rnr_timer = 1; + uint8_t retry_count = 7; + uint8_t rnr_retry_count = 7; + bool aggregation_enabled = false; + uint32_t aggregation_max_bytes = RDMA_DATAPATH_CAPACITY; + uint32_t aggregation_max_frames = 256; + uint32_t send_queue_capacity = 4096; [[nodiscard]] bool apply_environment(); [[nodiscard]] bool valid() const; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h index 9f51f9e53..fac9fac9f 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h @@ -331,12 +331,7 @@ class SocketTransport : public Transport public: // Multi-engine ctor: io_engines independent epoll/kqueue instances, // with io_workers total workers (per-engine = io_workers / io_engines). - SocketTransport(uint32_t io_engines, uint32_t io_workers, BufferPool *pool = nullptr -#ifdef CROWDB_RPC_HAVE_RDMA - , - RdmaConfig rdma_config = {} -#endif - ); + SocketTransport(uint32_t io_engines, uint32_t io_workers, BufferPool *pool = nullptr, RdmaConfig rdma_config = {}); // Deprecated alias: maps to (1, num_workers). Kept for backward compat. SocketTransport(uint32_t num_workers, BufferPool *pool); ~SocketTransport() override; diff --git a/lib/crowdb-rpc/src/c_api.cpp b/lib/crowdb-rpc/src/c_api.cpp index cfe432dec..5c1127abf 100644 --- a/lib/crowdb-rpc/src/c_api.cpp +++ b/lib/crowdb-rpc/src/c_api.cpp @@ -36,12 +36,6 @@ // The opaque C handles are wrappers around the C++ objects. The struct // names match the forward declarations in c_api.h (crowdb_rpc_pool_s, etc.). -struct crowdb_rpc_pool_s -{ - crowdb::rpc::BufferPool *pool; - bool owns; -}; - // Opaque handle struct definitions are in c_api_internal.h (shared // with co_client.cpp). @@ -296,141 +290,6 @@ void crowdb_rpc_pool_destroy(crowdb_rpc_pool_t pool) } } -// ── Server ──────────────────────────────────────────────────────── - -crowdb_rpc_server_t crowdb_rpc_server_create(crowdb_rpc_pool_t pool) -{ - try { - crowdb::rpc::BufferPool *bp = (pool != nullptr) ? pool->pool : nullptr; - return new crowdb_rpc_server_s{new crowdb::rpc::RpcServer(bp, 1, 1)}; - } - catch (...) { - return nullptr; - } -} - -crowdb_rpc_server_t crowdb_rpc_server_create_with_workers(crowdb_rpc_pool_t pool, uint32_t num_workers) -{ - try { - crowdb::rpc::BufferPool *bp = (pool != nullptr) ? pool->pool : nullptr; - return new crowdb_rpc_server_s{new crowdb::rpc::RpcServer(bp, 1, num_workers)}; - } - catch (...) { - return nullptr; - } -} - -crowdb_rpc_server_t crowdb_rpc_server_create_with_engines(crowdb_rpc_pool_t pool, uint32_t io_engines, - uint32_t io_workers) -{ - try { - crowdb::rpc::BufferPool *bp = (pool != nullptr) ? pool->pool : nullptr; - return new crowdb_rpc_server_s{new crowdb::rpc::RpcServer(bp, io_engines, io_workers)}; - } - catch (...) { - return nullptr; - } -} - -void crowdb_rpc_server_set_send_queue_capacity(crowdb_rpc_server_t server, uint32_t capacity) -{ - try { - if (server == nullptr || capacity == 0) { - return; - } - server->server->transport()->set_send_queue_capacity(capacity); - } - catch (...) { - } -} - -void crowdb_rpc_server_set_tcp_nodelay(crowdb_rpc_server_t server, int enabled) -{ - try { - if (server == nullptr) { - return; - } - server->server->transport()->set_tcp_nodelay(enabled != 0); - } - catch (...) { - } -} - -void crowdb_rpc_server_set_quickack(crowdb_rpc_server_t server, int enabled) -{ - try { - if (server == nullptr) { - return; - } - server->server->transport()->set_quickack(enabled != 0); - } - catch (...) { - } -} - -void crowdb_rpc_server_set_event_write(crowdb_rpc_server_t server, int enabled) -{ - try { - if (server == nullptr) { - return; - } - server->server->transport()->set_event_write(enabled != 0); - } - catch (...) { - } -} - -void crowdb_rpc_server_destroy(crowdb_rpc_server_t server) -{ - try { - if (server == nullptr) { - return; - } - delete server->server; - delete server; - } - catch (...) { - } -} - -void crowdb_rpc_server_clear_handlers(crowdb_rpc_server_t server) -{ - try { - if (server == nullptr) { - return; - } - server->server->clear_handlers(); - } - catch (...) { - } -} - -static void copy_latency(crowdb_rpc_latency_stats_t *out, const crowdb::rpc::LatencyHistogram &h) -{ - out->count = h.count.load(std::memory_order_relaxed); - out->sum_ns = h.sum_ns.load(std::memory_order_relaxed); - out->min_ns = h.min_ns.load(std::memory_order_relaxed); - out->max_ns = h.max_ns.load(std::memory_order_relaxed); -} - -void crowdb_rpc_server_transport_stats(crowdb_rpc_server_t server, crowdb_rpc_transport_stats_t *out) -{ - try { - if (server == nullptr || out == nullptr) { - return; - } - auto *t = server->server->transport(); - if (t == nullptr) { - return; - } - auto &s = t->stats(); - copy_latency(&out->submit_to_writev, s.submit_to_writev); - out->send_queue_rejects = s.send_queue_rejects.load(std::memory_order_relaxed); - } - catch (...) { - } -} - void crowdb_rpc_client_get_counters(crowdb_rpc_client_t /*client*/, crowdb_rpc_client_counters_t *out) { try { @@ -447,59 +306,6 @@ void crowdb_rpc_client_get_counters(crowdb_rpc_client_t /*client*/, crowdb_rpc_c } } -crowdb_rpc_status crowdb_rpc_server_listen(crowdb_rpc_server_t server, const char *addr, int port) -{ - try { - if (server == nullptr || addr == nullptr) { - return CROWDB_RPC_ERR_INVALID_ARG; - } - if (!server->server->listen(addr, port)) { - return CROWDB_RPC_ERR_CONN_ERROR; - } - return CROWDB_RPC_OK; - } - catch (...) { - return CROWDB_RPC_ERR_CONN_ERROR; - } -} - -void crowdb_rpc_server_start(crowdb_rpc_server_t server) -{ - try { - if (server == nullptr) { - return; - } - server->server->start(); - } - catch (...) { - } -} - -void crowdb_rpc_server_stop(crowdb_rpc_server_t server) -{ - try { - if (server == nullptr) { - return; - } - server->server->stop(); - } - catch (...) { - } -} - -int crowdb_rpc_server_port(crowdb_rpc_server_t server) -{ - try { - if (server == nullptr) { - return 0; - } - return server->server->listen_port(); - } - catch (...) { - return 0; - } -} - // ── Caller ──────────────────────────────────────────────────────── crowdb_rpc_client_t crowdb_rpc_client_create(void) diff --git a/lib/crowdb-rpc/src/c_api/server.cpp b/lib/crowdb-rpc/src/c_api/server.cpp new file mode 100644 index 000000000..0da47b585 --- /dev/null +++ b/lib/crowdb-rpc/src/c_api/server.cpp @@ -0,0 +1,205 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/server/server.h" + +#include "crowdb-rpc/c_api_internal.h" + +// ── Server ──────────────────────────────────────────────────────── + +crowdb_rpc_server_t crowdb_rpc_server_create(crowdb_rpc_pool_t pool) +{ + try { + crowdb::rpc::BufferPool *bp = (pool != nullptr) ? pool->pool : nullptr; + return new crowdb_rpc_server_s{new crowdb::rpc::RpcServer(bp, 1, 1)}; + } + catch (...) { + return nullptr; + } +} + +crowdb_rpc_server_t crowdb_rpc_server_create_with_workers(crowdb_rpc_pool_t pool, uint32_t num_workers) +{ + try { + crowdb::rpc::BufferPool *bp = (pool != nullptr) ? pool->pool : nullptr; + return new crowdb_rpc_server_s{new crowdb::rpc::RpcServer(bp, 1, num_workers)}; + } + catch (...) { + return nullptr; + } +} + +crowdb_rpc_server_t crowdb_rpc_server_create_with_engines(crowdb_rpc_pool_t pool, uint32_t io_engines, + uint32_t io_workers) +{ + return crowdb_rpc_server_create_with_layout(pool, io_engines, io_workers, 0, crowdb::rpc::RDMA_DATAPATH_CAPACITY); +} + +crowdb_rpc_server_t crowdb_rpc_server_create_with_layout(crowdb_rpc_pool_t pool, uint32_t io_engines, + uint32_t io_workers, uint32_t wire_layout, + uint32_t data_capacity) +{ + try { + const auto config = + crowdb::rpc::RdmaConfig::for_layout(static_cast(wire_layout), data_capacity); + if (!config.valid() || io_engines == 0 || io_workers == 0 || io_workers % io_engines != 0) + return nullptr; + crowdb::rpc::BufferPool *bp = (pool != nullptr) ? pool->pool : nullptr; + return new crowdb_rpc_server_s{new crowdb::rpc::RpcServer(bp, io_engines, io_workers, config)}; + } + catch (...) { + return nullptr; + } +} + +void crowdb_rpc_server_set_send_queue_capacity(crowdb_rpc_server_t server, uint32_t capacity) +{ + try { + if (server == nullptr || capacity == 0) { + return; + } + server->server->transport()->set_send_queue_capacity(capacity); + } + catch (...) { + } +} + +void crowdb_rpc_server_set_tcp_nodelay(crowdb_rpc_server_t server, int enabled) +{ + try { + if (server == nullptr) { + return; + } + server->server->transport()->set_tcp_nodelay(enabled != 0); + } + catch (...) { + } +} + +void crowdb_rpc_server_set_quickack(crowdb_rpc_server_t server, int enabled) +{ + try { + if (server == nullptr) { + return; + } + server->server->transport()->set_quickack(enabled != 0); + } + catch (...) { + } +} + +void crowdb_rpc_server_set_event_write(crowdb_rpc_server_t server, int enabled) +{ + try { + if (server == nullptr) { + return; + } + server->server->transport()->set_event_write(enabled != 0); + } + catch (...) { + } +} + +void crowdb_rpc_server_destroy(crowdb_rpc_server_t server) +{ + try { + if (server == nullptr) { + return; + } + delete server->server; + delete server; + } + catch (...) { + } +} + +void crowdb_rpc_server_clear_handlers(crowdb_rpc_server_t server) +{ + try { + if (server == nullptr) { + return; + } + server->server->clear_handlers(); + } + catch (...) { + } +} + +static void copy_latency(crowdb_rpc_latency_stats_t *out, const crowdb::rpc::LatencyHistogram &h) +{ + out->count = h.count.load(std::memory_order_relaxed); + out->sum_ns = h.sum_ns.load(std::memory_order_relaxed); + out->min_ns = h.min_ns.load(std::memory_order_relaxed); + out->max_ns = h.max_ns.load(std::memory_order_relaxed); +} + +void crowdb_rpc_server_transport_stats(crowdb_rpc_server_t server, crowdb_rpc_transport_stats_t *out) +{ + try { + if (server == nullptr || out == nullptr) { + return; + } + auto *t = server->server->transport(); + if (t == nullptr) { + return; + } + auto &s = t->stats(); + copy_latency(&out->submit_to_writev, s.submit_to_writev); + out->send_queue_rejects = s.send_queue_rejects.load(std::memory_order_relaxed); + } + catch (...) { + } +} + +crowdb_rpc_status crowdb_rpc_server_listen(crowdb_rpc_server_t server, const char *addr, int port) +{ + try { + if (server == nullptr || addr == nullptr) { + return CROWDB_RPC_ERR_INVALID_ARG; + } + if (!server->server->listen(addr, port)) { + return CROWDB_RPC_ERR_CONN_ERROR; + } + return CROWDB_RPC_OK; + } + catch (...) { + return CROWDB_RPC_ERR_CONN_ERROR; + } +} + +void crowdb_rpc_server_start(crowdb_rpc_server_t server) +{ + try { + if (server == nullptr) { + return; + } + server->server->start(); + } + catch (...) { + } +} + +void crowdb_rpc_server_stop(crowdb_rpc_server_t server) +{ + try { + if (server == nullptr) { + return; + } + server->server->stop(); + } + catch (...) { + } +} + +int crowdb_rpc_server_port(crowdb_rpc_server_t server) +{ + try { + if (server == nullptr) { + return 0; + } + return server->server->listen_port(); + } + catch (...) { + return 0; + } +} diff --git a/lib/crowdb-rpc/src/server/server.cpp b/lib/crowdb-rpc/src/server/server.cpp index 44958b85e..9e9a88d2f 100644 --- a/lib/crowdb-rpc/src/server/server.cpp +++ b/lib/crowdb-rpc/src/server/server.cpp @@ -39,14 +39,14 @@ std::string listener_key(const std::string &addr, int port) } } // namespace -RpcServer::RpcServer(BufferPool *pool, uint32_t io_engines, uint32_t io_workers) +RpcServer::RpcServer(BufferPool *pool, uint32_t io_engines, uint32_t io_workers, RdmaConfig rdma_config) : pool_(pool), owns_pool_(pool == nullptr) { if (pool_ == nullptr) { pool_ = new SystemBufferPool(); } - transport_ = std::make_unique(io_engines, io_workers, pool_); + transport_ = std::make_unique(io_engines, io_workers, pool_, std::move(rdma_config)); } RpcServer::~RpcServer() diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp index 0f208dfdf..1d91284fe 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp @@ -14,6 +14,17 @@ namespace crowdb::rpc { +RdmaConfig RdmaConfig::for_layout(RdmaWireLayout layout, uint32_t data_capacity) +{ + RdmaConfig config; + config.wire_layout = layout; + const uint64_t wire = + static_cast(data_capacity) + (layout == RdmaWireLayout::ControlData ? RDMA_CONTROL_CAPACITY : 0); + config.wire_slot_capacity = wire > UINT32_MAX ? 0 : static_cast(wire); + config.receive_arena_bytes = static_cast(config.posted_receive_target) * config.wire_slot_capacity; + return config; +} + bool RdmaConfig::apply_environment() { auto read = [](const char *name, T &destination) { diff --git a/lib/crowdb-rpc/src/transport/socket_transport.cpp b/lib/crowdb-rpc/src/transport/socket_transport.cpp index c3da9a734..376032f14 100644 --- a/lib/crowdb-rpc/src/transport/socket_transport.cpp +++ b/lib/crowdb-rpc/src/transport/socket_transport.cpp @@ -539,12 +539,7 @@ static bool on_writable_impl(Connection *conn, int fd, TransportStats *stats) // ── SocketTransport ─────────────────────────────────────────────── -SocketTransport::SocketTransport(uint32_t io_engines, uint32_t io_workers, BufferPool *pool -#ifdef CROWDB_RPC_HAVE_RDMA - , - RdmaConfig rdma_config -#endif - ) +SocketTransport::SocketTransport(uint32_t io_engines, uint32_t io_workers, BufferPool *pool, RdmaConfig rdma_config) : pool_(pool) { assert(io_engines >= 1 && io_workers >= 1); @@ -559,9 +554,9 @@ SocketTransport::SocketTransport(uint32_t io_engines, uint32_t io_workers, Buffe owned_pool_ = std::make_unique(); pool_ = owned_pool_.get(); } -#ifdef CROWDB_RPC_HAVE_RDMA rdma_config_ = std::move(rdma_config); rdma_config_.require_tcp_preflight = true; +#ifdef CROWDB_RPC_HAVE_RDMA if (const char *enabled = std::getenv("CROWDB_RDMA_ENABLED"); enabled != nullptr && std::strcmp(enabled, "1") == 0) { rdma_transport_ = std::make_unique(rdma_config_, pool_); From e06889dadb1f4265537566e5d01cfb69186c2447 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 04:12:51 +0800 Subject: [PATCH 043/258] Retain DiskIO buffers and transport owners through asynchronous completion --- app/crowdb-diskio/src/dio_main.cpp | 4 +- app/crowdb-diskio/src/rpc/dio_fsync.cpp | 59 ++++ app/crowdb-diskio/src/rpc/dio_read.cpp | 110 ++++++++ app/crowdb-diskio/src/rpc/dio_read_buffer.h | 91 ++++++ app/crowdb-diskio/src/rpc/dio_request.h | 51 ++++ app/crowdb-diskio/src/rpc/dio_server.cpp | 211 ++------------ app/crowdb-diskio/src/rpc/dio_server.h | 29 +- .../src/rpc/dio_server_write.cpp | 40 +-- .../tests/common/test_dio_hardware.h | 266 ++++++++++++++++++ app/crowdb-diskio/tests/dio_hardware_test.cpp | 154 ++++++++++ app/crowdb-diskio/tests/dio_lifetime_test.cpp | 252 +++++++++++++++++ app/crowdb-diskio/tests/dio_tcp_test.cpp | 52 ++++ doc/backlog/R240-rdma-datapath-stream.md | 3 + doc/design/diskio/design-crowdb-diskio.md | 19 ++ doc/design/rpc/design-crowdb-rpc-rdma.md | 25 +- doc/working/plan-rdma-datapath-stream.md | 84 +++++- lib/crowdb-diskio-client/src/semantic.rs | 25 +- .../src/semantic/connection.rs | 18 +- .../src/semantic/write.rs | 3 + .../tests/rdma_failure_test.rs | 155 ++++++++++ lib/crowdb-rpc/ffi/src/server.rs | 7 + lib/crowdb-rpc/ffi/src/sys.rs | 1 + .../ffi/tests/transport_config_test.rs | 3 + lib/crowdb-rpc/include/crowdb-rpc/c_api.h | 2 + .../include/crowdb-rpc/server/server.h | 11 +- lib/crowdb-rpc/src/c_api/transport.cpp | 27 +- lib/crowdb-rpc/src/server/server.cpp | 13 +- 27 files changed, 1459 insertions(+), 256 deletions(-) create mode 100644 app/crowdb-diskio/src/rpc/dio_fsync.cpp create mode 100644 app/crowdb-diskio/src/rpc/dio_read.cpp create mode 100644 app/crowdb-diskio/src/rpc/dio_read_buffer.h create mode 100644 app/crowdb-diskio/src/rpc/dio_request.h create mode 100644 app/crowdb-diskio/tests/common/test_dio_hardware.h create mode 100644 app/crowdb-diskio/tests/dio_hardware_test.cpp create mode 100644 app/crowdb-diskio/tests/dio_lifetime_test.cpp create mode 100644 app/crowdb-diskio/tests/dio_tcp_test.cpp create mode 100644 lib/crowdb-diskio-client/tests/rdma_failure_test.rs diff --git a/app/crowdb-diskio/src/dio_main.cpp b/app/crowdb-diskio/src/dio_main.cpp index 6fbdff432..61b9587de 100644 --- a/app/crowdb-diskio/src/dio_main.cpp +++ b/app/crowdb-diskio/src/dio_main.cpp @@ -217,8 +217,8 @@ int main(int argc, char *argv[]) disk_set->size()); std::fflush(stdout); - auto *transport = server.transport(); - auto dio_server = + auto transport = server.shared_transport(); + auto dio_server = std::make_unique(disk_set, transport, cfg.max_write_request_age_ms, cfg.max_clock_skew_ms); dio_server->register_handlers(server); diff --git a/app/crowdb-diskio/src/rpc/dio_fsync.cpp b/app/crowdb-diskio/src/rpc/dio_fsync.cpp new file mode 100644 index 000000000..17986608c --- /dev/null +++ b/app/crowdb-diskio/src/rpc/dio_fsync.cpp @@ -0,0 +1,59 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-common/metrics/metrics.h" +#include "rpc/dio_request.h" +#include "rpc/dio_server.h" + +#include + +#include +#include +#include +#include +#include + +namespace crowdb::diskio +{ +namespace dproto = crowdb::diskio::proto; +namespace rproto = crowdb::rpc::proto; + +crowdb::rpc::OutFrame *DiskioServer::handle_fsync(crowdb::rpc::Frame *request, crowdb::rpc::Connection *conn) +{ + uint64_t req_id = request->request_id; + uint64_t create_nano = request->rpc_create_nano; + uint16_t msg_type = static_cast(rproto::FBMsgType_EDiskFsyncResponse); + + auto *fb_req = decode_disk_request(*request); + if (fb_req == nullptr) { + delete request; + send_error_response(conn, req_id, create_nano, msg_type, static_cast(dproto::FBDiskIoRetCode_IoError)); + return nullptr; + } + DiskId did = parse_disk_id(fb_req->disk_id()); + + auto disk = disk_set_->find_disk(did); + if (disk == nullptr) { + delete request; + send_error_response(conn, req_id, create_nano, msg_type, + static_cast(dproto::FBDiskIoRetCode_DiskNotExist)); + return nullptr; + } + + delete request; + + Disk *disk_ptr = disk.get(); + auto started = std::chrono::steady_clock::now(); + disk_ptr->engine()->submit_fsync(disk_ptr, [transport = transport_, connection = conn->weak_from_this().lock(), + latency = fsync_latency_, req_id, create_nano, msg_type, started, + disk = std::move(disk)](int res) { + latency->observe(elapsed_nanos(started)); + int16_t ret_code = (res < 0) ? static_cast(dproto::FBDiskIoRetCode_IoError) + : static_cast(dproto::FBDiskIoRetCode_Success); + send_response(transport, connection, req_id, create_nano, msg_type, ret_code); + }); + + return nullptr; +} + +} // namespace crowdb::diskio diff --git a/app/crowdb-diskio/src/rpc/dio_read.cpp b/app/crowdb-diskio/src/rpc/dio_read.cpp new file mode 100644 index 000000000..88f7698d5 --- /dev/null +++ b/app/crowdb-diskio/src/rpc/dio_read.cpp @@ -0,0 +1,110 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-common/metrics/metrics.h" +#include "rpc/dio_read_buffer.h" +#include "rpc/dio_request.h" +#include "rpc/dio_server.h" + +#include + +#include +#include +#include +#include +#include + +namespace crowdb::diskio +{ +namespace dproto = crowdb::diskio::proto; +namespace rproto = crowdb::rpc::proto; + +crowdb::rpc::OutFrame *DiskioServer::handle_read(crowdb::rpc::Frame *request, crowdb::rpc::Connection *conn) +{ + uint64_t req_id = request->request_id; + uint64_t create_nano = request->rpc_create_nano; + uint16_t msg_type = static_cast(rproto::FBMsgType_EDiskReadResponse); + + auto *fb_req = decode_disk_request(*request); + if (fb_req == nullptr || (request->data_buf != nullptr && request->data_buf->len != 0)) { + delete request; + send_error_response(conn, req_id, create_nano, msg_type, static_cast(dproto::FBDiskIoRetCode_IoError)); + return nullptr; + } + DiskId did = parse_disk_id(fb_req->disk_id()); + uint32_t zone_index = fb_req->zone_index(); + uint64_t zone_offset = fb_req->zone_offset(); + uint32_t size = fb_req->size(); + uint64_t test_pattern_offset = fb_req->test_pattern_offset(); + + auto disk = disk_set_->find_disk(did); + if (disk == nullptr) { + delete request; + send_error_response(conn, req_id, create_nano, msg_type, + static_cast(dproto::FBDiskIoRetCode_DiskNotExist)); + return nullptr; + } + + Zone *zone = disk->find_zone(zone_index); + if (zone == nullptr) { + delete request; + send_error_response(conn, req_id, create_nano, msg_type, + static_cast(dproto::FBDiskIoRetCode_ZoneNotExist)); + return nullptr; + } + if (!valid_disk_range(*zone, zone_offset, size)) { + delete request; + send_error_response(conn, req_id, create_nano, msg_type, static_cast(dproto::FBDiskIoRetCode_IoError)); + return nullptr; + } + off_t phys_offset = zone->base_offset + static_cast(zone_offset); + + const bool registered = conn->path() == crowdb::rpc::ConnectionPath::Rdma; + auto *root = request->data_buf == nullptr ? nullptr : request->data_buf->allocation_owner; + const bool receive_owned = + root != nullptr && root->type == crowdb::rpc::BufferType::Registered && root->registration_handle != 0 && + root->last_release_cb != nullptr && + root->ownership.load(std::memory_order_acquire) == crowdb::rpc::BufferOwnership::CallerOwned; + // Control has been decoded, and READ carries no input payload. Retain its + // existing receive data region until disk and response SEND both finish. + auto *read_buf = registered ? (receive_owned ? root->slice(0, size) : nullptr) : conn->pool()->alloc(size); + if (read_buf == nullptr) { + delete request; + send_error_response(conn, req_id, create_nano, msg_type, static_cast(dproto::FBDiskIoRetCode_IoError)); + return nullptr; + } + + DiskReadBuffer io{.buffer = read_buf, .size = size, .offset = phys_offset, .io_size = size, .aligned = {}}; + const auto aligned = io.prepare_alignment(*disk, *zone); + delete request; + if (aligned != IoRetCode::Success) { + read_buf->release(); + send_error_response(conn, req_id, create_nano, msg_type, static_cast(aligned)); + return nullptr; + } + if (!io.begin()) { + conn->close(); + return nullptr; + } + Disk *disk_ptr = disk.get(); + auto started = std::chrono::steady_clock::now(); + auto *io_data = io.data(); + disk_ptr->engine()->submit_read(disk_ptr, io.offset, io_data, io.io_size, test_pattern_offset, + [transport = transport_, connection = conn->weak_from_this().lock(), + latency = read_latency_, req_id, create_nano, msg_type, io = std::move(io), + disk = std::move(disk), started](int result) mutable { + latency->observe(elapsed_nanos(started)); + const auto completed = io.finish(result); + if (!completed.valid_owner) { + if (connection != nullptr) + connection->close(); + return; + } + send_response(transport, connection, req_id, create_nano, msg_type, + static_cast(completed.code), completed.data); + }); + + return nullptr; +} + +} // namespace crowdb::diskio diff --git a/app/crowdb-diskio/src/rpc/dio_read_buffer.h b/app/crowdb-diskio/src/rpc/dio_read_buffer.h new file mode 100644 index 000000000..71131a074 --- /dev/null +++ b/app/crowdb-diskio/src/rpc/dio_read_buffer.h @@ -0,0 +1,91 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/buffer.h" +#include "disk/disk.h" + +#include +#include +#include +#include + +namespace crowdb::diskio +{ +// One disk-read destination, including alignment-edge staging when required. +// Its asynchronous callback must finish ownership before publishing/releasing. +struct DiskReadBuffer +{ + crowdb::rpc::Buffer *buffer; + uint32_t size; + off_t offset; + size_t io_size; + size_t prefix = 0; + std::shared_ptr aligned; + crowdb::rpc::BufferOwnership previous_owner = crowdb::rpc::BufferOwnership::Available; + + IoRetCode prepare_alignment(const Disk &disk, const Zone &zone) + { + if (!disk.is_o_direct()) + return IoRetCode::Success; + const size_t alignment = std::max(4096, disk.block_size()); + if ((alignment & (alignment - 1)) != 0 || alignment > static_cast(std::numeric_limits::max()) || + static_cast(offset) + size > static_cast(std::numeric_limits::max()) - alignment) + return IoRetCode::InvalidAlignment; + prefix = static_cast(offset) % alignment; + offset -= static_cast(prefix); + io_size = ((prefix + size + alignment - 1) / alignment) * alignment; + if (offset < zone.base_offset || io_size > static_cast(zone.capacity - (offset - zone.base_offset))) + return IoRetCode::InvalidAlignment; + if (prefix != 0 || io_size != size || reinterpret_cast(buffer->payload_data()) % alignment != 0) { + aligned = + std::shared_ptr(static_cast(std::aligned_alloc(alignment, io_size)), std::free); + if (aligned == nullptr) + return IoRetCode::IoError; + } + return IoRetCode::Success; + } + + bool begin() + { + using crowdb::rpc::BufferOwnership; + previous_owner = buffer->ownership.load(std::memory_order_acquire); + if ((previous_owner == BufferOwnership::Available || previous_owner == BufferOwnership::CallerOwned) && + buffer->transition(previous_owner, BufferOwnership::DiskOwned)) + return true; + (void)buffer->quarantine(); + buffer->release(); + return false; + } + + uint8_t *data() const + { + return aligned ? aligned.get() : buffer->payload_data(); + } + + struct Result + { + IoRetCode code; + crowdb::rpc::Buffer *data; + bool valid_owner; + }; + + Result finish(int result) + { + if (!buffer->transition(crowdb::rpc::BufferOwnership::DiskOwned, previous_owner)) { + (void)buffer->quarantine(); + buffer->release(); + return {IoRetCode::IoError, nullptr, false}; + } + if (result < 0 || static_cast(result) < io_size) { + buffer->release(); + return {result < 0 ? IoRetCode::IoError : IoRetCode::PartialWrite, nullptr, true}; + } + if (aligned) + std::memcpy(buffer->payload_data(), aligned.get() + prefix, size); + buffer->len = size; + return {IoRetCode::Success, buffer, true}; + } +}; +} // namespace crowdb::diskio diff --git a/app/crowdb-diskio/src/rpc/dio_request.h b/app/crowdb-diskio/src/rpc/dio_request.h new file mode 100644 index 000000000..e40d6437d --- /dev/null +++ b/app/crowdb-diskio/src/rpc/dio_request.h @@ -0,0 +1,51 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/framing.h" +#include "disk/disk.h" + +#include +#include + +#include +#include + +namespace crowdb::diskio +{ +template const Message *decode_disk_request(const crowdb::rpc::Frame &request) +{ + if (request.control.size() < sizeof(flatbuffers::uoffset_t)) + return nullptr; + flatbuffers::Verifier verifier(request.control.data(), request.control.size()); + return verifier.VerifyBuffer(nullptr) ? flatbuffers::GetRoot(request.control.data()) : nullptr; +} + +inline DiskId parse_disk_id(const crowdb::rpc::proto::FBInt128 *id) +{ + return id == nullptr ? DiskId{0, 0} : DiskId{id->high(), id->low()}; +} + +inline bool valid_disk_range(const Zone &zone, uint64_t offset, uint32_t size) +{ + if (zone.base_offset < 0 || zone.capacity < 0 || offset > static_cast(zone.capacity) || + size > static_cast(zone.capacity) - offset) + return false; + const auto remaining = static_cast(std::numeric_limits::max() - zone.base_offset); + return offset <= remaining && size <= remaining - offset; +} + +inline uint64_t elapsed_nanos(std::chrono::steady_clock::time_point started) +{ + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()); +} + +inline uint64_t unix_time_ms() +{ + return static_cast( + std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) + .count()); +} +} // namespace crowdb::diskio diff --git a/app/crowdb-diskio/src/rpc/dio_server.cpp b/app/crowdb-diskio/src/rpc/dio_server.cpp index 830d43bdd..0cf07e166 100644 --- a/app/crowdb-diskio/src/rpc/dio_server.cpp +++ b/app/crowdb-diskio/src/rpc/dio_server.cpp @@ -7,6 +7,7 @@ #include "crowdb-rpc/server/message.h" #include "crowdb-rpc/server/server.h" #include "disk/disk.h" +#include "rpc/dio_request.h" #include #include @@ -50,18 +51,12 @@ crowdb::common::metrics::LatencyHistogram &engine_fsync_latency_metric() return *metric; } -uint64_t elapsed_nanos(std::chrono::steady_clock::time_point started) -{ - return static_cast( - std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()); -} - } // namespace -DiskioServer::DiskioServer(std::shared_ptr disk_set, crowdb::rpc::Transport *transport, +DiskioServer::DiskioServer(std::shared_ptr disk_set, std::shared_ptr transport, uint64_t max_write_request_age_ms, uint64_t max_clock_skew_ms) : disk_set_(std::move(disk_set)), - transport_(transport), + transport_(std::move(transport)), aligned_writer_(), read_latency_(&engine_read_latency_metric()), write_latency_(&engine_write_latency_metric()), @@ -95,193 +90,41 @@ crowdb::rpc::Buffer *DiskioServer::build_response_ctrl(crowdb::rpc::BufferPool * if (buf == nullptr) { return nullptr; } - std::memcpy(buf->data, fbb.GetBufferPointer(), size); - buf->write(buf->data, size); + buf->write(fbb.GetBufferPointer(), size); return buf; } void DiskioServer::send_error_response(crowdb::rpc::Connection *conn, uint64_t request_id, uint64_t rpc_create_nano, uint16_t msg_type, int16_t ret_code) { - auto *pool = conn->pool(); - auto *ctrl = build_response_ctrl(pool, request_id, rpc_create_nano, ret_code, msg_type); - auto *out = crowdb::rpc::build_out_frame(request_id, msg_type, ctrl, nullptr); - out->create_nano = static_cast(std::chrono::steady_clock::now().time_since_epoch().count()); - transport_->submit(conn, out); -} - -static DiskId parse_disk_id(const crowdb::rpc::proto::FBInt128 *fb_id) -{ - if (fb_id == nullptr) { - return {0, 0}; - } - return {fb_id->high(), fb_id->low()}; -} - -crowdb::rpc::OutFrame *DiskioServer::handle_read(crowdb::rpc::Frame *request, crowdb::rpc::Connection *conn) -{ - uint64_t req_id = request->request_id; - uint64_t create_nano = request->rpc_create_nano; - uint16_t msg_type = static_cast(rproto::FBMsgType_EDiskReadResponse); - - auto *fb_req = ::flatbuffers::GetRoot(request->control.data()); - if (fb_req == nullptr || request->control.size() < 4) { - delete request; - send_error_response(conn, req_id, create_nano, msg_type, static_cast(dproto::FBDiskIoRetCode_IoError)); - return nullptr; - } - DiskId did = parse_disk_id(fb_req->disk_id()); - uint32_t zone_index = fb_req->zone_index(); - uint64_t zone_offset = fb_req->zone_offset(); - uint32_t size = fb_req->size(); - uint64_t test_pattern_offset = fb_req->test_pattern_offset(); - - auto disk = disk_set_->find_disk(did); - if (disk == nullptr) { - delete request; - send_error_response(conn, req_id, create_nano, msg_type, - static_cast(dproto::FBDiskIoRetCode_DiskNotExist)); - return nullptr; - } - - Zone *zone = disk->find_zone(zone_index); - if (zone == nullptr) { - delete request; - send_error_response(conn, req_id, create_nano, msg_type, - static_cast(dproto::FBDiskIoRetCode_ZoneNotExist)); - return nullptr; - } - if (zone->base_offset < 0 || zone->capacity < 0 || zone_offset > static_cast(zone->capacity) || - size > static_cast(zone->capacity) - zone_offset || - zone_offset > static_cast(std::numeric_limits::max() - zone->base_offset)) { - delete request; - send_error_response(conn, req_id, create_nano, msg_type, static_cast(dproto::FBDiskIoRetCode_IoError)); - return nullptr; - } - off_t phys_offset = zone->base_offset + static_cast(zone_offset); - - auto *pool = conn->pool(); - auto *read_buf = pool->alloc(size); - if (read_buf == nullptr) { - delete request; - send_error_response(conn, req_id, create_nano, msg_type, static_cast(dproto::FBDiskIoRetCode_IoError)); - return nullptr; - } - - auto io_offset = phys_offset; - size_t io_size = size; - size_t prefix = 0; - std::shared_ptr aligned_data; - if (disk->is_o_direct()) { - const size_t alignment = std::max(4096, disk->block_size()); - if ((alignment & (alignment - 1)) != 0 || alignment > static_cast(std::numeric_limits::max()) || - static_cast(phys_offset) + size > - static_cast(std::numeric_limits::max()) - alignment) { - read_buf->release(); - delete request; - send_error_response(conn, req_id, create_nano, msg_type, - static_cast(dproto::FBDiskIoRetCode_InvalidAlignment)); - return nullptr; - } - prefix = static_cast(phys_offset) % alignment; - io_offset = phys_offset - static_cast(prefix); - io_size = ((prefix + size + alignment - 1) / alignment) * alignment; - if (io_offset < zone->base_offset || - io_size > static_cast(zone->capacity - (io_offset - zone->base_offset))) { - read_buf->release(); - delete request; - send_error_response(conn, req_id, create_nano, msg_type, - static_cast(dproto::FBDiskIoRetCode_InvalidAlignment)); - return nullptr; - } - aligned_data = std::shared_ptr(static_cast(std::aligned_alloc(alignment, io_size)), - [](uint8_t *data) { std::free(data); }); - if (aligned_data == nullptr) { - read_buf->release(); - delete request; - send_error_response(conn, req_id, create_nano, msg_type, - static_cast(dproto::FBDiskIoRetCode_IoError)); - return nullptr; - } - } - - delete request; - - Disk *disk_ptr = disk.get(); - auto started = std::chrono::steady_clock::now(); - auto *io_data = aligned_data ? aligned_data.get() : read_buf->data; - disk_ptr->engine()->submit_read(disk_ptr, io_offset, io_data, io_size, test_pattern_offset, - [this, conn, req_id, create_nano, msg_type, read_buf, size, io_size, prefix, - aligned_data = std::move(aligned_data), disk = std::move(disk), started](int res) { - read_latency_->observe(elapsed_nanos(started)); - int16_t ret_code = static_cast(dproto::FBDiskIoRetCode_Success); - crowdb::rpc::Buffer *data = nullptr; - if (res < 0) { - ret_code = static_cast(dproto::FBDiskIoRetCode_IoError); - } - else if (static_cast(res) < io_size) { - ret_code = static_cast(dproto::FBDiskIoRetCode_PartialWrite); - } - else { - if (aligned_data) { - std::memcpy(read_buf->data, aligned_data.get() + prefix, size); - } - read_buf->len = size; - data = read_buf; - } - auto *pool = conn->pool(); - auto *ctrl = build_response_ctrl(pool, req_id, create_nano, ret_code, msg_type); - auto *out = crowdb::rpc::build_out_frame(req_id, msg_type, ctrl, data); - out->create_nano = static_cast( - std::chrono::steady_clock::now().time_since_epoch().count()); - transport_->submit(conn, out); - if (data == nullptr) { - read_buf->release(); - } - }); - - return nullptr; + send_response(transport_, conn->weak_from_this().lock(), request_id, rpc_create_nano, msg_type, ret_code); } -crowdb::rpc::OutFrame *DiskioServer::handle_fsync(crowdb::rpc::Frame *request, crowdb::rpc::Connection *conn) +void DiskioServer::send_response(const std::shared_ptr &transport, + const std::shared_ptr &connection, uint64_t request_id, + uint64_t created, uint16_t type, int16_t code, crowdb::rpc::Buffer *data, + bool sync_completed) { - uint64_t req_id = request->request_id; - uint64_t create_nano = request->rpc_create_nano; - uint16_t msg_type = static_cast(rproto::FBMsgType_EDiskFsyncResponse); - - auto *fb_req = ::flatbuffers::GetRoot(request->control.data()); - if (fb_req == nullptr || request->control.size() < 4) { - delete request; - send_error_response(conn, req_id, create_nano, msg_type, static_cast(dproto::FBDiskIoRetCode_IoError)); - return nullptr; + const auto release = [](crowdb::rpc::Buffer *buffer) { + if (buffer != nullptr) + buffer->release(); + }; + std::unique_ptr payload(data, release); + if (connection == nullptr || !connection->is_open()) + return; + auto *control = build_response_ctrl(connection->pool(), request_id, created, code, type, sync_completed); + if (control == nullptr) { + connection->close(); + return; } - DiskId did = parse_disk_id(fb_req->disk_id()); - - auto disk = disk_set_->find_disk(did); - if (disk == nullptr) { - delete request; - send_error_response(conn, req_id, create_nano, msg_type, - static_cast(dproto::FBDiskIoRetCode_DiskNotExist)); - return nullptr; + auto *frame = crowdb::rpc::build_out_frame(request_id, type, control, payload.release()); + frame->create_nano = static_cast(std::chrono::steady_clock::now().time_since_epoch().count()); + if (!transport->submit(connection.get(), frame)) { + control->release(); + frame->release_data(); + delete frame; + connection->close(); } - - delete request; - - Disk *disk_ptr = disk.get(); - auto started = std::chrono::steady_clock::now(); - disk_ptr->engine()->submit_fsync( - disk_ptr, [this, conn, req_id, create_nano, msg_type, started, disk = std::move(disk)](int res) { - fsync_latency_->observe(elapsed_nanos(started)); - int16_t ret_code = (res < 0) ? static_cast(dproto::FBDiskIoRetCode_IoError) - : static_cast(dproto::FBDiskIoRetCode_Success); - auto *pool = conn->pool(); - auto *ctrl = build_response_ctrl(pool, req_id, create_nano, ret_code, msg_type); - auto *out = crowdb::rpc::build_out_frame(req_id, msg_type, ctrl, nullptr); - out->create_nano = static_cast(std::chrono::steady_clock::now().time_since_epoch().count()); - transport_->submit(conn, out); - }); - - return nullptr; } void DiskioServer::register_handlers(crowdb::rpc::RpcServer &server) diff --git a/app/crowdb-diskio/src/rpc/dio_server.h b/app/crowdb-diskio/src/rpc/dio_server.h index 5eaedb7e0..90a0b447f 100644 --- a/app/crowdb-diskio/src/rpc/dio_server.h +++ b/app/crowdb-diskio/src/rpc/dio_server.h @@ -39,7 +39,7 @@ namespace crowdb::diskio class DiskioServer { public: - DiskioServer(std::shared_ptr disk_set, crowdb::rpc::Transport *transport, + DiskioServer(std::shared_ptr disk_set, std::shared_ptr transport, uint64_t max_write_request_age_ms = protocol::kMaxWriteRequestAgeMs, uint64_t max_clock_skew_ms = protocol::kMaxWriteClockSkewMs); @@ -55,25 +55,30 @@ class DiskioServer void register_handlers(crowdb::rpc::RpcServer &server); private: - std::shared_ptr disk_set_; - crowdb::rpc::Transport *transport_; - AlignedWriter aligned_writer_; - crowdb::common::metrics::LatencyHistogram *read_latency_; - crowdb::common::metrics::LatencyHistogram *write_latency_; - crowdb::common::metrics::LatencyHistogram *fsync_latency_; - uint64_t max_write_request_age_ms_; - uint64_t max_clock_skew_ms_; + std::shared_ptr disk_set_; + std::shared_ptr transport_; + AlignedWriter aligned_writer_; + crowdb::common::metrics::LatencyHistogram *read_latency_; + crowdb::common::metrics::LatencyHistogram *write_latency_; + crowdb::common::metrics::LatencyHistogram *fsync_latency_; + uint64_t max_write_request_age_ms_; + uint64_t max_clock_skew_ms_; // Build a response control buffer for a diskio response msg_type. // ret_code is a proto::FBDiskIoRetCode value (int16_t to avoid // pulling the generated header into this .h). - crowdb::rpc::Buffer *build_response_ctrl(crowdb::rpc::BufferPool *pool, uint64_t request_id, - uint64_t rpc_create_nano, int16_t ret_code, uint16_t msg_type, - bool sync_completed = false); + static crowdb::rpc::Buffer *build_response_ctrl(crowdb::rpc::BufferPool *pool, uint64_t request_id, + uint64_t rpc_create_nano, int16_t ret_code, uint16_t msg_type, + bool sync_completed = false); // Send an error response synchronously (disk not found, parse error). void send_error_response(crowdb::rpc::Connection *conn, uint64_t request_id, uint64_t rpc_create_nano, uint16_t msg_type, int16_t ret_code); + + static void send_response(const std::shared_ptr &transport, + const std::shared_ptr &connection, uint64_t request_id, + uint64_t created, uint16_t type, int16_t code, crowdb::rpc::Buffer *data = nullptr, + bool sync_completed = false); }; } // namespace crowdb::diskio diff --git a/app/crowdb-diskio/src/rpc/dio_server_write.cpp b/app/crowdb-diskio/src/rpc/dio_server_write.cpp index 01f135c4a..1b6d8e9d1 100644 --- a/app/crowdb-diskio/src/rpc/dio_server_write.cpp +++ b/app/crowdb-diskio/src/rpc/dio_server_write.cpp @@ -125,30 +125,36 @@ crowdb::rpc::OutFrame *DiskioServer::handle_write(crowdb::rpc::Frame *request, c send_error_response(conn, req_id, create_nano, msg_type, static_cast(dproto::FBDiskIoRetCode_IoError)); return nullptr; } - auto data = std::shared_ptr(data_buf, [](crowdb::rpc::Buffer *buffer) { + using crowdb::rpc::BufferOwnership; + const auto previous_owner = data_buf == nullptr ? BufferOwnership::Available + : data_buf->ownership.load(std::memory_order_acquire); + if (data_buf != nullptr && + ((previous_owner != BufferOwnership::CallerOwned && previous_owner != BufferOwnership::Available) || + !data_buf->transition(previous_owner, BufferOwnership::DiskOwned))) { + (void)data_buf->quarantine(); + data_buf->release(); + conn->close(); + return nullptr; + } + auto data = std::shared_ptr(data_buf, [previous_owner](crowdb::rpc::Buffer *buffer) { if (buffer != nullptr) { + if (!buffer->transition(BufferOwnership::DiskOwned, previous_owner)) { + (void)buffer->quarantine(); + } buffer->release(); } }); - auto complete = [this, connection = conn->shared_from_this(), data, req_id, create_nano, + auto complete = [transport = transport_, connection = conn->shared_from_this(), data, req_id, create_nano, msg_type](int16_t code, bool synchronized) { - auto *control = build_response_ctrl(connection->pool(), req_id, create_nano, code, msg_type, synchronized); - if (control == nullptr) { - return; - } - auto *out = crowdb::rpc::build_out_frame(req_id, msg_type, control, nullptr); - out->create_nano = static_cast(std::chrono::steady_clock::now().time_since_epoch().count()); - if (!transport_->submit(connection.get(), out)) { - control->release(); - delete out; - } + send_response(transport, connection, req_id, create_nano, msg_type, code, nullptr, synchronized); }; uint64_t ordering_phys_offset = zone->base_offset + ordering_zone_offset; auto started = std::chrono::steady_clock::now(); aligned_writer_.submit_ordered( - disk, phys_offset, data_buf != nullptr ? data_buf->data : nullptr, size, ordering_phys_offset, - [this, disk, complete = std::move(complete), sync, size, started](int res) mutable { - write_latency_->observe(elapsed_nanos(started)); + disk, phys_offset, data_buf != nullptr ? data_buf->payload_data() : nullptr, size, ordering_phys_offset, + [latency = write_latency_, flush_latency = fsync_latency_, disk, complete = std::move(complete), + sync, size, started](int res) mutable { + latency->observe(elapsed_nanos(started)); if (res == -ETIMEDOUT) { complete(static_cast(dproto::FBDiskIoRetCode_OldRequest), false); } @@ -164,8 +170,8 @@ crowdb::rpc::OutFrame *DiskioServer::handle_write(crowdb::rpc::Frame *request, c else { auto flush_started = std::chrono::steady_clock::now(); disk->engine()->submit_fsync( - disk.get(), [this, disk, complete = std::move(complete), flush_started](int result) mutable { - fsync_latency_->observe(elapsed_nanos(flush_started)); + disk.get(), [flush_latency, disk, complete = std::move(complete), flush_started](int result) mutable { + flush_latency->observe(elapsed_nanos(flush_started)); complete(static_cast(result < 0 ? dproto::FBDiskIoRetCode_FsyncFailed : dproto::FBDiskIoRetCode_Success), result >= 0); diff --git a/app/crowdb-diskio/tests/common/test_dio_hardware.h b/app/crowdb-diskio/tests/common/test_dio_hardware.h new file mode 100644 index 000000000..ffde27412 --- /dev/null +++ b/app/crowdb-diskio/tests/common/test_dio_hardware.h @@ -0,0 +1,266 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-common/runtime_path.h" +#include "crowdb-rpc/client/client.h" +#include "disk/block_disk.h" +#include "engine/uring/uring_engine.h" +#include "rpc/dio_server.h" + +#include +#include +#include + +#if defined(CROWDB_HAVE_LIBURING) +# include +# include + +# include +# include +# include +# include + +namespace crowdb::diskio +{ +class TestObservedEngine final : public IoEngine +{ + public: + struct ParkedWrite + { + Disk *disk; + off_t offset; + const uint8_t *data; + size_t size; + std::function done; + }; + + UringEngine io{32}; + std::atomic write_pointer{0}, read_pointer{0}; + std::atomic reads{0}; + std::atomic hold_writes{false}; + std::atomic parked{nullptr}; + std::shared_ptr> completed_writes = std::make_shared>(0); + + void submit_write(Disk *disk, off_t offset, const uint8_t *data, size_t size, + std::function done) override + { + write_pointer.store(reinterpret_cast(data)); + ParkedWrite write{disk, offset, data, size, std::move(done)}; + if (hold_writes.load(std::memory_order_acquire)) { + parked.store(new ParkedWrite(std::move(write)), std::memory_order_release); + return; + } + start_write(std::move(write)); + } + + void submit_read(Disk *disk, off_t offset, uint8_t *data, size_t size, uint64_t pattern, + std::function done) override + { + read_pointer.store(reinterpret_cast(data)); + reads.fetch_add(1); + io.submit_read(disk, offset, data, size, pattern, std::move(done)); + } + + void submit_fsync(Disk *disk, std::function done) override + { + io.submit_fsync(disk, std::move(done)); + } + + void resume_write() + { + auto pending = std::unique_ptr(parked.exchange(nullptr, std::memory_order_acq_rel)); + if (pending != nullptr) + start_write(std::move(*pending)); + } + + private: + void start_write(ParkedWrite write) + { + io.submit_write(write.disk, write.offset, write.data, write.size, + [done = std::move(write.done), completed = completed_writes](int result) { + done(result); + completed->fetch_add(1, std::memory_order_release); + }); + } +}; + +struct TestDiskCall +{ + std::atomic callbacks{0}; + crowdb_rpc_status status = CROWDB_RPC_ERR_CONN_ERROR; + crowdb_rpc_buffer_t control = nullptr, data = nullptr; + + ~TestDiskCall() + { + release_data(); + if (control != nullptr) + crowdb_rpc_buffer_release(control); + } + + void release_data() + { + if (data != nullptr) + crowdb_rpc_buffer_release(std::exchange(data, nullptr)); + } + + static void complete(uint64_t, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, crowdb_rpc_status status, + void *context) + { + const auto owner = + std::unique_ptr>(static_cast *>(context)); + auto &call = **owner; + call.control = control; + call.data = data; + call.status = status; + call.callbacks.fetch_add(1, std::memory_order_release); + } +}; + +class TestDioHardware : public ::testing::Test +{ + protected: + bool use_rdma = true; + bool allow_tcp_fallback = false; + static constexpr uint32_t BLOCK = 1 << 20; + std::shared_ptr engine; + std::shared_ptr disks; + std::unique_ptr server, client; + std::unique_ptr service; + std::shared_ptr peer; + crowdb::rpc::RpcClient rpc; + std::atomic receive_pointer{0}; + std::atomic read_receive_pointer{0}; + std::filesystem::path path; + + void SetUp() override + { + const auto *address = use_rdma ? std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR") : "127.0.0.1"; + if (use_rdma && (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr)) + GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; +# ifndef CROWDB_RPC_HAVE_RDMA + if (use_rdma) + GTEST_SKIP() << "RDMA support not compiled"; +# endif + path = crowdb::common::test_runtime_path("diskio-rdma") / "block.bin"; + const auto fd = ::open(path.c_str(), O_CREAT | O_TRUNC | O_RDWR, 0600); + ASSERT_GE(fd, 0); + const auto sized = ::ftruncate(fd, 4 * BLOCK); + ::close(fd); + ASSERT_EQ(sized, 0); + engine = std::make_shared(); + auto disk = std::make_shared( + DiskId{ + 7, 9 + }, + path.string(), engine, std::vector{{.zone_index = 0, .base_offset = 0, .capacity = 4 * BLOCK}}, true); + ASSERT_GE(disk->fd(), 0); + ASSERT_TRUE(disk->is_o_direct()); + disks = std::make_shared(); + disks->add(std::move(disk)); + crowdb::rpc::RdmaConfig config = + crowdb::rpc::RdmaConfig::for_layout(crowdb::rpc::RdmaWireLayout::ControlData, BLOCK); + config.use_environment = false; + config.sq_depth = config.rq_depth = config.posted_receive_target = 4; + config.cq_depth = 8; + config.send_arena_bytes = 4 * BLOCK; + config.receive_arena_bytes = uint64_t{4} * (BLOCK + 1024); + config.source_address = address; + if (const auto *device = std::getenv("CROWDB_RDMA_DEVICE")) + config.device = device; + server = std::make_unique(); + ASSERT_TRUE(server->configure_rdma(config, use_rdma, allow_tcp_fallback, use_rdma)); + ASSERT_TRUE(server->listen(address, 0)); + service = std::make_unique(disks, server->shared_transport()); + service->register_handlers(*server); + const auto write_type = static_cast(crowdb::rpc::proto::FBMsgType_EDiskWriteRequest); + server->register_handler(write_type, [this](crowdb::rpc::Frame *frame, crowdb::rpc::Connection *connection) { + receive_pointer.store(reinterpret_cast(frame->data_buf->payload_data())); + return service->handle_write(frame, connection); + }); + const auto read_type = static_cast(crowdb::rpc::proto::FBMsgType_EDiskReadRequest); + server->register_handler(read_type, [this](crowdb::rpc::Frame *frame, crowdb::rpc::Connection *connection) { + read_receive_pointer.store( + frame->data_buf == nullptr ? 0 : reinterpret_cast(frame->data_buf->payload_data())); + return service->handle_read(frame, connection); + }); + server->start(); + client = std::make_unique(); + ASSERT_TRUE(client->configure_rdma(config, use_rdma, allow_tcp_fallback, false)); + client->start(); + peer = client->transport()->connect(address, server->listen_port()); + ASSERT_NE(peer, nullptr); + ASSERT_EQ(peer->path(), use_rdma ? crowdb::rpc::ConnectionPath::Rdma : crowdb::rpc::ConnectionPath::Tcp); + rpc.attach(peer.get()); + } + + void TearDown() override + { + if (engine != nullptr) + engine->resume_write(); + if (peer != nullptr) { + peer->close(); + rpc.fail_all(peer.get(), crowdb::rpc::RpcError::ConnectionClosed); + } + if (client != nullptr) + client->stop(); + if (server != nullptr) + server->stop(); + service.reset(); + disks.reset(); + engine.reset(); + if (!path.empty()) + std::filesystem::remove(path); + } + + template bool wait_for(Predicate predicate) + { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!predicate() && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + return predicate(); + } + + std::shared_ptr call(uint64_t id, uint32_t index, bool writing) + { + flatbuffers::FlatBufferBuilder builder; + crowdb::rpc::proto::FBInt128 disk(7, 9); + uint16_t type; + crowdb::rpc::Buffer *data = nullptr; + if (writing) { + const auto now = static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); + builder.Finish( + proto::CreateFBDiskWriteRequest(builder, id, 0, &disk, 0, uint64_t{index} * BLOCK, BLOCK, 0, now)); + type = static_cast(crowdb::rpc::proto::FBMsgType_EDiskWriteRequest); + data = client->transport()->allocate_producer_buffer(BLOCK, use_rdma); + if (data == nullptr) + return nullptr; + for (uint32_t byte = 0; byte < BLOCK; ++byte) + data->data[byte] = static_cast(byte * 131U + index); + data->len = BLOCK; + } + else { + builder.Finish(proto::CreateFBDiskReadRequest(builder, id, 0, &disk, 0, uint64_t{index} * BLOCK, BLOCK)); + type = static_cast(crowdb::rpc::proto::FBMsgType_EDiskReadRequest); + } + auto *control = client->pool()->alloc(builder.GetSize()); + if (control == nullptr) { + if (data != nullptr) + data->release(); + return nullptr; + } + control->write(builder.GetBufferPointer(), builder.GetSize()); + auto result = std::make_shared(); + auto *context = new std::shared_ptr(result); + if (!rpc.send(client->transport(), peer.get(), id, control, data, type, TestDiskCall::complete, context)) { + delete context; + return nullptr; + } + return result; + } +}; +} // namespace crowdb::diskio +#endif diff --git a/app/crowdb-diskio/tests/dio_hardware_test.cpp b/app/crowdb-diskio/tests/dio_hardware_test.cpp new file mode 100644 index 000000000..7934ab86d --- /dev/null +++ b/app/crowdb-diskio/tests/dio_hardware_test.cpp @@ -0,0 +1,154 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "common/test_dio_hardware.h" + +#include + +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_HAVE_LIBURING) +namespace crowdb::diskio +{ +namespace +{ +folly::dynamic test_status(const std::string &snapshot) +{ + folly::json::serialization_opts options; + // Capability generations span all uint64_t values. Preserve decimal + // counters exactly instead of forcing the whole snapshot through int64_t. + options.parse_numbers_as_strings = true; + return folly::parseJson(snapshot, options)["rdma"]; +} +} // namespace + +class TestDioFailure : public TestDioHardware +{ + protected: + void SetUp() override + { + allow_tcp_fallback = true; + TestDioHardware::SetUp(); + } +}; + +TEST_F(TestDioHardware, DirectBlockRoundTripKeepsRegisteredPointersAndBoundsHeldResponses) +{ + for (uint32_t index = 0; index < 3; ++index) { + auto written = call(index + 1, index, true); + ASSERT_NE(written, nullptr); + ASSERT_TRUE(wait_for([&] { return written->callbacks.load(std::memory_order_acquire) == 1; })); + ASSERT_EQ(written->status, CROWDB_RPC_OK); + ASSERT_NE(written->control, nullptr); + const auto *response = + flatbuffers::GetRoot(crowdb_rpc_buffer_data(written->control)); + ASSERT_EQ(response->ret_code(), proto::FBDiskIoRetCode_Success); + EXPECT_EQ(engine->write_pointer.load(), receive_pointer.load()); + EXPECT_EQ(engine->write_pointer.load() % 4096, 0U); + } + const auto registered_before = server->transport()->status_json(); + std::vector> reads; + for (uint32_t index = 0; index < 8; ++index) { + auto read = call(index + 100, index % 3, false); + ASSERT_NE(read, nullptr); + reads.push_back(std::move(read)); + } + ASSERT_TRUE(wait_for([&] { return engine->reads.load() == 8; })); + ASSERT_TRUE(wait_for([&] { + return std::count_if(reads.begin(), reads.end(), [](const auto &read) { + return read->callbacks.load(std::memory_order_acquire) == 1; + }) == 4; + })); + EXPECT_EQ(engine->read_pointer.load() % 4096, 0U); + EXPECT_EQ(engine->read_pointer.load(), read_receive_pointer.load()); + const auto before = test_status(registered_before); + const auto held = test_status(server->transport()->status_json()); + EXPECT_EQ(held["registered_bytes"], before["registered_bytes"]); + EXPECT_EQ(held["registered_bytes_high_water"], before["registered_bytes_high_water"]); + // Four caller-held response leases consume all client RQ slots. The next + // four replies retain the server's own receive data regions in SEND. + std::vector released(reads.size(), false); + size_t remaining = reads.size(); + while (remaining != 0) { + size_t index = reads.size(); + ASSERT_TRUE(wait_for([&] { + for (size_t candidate = 0; candidate < reads.size(); ++candidate) { + if (!released[candidate] && reads[candidate]->callbacks.load(std::memory_order_acquire) == 1) { + index = candidate; + return true; + } + } + return false; + })); + auto &read = reads[index]; + ASSERT_EQ(read->status, CROWDB_RPC_OK); + ASSERT_NE(read->control, nullptr); + const auto *response = flatbuffers::GetRoot(crowdb_rpc_buffer_data(read->control)); + ASSERT_EQ(response->ret_code(), proto::FBDiskIoRetCode_Success); + ASSERT_NE(read->data, nullptr); + ASSERT_EQ(crowdb_rpc_buffer_len(read->data), BLOCK); + const auto *bytes = crowdb_rpc_buffer_data(read->data); + std::vector expected(BLOCK); + for (uint32_t byte = 0; byte < BLOCK; ++byte) + expected[byte] = static_cast(byte * 131U + index % 3); + ASSERT_EQ(std::memcmp(bytes, expected.data(), BLOCK), 0); + read->release_data(); + released[index] = true; + --remaining; + } + const auto client_status = client->transport()->status_json(); + const auto server_status = server->transport()->status_json(); + const auto after = test_status(server_status); + EXPECT_EQ(after["registered_bytes"], before["registered_bytes"]); + EXPECT_EQ(after["registered_bytes_high_water"], before["registered_bytes_high_water"]); + for (const auto &status : {client_status, server_status}) { + EXPECT_NE(status.find("\"copied_payload_bytes\":0"), std::string::npos); + EXPECT_NE(status.find("\"fallback_count\":0"), std::string::npos); + EXPECT_NE(status.find("\"receive_post_errors\":0"), std::string::npos); + EXPECT_NE(status.find("\"send_errors\":0"), std::string::npos); + EXPECT_NE(status.find("\"protocol_errors\":0"), std::string::npos); + EXPECT_NE(status.find("\"cq_errors\":0"), std::string::npos); + } + EXPECT_NE(client_status.find("\"direct_payload_bytes\":3145728"), std::string::npos); + EXPECT_NE(server_status.find("\"direct_payload_bytes\":8388608"), std::string::npos); + RecordProperty("client_transport", client_status); + RecordProperty("server_transport_before_reads", registered_before); + RecordProperty("server_transport", server_status); + RecordProperty("topology", "single_process_hca_local_loop"); +} + +TEST_F(TestDioFailure, PeerCloseAndServerDestructionRetainPendingDiskDataWithoutFallback) +{ + engine->hold_writes.store(true, std::memory_order_release); + auto written = call(41, 0, true); + ASSERT_NE(written, nullptr); + ASSERT_TRUE(wait_for([&] { return engine->parked.load(std::memory_order_acquire) != nullptr; })); + peer->close(); + ASSERT_TRUE(wait_for([&] { return written->callbacks.load(std::memory_order_acquire) == 1; })); + EXPECT_EQ(written->status, CROWDB_RPC_ERR_CONN_CLOSED); + std::weak_ptr retained = server->shared_transport(); + server->stop(); + server.reset(); + ASSERT_FALSE(retained.expired()); + const auto *pending = engine->parked.load(std::memory_order_acquire); + ASSERT_NE(pending, nullptr); + EXPECT_EQ(reinterpret_cast(pending->data), receive_pointer.load()); + EXPECT_EQ(pending->data[0], 0U); + EXPECT_EQ(pending->data[BLOCK - 1], static_cast((BLOCK - 1) * 131U)); + engine->resume_write(); + ASSERT_TRUE(wait_for([&] { return engine->completed_writes->load(std::memory_order_acquire) == 1; })); + EXPECT_EQ(written->callbacks.load(), 1U); + EXPECT_NE(client->transport()->status_json().find("\"fallback_count\":0"), std::string::npos); + RecordProperty("client_transport", client->transport()->status_json()); + RecordProperty("expected_connection_errors", 1); + const auto fd = ::open(path.c_str(), O_RDONLY); + ASSERT_GE(fd, 0); + std::vector stored(BLOCK); + const auto bytes = ::pread(fd, stored.data(), stored.size(), 0); + ::close(fd); + ASSERT_EQ(bytes, BLOCK); + for (uint32_t index = 0; index < BLOCK; ++index) + ASSERT_EQ(stored[index], static_cast(index * 131U)); + service.reset(); + EXPECT_TRUE(retained.expired()); +} +} // namespace crowdb::diskio +#endif diff --git a/app/crowdb-diskio/tests/dio_lifetime_test.cpp b/app/crowdb-diskio/tests/dio_lifetime_test.cpp new file mode 100644 index 000000000..6d53adf70 --- /dev/null +++ b/app/crowdb-diskio/tests/dio_lifetime_test.cpp @@ -0,0 +1,252 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rpc/dio_server.h" + +#include +#include + +#include +#include +#include +#include +#include + +namespace +{ +using namespace crowdb::diskio; +using namespace crowdb::rpc; +namespace proto = crowdb::diskio::proto; + +class TestDeferredEngine final : public IoEngine +{ + public: + struct Pending + { + const uint8_t *data; + uint8_t *output; + size_t size; + std::function done; + }; + + std::vector pending; + + void submit_write(Disk *, off_t, const uint8_t *data, size_t size, std::function done) override + { + pending.push_back({data, nullptr, size, std::move(done)}); + } + + void submit_read(Disk *, off_t, uint8_t *data, size_t size, uint64_t, std::function done) override + { + pending.push_back({nullptr, data, size, std::move(done)}); + } + + void submit_fsync(Disk *, std::function done) override + { + pending.push_back({nullptr, nullptr, 0, std::move(done)}); + } + + void complete(int result) + { + ASSERT_FALSE(pending.empty()); + auto io = std::move(pending.front()); + pending.erase(pending.begin()); + if (io.output != nullptr && result > 0) + std::memset(io.output, 0x5a, std::min(io.size, static_cast(result))); + io.done(result); + } +}; + +class TestDeferredDisk final : public Disk +{ + public: + explicit TestDeferredDisk(IoEngine *engine) : engine_(engine) + { + zones_.push_back({.zone_index = 0, .base_offset = 0, .capacity = 8 << 20}); + } + + DiskType type() const override + { + return DiskType::Mem; + } + + int fd() const override + { + return 1; + } + + bool is_o_direct() const override + { + return false; + } + + size_t block_size() const override + { + return 1; + } + + IoEngine *engine() override + { + return engine_; + } + + DiskId id() const override + { + return {7, 9}; + } + + Zone *find_zone(uint32_t index) override + { + return index == 0 ? &zones_[0] : nullptr; + } + + private: + IoEngine *engine_; +}; + +class DioLifetimeTest : public ::testing::Test +{ + protected: + TestDeferredEngine engine; + std::shared_ptr disks = std::make_shared(); + std::unique_ptr server; + std::unique_ptr service; + std::shared_ptr connection; + + void SetUp() override + { + disks->add(std::make_shared(&engine)); + server = std::make_unique(); + ASSERT_TRUE(server->configure_rdma({}, false, false, false)); + connection = std::make_shared(1, "delayed", server->pool()); + // The unit fixture invokes handlers directly and records I/O only. + // Completion must safely discard a reply to this closed connection. + connection->close(); + service = std::make_unique(disks, server->shared_transport()); + } + + Frame *write(Buffer *data, uint64_t offset = 0, uint32_t size = 16, uint64_t ordering = 0) + { + flatbuffers::FlatBufferBuilder builder; + crowdb::rpc::proto::FBInt128 disk(7, 9); + const auto now = static_cast( + std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) + .count()); + auto request = proto::CreateFBDiskWriteRequest(builder, 17, 0, &disk, 0, offset, size, ordering, now); + builder.Finish(request); + auto *frame = new Frame; + frame->request_id = 17; + frame->control.assign(builder.GetBufferPointer(), builder.GetBufferPointer() + builder.GetSize()); + frame->data_buf = data; + return frame; + } + + Frame *read(uint32_t size) + { + flatbuffers::FlatBufferBuilder builder; + crowdb::rpc::proto::FBInt128 disk(7, 9); + builder.Finish(proto::CreateFBDiskReadRequest(builder, 19, 0, &disk, 0, 0, size)); + auto *frame = new Frame; + frame->request_id = 19; + frame->control.assign(builder.GetBufferPointer(), builder.GetBufferPointer() + builder.GetSize()); + return frame; + } +}; + +TEST_F(DioLifetimeTest, DiskOwnsExactPayloadUntilCompletionWithoutReplacementAdmission) +{ + RegisteredBufferPool buffers(4096, 4); + uint32_t releases = 0; + for (uint32_t index = 0; index < 4; ++index) { + auto *data = buffers.alloc(32); + ASSERT_NE(data, nullptr); + data->write("prefix!registered data for disk", 30); + data->view_offset = 7; + data->len = 16; + data->ownership.store(BufferOwnership::CallerOwned); + data->last_release_ctx = &releases; + data->last_release_cb = [](void *context, Buffer *buffer) { + ++*static_cast(context); + EXPECT_TRUE(buffer->transition(BufferOwnership::CallerOwned, BufferOwnership::Available)); + buffer->reset_view(); + buffer->release(); + }; + const auto *payload = data->payload_data(); + EXPECT_EQ(service->handle_write(write(data), connection.get()), nullptr); + ASSERT_EQ(engine.pending.size(), index + 1); + EXPECT_EQ(engine.pending.back().data, payload); + EXPECT_EQ(data->ownership.load(), BufferOwnership::DiskOwned); + EXPECT_EQ(std::memcmp(payload, "registered data ", 16), 0); + } + EXPECT_EQ(releases, 0U); + EXPECT_EQ(buffers.alloc(1), nullptr); + engine.complete(16); + EXPECT_EQ(releases, 1U); + auto *returned = buffers.alloc(1); + ASSERT_NE(returned, nullptr); + EXPECT_EQ(buffers.alloc(1), nullptr); + returned->release(); + engine.complete(8); + engine.complete(-EIO); + engine.complete(-ECANCELED); + EXPECT_EQ(releases, 4U); + EXPECT_EQ(buffers.committed_bytes(), 4096U); + EXPECT_EQ(buffers.quarantined_bytes(), 0U); +} + +TEST_F(DioLifetimeTest, WriteCompletionRetainsTransportAndConnectionAfterBothFacadesDie) +{ + std::weak_ptr transport = server->shared_transport(); + std::weak_ptr peer = connection; + auto *data = server->pool()->alloc(16); + ASSERT_NE(data, nullptr); + data->write("delayed I/O data", 16); + service->handle_write(write(data), connection.get()); + ASSERT_EQ(engine.pending.size(), 1U); + service.reset(); + server.reset(); + connection.reset(); + EXPECT_FALSE(transport.expired()); + EXPECT_FALSE(peer.expired()); + EXPECT_EQ(std::memcmp(engine.pending.front().data, "delayed I/O data", 16), 0); + engine.complete(16); + EXPECT_TRUE(peer.expired()); + EXPECT_TRUE(transport.expired()); +} + +TEST_F(DioLifetimeTest, ReadCompletionRetainsOutputAndTransportAfterBothFacadesDie) +{ + std::weak_ptr transport = server->shared_transport(); + service->handle_read(read(4096), connection.get()); + ASSERT_EQ(engine.pending.size(), 1U); + ASSERT_NE(engine.pending.front().output, nullptr); + service.reset(); + server.reset(); + connection.reset(); + EXPECT_FALSE(transport.expired()); + engine.complete(4096); + EXPECT_TRUE(transport.expired()); +} + +TEST_F(DioLifetimeTest, MalformedControlAndPayloadOrZoneBoundsNeverReachDisk) +{ + auto *malformed = new Frame; + malformed->control.assign(16, 0xff); + service->handle_write(malformed, connection.get()); + auto *short_data = server->pool()->alloc(8); + ASSERT_NE(short_data, nullptr); + short_data->len = 8; + service->handle_write(write(short_data), connection.get()); + for (auto offset : {uint64_t{8 << 20}, UINT64_MAX}) { + auto *data = server->pool()->alloc(16); + ASSERT_NE(data, nullptr); + data->len = 16; + service->handle_write(write(data, offset), connection.get()); + } + auto *data = server->pool()->alloc(16); + ASSERT_NE(data, nullptr); + data->len = 16; + service->handle_write(write(data, 0, 16, 1), connection.get()); + EXPECT_TRUE(engine.pending.empty()); +} +} // namespace diff --git a/app/crowdb-diskio/tests/dio_tcp_test.cpp b/app/crowdb-diskio/tests/dio_tcp_test.cpp new file mode 100644 index 000000000..41d37e16f --- /dev/null +++ b/app/crowdb-diskio/tests/dio_tcp_test.cpp @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "common/test_dio_hardware.h" + +#ifdef CROWDB_HAVE_LIBURING +namespace crowdb::diskio +{ +class TestDioTcp : public TestDioHardware +{ + protected: + void SetUp() override + { + use_rdma = false; + TestDioHardware::SetUp(); + } +}; + +TEST_F(TestDioTcp, DirectBlockCorpusMatchesRdmaBytesWithExplicitTcp) +{ + for (uint32_t index = 0; index < 3; ++index) { + auto written = call(index + 1, index, true); + ASSERT_NE(written, nullptr); + ASSERT_TRUE(wait_for([&] { return written->callbacks.load(std::memory_order_acquire) == 1; })); + ASSERT_EQ(written->status, CROWDB_RPC_OK); + ASSERT_NE(written->control, nullptr); + const auto *write_response = + flatbuffers::GetRoot(crowdb_rpc_buffer_data(written->control)); + ASSERT_EQ(write_response->ret_code(), proto::FBDiskIoRetCode_Success); + auto read = call(index + 100, index, false); + ASSERT_NE(read, nullptr); + ASSERT_TRUE(wait_for([&] { return read->callbacks.load(std::memory_order_acquire) == 1; })); + ASSERT_EQ(read->status, CROWDB_RPC_OK); + ASSERT_NE(read->control, nullptr); + const auto *read_response = + flatbuffers::GetRoot(crowdb_rpc_buffer_data(read->control)); + ASSERT_EQ(read_response->ret_code(), proto::FBDiskIoRetCode_Success); + ASSERT_NE(read->data, nullptr); + ASSERT_EQ(crowdb_rpc_buffer_len(read->data), BLOCK); + std::vector expected(BLOCK); + for (uint32_t byte = 0; byte < BLOCK; ++byte) + expected[byte] = static_cast(byte * 131U + index); + ASSERT_EQ(std::memcmp(crowdb_rpc_buffer_data(read->data), expected.data(), BLOCK), 0); + } + EXPECT_NE(client->transport()->status_json().find("\"rdma\":null"), std::string::npos); + EXPECT_NE(server->transport()->status_json().find("\"rdma\":null"), std::string::npos); + RecordProperty("client_transport", client->transport()->status_json()); + RecordProperty("server_transport", server->transport()->status_json()); + RecordProperty("topology", "single_process_tcp_loopback"); +} +} // namespace crowdb::diskio +#endif diff --git a/doc/backlog/R240-rdma-datapath-stream.md b/doc/backlog/R240-rdma-datapath-stream.md index 5873496ad..227818acb 100644 --- a/doc/backlog/R240-rdma-datapath-stream.md +++ b/doc/backlog/R240-rdma-datapath-stream.md @@ -35,6 +35,9 @@ I/O still uses it. byte budget. A held or quarantined lease cannot be replaced outside it. 5. Preserve R239 fallback: before payload submission the operation may use TCP; after partial RDMA transfer the stream fails without TCP byte splicing. + The semantic DiskIO client must also stop automatic reconnect/retry after + an accepted RDMA operation fails. Surface its typed failure or uncertain + write/durability outcome; preserve pre-submit rejection and TCP retry policy. ## Dependencies diff --git a/doc/design/diskio/design-crowdb-diskio.md b/doc/design/diskio/design-crowdb-diskio.md index e4e3eed4d..5abbf20fd 100644 --- a/doc/design/diskio/design-crowdb-diskio.md +++ b/doc/design/diskio/design-crowdb-diskio.md @@ -675,6 +675,11 @@ Each request's control message is a flatbuffer with `ordering_zone_offset`, `write_create_time_ms`, a `sync` option, and a raw data payload of exactly `size` bytes. The write handler validates the FlatBuffer, zone bounds and payload length before IO. The handler: + +Async callbacks retain the disk, connection, buffer and shared transport +independently of the server facade. Exact registered payload views remain +DiskOwned through write and synchronization completion; a closed connection +suppresses response submission without releasing active disk data. 1. Resolves `disk_id` to a `Disk` via `DiskSet`. 2. Computes the physical offset: `zone.base_offset + zone_offset`. 3. Calls `AlignedWriter::submit_ordered` for writes or `IoEngine::read`/`fsync` for @@ -692,6 +697,15 @@ The data payload is passed from the crowdb-rpc frame decoder to writes use one aligned server buffer. The read response includes the raw data payload. +The negotiated RDMA control/data layout supplies a separately aligned 1 MiB +data region. Eligible aligned writes hand this registered receive storage to +the disk engine and retain DiskOwned state until completion. A READ reuses its +request's reserved receive data allocation as output, retaining it through disk +and response SEND completion. Client response leases stay caller-owned until +their final release. These retained receive allocations propagate disk and +consumer pressure through the bounded receive window. Alignment edges retain +the existing explicit staging path; TCP keeps the same result/error mapping. + Related writes are serialized by `ordering_zone_offset` within `AlignedWriter`. This ordering protects partial-block read-modify-write state; it does not validate allocation ownership. Write requests carry only the @@ -771,6 +785,11 @@ proof of synchronization. Independent fsync remains available and can be repeated after an ambiguous response. The lifecycle no-reuse window is what makes an identical write retry safe. +Failure of an accepted RDMA read, write or fsync is terminal for that operation: +the client does not reconnect and replay it through a TCP fallback connection. +It returns the typed transport error, ambiguous write outcome or durability +failure. Known pre-submit rejections retain the existing retry handling. + Status exposes route generation and age, topology and pool counts, healthy connections, operation/retry/backpressure/ambiguity counters, and operation latencies without exposing raw transport handles. The only production escape diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index 99d797439..264c53ee9 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -205,6 +205,10 @@ state owned by every outstanding allocation. Destroying the facade therefore does not invalidate a TCP/application buffer still held by an asynchronous consumer. Clones share one allocation charge; its final release returns that charge. The transport owns any default application-pool facade it creates. +`RpcServer` and asynchronous service callbacks share the transport owner. +Stopping the server closes admission and connections; outstanding disk +callbacks retain their connection, transport and buffer until completion. +An externally supplied application pool must outlive these retained users. `allocate_producer_buffer` lends storage from the device-owned send arena after CM context adoption publishes that arena. Its registration charge and arena @@ -305,6 +309,12 @@ completions cannot publish data into a reused buffer. QP, CM, and CQ errors move the connection to draining or failed state and quarantine uncertain buffers. +The semantic DiskIO client treats failure of an accepted RDMA read, write or +fsync as terminal for that operation. It does not reconnect and replay that +uncertain operation through TCP. Writes report their ambiguous outcome and +fsync preserves durability failure mapping. Known pre-submit rejection and +the existing TCP retry policy retain their separate handling. + `Connection::close()` also wakes its assigned CQ through a private transport notifier. The CQ owner requests `rdma_disconnect` once, guarded by the same per-connection operation admission that protects posting and CM-ID lifetime. @@ -319,7 +329,20 @@ retained buffer leases continue to own their arena and device resources. For S3/HTTP and DiskIO, HTTP or ChunkIO fills the Access Server registered send buffer; DiskIO receives into its own registered buffer and submits that buffer directly to asynchronous disk I/O. DiskIO returns it to the receive pool only -after disk completion. +after disk completion. Disk requests validate their complete control buffer, +payload length and physical zone bounds before submission. Disk callbacks use +the exact view address and retain their owners through short I/O, errors and +server shutdown. + +A direct-layout READ reuses the request's reserved receive data allocation as +its disk destination. READ has no input payload and its control has already +been validated. The registered data region remains owned through disk and +response SEND completion, then becomes eligible for repost. This makes RQ +admission bound asynchronous reads as well as writes. A slow client retains +response receive leases; exhausted client receive admission retains server +SEND sources and eventually stops new request receives. Eligible aligned +O_DIRECT reads use that same allocation; alignment edges use explicit staging. +TCP uses the ordinary application pool and the same disk result mapping. For Access Server and Chunk-KV, the existing access and aggregation layer selects the 64 KiB or 256 KiB class. RDMA transports each aggregate with the diff --git a/doc/working/plan-rdma-datapath-stream.md b/doc/working/plan-rdma-datapath-stream.md index 4a89f3d2d..276f9c8b3 100644 --- a/doc/working/plan-rdma-datapath-stream.md +++ b/doc/working/plan-rdma-datapath-stream.md @@ -184,7 +184,7 @@ it does not add application aggregation or a flush timer. (zero exit with style warnings), and 146 TCP-only CTest items also pass (one HCA-only item skipped). This closes producer wiring, while disk lifetime and complete HTTP/service/OCI acceptance remain below. -- [~] **Close DiskIO handoff and read lifetime**: use exact view pointer/length, +- [x] **Close DiskIO handoff and read lifetime**: use exact view pointer/length, validate request bounds before disk submission, transition data to DiskOwned until the callback returns, and retain connection state across delayed I/O. Registered read storage must cover engine completion and SEND completion; @@ -200,7 +200,71 @@ it does not add application aggregation or a flush timer. destructor drains its own queued operations; its block-size-one fast path, reads and fsync callbacks must remain independently safe after the service facade is destroyed. No new completion-path mutex is needed. -- [ ] **Verify real service flows**: run focused baseline and changed tests, + For a direct-layout READ, reuse the request's already reserved receive data + allocation as disk output, instead of allocating another producer buffer. + The control is already copied/verified and the request has no input payload. + Take a capacity-checked view of that canonical receive allocation, retain it + across disk and response SEND completion, then repost it. This uses the same + registered region and naturally bounds asynchronous reads by RQ admission; + it also propagates a slow response consumer back through server SEND and + request receive credits. Validate registered receive ownership before reuse; + keep the existing system-pool path for TCP and alignment-edge staging only + when the disk requires it. The first focused test build uses an incorrect + field spelling `data_offset`; the actual buffer field is `view_offset`. + Correct only the test field and rebuild before running the cases. + Twelve focused lifecycle/alignment cases pass. The first real HCA plus + O_DIRECT/io_uring case passes in 50 ms: three ordered 1 MiB writes reconstruct + 3 MiB exactly; disk write and read pointers equal the registered receive + pointers and are 4096-byte aligned. Four retained client responses consume + its RQ, the following four responses retain server receive data in SEND, + and releasing caller leases resumes all eight with zero copy/fallback and + no CQ/protocol/SEND/repost errors. This functional fixture uses one process + with actual client/server QPs on the local HCA; it is not a throughput sample + or the separate two-container end-to-end acceptance. + Inspection finds an additional application retry seam: `DiskioClient` retries + errors from an accepted RPC future through `prepare_retry`, which can create + a TCP fallback connection after an uncertain RDMA operation. Add an immutable + typed connection-path getter (also valid after close), stop accepted-RDMA + read/write/fsync retries, and return the existing typed/ambiguous/durability + errors. Keep known pre-submit rejection and established TCP retry behavior. + Add optional explicit `RdmaConfig` to the semantic DiskIO client so its data + transport can be configured independently of metadata clients and tested + without process-wide environment mutation. Add a real-HCA semantic-client + case asserting one accepted write, zero retries/reconnects, and ambiguous + failure after the peer closes before its reply. + The completed 15-case native run passes in 112 ms, including delayed disk + ownership, actual HCA/O_DIRECT round trips, peer-close/server-destruction, + held response leases, constant registered-memory high water, and TCP byte + parity. Evidence: `.crowdb-runtime/rdma-check/artifacts/r240-direct/diskio-hca.{log,xml}`. + The new status assertion initially fails because Folly's default JSON parser + converts a random unsigned 64-bit capability generation through signed + `int64_t`. Preserve the failing log/XML as `diskio-hca-parser-failure.*`; + parse numeric tokens as strings in this test's snapshot helper so exact + memory-counter comparisons retain full precision. No production generation + or lifecycle assertion changes. The corrected run has zero failures. + All nine semantic DiskIO tests pass after extracting write/fsync into + `semantic/write.rs`; three actual-HCA cases cover accepted read/write/fsync + failure with fallback enabled, exactly one accepted operation and no retry + or reconnect. Evidence: `diskio-rust.log` in the same artifact directory. + The first Rust lint run flags the new fault helper's 114-line function; + extract only its cold `RdmaConfig` builder. Workspace all-feature clippy and + rs-lint then pass. C++ formatting needs repeated passes to stabilize the + pre-existing numeric-config macro; stabilize its whitespace, rebuild, and + rerun the 203-item RPC CTest set, all 71 native DiskIO cases with HCA enabled, + and all nine semantic cases. All pass. Full C++/FFI gate evidence is + `.crowdb-runtime/artifacts/unit/cpp-e70af11a38a443f6a7b660245507292a/`; + 146 no-verbs items also pass (one HCA-only item skips), and scoped tree-lint + exits zero with style warnings. All 125 ChunkIO tests pass. The first + full Access gate reaches its 60-second shell bound while compiling test + executables, before running any case; complete `--no-run` separately. The + compiled suite also exceeds the aggregate shell bound across its independent + test programs, with no failed assertion before termination. Run it with + `CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="timeout 60s"`, giving each + existing test program the same 60-second execution bound. Request deadlines + and assertions remain unchanged. Preserve the aggregate-limit log as + `access-suite-shell-limit.log`. The complete per-program-bounded Access + suite passes with zero failures; its complete log is `access.log`. +- [~] **Verify real service flows**: run focused baseline and changed tests, then real DiskIO plus HTTP/ChunkIO upload/download of objects larger than 1 MiB on the prepared HCA. Assert direct-payload counters and same data addresses at producer/SEND and receive/disk handoff, exact bytes and existing @@ -208,6 +272,19 @@ it does not add application aggregation or a flush timer. per-case manifests, settings, errors and complete logs. Run required fmt, clippy/tree-lint and full affected gates before cleanup; then use R241's own plan for Chunk-KV integration. + Use a service fixture that runs a real Hyper HTTP socket, the production S3 + body-to-ChunkIO transfer function, mirror ChunkIO framing/read verification, + and a separate real DiskIO executable with O_DIRECT/io_uring. Inject only + chunk placement/catalog state, keeping namespace/authentication outside this + datapath case. A `test-util` entry delegates to the production upload function + rather than reimplementing its ownership/pressure loop. Build the normal + routed disk adapter from an already connected semantic client so the fixture + shares production submit/read/error behavior. Use two replicas and an object + over 3 MiB with a non-frame-aligned tail; compare exact download and locations + across strict RDMA and explicit TCP. Assert native direct/copy/fallback and + error counters in machine-readable artifacts. Run the same fixture in two + non-root host-network containers sharing the one configured HCA, recording + this local-loop topology and pinned runtime image and executable identities. ## Phase 1: map existing data flow @@ -293,7 +370,8 @@ E2E tests: - `lib/crowdb-chunk-client/` - `lib/crowdb-diskio-client/` - `lib/crowdb-rpc/` -- `lib/crowdb-chunkio/` or the existing ChunkIO owner module +- `app/crowdb-diskio/src/rpc/` and its `tests/` +- `lib/crowdb-chunk-client/src/disk_io/`, `client.rs`, and existing chunk writers - relevant S3/DiskIO integration tests and OCI E2E runner ## Verification gates diff --git a/lib/crowdb-diskio-client/src/semantic.rs b/lib/crowdb-diskio-client/src/semantic.rs index 4cba99f9e..b331168bf 100644 --- a/lib/crowdb-diskio-client/src/semantic.rs +++ b/lib/crowdb-diskio-client/src/semantic.rs @@ -17,7 +17,7 @@ use crowdb_rpc_ffi::{ ConnectionPoolError, ConnectionPoolIndex, OwnedClientRoute, RpcError, RpcServer, SelectedConnection, }; -use crate::client::{DiskIoRetCode, WireClient, WireError, WireWriteTarget}; +use crate::client::{DiskIoRetCode, WireClient, WireError}; use crate::topology::{self, DiskRoute}; use crate::{DiskId, DiskioError, DiskioResult, DiskioStatus, SegmentTarget}; @@ -87,6 +87,8 @@ impl OperationOptions { /// Discovery, pool, and retry bounds for a semantic client. #[derive(Debug, Clone)] pub struct DiskioClientConfig { + /// Explicit data-transport configuration; `None` retains native environment defaults. + pub rdma: Option, pub management_seeds: Vec, pub normal_connections_per_endpoint: usize, pub priority_connections_per_endpoint: usize, @@ -128,6 +130,7 @@ impl NativeDiskIoRoutes { impl Default for DiskioClientConfig { fn default() -> Self { Self { + rdma: None, management_seeds: Vec::new(), normal_connections_per_endpoint: 2, priority_connections_per_endpoint: 1, @@ -338,16 +341,7 @@ impl DiskioClient { #[cfg(feature = "test-util")] pub fn connect_for_tests(routes: Vec, config: DiskioClientConfig) -> DiskioResult { config.validate(false)?; - let server = Arc::new( - RpcServer::with_rdma_layout( - None, - 1, - config.rpc_workers, - crowdb_rpc_ffi::RdmaWireLayout::ControlData, - 1 << 20, - ) - .map_err(|error| DiskioError::TransportUnavailable(format!("create RPC client: {error}")))?, - ); + let server = Self::create_rpc_server(&config)?; server.set_send_queue_capacity(config.send_queue_capacity); server .listen("127.0.0.1", 0) @@ -550,7 +544,11 @@ impl DiskioClient { let result = self.await_read(future, length, options.deadline).await; match result { Ok(data) => return Ok(data), - Err(error) if error.is_retryable_read() && attempt + 1 < self.config.retry_attempts => { + Err(error) + if !selected.connection().is_rdma() + && error.is_retryable_read() + && attempt + 1 < self.config.retry_attempts => + { self.refresh_topology_after_read_failure().await; self.prepare_retry(&route, options.lane, &selected, &mut backoff, options.deadline) .await?; @@ -740,6 +738,9 @@ impl DiskioClient { } Err(error) => { last_error = Some(error); + if selected.connection().is_rdma() { + break; + } if attempt + 1 < self.config.retry_attempts && self .prepare_retry(&route, options.lane, &selected, &mut backoff, options.deadline) diff --git a/lib/crowdb-diskio-client/src/semantic/connection.rs b/lib/crowdb-diskio-client/src/semantic/connection.rs index b4092d71c..70209da20 100644 --- a/lib/crowdb-diskio-client/src/semantic/connection.rs +++ b/lib/crowdb-diskio-client/src/semantic/connection.rs @@ -12,6 +12,19 @@ use crowdb_rpc_ffi::{ConnectionPoolIndex, RdmaWireLayout, RpcServer}; use std::sync::{atomic::AtomicU64, Arc}; impl DiskioClient { + pub(super) fn create_rpc_server(config: &DiskioClientConfig) -> DiskioResult> { + let server = Arc::new( + RpcServer::with_rdma_layout(None, 1, config.rpc_workers, RdmaWireLayout::ControlData, 1 << 20) + .map_err(|error| DiskioError::TransportUnavailable(format!("create RPC client: {error}")))?, + ); + if let Some(rdma) = &config.rdma { + server.configure_rdma(rdma).map_err(|error| { + DiskioError::TransportUnavailable(format!("configure DiskIO RPC client: {error}")) + })?; + } + Ok(server) + } + #[cfg(feature = "test-util")] /// # Errors /// Uses production scoped discovery against injected real Group-0 clients. @@ -83,10 +96,7 @@ impl DiskioClient { scope: Option, ) -> DiskioResult { config.validate(false)?; - let server = Arc::new( - RpcServer::with_rdma_layout(None, 1, config.rpc_workers, RdmaWireLayout::ControlData, 1 << 20) - .map_err(|error| DiskioError::TransportUnavailable(format!("create RPC client: {error}")))?, - ); + let server = Self::create_rpc_server(&config)?; server.set_send_queue_capacity(config.send_queue_capacity); server .listen("127.0.0.1", 0) diff --git a/lib/crowdb-diskio-client/src/semantic/write.rs b/lib/crowdb-diskio-client/src/semantic/write.rs index d185fc8cc..e2027dc91 100644 --- a/lib/crowdb-diskio-client/src/semantic/write.rs +++ b/lib/crowdb-diskio-client/src/semantic/write.rs @@ -116,6 +116,9 @@ impl DiskioClient { Err(error) => { only_backpressure = false; last_error = Some(error); + if selected.connection().is_rdma() { + break; + } if attempt + 1 < attempts && self .prepare_retry(&route, options.lane, &selected, &mut backoff, options.deadline) diff --git a/lib/crowdb-diskio-client/tests/rdma_failure_test.rs b/lib/crowdb-diskio-client/tests/rdma_failure_test.rs new file mode 100644 index 000000000..d7d4e02b0 --- /dev/null +++ b/lib/crowdb-diskio-client/tests/rdma_failure_test.rs @@ -0,0 +1,155 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use bytes::Bytes; +use crowdb_diskio_client::{ + DiskId, DiskioClient, DiskioClientConfig, DiskioError, Durability, SegmentTarget, TestDiskRoute, +}; +use crowdb_protocol::fb::FBMsgType; +use crowdb_rpc_ffi::{RdmaConfig, RdmaWireLayout, RpcServer}; + +#[derive(Clone, Copy)] +enum TestOperation { + Read, + Write, + Fsync, +} + +fn test_rdma_config(address: &str) -> RdmaConfig { + RdmaConfig { + enabled: true, + listen_enabled: true, + allow_tcp_fallback: true, + wire_layout: RdmaWireLayout::ControlData, + wire_slot_capacity: (1 << 20) + 1024, + sq_depth: 4, + rq_depth: 4, + posted_receive_target: 4, + cq_depth: 16, + send_arena_bytes: 4 << 20, + receive_arena_bytes: 4 * ((1 << 20) + 1024), + source_address: address.to_owned(), + device: std::env::var("CROWDB_RDMA_DEVICE").unwrap_or_default(), + ..RdmaConfig::default() + } +} + +async fn assert_accepted_rdma_failure(operation: TestOperation) { + if std::env::var_os("CROWDB_RDMA_E2E").is_none() { + return; + } + let address = std::env::var("CROWDB_RDMA_E2E_LISTEN_ADDR").unwrap(); + let config = test_rdma_config(&address); + let server = RpcServer::new(None); + server.configure_rdma(&config).unwrap(); + server.listen(&address, 0).unwrap(); + let received = Arc::new(AtomicU64::new(0)); + let observed = Arc::clone(&received); + let message = match operation { + TestOperation::Read => FBMsgType::EDiskReadRequest, + TestOperation::Write => FBMsgType::EDiskWriteRequest, + TestOperation::Fsync => FBMsgType::EDiskFsyncRequest, + }; + server.register_handler(u16::try_from(message.0).unwrap(), move |request| { + if matches!(operation, TestOperation::Write) { + assert_eq!(request.data().unwrap(), b"accepted write"); + } else { + assert!(request.data().is_none()); + } + observed.fetch_add(1, Ordering::Release); + // Drop the receive lease but deliberately send no durable reply. + }); + server.start(); + let client = Arc::new( + DiskioClient::connect_for_tests( + vec![TestDiskRoute { + disk_id: DiskId::new(7, 9), + rack_id: 1, + node_id: 1, + disk_group_id: 1, + instance_id: 1, + endpoint: format!("{address}:{}", server.port()), + }], + DiskioClientConfig { + rdma: Some(RdmaConfig { + listen_enabled: false, + ..config + }), + normal_connections_per_endpoint: 1, + priority_connections_per_endpoint: 1, + rpc_workers: 1, + retry_attempts: 3, + ..DiskioClientConfig::default() + }, + ) + .unwrap(), + ); + let baseline = client.status(); + let writer = Arc::clone(&client); + let pending = tokio::spawn(async move { + let target = SegmentTarget::new(DiskId::new(7, 9), 0, 0, 1, 1 << 20).unwrap(); + match operation { + TestOperation::Read => writer + .read(target, 0, 14, writer.normal_options()) + .await + .map(|_| ()), + TestOperation::Write => { + writer + .write( + target, + 0, + Bytes::from_static(b"accepted write"), + Durability::Buffered, + writer.normal_options(), + ) + .await + } + TestOperation::Fsync => writer.fsync(DiskId::new(7, 9), writer.normal_options()).await, + } + }); + tokio::time::timeout(Duration::from_secs(2), async { + while received.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + server.stop(); + let error = tokio::time::timeout(Duration::from_secs(2), pending) + .await + .unwrap() + .unwrap() + .unwrap_err(); + if matches!(operation, TestOperation::Write) { + assert!(matches!(error, DiskioError::AmbiguousWrite(_)), "{error:?}"); + } else { + assert!(matches!(error, DiskioError::TransportUnavailable(_)), "{error:?}"); + } + let status = client.status(); + assert_eq!(received.load(Ordering::Acquire), 1); + assert_eq!(status.retries, baseline.retries); + assert_eq!(status.reconnect_attempts, baseline.reconnect_attempts); + assert_eq!( + status.ambiguous_writes, + baseline.ambiguous_writes + u64::from(matches!(operation, TestOperation::Write)) + ); +} + +#[tokio::test] +async fn accepted_rdma_write_failure_does_not_reconnect_or_replay_on_tcp() { + assert_accepted_rdma_failure(TestOperation::Write).await; +} + +#[tokio::test] +async fn accepted_rdma_read_failure_does_not_reconnect_or_retry_on_tcp() { + assert_accepted_rdma_failure(TestOperation::Read).await; +} + +#[tokio::test] +async fn accepted_rdma_fsync_failure_does_not_reconnect_or_retry_on_tcp() { + assert_accepted_rdma_failure(TestOperation::Fsync).await; +} diff --git a/lib/crowdb-rpc/ffi/src/server.rs b/lib/crowdb-rpc/ffi/src/server.rs index dc8607404..a2cd0bf6c 100644 --- a/lib/crowdb-rpc/ffi/src/server.rs +++ b/lib/crowdb-rpc/ffi/src/server.rs @@ -397,6 +397,13 @@ impl Connection { unsafe { sys::crowdb_rpc_conn_is_open(self.inner.handle) != 0 } } + /// Selected RDMA path, retained even after this connection closes. + #[must_use] + pub fn is_rdma(&self) -> bool { + // SAFETY: this owned handle retains immutable path metadata. + unsafe { sys::crowdb_rpc_conn_is_rdma(self.inner.handle) != 0 } + } + /// Whether TCP was refused before the peer accepted this connection. /// Established connection loss and timeouts do not prove an unsent request. #[must_use] diff --git a/lib/crowdb-rpc/ffi/src/sys.rs b/lib/crowdb-rpc/ffi/src/sys.rs index 1eb758b14..7a0aa9278 100644 --- a/lib/crowdb-rpc/ffi/src/sys.rs +++ b/lib/crowdb-rpc/ffi/src/sys.rs @@ -323,6 +323,7 @@ extern "C" { pub fn crowdb_rpc_conn_destroy(conn: crowdb_rpc_conn_t); pub fn crowdb_rpc_conn_is_open(conn: crowdb_rpc_conn_t) -> c_int; + pub fn crowdb_rpc_conn_is_rdma(conn: crowdb_rpc_conn_t) -> c_int; pub fn crowdb_rpc_conn_connect_refused(conn: crowdb_rpc_conn_t) -> c_int; pub fn crowdb_rpc_conn_close(conn: crowdb_rpc_conn_t); diff --git a/lib/crowdb-rpc/ffi/tests/transport_config_test.rs b/lib/crowdb-rpc/ffi/tests/transport_config_test.rs index 82cc3415a..c1eca2064 100644 --- a/lib/crowdb-rpc/ffi/tests/transport_config_test.rs +++ b/lib/crowdb-rpc/ffi/tests/transport_config_test.rs @@ -103,6 +103,7 @@ async fn status_control_rpc_returns_owned_versioned_json() { let connection_status: Value = serde_json::from_slice(&connection.transport_status_json().unwrap()).unwrap(); assert_eq!(connection_status["path"], "tcp"); + assert!(!connection.is_rdma()); server.stop(); let closed: Value = serde_json::from_slice(&connection.transport_status_json().unwrap()).unwrap(); assert_eq!(closed["open"], false); @@ -145,6 +146,7 @@ fn hardware_qpair_status_retains_immutable_provider_values_after_close() { let connection = client.connect(&address, server.port()).unwrap(); let info: Value = serde_json::from_slice(&connection.transport_status_json().unwrap()).unwrap(); assert_eq!(info["path"], "rdma"); + assert!(connection.is_rdma()); assert!(info["info"]["qpair_number"].as_u64().unwrap() > 0); assert!(info["info"]["effective_sq_depth"].as_u64().unwrap() >= 8); assert_eq!(info["info"]["peer_receive_slot_capacity"], 1024); @@ -153,4 +155,5 @@ fn hardware_qpair_status_retains_immutable_provider_values_after_close() { let closed: Value = serde_json::from_slice(&connection.transport_status_json().unwrap()).unwrap(); assert_eq!(closed["open"], false); assert_eq!(closed["info"], info["info"]); + assert!(connection.is_rdma()); } diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h index 381d21180..e24e8b8f7 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h @@ -290,6 +290,8 @@ void crowdb_rpc_conn_destroy(crowdb_rpc_conn_t conn); // Return non-zero while the underlying transport connection is open. int crowdb_rpc_conn_is_open(crowdb_rpc_conn_t conn); +// Selected path remains readable after the connection closes. +int crowdb_rpc_conn_is_rdma(crowdb_rpc_conn_t conn); // True only for a TCP refusal before peer acceptance; resets remain ambiguous. int crowdb_rpc_conn_connect_refused(crowdb_rpc_conn_t conn); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/server/server.h b/lib/crowdb-rpc/include/crowdb-rpc/server/server.h index 356f3645e..ba363afc2 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/server/server.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/server/server.h @@ -77,6 +77,14 @@ class RpcServer return transport_.get(); } + // Async service callbacks retain the stopped transport and default pool + // until their final I/O completion, independently of this server facade. + // An externally supplied pool must outlive all retained transport users. + std::shared_ptr shared_transport() const + { + return transport_; + } + // The buffer pool (for handlers that allocate response buffers). BufferPool *pool() { @@ -92,8 +100,7 @@ class RpcServer private: BufferPool *pool_; - bool owns_pool_; - std::unique_ptr transport_; + std::shared_ptr transport_; HandlerRegistry handlers_; RpcClient *request_client_{nullptr}; // server-initiated request-response diff --git a/lib/crowdb-rpc/src/c_api/transport.cpp b/lib/crowdb-rpc/src/c_api/transport.cpp index faad622b9..c3919f100 100644 --- a/lib/crowdb-rpc/src/c_api/transport.cpp +++ b/lib/crowdb-rpc/src/c_api/transport.cpp @@ -6,15 +6,24 @@ using namespace crowdb::rpc; -#define CONFIG_NUMBERS(X) \ - X(cq_count) \ - X(cq_depth) \ - X(sq_depth) \ - X(rq_depth) \ - X(posted_receive_target) \ - X(wire_slot_capacity) X(poll_batch) X(verbs_poll_batch) X(poll_empty_budget) X(send_queue_capacity) \ - X(aggregation_max_bytes) X(aggregation_max_frames) X(send_arena_bytes) X(receive_arena_bytes) \ - X(registered_memory_budget_bytes) +extern "C" int crowdb_rpc_conn_is_rdma(crowdb_rpc_conn_t connection) +{ + return connection != nullptr && connection->conn != nullptr && connection->conn->path() == ConnectionPath::Rdma; +} + +#define CONFIG_NUMBERS(X) \ + X(cq_count) \ + X(cq_depth) \ + X(sq_depth) \ + X(rq_depth) \ + X(posted_receive_target) \ + X(wire_slot_capacity) \ + X(poll_batch) \ + X(verbs_poll_batch) \ + X(poll_empty_budget) \ + X(send_queue_capacity) \ + X(aggregation_max_bytes) \ + X(aggregation_max_frames) X(send_arena_bytes) X(receive_arena_bytes) X(registered_memory_budget_bytes) extern "C" void crowdb_rpc_rdma_config_init(crowdb_rpc_rdma_config_t *config) { diff --git a/lib/crowdb-rpc/src/server/server.cpp b/lib/crowdb-rpc/src/server/server.cpp index 9e9a88d2f..609578df6 100644 --- a/lib/crowdb-rpc/src/server/server.cpp +++ b/lib/crowdb-rpc/src/server/server.cpp @@ -39,14 +39,10 @@ std::string listener_key(const std::string &addr, int port) } } // namespace -RpcServer::RpcServer(BufferPool *pool, uint32_t io_engines, uint32_t io_workers, RdmaConfig rdma_config) - : pool_(pool), - owns_pool_(pool == nullptr) +RpcServer::RpcServer(BufferPool *pool, uint32_t io_engines, uint32_t io_workers, RdmaConfig rdma_config) : pool_(pool) { - if (pool_ == nullptr) { - pool_ = new SystemBufferPool(); - } - transport_ = std::make_unique(io_engines, io_workers, pool_, std::move(rdma_config)); + transport_ = std::make_shared(io_engines, io_workers, pool_, std::move(rdma_config)); + pool_ = transport_->pool(); } RpcServer::~RpcServer() @@ -57,9 +53,6 @@ RpcServer::~RpcServer() // transport pointer returns 0 instead of accessing freed memory. alive_->store(false, std::memory_order_release); transport_.reset(); - if (owns_pool_) { - delete pool_; - } } bool RpcServer::listen(const std::string &addr, int port) From c99657931f69419ce32fa13ad7bad18a433f64f8 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 04:32:09 +0800 Subject: [PATCH 044/258] Verify native HTTP chunk transfers through direct RDMA DiskIO --- app/crowdb-access-server/src/s3.rs | 2 + app/crowdb-access-server/src/s3/operations.rs | 2 +- .../src/s3/operations/upload.rs | 6 +- .../src/s3/test_support.rs | 31 +++ .../tests/common/rdma_datapath/allocator.rs | 145 +++++++++++++ .../tests/common/rdma_datapath/http.rs | 113 ++++++++++ .../tests/rdma_datapath_test.rs | 187 ++++++++++++++++ app/crowdb-diskio/src/dio_main.cpp | 7 +- doc/design/rpc/rpc-flow-analysis.md | 51 +++++ doc/working/plan-rdma-datapath-stream.md | 89 ++++++-- .../src/disk_io/routing.rs | 6 + lib/crowdb-diskio-client/src/semantic.rs | 10 + tools/rdma/run-datapath.py | 205 ++++++++++++++++++ 13 files changed, 833 insertions(+), 21 deletions(-) create mode 100644 app/crowdb-access-server/src/s3/test_support.rs create mode 100644 app/crowdb-access-server/tests/common/rdma_datapath/allocator.rs create mode 100644 app/crowdb-access-server/tests/common/rdma_datapath/http.rs create mode 100644 app/crowdb-access-server/tests/rdma_datapath_test.rs create mode 100644 tools/rdma/run-datapath.py diff --git a/app/crowdb-access-server/src/s3.rs b/app/crowdb-access-server/src/s3.rs index 816fcd151..bde4a31cd 100644 --- a/app/crowdb-access-server/src/s3.rs +++ b/app/crowdb-access-server/src/s3.rs @@ -27,6 +27,8 @@ mod inspection; pub use inspection::{ObjectInspector, OBJECT_LOCATIONS_PATH}; mod health; mod operations; +#[cfg(feature = "test-util")] +pub mod test_support; pub use health::{AccessHealthHandler, HealthObserver}; pub use crate::http_receive::install_body_receive_provider; diff --git a/app/crowdb-access-server/src/s3/operations.rs b/app/crowdb-access-server/src/s3/operations.rs index 846559786..47d7a6f90 100644 --- a/app/crowdb-access-server/src/s3/operations.rs +++ b/app/crowdb-access-server/src/s3/operations.rs @@ -43,7 +43,7 @@ mod delete; mod multipart; mod upload; -use upload::write_object_body; +pub(crate) use upload::write_object_body; const DEFAULT_LIST_LIMIT: usize = 1_000; const DEFAULT_LIST_SCAN_BYTES: usize = 4 * 1024 * 1024; diff --git a/app/crowdb-access-server/src/s3/operations/upload.rs b/app/crowdb-access-server/src/s3/operations/upload.rs index 1980416b8..b35fa36b0 100644 --- a/app/crowdb-access-server/src/s3/operations/upload.rs +++ b/app/crowdb-access-server/src/s3/operations/upload.rs @@ -20,7 +20,7 @@ use crate::upload_flow::body_encoding::{UploadBody, UploadEncodingError}; use crate::upload_flow::digest_pipe::DigestPipe; use crate::upload_flow::{drive_transfer, write_buffers, OfferStatus, UploadBuffer, WriteFlow}; -use super::ObjectWriter; +use crowdb_chunk_client::ChunkIoWriter; const TARGET_BUFFER_BYTES: usize = 1024 * 1024; @@ -32,9 +32,9 @@ fn failed(code: PutErrorCode, message: &impl ToString) -> PutOutcome { } #[allow(clippy::too_many_arguments)] -pub(super) async fn write_object_body( +pub(crate) async fn write_object_body( body: &mut UploadBody, - writer: &mut ObjectWriter, + writer: &mut W, native_receiver: Option<&NativeBodyReceiver>, declared_length: Option, expected_content_md5: Option<&str>, diff --git a/app/crowdb-access-server/src/s3/test_support.rs b/app/crowdb-access-server/src/s3/test_support.rs new file mode 100644 index 000000000..2a5646ef9 --- /dev/null +++ b/app/crowdb-access-server/src/s3/test_support.rs @@ -0,0 +1,31 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Production HTTP body transfer with injected storage and placement. + +use crowdb_access_s3::native_buffer::NativeBodyAllocator; +use crowdb_chunk_client::ChunkIoWriter; +use hyper::body::Incoming; +use hyper::Request; + +/// Exercise the production S3 body transfer through a real HTTP listener. +/// +/// # Errors +/// Returns the same framing, integrity or write failure as the production flow. +pub async fn write_large_body_for_tests( + mut request: Request, + writer: &mut impl ChunkIoWriter, + allocator: &NativeBodyAllocator, +) -> Result<(String, Vec), String> { + let receiver = crate::http_receive::install_native_body_receive_provider(&mut request, allocator); + receiver.enable_owner_handoff(); + let (parts, incoming) = request.into_parts(); + let mut body = + crate::upload_flow::body_encoding::UploadBody::new(incoming, &parts.headers, None, 64 << 20) + .map_err(|error| error.to_string())?; + body.defer_md5(); + let length = body.decoded_length(); + super::operations::write_object_body(&mut body, writer, Some(&receiver), length, None, None, 4, None) + .await + .map_err(|error| format!("{error:?}")) +} diff --git a/app/crowdb-access-server/tests/common/rdma_datapath/allocator.rs b/app/crowdb-access-server/tests/common/rdma_datapath/allocator.rs new file mode 100644 index 000000000..d5a3d415c --- /dev/null +++ b/app/crowdb-access-server/tests/common/rdma_datapath/allocator.rs @@ -0,0 +1,145 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Inject only placement; payload, framing and disk I/O use production code. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use arc_swap::ArcSwap; +use async_trait::async_trait; +use crowdb_chunk_client::{ChunkAllocator, IoError, Result}; +use crowdb_protocol::chunkdb::rpc::*; +use crowdb_protocol::common::{ChunkId, DiskId}; +use crowdb_protocol::diskdb::rpc::Segment; + +#[derive(Default)] +pub struct TestAllocator { + next_segment: AtomicU64, + chunks: ArcSwap>, +} + +impl TestAllocator { + fn update(&self, action: impl Fn(&mut HashMap<(u64, u64), Chunk>) -> T) -> T { + loop { + let current = self.chunks.load_full(); + let mut next = (*current).clone(); + let result = action(&mut next); + let previous = self.chunks.compare_and_swap(¤t, Arc::new(next)); + if Arc::ptr_eq(&previous, ¤t) { + return result; + } + } + } + + fn strip(&self, id: ChunkId, sequence: u32) -> ChunkStrip { + let unit_offset = self.next_segment.fetch_add(256, Ordering::Relaxed); + ChunkStrip { + chunk_offset: sequence * 1024, + strip_sequence: sequence, + unit_kb: 4, + capacity: 1024, + strip_type: StripType::Mirror as i32, + strip: Some(Strip::MirrorStrip(MirrorStrip { + segments: (1..=2) + .map(|high| Segment { + disk_id: Some(DiskId { high, low: 0 }), + owner_chunk: Some(id), + unit_offset, + unit_count: 256, + ..Segment::default() + }) + .collect(), + })), + ..ChunkStrip::default() + } + } +} + +#[async_trait] +impl ChunkAllocator for TestAllocator { + async fn allocate_chunk(&self, req: AllocateChunkRequest) -> Result { + assert_eq!(req.strip_type, StripType::Mirror as i32); + assert_eq!(req.copy_count, 2); + let id = req.chunk_id.unwrap(); + let strips: Vec<_> = (0..req.strip_count.max(1)) + .map(|sequence| self.strip(id, sequence)) + .collect(); + let chunk = Chunk { + id: Some(id), + modify_ts: 1, + state: ChunkState::Active as i32, + capacity: strips.iter().map(|strip| strip.capacity).sum(), + next_strip_sequence: u32::try_from(strips.len()).unwrap(), + strips, + chunk_type: req.chunk_type, + writer_epoch: req.writer_epoch, + ..Chunk::default() + }; + self.update(|chunks| chunks.insert((id.high, id.low), chunk.clone())); + Ok(AllocateChunkResponse { chunk: Some(chunk) }) + } + + async fn append_chunk(&self, req: AppendChunkRequest) -> Result { + let id = req.chunk_id.unwrap(); + self.update(|chunks| { + let chunk = chunks.get_mut(&(id.high, id.low)).unwrap(); + assert_eq!(chunk.modify_ts, req.modify_ts); + let first = chunk.next_strip_sequence; + let strips: Vec<_> = (0..req.strip_count.max(1)) + .map(|offset| self.strip(id, first + offset)) + .collect(); + chunk.capacity += strips.iter().map(|strip| strip.capacity).sum::(); + chunk.next_strip_sequence += u32::try_from(strips.len()).unwrap(); + chunk.modify_ts += 1; + chunk.strips.extend(strips.iter().cloned()); + Ok(AppendChunkResponse { + modify_ts: chunk.modify_ts, + strips, + chunk: None, + }) + }) + } + + async fn seal_chunk(&self, req: SealChunkRequest) -> Result { + let id = req.chunk_id.unwrap(); + self.update(|chunks| { + let chunk = chunks.get_mut(&(id.high, id.low)).unwrap(); + chunk.state = ChunkState::Sealed as i32; + chunk.sealed_length = req.seal_length; + for strip in &mut chunk.strips { + strip.sealed_length = req + .seal_length + .saturating_sub(strip.chunk_offset) + .min(strip.capacity); + } + Ok(SealChunkResponse { + chunk: Some(chunk.clone()), + }) + }) + } + + async fn delete_chunk(&self, req: DeleteChunkRequest) -> Result { + let id = req.chunk_id.unwrap(); + self.update(|chunks| { + let chunk = chunks.get_mut(&(id.high, id.low)).unwrap(); + chunk.state = ChunkState::Deleted as i32; + Ok(DeleteChunkResponse { + chunk: Some(chunk.clone()), + }) + }) + } + + async fn query_chunk(&self, req: QueryChunkRequest) -> Result { + let id = req.chunk_id.unwrap(); + Ok(QueryChunkResponse { + chunk: self.chunks.load().get(&(id.high, id.low)).cloned(), + layout_validity_ms: 60_000, + }) + } + + async fn update_chunk_strip(&self, _req: UpdateChunkStripRequest) -> Result { + Err(IoError::Unsupported("fixture does not repair placement".into())) + } +} diff --git a/app/crowdb-access-server/tests/common/rdma_datapath/http.rs b/app/crowdb-access-server/tests/common/rdma_datapath/http.rs new file mode 100644 index 000000000..4ab200f7c --- /dev/null +++ b/app/crowdb-access-server/tests/common/rdma_datapath/http.rs @@ -0,0 +1,113 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use arc_swap::ArcSwap; +use crowdb_access_s3::native_buffer::NativeBodyAllocator; +use crowdb_access_server::s3::{HandlerFuture, ResponseBody, S3HttpHandler}; +use crowdb_chunk_client::{ChunkClientConfig, ChunkIoClient, ChunkIoWriter, LargeWritePolicy}; +use crowdb_common::ec::EcScheme; +use crowdb_protocol::chunkdb::rpc::{ChunkType, Location}; +use futures::stream; +use http_body_util::{BodyExt, Full, StreamBody}; +use hyper::body::{Bytes, Frame, Incoming}; +use hyper::{Method, Request, Response}; + +pub struct TestHttp { + chunks: Arc, + pub allocator: Arc, + pub locations: Arc>>, +} + +impl TestHttp { + pub fn new(chunks: Arc) -> Self { + let allocator = NativeBodyAllocator::new(2 << 20, 1 << 20) + .unwrap() + .with_producer_allocator(chunks.producer_allocator().unwrap()) + .unwrap(); + Self { + chunks, + allocator: Arc::new(allocator), + locations: Arc::new(ArcSwap::from_pointee(Vec::new())), + } + } + + async fn put( + chunks: Arc, + allocator: Arc, + locations: Arc>>, + request: Request, + ) -> Response { + let size = request.headers()["content-length"] + .to_str() + .unwrap() + .parse() + .unwrap(); + let mut config = ChunkClientConfig::new(ChunkType::S3); + config.large_mirror_copies = Some(2); + config.max_chunk_size = 2 << 20; + config.large_parallel_strip_writes = 2; + let policy = LargeWritePolicy { + ec_scheme: EcScheme::new(1, 1), + client: Arc::new(config), + }; + let mut writer = chunks.prepare_large_write(Some(size), policy); + writer.wait_until_prepared().await.unwrap(); + let (etag, _) = crowdb_access_server::s3::test_support::write_large_body_for_tests( + request, + &mut writer, + &allocator, + ) + .await + .unwrap(); + locations.store(Arc::new(writer.on_finish().await.unwrap())); + Response::builder().header("etag", etag).body(empty()).unwrap() + } + + fn get(&self) -> Response { + let read = self.chunks.read_stream(&self.locations.load()).unwrap(); + let body = stream::unfold((read, 0), |(mut read, delivered)| async move { + if delivered == 1 && std::env::var("CROWDB_RDMA_DATAPATH_FAULT").as_deref() == Ok("1") { + let marker = std::env::var("CROWDB_RDMA_DATAPATH_PAUSED").unwrap(); + let resume = std::env::var("CROWDB_RDMA_DATAPATH_RESUME").unwrap(); + std::fs::write(marker, b"verified frame delivered").unwrap(); + while !std::path::Path::new(&resume).exists() { + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + } + read.next_chunk().await.map(|result| { + ( + result + .map(Frame::data) + .map_err(|error| -> crowdb_access_server::s3::BoxError { + eprintln!("datapath HTTP download failed: {error:?}"); + error.to_string().into() + }), + (read, delivered + 1), + ) + }) + }); + Response::builder() + .body(BodyExt::boxed(StreamBody::new(Box::pin(body)))) + .unwrap() + } +} + +impl S3HttpHandler for TestHttp { + fn handle(&self, request: Request) -> HandlerFuture { + if request.method() == Method::PUT { + let chunks = Arc::clone(&self.chunks); + let allocator = Arc::clone(&self.allocator); + let locations = Arc::clone(&self.locations); + Box::pin(async move { Ok(Self::put(chunks, allocator, locations, request).await) }) + } else { + let response = self.get(); + Box::pin(async move { Ok(response) }) + } + } +} + +fn empty() -> ResponseBody { + Full::new(Bytes::new()).map_err(|never| match never {}).boxed() +} diff --git a/app/crowdb-access-server/tests/rdma_datapath_test.rs b/app/crowdb-access-server/tests/rdma_datapath_test.rs new file mode 100644 index 000000000..972ed6280 --- /dev/null +++ b/app/crowdb-access-server/tests/rdma_datapath_test.rs @@ -0,0 +1,187 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/rdma_datapath/allocator.rs"] +mod allocator; +#[path = "common/rdma_datapath/http.rs"] +mod http; + +use std::sync::Arc; +use std::time::Duration; + +use crowdb_chunk_client::{ChunkIoClient, RoutedDiskWriter}; +use crowdb_diskio_client::{DiskId, DiskioClient, DiskioClientConfig, TestDiskRoute}; +use crowdb_protocol::chunkdb::rpc::ChunkType; +use crowdb_rpc_ffi::{RdmaConfig, RdmaWireLayout}; +use futures::StreamExt; +use sha2::{Digest, Sha256}; + +fn test_rdma_config(enabled: bool) -> RdmaConfig { + RdmaConfig { + enabled, + allow_tcp_fallback: false, + listen_enabled: false, + wire_layout: RdmaWireLayout::ControlData, + wire_slot_capacity: (1 << 20) + 1024, + sq_depth: 4, + rq_depth: 4, + posted_receive_target: 4, + cq_depth: 16, + send_arena_bytes: 4 << 20, + receive_arena_bytes: 4 * ((1 << 20) + 1024), + device: std::env::var("CROWDB_RDMA_DEVICE").unwrap_or_default(), + source_address: std::env::var("CROWDB_RDMA_SOURCE_ADDR").unwrap_or_default(), + ..RdmaConfig::default() + } +} + +fn test_client() -> Arc { + let endpoint = std::env::var("CROWDB_RDMA_DATAPATH_ENDPOINT").unwrap(); + let enabled = std::env::var("CROWDB_RDMA_DATAPATH_TRANSPORT").unwrap() == "rdma"; + Arc::new( + DiskioClient::connect_for_tests( + (1..=2) + .map(|high| TestDiskRoute { + disk_id: DiskId::new(high, 0), + rack_id: 1, + node_id: 1, + disk_group_id: 1, + instance_id: 1, + endpoint: endpoint.clone(), + }) + .collect(), + DiskioClientConfig { + rdma: Some(test_rdma_config(enabled)), + normal_connections_per_endpoint: 1, + priority_connections_per_endpoint: 1, + rpc_workers: 1, + ..DiskioClientConfig::default() + }, + ) + .unwrap(), + ) +} + +async fn test_download(http: &reqwest::Client, url: &str, bytes: &[u8]) -> (usize, Option, bool) { + let get = http.get(url).send().await.unwrap(); + assert_eq!(get.status(), 200); + let fault = std::env::var("CROWDB_RDMA_DATAPATH_FAULT").as_deref() == Ok("1"); + let mut download = get.bytes_stream(); + let mut downloaded = Vec::new(); + let mut download_error = None; + while let Some(result) = download.next().await { + match result { + Ok(data) => downloaded.extend_from_slice(&data), + Err(error) => { + download_error = Some(error.to_string()); + break; + } + } + } + if fault { + assert!(download_error.is_some()); + assert!(!downloaded.is_empty() && downloaded.len() < bytes.len()); + assert_eq!(downloaded, bytes[..downloaded.len()]); + } else { + assert!(download_error.is_none(), "{download_error:?}"); + assert_eq!(downloaded, bytes); + } + (downloaded.len(), download_error, fault) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires an owned DiskIO process; run tools/rdma/run-datapath.py"] +async fn http_chunkio_round_trip_uses_selected_transport_and_exact_framed_bytes() { + let client = test_client(); + let chunks = Arc::new(ChunkIoClient::from_parts( + Arc::new(allocator::TestAllocator::default()), + Arc::new(RoutedDiskWriter::from_client(Arc::clone(&client))), + ChunkType::S3, + )); + let handler = Arc::new(http::TestHttp::new(Arc::clone(&chunks))); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (shutdown, stopped) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(crowdb_access_server::s3::serve( + listener, + handler.clone(), + async { + let _ = stopped.await; + }, + )); + let bytes: Vec<_> = (0..3 * (1 << 20) + 12_345) + .map(|index| u8::try_from(index % 251).unwrap()) + .collect(); + let http = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build() + .unwrap(); + let url = format!("http://{address}/datapath/object"); + let put = http + .put(&url) + .body(bytes.clone()) + .send() + .await + .unwrap_or_else(|error| { + panic!( + "HTTP PUT failed: {error}; allocator={:?}; transport={}", + handler.allocator.metrics_snapshot(), + String::from_utf8_lossy(&client.transport_status_json().unwrap()) + ); + }); + assert_eq!(put.status(), 200); + let etag = put.headers()["etag"].to_str().unwrap().to_owned(); + let (downloaded_bytes, download_error, fault) = test_download(&http, &url, &bytes).await; + let locations = handler.locations.load_full(); + assert!( + locations.len() >= 2, + "corpus must cross chunk and message boundaries" + ); + let mut cursor = 0; + for location in locations.iter() { + assert_eq!(location.logical_offset, cursor); + cursor += location.logical_length; + } + assert_eq!(cursor, u64::try_from(bytes.len()).unwrap()); + let status: serde_json::Value = serde_json::from_slice(&client.transport_status_json().unwrap()).unwrap(); + if std::env::var("CROWDB_RDMA_DATAPATH_TRANSPORT").unwrap() == "rdma" { + let rdma = &status["rdma"]; + assert!(rdma["direct_payload_bytes"].as_u64().unwrap() >= 2 * cursor); + for field in [ + "copied_payload_bytes", + "fallback_count", + "cq_errors", + "protocol_errors", + "send_errors", + "receive_post_errors", + ] { + assert_eq!(rdma[field], 0, "{field}: {status}"); + } + } else { + assert!(status["rdma"].is_null()); + } + assert_eq!(client.status().retries, 0); + let metrics = handler.allocator.metrics_snapshot(); + assert!(metrics.peak_retained_bytes <= 2 << 20); + assert!(metrics.allocations >= 4); + let result = serde_json::json!({ + "effective_uid": std::fs::read_to_string("/proc/self/status").unwrap().lines() + .find_map(|line| line.strip_prefix("Uid:")).unwrap().split_whitespace().nth(1).unwrap().parse::().unwrap(), + "payload_bytes": bytes.len(), "sha256": format!("{:x}", Sha256::digest(&bytes)), + "etag": etag, "locations": *locations, "transport": status, + "http_native": {"direct_bytes": metrics.direct_bytes, "prefix_copy_bytes": metrics.prefix_copy_bytes, + "peak_retained_bytes": metrics.peak_retained_bytes, "allocations": metrics.allocations, + "backpressure_events": metrics.backpressure_events}, + "errors": u64::from(fault), "expected_errors": u64::from(fault), "verification_errors": 0, + "downloaded_bytes": downloaded_bytes, "download_error": download_error, + "topology": std::env::var("CROWDB_RDMA_DATAPATH_TOPOLOGY").unwrap(), + }); + std::fs::write( + std::env::var("CROWDB_RDMA_DATAPATH_RESULT").unwrap(), + serde_json::to_vec_pretty(&result).unwrap(), + ) + .unwrap(); + shutdown.send(()).unwrap(); + server.await.unwrap().unwrap(); + chunks.shutdown_small_writes().await.unwrap(); +} diff --git a/app/crowdb-diskio/src/dio_main.cpp b/app/crowdb-diskio/src/dio_main.cpp index 61b9587de..d86744bce 100644 --- a/app/crowdb-diskio/src/dio_main.cpp +++ b/app/crowdb-diskio/src/dio_main.cpp @@ -116,13 +116,16 @@ static std::shared_ptr create_engine(const crowdb::dis #ifdef CROWDB_HAVE_LIBURING try { auto engine = std::make_shared(cfg.sq_entries); + std::printf("crowdb-diskio engine: io_uring\n"); return engine; } catch (const std::exception &e) { std::fprintf(stderr, "warning: uring engine creation failed (%s), falling back to blocking\n", e.what()); } #endif - return std::make_shared(cfg.thread_pool_size); + auto engine = std::make_shared(cfg.thread_pool_size); + std::printf("crowdb-diskio engine: blocking\n"); + return engine; } // Build the DiskSet from config. Disks with a non-empty path are @@ -223,6 +226,7 @@ int main(int argc, char *argv[]) dio_server->register_handlers(server); server.start(); + std::printf("crowdb-diskio transport ready: %s\n", transport->status_json().c_str()); // Scheduled executor for periodic tasks (group-0 sync, etc.). // The main loop polls run_due_tasks() every ~100ms. @@ -284,6 +288,7 @@ int main(int argc, char *argv[]) if (group0_sync != nullptr) { group0_sync->stop(); } + std::printf("crowdb-diskio transport final: %s\n", transport->status_json().c_str()); server.stop(); disk_set->shutdown(); diff --git a/doc/design/rpc/rpc-flow-analysis.md b/doc/design/rpc/rpc-flow-analysis.md index 1426ece05..8130e00c6 100644 --- a/doc/design/rpc/rpc-flow-analysis.md +++ b/doc/design/rpc/rpc-flow-analysis.md @@ -160,6 +160,57 @@ require exactly one failed completion, retained RDMA path and no TCP payload. The sections below retain earlier 20-second TCP references. `Nagle` means TCP coalescing is enabled; old histogram precision and measurement scopes differ. +### HTTP/ChunkIO and DiskIO functional acceptance — 2026-10-23 + +`tools/rdma/run-datapath.py` runs a real Hyper HTTP socket, the production S3 +body transfer, ChunkIO framing/mirror writes and verified reads, and a separate +DiskIO executable over O_DIRECT files. Only chunk placement/catalog state is +injected; this case does not exercise namespace/authentication or Group-0. +The corpus is 3,158,073 bytes, with an unaligned tail and two chunk locations. +All successful TCP/RDMA profiles match SHA256, ETag and physical/logical +location geometry. TPS and latency percentiles are not measured by this +functional fixture; the RPC throughput runner above remains the common load +measurement. + +| Topology | Path | Disk engine | Downloaded bytes | Upload direct bytes | Download direct bytes | Transport copy bytes | TCP fallback | Expected errors | Verification errors | +|--------------------------|------|-----------------|------------------|---------------------|-----------------------|----------------------|--------------|-----------------|---------------------| +| Two native processes | RDMA | io_uring | 3,158,073 | 6,319,478 | 3,159,739 | 0 | 0 | 0 | 0 | +| Two native processes | TCP | io_uring | 3,158,073 | n/a | n/a | n/a | n/a | 0 | 0 | +| Two non-root containers | RDMA | BlockingEngine | 3,158,073 | 6,319,478 | 3,159,739 | 0 | 0 | 0 | 0 | +| Two non-root containers | TCP | BlockingEngine | 3,158,073 | n/a | n/a | n/a | n/a | 0 | 0 | +| Containers / peer close | RDMA | BlockingEngine | 3,144,096 prefix | 6,319,478 | 3,145,728 | 0 | 0 | 1 | 0 | + +The failure case gates HTTP delivery after one verified frame, closes DiskIO, +then resumes delivery. The body terminates with an error and never publishes a +successful complete object. Expected attempts to unavailable read replicas are +visible in the raw preflight counters. Accepted RPC retry counts remain zero. +The upload direct-byte count includes two replicas and physical frame overhead. +HTTP's existing prefetched prefix copy is reported separately (8,098 bytes in +the native RDMA case); transport payload copies remain zero. + +Shared RDMA settings: two QPs per peer, one CQ and polling/event thread, +`sq_depth = rq_depth = posted_receive_target = 4`, requested `cq_depth = 16` +(provider 31), `wire_slot_capacity = 1049600`, `control_slot_capacity = 1024`, +`data_slot_capacity = 1048576`, `send_arena_bytes = 4194304` and +`receive_arena_bytes = 4198400`. Transport aggregation and TCP fallback are +disabled. One socket worker serves each peer. HTTP native admission peaks at +2 MiB. Native disk I/O uses `sq_entries = 32`; default OCI seccomp rejects +io_uring, so the service selects its asynchronous BlockingEngine. This disk +engine selection is independent of the RDMA/TCP path. + +Both containers use UID 1000, host networking and the same host/HCA, with +independent QPs. The pinned runtime image is +`sha256:e3c9c5c6d5fc3424ac3f22cf4cdcc9b1c872f47336484f54962b2541c4ed6cef`. +Manifests retain binary hashes, source/dirty state, actual server identity, +engine, settings, complete logs and raw counters. Artifacts under +`.crowdb-runtime/rdma-check/artifacts/rdma-datapath/` are: + +- `dd26085b76ef40f4bb61fe05813c9b17`: native RDMA; +- `73f4fd013ea048338b7d3a18dc7bf53b`: native TCP; +- `f7c807fcfbaa496ca02c4dd38d37e140`: shared-HCA OCI RDMA; +- `e2de510290114a8ab2e59780235f0110`: OCI TCP without RDMA devices; +- `24d2205fc7d94bbf92a52df176a2af8b`: OCI partial-download peer close. + ### Linux — 2026-08-27 AMD Ryzen 9 5950X, 16c/32t, x86_64, Linux 6.8. Single engine; coroutine diff --git a/doc/working/plan-rdma-datapath-stream.md b/doc/working/plan-rdma-datapath-stream.md index 276f9c8b3..2967f0c7a 100644 --- a/doc/working/plan-rdma-datapath-stream.md +++ b/doc/working/plan-rdma-datapath-stream.md @@ -264,7 +264,7 @@ it does not add application aggregation or a flush timer. and assertions remain unchanged. Preserve the aggregate-limit log as `access-suite-shell-limit.log`. The complete per-program-bounded Access suite passes with zero failures; its complete log is `access.log`. -- [~] **Verify real service flows**: run focused baseline and changed tests, +- [x] **Verify real service flows**: run focused baseline and changed tests, then real DiskIO plus HTTP/ChunkIO upload/download of objects larger than 1 MiB on the prepared HCA. Assert direct-payload counters and same data addresses at producer/SEND and receive/disk handoff, exact bytes and existing @@ -285,56 +285,113 @@ it does not add application aggregation or a flush timer. error counters in machine-readable artifacts. Run the same fixture in two non-root host-network containers sharing the one configured HCA, recording this local-loop topology and pinned runtime image and executable identities. + The first real HTTP fixture stalls before any DiskIO RPC. Instrumentation + confirms placement and registered producer allocation succeed, then two + owners fill the 2 MiB HTTP budget without owner handoff. The new test wrapper + omitted `enable_owner_handoff()`, which production PUT/UploadPart already + call. Initialize handoff and deferred MD5 exactly as production does; retain + the original 10-second HTTP deadline and all byte/copy assertions. Remove + temporary per-frame diagnostics. Failed evidence is preserved under + `rdma-datapath/bff1ae9a76714199a7dabe9da47edde8`, + `391b25b30b174bb5bf6b456ac67a2f5a`, and `fc50a99060c84e33a99118fc966c2cf0`. + Upload then completes with actual RDMA writes, but GET fails before issuing a + disk read: the injected catalog seals the chunk without sealing its strips, + so the production reader correctly rejects zero durable strip bytes. Mirror + the production seal handler's bounded strip lengths in the injected catalog; + retain normal reader availability/integrity checks. Failed logs remain in + `ba2b03c512bb4989af8957434422ff1a` and `5f8600b20b364ea5bb2e8ce00d29c3ab`. + Native RDMA and TCP now pass the 3,158,073-byte corpus across two chunks. + Two non-root shared-HCA containers also pass. Registered transport payload + copies and TCP fallback are zero; HTTP's bounded prefetched header/body + prefix copy remains separately reported. OCI's default seccomp denies + io_uring, so that profile uses the existing asynchronous BlockingEngine; + report the actual engine explicitly, while native uses io_uring. + Add a partial-download failure variant: the HTTP fixture publishes one + verified frame, then pauses before further stream delivery. The runner + closes DiskIO and releases that gate. The HTTP body must fail as one object, + with no successful full download or TCP suffix. Preserve exact expected + failure counts separately from verification errors in its result artifact. + The completed five-profile matrix passes with identical SHA256, ETag and + physical/logical location geometry for all four successful profiles. Actual + upload direct payload is 6,319,478 bytes across two replicas; download direct + payload is 3,159,739 bytes. Transport payload copies/fallback/errors are zero. + Final artifacts under `.crowdb-runtime/rdma-check/artifacts/rdma-datapath/`: + native RDMA `dd26085b76ef40f4bb61fe05813c9b17`, native TCP + `73f4fd013ea048338b7d3a18dc7bf53b`, OCI RDMA + `f7c807fcfbaa496ca02c4dd38d37e140`, OCI TCP + `e2de510290114a8ab2e59780235f0110`, OCI peer-close + `24d2205fc7d94bbf92a52df176a2af8b`. OCI uses inspected UID 1000, + host networking and pinned image + `sha256:e3c9c5c6d5fc3424ac3f22cf4cdcc9b1c872f47336484f54962b2541c4ed6cef`. + The failed download returns 3,144,096 verified prefix bytes then one HTTP + body error, with zero verification errors/copy/fallback. Expected subsequent + reader attempts to unavailable replicas are retained in preflight counters; + the semantic client performs no accepted-RPC replay. Both peers use two QPs, + one CQ/poller, `sq_depth = rq_depth = posted_receive_target = 4`, + `cq_depth = 16` (provider 31), 4 MiB send arena, 4,198,400-byte receive arena, + and 1,049,600-byte control/data wire slots. HTTP admission peaks at 2 MiB. + Native disk engine is io_uring with `sq_entries = 32`; OCI uses BlockingEngine + under default seccomp. These are functional tests, not TPS measurements. + Final full C++/FFI gate passes at + `.crowdb-runtime/artifacts/unit/cpp-2032c286bd054ca089f560b5b6fced54/`. + All 278 affected Rust tests pass (three external/owned-fixture cases ignored + by the default suite and exercised separately where applicable). The semantic + read/write/fsync fault tests run on the actual HCA. Workspace all-feature + clippy, rs-lint, Rust/C++ format, scoped tree-lint (zero exit with existing + style warnings) and Python compile checks pass. Final environment-sanitized + OCI fault run is `fef1971b25904a9abff2cec49d6a9bed`, with one expected error + and zero verification errors consistently in both manifest and result. + Remaining work is requirement cleanup and advancing to the existing R241 plan. ## Phase 1: map existing data flow -- [ ] **Trace upload flow**: identify where HTTP body bytes enter ChunkIO, +- [x] **Trace upload flow**: identify where HTTP body bytes enter ChunkIO, become aggregated buffers, are submitted to DiskIO, and are released after durable completion. Files: `app/crowdb-access-server/`, `lib/crowdb-chunk-client/`, `lib/crowdb-diskio-client/`, and ChunkIO designs. -- [ ] **Trace download flow**: identify how SDK/HTTP consumers receive data, +- [x] **Trace download flow**: identify how SDK/HTTP consumers receive data, how read-ahead aggregation chooses message boundaries, and when a caller releases each buffer. -- [ ] **Define transport adapter**: add the smallest Access/DiskIO adapter that +- [x] **Define transport adapter**: add the smallest Access/DiskIO adapter that accepts an already formed buffer, actual length, request identity, sequence, and logical offset. Do not add a second flush timer or aggregation queue. ## Phase 2: upload to DiskIO -- [ ] **Allocate 1 MiB datapath class**: select the configured 1 MiB data +- [x] **Allocate 1 MiB datapath class**: select the configured 1 MiB data capacity plus 1 KiB control buffer for S3/HTTP/DiskIO streams; the configured total wire slot includes both regions and the header. -- [ ] **Fill registered send buffer**: make the HTTP/ChunkIO producer write +- [x] **Fill registered send buffer**: make the HTTP/ChunkIO producer write directly into the send-owned registered buffer and submit its actual length. The producer must stop when the send/registered budget is exhausted. -- [ ] **Receive ownership handoff**: on RDMA receive completion, transfer the +- [x] **Receive ownership handoff**: on RDMA receive completion, transfer the DiskIO buffer to the asynchronous disk writer. Keep request, sequence, logical offset, and expected length attached to the write. -- [ ] **Repost after disk completion**: only reset and repost the receive buffer +- [x] **Repost after disk completion**: only reset and repost the receive buffer after the disk write and integrity result are complete. Propagate disk errors to the stream and do not silently repost corrupted data. ## Phase 3: download to SDK/HTTP -- [ ] **Prepost receive buffers**: use the R238 pool and R239 receive window; +- [x] **Prepost receive buffers**: use the R238 pool and R239 receive window; publish a `BufferLease` only after header/class/length validation. -- [ ] **Preserve caller ownership**: keep the lease caller-owned until explicit +- [x] **Preserve caller ownership**: keep the lease caller-owned until explicit release. Do not let CQ completion or a stream callback recycle it. -- [ ] **Implement multi-message assembly**: deliver messages in sequence and +- [x] **Implement multi-message assembly**: deliver messages in sequence and logical offset order, preserving end/error flags and cancellation behavior. -- [ ] **Keep TCP parity**: use the same access result and error mapping when the +- [x] **Keep TCP parity**: use the same access result and error mapping when the stream selects TCP before RDMA payload submission. ## Phase 4: pressure and failure behavior -- [ ] **Disk queue pressure**: stop reposting when the asynchronous disk queue +- [x] **Disk queue pressure**: stop reposting when the asynchronous disk queue reaches its byte bound; let receive-window exhaustion backpressure the sender. -- [ ] **SDK pressure**: stop publishing new leases when held bytes reach the +- [x] **SDK pressure**: stop publishing new leases when held bytes reach the configured limit; resume after release. -- [ ] **Partial stream failure**: fail the logical object/request after partial +- [x] **Partial stream failure**: fail the logical object/request after partial RDMA transfer and quarantine uncertain buffers; never append TCP bytes. -- [ ] **Cancellation**: stop new aggregation submissions, drain known WRs, +- [x] **Cancellation**: stop new aggregation submissions, drain known WRs, preserve in-flight buffer lifetimes, and return only buffers proven safe. ## Tests diff --git a/lib/crowdb-chunk-client/src/disk_io/routing.rs b/lib/crowdb-chunk-client/src/disk_io/routing.rs index 0985a71e3..ff8baedb2 100644 --- a/lib/crowdb-chunk-client/src/disk_io/routing.rs +++ b/lib/crowdb-chunk-client/src/disk_io/routing.rs @@ -21,6 +21,12 @@ pub struct RoutedDiskWriter { } impl RoutedDiskWriter { + /// Reuse an already connected semantic client in an embedded storage stack. + #[must_use] + pub fn from_client(client: Arc) -> Self { + Self { client } + } + /// Discover live DiskIO owners and connect to their endpoints. pub async fn connect(service: &ServiceRegistryClient, hardware: &HardwareClient) -> Result { Self::connect_with_connections(service, hardware, 1).await diff --git a/lib/crowdb-diskio-client/src/semantic.rs b/lib/crowdb-diskio-client/src/semantic.rs index b331168bf..a769e9db0 100644 --- a/lib/crowdb-diskio-client/src/semantic.rs +++ b/lib/crowdb-diskio-client/src/semantic.rs @@ -317,6 +317,16 @@ impl std::fmt::Debug for DiskioClient { } impl DiskioClient { + /// Snapshot the owned RPC transport's versioned settings and counters. + /// + /// # Errors + /// Returns an error if the native snapshot cannot be allocated. + pub fn transport_status_json(&self) -> DiskioResult> { + self.server + .transport_status_json() + .map_err(|error| DiskioError::TransportUnavailable(error.to_string())) + } + /// Producer storage uses the same device arena as the discovered routes. /// An exhausted active arena returns no allocation, including in mixed /// TCP/RDMA routing; it never expands the native registered budget. diff --git a/tools/rdma/run-datapath.py b/tools/rdma/run-datapath.py new file mode 100644 index 000000000..b725e15ef --- /dev/null +++ b/tools/rdma/run-datapath.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""HTTP/ChunkIO to a separate O_DIRECT DiskIO process, native or two OCI containers.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import socket +import subprocess +import sys +import time +import uuid + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tools/benchmark")) +from rpc_bench_process import Runtime, start # noqa: E402 + + +def build(pixi, output): + subprocess.run([pixi, "run", "--", "cmake", "--build", "app/crowdb-diskio/build", + "-j32", "--target", "crowdb-diskio"], cwd=ROOT, check=True) + result = subprocess.run([pixi, "run", "--", "cargo", "test", "-p", "crowdb-access-server", + "--test", "rdma_datapath_test", "--no-run", "--message-format=json"], + cwd=ROOT, capture_output=True, text=True) + (output / "build.jsonl").write_text(result.stdout) + print(result.stderr, end="", flush=True) + executable = None + for line in result.stdout.splitlines(): + item = json.loads(line) + if item.get("reason") == "compiler-message": + print(item["message"].get("rendered", ""), end="", flush=True) + if item.get("target", {}).get("name") == "rdma_datapath_test" and item.get("executable"): + executable = Path(item["executable"]) + result.check_returncode() + if executable is None: + raise RuntimeError("Cargo did not identify the datapath test executable") + return executable + + +def status(log, marker): + for line in log.read_text().splitlines(): + if line.startswith(marker): + return json.loads(line.removeprefix(marker)) + return None + + +def ready(server, log, rdma): + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if server.poll() is not None: + raise RuntimeError(f"DiskIO exited before readiness: {server.returncode}") + observed = status(log, "crowdb-diskio transport ready: ") + if observed is not None: + if observed["allow_tcp_fallback"]: + raise RuntimeError("datapath fixture must disable TCP fallback") + if rdma and not (observed["rdma"]["ready"] and observed["rdma"]["listen_active"]): + raise RuntimeError("DiskIO did not publish an active RDMA listener") + if not rdma and observed["rdma"] is not None: + raise RuntimeError("TCP-only DiskIO unexpectedly enabled RDMA") + return observed + time.sleep(0.02) + raise RuntimeError("DiskIO readiness deadline expired") + + +def checked_result(output, transport, final, fault): + result = json.loads((output / "client.json").read_text()) + if result["errors"] != int(fault) or result["verification_errors"] != 0 or result["payload_bytes"] <= 3 << 20: + raise RuntimeError("datapath corpus or error count is invalid") + if transport == "rdma": + rdma = final["rdma"] + for field in ("copied_payload_bytes", "fallback_count", "cq_errors", "protocol_errors", + "send_errors", "receive_post_errors"): + if rdma[field] != 0: + raise RuntimeError(f"DiskIO {field}={rdma[field]}") + if rdma["direct_payload_bytes"] < (1 if fault else result["payload_bytes"]): + raise RuntimeError("DiskIO download did not use direct RDMA payloads") + elif final["rdma"] is not None: + raise RuntimeError("TCP-only DiskIO attempted RDMA") + result["server_transport"] = final + result["disk_engine"] = next((line.removeprefix("crowdb-diskio engine: ") + for line in (output / "server.log").read_text().splitlines() + if line.startswith("crowdb-diskio engine: ")), None) + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--transport", choices=("tcp", "rdma"), required=True) + parser.add_argument("--topology", choices=("native", "containers"), default="native") + parser.add_argument("--address", default=os.environ.get("CROWDB_RDMA_SOURCE_ADDR", "10.200.1.1")) + parser.add_argument("--device", default=os.environ.get("CROWDB_RDMA_DEVICE", "rocep35s0f0")) + parser.add_argument("--image", default="crowdb-node:dev") + parser.add_argument("--fault-disconnect", action="store_true") + args = parser.parse_args() + pixi = shutil.which("pixi") + if pixi is None: + raise RuntimeError("pixi is required") + root = Path(os.environ.get("CROWDB_RUNTIME_ROOT", ROOT / ".crowdb-runtime")) + output = (root / "artifacts/rdma-datapath" / uuid.uuid4().hex).resolve() + output.mkdir(parents=True) + client_binary = build(pixi, output) + server_binary = ROOT / "app/crowdb-diskio/build/crowdb-diskio" + if args.topology == "containers": + args.image = subprocess.check_output([pixi, "run", "--", "docker", "image", "inspect", + args.image, "--format", "{{.Id}}"], text=True).strip() + rdma = args.transport == "rdma" + address = args.address if rdma else "127.0.0.1" + with socket.socket() as reservation: + reservation.bind((address, 0)) + port = reservation.getsockname()[1] + config = ["[server]", f'bind_address = "{address}"', f"listen_port = {port}", "rpc_workers = 1", + "o_direct = true", "[engine]", "sq_entries = 32", "[metrics]", "interval_secs = 0"] + for high in (1, 2): + disk = output / f"disk-{high}.data" + with disk.open("wb") as stream: + stream.truncate(64 << 20) + config += ["[[disk]]", f'id = "{high}:0"', f'path = "{disk}"', f"zone_capacity = {64 << 20}"] + config_path = output / "diskio.toml" + config_path.write_text("\n".join(config) + "\n") + env = {key: value for key, value in os.environ.items() if not key.startswith("CROWDB_RDMA_")} + env.update(CROWDB_RUNTIME_ROOT=str(output), CROWDB_RDMA_ENABLED=str(int(rdma)), + CROWDB_RDMA_ALLOW_TCP_FALLBACK="0", CROWDB_RDMA_DEVICE=args.device, + CROWDB_RDMA_SOURCE_ADDR=address, CROWDB_RDMA_CQ_COUNT="1", CROWDB_RDMA_CQ_DEPTH="16", + CROWDB_RDMA_WIRE_LAYOUT="1", CROWDB_RDMA_PORT="1", + CROWDB_RDMA_SQ_DEPTH="4", CROWDB_RDMA_RQ_DEPTH="4", CROWDB_RDMA_POSTED_RECEIVE_TARGET="4", + CROWDB_RDMA_SEND_ARENA_BYTES=str(4 << 20), CROWDB_RDMA_RECEIVE_ARENA_BYTES=str(4 * ((1 << 20) + 1024)), + CROWDB_RDMA_AGGREGATION_ENABLED="0", CROWDB_RDMA_DATAPATH_ENDPOINT=f"{address}:{port}", + CROWDB_RDMA_DATAPATH_TRANSPORT=args.transport, CROWDB_RDMA_DATAPATH_RESULT=str(output / "client.json"), + CROWDB_RDMA_DATAPATH_FAULT=str(int(args.fault_disconnect)), + CROWDB_RDMA_DATAPATH_PAUSED=str(output / "download-paused"), + CROWDB_RDMA_DATAPATH_RESUME=str(output / "download-resume"), + CROWDB_RDMA_DATAPATH_TOPOLOGY=args.topology + ("_hca_local_loop" if rdma else "_tcp_loopback")) + topology = args.topology if rdma or args.topology == "native" else "containers-tcp" + runtime = Runtime(ROOT, output, pixi, topology, args.image) + manifest = {"transport": args.transport, "topology": env["CROWDB_RDMA_DATAPATH_TOPOLOGY"], + "image": args.image if args.topology == "containers" else None, + "source": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip(), + "source_dirty": bool(subprocess.check_output(["git", "status", "--porcelain"], cwd=ROOT, text=True)), + "binaries": {str(binary): hashlib.sha256(binary.read_bytes()).hexdigest() + for binary in (server_binary, client_binary)}, "errors": None, + "expected_errors": int(args.fault_disconnect), "verification_errors": None} + server = client = None + client_log_printed = False + server_log, client_log = output / "server.log", output / "client.log" + try: + with server_log.open("w") as stream: + server = start(runtime.command([server_binary, "--config", config_path], env, "server"), env, stream, ROOT) + manifest["server_startup"] = ready(server, server_log, rdma) + if args.topology == "containers": + info = json.loads(subprocess.check_output([pixi, "run", "--", "docker", "inspect", + runtime.names["server"]], text=True))[0] + manifest["server_container"] = { + "image": info["Image"], "user": info["Config"]["User"], + "network": info["HostConfig"]["NetworkMode"], "devices": info["HostConfig"]["Devices"], + } + if info["Config"]["User"].split(":")[0] in ("", "0", "root"): + raise RuntimeError("DiskIO container must run as non-root") + with client_log.open("w") as stream: + client = start(runtime.command([client_binary, "--ignored", "--exact", + "http_chunkio_round_trip_uses_selected_transport_and_exact_framed_bytes", "--nocapture"], + env, "client"), env, stream, ROOT) + if args.fault_disconnect: + deadline = time.monotonic() + 10 + while not (output / "download-paused").exists(): + if client.poll() is not None or time.monotonic() >= deadline: + raise RuntimeError("download did not reach its verified partial-result gate") + time.sleep(0.01) + runtime.stop(server, "server") + (output / "download-resume").write_text("peer stopped\n") + code = client.wait(timeout=45) + print(client_log.read_text(), end="", flush=True) + client_log_printed = True + if code != 0: + raise RuntimeError(f"datapath client failed: {code}") + if args.topology == "containers" and json.loads((output / "client.json").read_text())["effective_uid"] == 0: + raise RuntimeError("HTTP/ChunkIO container must run as non-root") + runtime.stop(server, "server") + final = status(server_log, "crowdb-diskio transport final: ") + if final is None: + raise RuntimeError("DiskIO did not retain its final transport counters") + checked_result(output, args.transport, final, args.fault_disconnect) + manifest["errors"] = int(args.fault_disconnect) + manifest["verification_errors"] = 0 + except Exception as error: + manifest["verification_errors"] = 1 + manifest["failure"] = str(error) + raise + finally: + for process, side in ((client, "client"), (server, "server")): + if process is not None: + runtime.stop(process, side) + if not client_log_printed and client_log.exists(): + print(client_log.read_text(), end="", flush=True) + if server_log.exists(): + print(server_log.read_text(), end="", flush=True) + (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + print(f"Artifacts: {output}", flush=True) + + +if __name__ == "__main__": + main() From 075ded4742db080329531d85d189480d68a2c592 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 04:33:42 +0800 Subject: [PATCH 045/258] Accept the registered HTTP and DiskIO datapath verification --- doc/backlog/R240-rdma-datapath-stream.md | 70 ---- doc/backlog/R5-rdma-alloc.md | 12 +- doc/backlog/backlog.md | 6 +- doc/design/rpc/rpc-flow-analysis.md | 13 +- doc/working/plan-rdma-datapath-stream.md | 457 ----------------------- doc/working/plan-rdma.md | 38 +- 6 files changed, 35 insertions(+), 561 deletions(-) delete mode 100644 doc/backlog/R240-rdma-datapath-stream.md delete mode 100644 doc/working/plan-rdma-datapath-stream.md diff --git a/doc/backlog/R240-rdma-datapath-stream.md b/doc/backlog/R240-rdma-datapath-stream.md deleted file mode 100644 index 227818acb..000000000 --- a/doc/backlog/R240-rdma-datapath-stream.md +++ /dev/null @@ -1,70 +0,0 @@ - - - -### R240: Access server — RDMA datapath stream - -## Problem - -The S3/HTTP and DiskIO path currently has no RDMA stream contract. The path -must preserve existing ChunkIO aggregation, use a 1 MiB data capacity, and -transfer ownership from HTTP input to DiskIO without an intermediate CPU -payload copy. DiskIO must not recycle a receive buffer while asynchronous disk -I/O still uses it. - -## Solution - -1. Feed already aggregated ChunkIO buffers to the common transport without - another application aggregation queue or flush timer. Ordinary RPC transport - coalescing remains bounded by both peers' wire-slot admission. -2. Use a 1 KiB control region plus a 1 MiB data region for the direct datapath. - `wire_slot_capacity` bounds their total, including the RPC header. Negotiate - this layout before payload submission; reject incompatible RDMA layouts - before selecting the path. Direct messages use separate registered SGEs for - control and immutable payload, so mirror fanout can retain the same payload - without modifying its bytes or copying it behind different request headers. - A logical object is an ordered sequence of messages when larger than one - message. This layout sends one application message per SEND; the existing - contiguous layout retains bounded complete-frame aggregation. -3. Let Access Server fill its registered send buffer from the HTTP path. Let - DiskIO own its registered receive buffer after CQ completion and submit that - buffer directly to asynchronous disk I/O. -4. Repost a DiskIO receive buffer only after disk completion. Apply bounded - receive-window backpressure when DiskIO or the SDK consumer is slow. - Receive data starts at the registered data-region base, preserving disk - alignment. Control storage and data storage share the configured registered - byte budget. A held or quarantined lease cannot be replaced outside it. -5. Preserve R239 fallback: before payload submission the operation may use TCP; - after partial RDMA transfer the stream fails without TCP byte splicing. - The semantic DiskIO client must also stop automatic reconnect/retry after - an accepted RDMA operation fails. Surface its typed failure or uncertain - write/durability outcome; preserve pre-submit rejection and TCP retry policy. - -## Dependencies - -- Accepted [registered buffers and RDMA RPC](../design/rpc/design-crowdb-rpc-rdma.md). -- Existing ChunkIO aggregation, Access Server S3, DiskIO asynchronous I/O, and - integrity contracts. - -## Acceptance - -- Given an aggregated HTTP upload, when it is sent to DiskIO, the receiver - writes the registered buffer directly and reposts it only after disk - completion. Invariant: buffer lifetime covers disk I/O. E2E test. -- Given an object larger than 1 MiB, when it is transferred, ordered messages - reconstruct the exact object using sequence and logical offset. Invariant: - stream ordering. E2E test. -- Given a full DiskIO write queue, when receive buffers are consumed, sender - backpressure remains bounded and no replacement buffer is allocated. - Invariant: bounded receive window. Integration test. -- Given an RDMA failure after payload submission, when the operation ends, no - TCP bytes are appended to the partial object. Invariant: fallback isolation. - E2E test. - -Run: - -```bash -pixi run -- cargo fmt --all -- --check -pixi run -- cargo test -p crowdb-access-server --all-targets -pixi run -- cargo test -p crowdb-diskio-client --all-targets -pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings -``` diff --git a/doc/backlog/R5-rdma-alloc.md b/doc/backlog/R5-rdma-alloc.md index c1b2bd097..238a1f209 100644 --- a/doc/backlog/R5-rdma-alloc.md +++ b/doc/backlog/R5-rdma-alloc.md @@ -5,9 +5,9 @@ ## Status -The registered host-buffer foundation and common RDMA RPC transport are -accepted. Remaining child requirements R240 through R242 retain the datapath, -Chunk-KV, and fat-client goals. +The registered host-buffer foundation, common RDMA RPC transport and +HTTP/ChunkIO/DiskIO datapath are accepted. Remaining child requirements R241 +and R242 retain the Chunk-KV and fat-client goals. ## Problem @@ -24,9 +24,9 @@ defined in [`design-crowdb-rpc-rdma.md`](../design/rpc/design-crowdb-rpc-rdma.md bounded 64 KiB/256 KiB/1 MiB classes, explicit ownership and quarantine, TCP preflight, per-CQ polling/event workers and one-way fallback. 2. Retain the verified transport and benchmark invariants when adding services. -3. Implement the S3/HTTP/DiskIO stream in - [R240](R240-rdma-datapath-stream.md), reusing existing ChunkIO aggregation - and 1 MiB datapath buffers. +3. Reuse the accepted [HTTP/ChunkIO/DiskIO stream](../design/rpc/design-crowdb-rpc-rdma.md#7-service-interactions), + preserving existing aggregation, aligned 1 MiB receive data, disk completion + ownership, caller leases and partial-result failure without a TCP suffix. 4. Implement Access Server to Chunk-KV integration in [R241](R241-rdma-chunk-kv-access.md), using 64 KiB metadata and 256 KiB aggregated result messages. diff --git a/doc/backlog/backlog.md b/doc/backlog/backlog.md index 585befbb1..49cd489c5 100644 --- a/doc/backlog/backlog.md +++ b/doc/backlog/backlog.md @@ -32,10 +32,6 @@ live in the linked document; implementation follows - **[R170](R170-s3-cuobject-rdma.md)** — optional cuObject RDMA data plane. **Area:** access server / S3 / DiskIO / RDMA. **Complexity:** High. **Status:** Deferred until the basic TCP S3 path is stable and measured. -- **[R240](R240-rdma-datapath-stream.md)** — RDMA S3/HTTP/DiskIO stream using - existing ChunkIO aggregation. **Area:** access server / S3 / DiskIO / RDMA. - **Complexity:** High. **Dependencies:** accepted registered buffers and RDMA - RPC. - **[R241](R241-rdma-chunk-kv-access.md)** — RDMA Access Server to Chunk-KV transport with bounded metadata and aggregate messages. **Area:** access server / Chunk-KV / RDMA. **Complexity:** High. **Dependencies:** accepted @@ -109,7 +105,7 @@ live in the linked document; implementation follows **Area:** crowdb-tree engine. **Complexity:** Medium. - **[R5](R5-rdma-alloc.md)** — RDMA host-buffer RPC and data transport umbrella. **Area:** RPC / access server / Chunk-KV / DiskIO / SDK / RDMA. - **Complexity:** High. **Status:** Foundation/transport accepted; R240–R242 remain. + **Complexity:** High. **Status:** Foundation/transport/datapath accepted; R241–R242 remain. - **[R60](R60-tree-scan-sibling-leaf-readahead.md)** — sibling-leaf readahead on cold scans. **Area:** crowdb-tree / scan / DiskIO. **Complexity:** Medium. **Status:** Deferred pending cold file/block-backed measurements. diff --git a/doc/design/rpc/rpc-flow-analysis.md b/doc/design/rpc/rpc-flow-analysis.md index 8130e00c6..4a6a75a48 100644 --- a/doc/design/rpc/rpc-flow-analysis.md +++ b/doc/design/rpc/rpc-flow-analysis.md @@ -201,15 +201,16 @@ engine selection is independent of the RDMA/TCP path. Both containers use UID 1000, host networking and the same host/HCA, with independent QPs. The pinned runtime image is `sha256:e3c9c5c6d5fc3424ac3f22cf4cdcc9b1c872f47336484f54962b2541c4ed6cef`. -Manifests retain binary hashes, source/dirty state, actual server identity, +All five profiles use clean source `ab1ffbfb`. Manifests retain binary hashes, +source/dirty state, actual server identity, engine, settings, complete logs and raw counters. Artifacts under `.crowdb-runtime/rdma-check/artifacts/rdma-datapath/` are: -- `dd26085b76ef40f4bb61fe05813c9b17`: native RDMA; -- `73f4fd013ea048338b7d3a18dc7bf53b`: native TCP; -- `f7c807fcfbaa496ca02c4dd38d37e140`: shared-HCA OCI RDMA; -- `e2de510290114a8ab2e59780235f0110`: OCI TCP without RDMA devices; -- `24d2205fc7d94bbf92a52df176a2af8b`: OCI partial-download peer close. +- `7852ce8aef7a49e6804b1f25241f8424`: native RDMA; +- `9235eeb5ad6f407dae9a8fab190d0f73`: native TCP; +- `5268f02d8ffb4e458113d39e6a5f9dd2`: shared-HCA OCI RDMA; +- `a09e1892158d4b1aa2499c39d926f3fb`: OCI TCP without RDMA devices; +- `227f2ddb147a4fedb5fff32ffe187eda`: OCI partial-download peer close. ### Linux — 2026-08-27 diff --git a/doc/working/plan-rdma-datapath-stream.md b/doc/working/plan-rdma-datapath-stream.md deleted file mode 100644 index 2967f0c7a..000000000 --- a/doc/working/plan-rdma-datapath-stream.md +++ /dev/null @@ -1,457 +0,0 @@ - - - -# RDMA datapath stream Plan - -Upstream: [R240](../backlog/R240-rdma-datapath-stream.md), -[root RDMA plan](plan-rdma.md), [registered buffers](../design/rpc/design-crowdb-rpc-rdma.md#4-registered-arenas-and-slot-ownership), -and [RDMA RPC transport](../design/rpc/design-crowdb-rpc-rdma.md). - -Goal: connect existing HTTP/ChunkIO aggregation to the RDMA RPC stream and -DiskIO receive loop without introducing another aggregation or CPU payload -copy. - -## Execution checkpoint - -The transport/foundation acceptance is complete at `55ffee38`; cleanup is -committed at `d6bedf94`. Preserve the verified contiguous-slot benchmark path. -The six existing DiskIO-client integration cases pass before service changes. -The full C++ gate already covers DiskIO engines and AlignedWriter. -The 31 focused ChunkIO configuration/mirror/stream cases and 11 Access Server -configuration/service cases also pass before production changes. - -Observed flow and the first missing seam: - -- `NativeBodyAllocator` / `NativeOwner` in `crowdb-access-s3` supplies Hyper's - body receive regions. `NativeFramedOwner` finalizes physical frame regions - in place and supplies immutable `Bytes` views. S3 `write_object_body` passes - those owners through `UploadBuffer::Framed` to - `LargeAsyncObjectWriter::on_framed_data`. Do not route this path through - `frame_tail` or introduce another HTTP assembly copy. -- `RoutedDiskWriter` delegates views to `DiskioClient::write_views`, then - `WireClient::write_segment_views` and `RpcClient::call_chain`. Existing - grouping preserves order and `ordering_zone_offset`. The current RDMA - `flush_connection` copies every view into a new contiguous wire slot. -- `DiskioServer::handle_write` clones `request->data_buf`, deletes the Frame, - and retains the clone through `AlignedWriter::submit_ordered` completion. - Its current `data_buf->data` access must honor `payload_data()` for views. - AlignedWriter passes aligned full blocks directly, with explicit edge-block - staging only when required by device alignment. -- Reads currently allocate from `conn->pool()` and may stage O_DIRECT data. - Allocate eligible direct reads from registered producer storage. Rust - `WireClient::await_read_response` already returns `Bytes::from_owner(Buffer)`; - the final caller release can therefore own receive repost admission. - -### Direct layout and implementation order - -The existing one-SGE path copies complete frames and remains the ordinary RPC -benchmark path. A zero-copy service producer needs an immutable registered -payload independent of each destination's control header: mirror targets have -different disk addresses and may send concurrently. Add an explicitly -negotiated control/data layout for this service path, using the QP's existing -two-SGE capability. This avoids rewriting payload prefixes during fanout. -The layout is an implementation decision for the required zero-copy path; -it does not add application aggregation or a flush timer. - -- [x] **Expose bounded producer allocation first**: add native transport and - C ABI allocation from the already device-owned send arena, with explicit - `require_registered` policy and no replacement allocation on exhaustion. - Publish the arena pointer only after CM/device adoption using release/acquire; - the existing operation gate pins shutdown. Add unique writable Rust storage - with an uninitialized-capacity view and an explicit initialization contract. - Test no-device allocation, hard bounds, pointer-preserving commit and real - registered storage retained across server destruction. This API alone does - not establish zero-copy SEND; direct layout and producer wiring follow. - The new ordinary-buffer lifetime test first reproduces a SIGSEGV in - `Buffer::release()` after the `SystemBufferPool` facade is destroyed (GDB - confirms the retained raw recycler pointer). Move ordinary pool admission - and recycling into a retained shared State, keeping its CAS admission and - avoiding a new lock. Outstanding buffers must remain releasable after their - facade is destroyed, as registered leases already do. Verify existing detach - and concurrent outstanding-admission tests alongside the new regression. - Use the admission reservation itself as the allocation's State lifetime - reference, plus one facade bit. This retains the prior one-CAS acquire and - one-counter-release hot path; final release and facade destruction atomically - choose the sole State destructor. Add their concurrent race test. - The first HCA producer case reached its readiness assertion before calling - `RpcServer::start`; the listener is created during start. Add that missing - setup action and rerun the case without changing deadlines or assertions. - The next run confirms listen readiness but rejects allocation: PD/arenas are - adopted on the first real CM connection, not on listen. Establish a real - client QP before the allocation assertions; also assert required allocation - is unavailable before that CM adoption. This matches route-before-producer - initialization in the planned service wiring. - The corrected real-HCA test passes in 14 ms: exact arena exhaustion, - no optional system replacement, release/reallocation, and a cloned lease - surviving server/CM teardown. Held receive and peer-close cases also pass. - The final production source passes full C++/FFI verification at - `.crowdb-runtime/artifacts/unit/cpp-9266930ffd4142d18fd3d5c5318082de/`, - all 141 no-verbs CTest cases (one HCA case skips there), both new Rust - producer tests, workspace clippy/rs-lint, formatting and scoped tree-lint. - Tree-lint exits zero with style warnings. The test's corrected startup/CM - setup is rebuilt and verified separately after that full gate. -- [x] **Add negotiated direct layout**: add `RdmaWireLayout::Contiguous` and - `ControlData`, defaulting to `Contiguous`. For `ControlData`, the control - region is `RDMA_CONTROL_CAPACITY` including the 14-byte RPC header; derive - data capacity from `wire_slot_capacity - RDMA_CONTROL_CAPACITY` and validate - it against the existing classes. The 1 MiB data default therefore has total - `wire_slot_capacity = 1049600`, rather than claiming 1 MiB of payload fits - in a 1 MiB total wire slot. Advertise layout in the versioned TCP capability - and bind it in the compact CM offer. A mismatched layout fails before payload - and follows the existing explicit fallback policy. Reject direct layout with - transport aggregation enabled or provider SGE capacity below two. Extend - typed C/Rust cold config, environment parsing and status with the same names. - Preserve the existing C ABI configuration initializer/struct size. Add a - versioned configure entry accepting the existing config plus `wire_layout`, - rather than writing new fields past a version-one caller's storage. - Files: `rdma_config.h`, capability/offer/preflight modules, config ABI/FFI, - `rdma_connection.cpp`, config/status tests. - Implementation and the first 38 focused cases pass, including real HCA - full 1 MiB direct echo, 4096-byte receive alignment, exact payload/copy - counters, all four receive leases held without repost, release-triggered - repost, and source/receive leases retained after both CM transports die. - A second HCA case proves layout mismatch does not attempt CM and only uses - TCP with explicit fallback enabled. Portable cases cover exact total wire - length, malformed/truncated control, unknown layout, two-SGE requirements, - version-two decode and CM identity binding. Full C++/FFI verification passes - at `.crowdb-runtime/artifacts/unit/cpp-90779b2686624240909737a33e7c2a99/`: - 543 Tree, 168 RPC and 64 DiskIO native cases; all FFI cases including the - versioned direct-layout configuration pass. TCP-only build has 146 CTest - items, zero failures, with the producer HCA case skipped as expected. - Workspace all-feature clippy, rs-lint, Rust/C++ formatting, scoped tree-lint - (exit zero with style warnings) and all four CLI benchmark report tests pass. - The CLI test command initially named `bench_rpc_test`; Cargo lists the actual - target as `rpc_bench_test`, which passes after correcting only the command. - Final real-HCA lease/producer/layout cases pass at - `.crowdb-runtime/rdma-check/artifacts/r240-direct/native-hca.{log,xml}`. -- [x] **Allocate and send immutable registered data**: expose bounded producer - allocation from the transport's device-owned arena. Provide a unique Rust - `WritableBuffer` with a `MaybeUninit` capacity view and consuming commit; - freeze into the existing owner-backed `Bytes` contract. Validate address - ranges against the owning arena before using an MR key; retain the exact - underlying allocation through SEND completion and propagate quarantine to - that allocation. Build the small per-request header/control in its own - registered region, pad that region to 1 KiB, and post control plus payload - as two SGEs. Keep one WR per complete RPC and count direct/copied payload - bytes in status. No range registry or new hot-path lock. Files: buffer arena, - RDMA pool/send/work-request modules, C ABI/FFI buffer producer API. -- [x] **Receive into aligned data storage**: prepost two registered SGEs in - direct layout, with 1 KiB control and a separately aligned data allocation. - Charge both against `receive_arena_bytes`; descriptor count covers both and - arithmetic is checked. Validate the complete control/header/length before - dispatch; retain data ownership through disk/caller completion. Release or - quarantine both regions exactly once on post/CQE/cancel/shutdown failure. - Files: `rdma_receive.cpp`, receive posting, WorkRequest and repost context, - real HCA ownership tests. Keep generic receive credit implicit in RQ slots. -- [x] **Connect the actual HTTP producer**: allow `NativeBodyAllocator` to - obtain its owner storage from the retained DiskIO producer allocator after - route setup. Preserve its existing byte admission, AtomicWaker pressure and - framed-owner API. Wire it through `ChunkIoClient` / `RoutedDiskWriter` / - S3 storage setup. A bounded native allocator failure pauses intake; it must - not silently allocate an unregistered replacement. Retain TCP allocation - when the service selects TCP. Cover owner views, slices and mirror fanout. - Add a cold `RpcServer` construction path receiving layout/data-capacity - defaults before native environment overrides. DiskIO server and its semantic - client select control/data with 1 MiB data capacity; ordinary RPC construction - retains contiguous defaults. Existing typed explicit configuration remains - authoritative. Do not set a process-wide direct-layout environment variable: - the Access Server's metadata transports must retain their own layout. - Expose a clonable producer allocator retaining the client's `Arc` - through `RoutedDiskWriter` and `ChunkIoClient`. `NativeOwner` may hold its - writable allocation as backing storage while its established disjoint-region - rules govern initialization and immutable framed views. Drop that backing - before waking allocator credit waiters, so a resumed receiver observes the - returned native admission. Native pool exhaustion is `WouldBlock`, not a - malloc replacement or a terminal upload error. Keep the existing AtomicWaker - release wakeups; add a bounded retry wake for native credits held by other - producers or asynchronous transport completion. - The producer seam and 12 existing native-owner cases plus two new native - admission/view cases pass. The new opt-in real HCA case also passes: Hyper's - provider regions fill one complete 1 MiB framed owner, `BufferChain` retains - those views, and native status reports exactly 1 MiB direct payload and zero - client transport copy/fallback. This is provider-to-HCA integration, not yet - the full HTTP/DiskIO end-to-end acceptance. The first native build after - splitting server C ABI functions fails because the pool handle definition - was local to the old translation unit; move it into the existing internal - opaque-handle header. The corrected native build exits zero before tests. - All DiskIO-client tests and all RPC FFI tests pass with the new constructor. - Workspace clippy catches a missing doc-code backtick around `DiskIO`; fix - only that documentation warning and rerun the gate. - Final full C++/FFI gate passes at - `.crowdb-runtime/artifacts/unit/cpp-cba652da52b248cca3721baebc228927/`. - All 31 focused ChunkIO and 11 Access Server config/service cases pass; - workspace all-feature clippy, rs-lint, Rust/C++ format, scoped tree-lint - (zero exit with style warnings), and 146 TCP-only CTest items also pass - (one HCA-only item skipped). This closes producer wiring, while disk lifetime - and complete HTTP/service/OCI acceptance remain below. -- [x] **Close DiskIO handoff and read lifetime**: use exact view pointer/length, - validate request bounds before disk submission, transition data to DiskOwned - until the callback returns, and retain connection state across delayed I/O. - Registered read storage must cover engine completion and SEND completion; - client receive storage remains held through returned Bytes. Preserve existing - alignment handling and short-I/O/error semantics. Test delayed I/O, queue - pressure, held caller buffers, error, cancellation and connection shutdown. - Retain `SocketTransport` and its default application pool through asynchronous - DiskIO callbacks: `RpcServer` owns a shared transport, and that transport owns - its default pool. Expose the shared transport for the DiskIO constructor; - completion closures retain it and their connection, with global metric - pointers and static response builders rather than raw `DiskioServer *`. - A stopped connection skips reply submission. The existing AlignedWriter - destructor drains its own queued operations; its block-size-one fast path, - reads and fsync callbacks must remain independently safe after the service - facade is destroyed. No new completion-path mutex is needed. - For a direct-layout READ, reuse the request's already reserved receive data - allocation as disk output, instead of allocating another producer buffer. - The control is already copied/verified and the request has no input payload. - Take a capacity-checked view of that canonical receive allocation, retain it - across disk and response SEND completion, then repost it. This uses the same - registered region and naturally bounds asynchronous reads by RQ admission; - it also propagates a slow response consumer back through server SEND and - request receive credits. Validate registered receive ownership before reuse; - keep the existing system-pool path for TCP and alignment-edge staging only - when the disk requires it. The first focused test build uses an incorrect - field spelling `data_offset`; the actual buffer field is `view_offset`. - Correct only the test field and rebuild before running the cases. - Twelve focused lifecycle/alignment cases pass. The first real HCA plus - O_DIRECT/io_uring case passes in 50 ms: three ordered 1 MiB writes reconstruct - 3 MiB exactly; disk write and read pointers equal the registered receive - pointers and are 4096-byte aligned. Four retained client responses consume - its RQ, the following four responses retain server receive data in SEND, - and releasing caller leases resumes all eight with zero copy/fallback and - no CQ/protocol/SEND/repost errors. This functional fixture uses one process - with actual client/server QPs on the local HCA; it is not a throughput sample - or the separate two-container end-to-end acceptance. - Inspection finds an additional application retry seam: `DiskioClient` retries - errors from an accepted RPC future through `prepare_retry`, which can create - a TCP fallback connection after an uncertain RDMA operation. Add an immutable - typed connection-path getter (also valid after close), stop accepted-RDMA - read/write/fsync retries, and return the existing typed/ambiguous/durability - errors. Keep known pre-submit rejection and established TCP retry behavior. - Add optional explicit `RdmaConfig` to the semantic DiskIO client so its data - transport can be configured independently of metadata clients and tested - without process-wide environment mutation. Add a real-HCA semantic-client - case asserting one accepted write, zero retries/reconnects, and ambiguous - failure after the peer closes before its reply. - The completed 15-case native run passes in 112 ms, including delayed disk - ownership, actual HCA/O_DIRECT round trips, peer-close/server-destruction, - held response leases, constant registered-memory high water, and TCP byte - parity. Evidence: `.crowdb-runtime/rdma-check/artifacts/r240-direct/diskio-hca.{log,xml}`. - The new status assertion initially fails because Folly's default JSON parser - converts a random unsigned 64-bit capability generation through signed - `int64_t`. Preserve the failing log/XML as `diskio-hca-parser-failure.*`; - parse numeric tokens as strings in this test's snapshot helper so exact - memory-counter comparisons retain full precision. No production generation - or lifecycle assertion changes. The corrected run has zero failures. - All nine semantic DiskIO tests pass after extracting write/fsync into - `semantic/write.rs`; three actual-HCA cases cover accepted read/write/fsync - failure with fallback enabled, exactly one accepted operation and no retry - or reconnect. Evidence: `diskio-rust.log` in the same artifact directory. - The first Rust lint run flags the new fault helper's 114-line function; - extract only its cold `RdmaConfig` builder. Workspace all-feature clippy and - rs-lint then pass. C++ formatting needs repeated passes to stabilize the - pre-existing numeric-config macro; stabilize its whitespace, rebuild, and - rerun the 203-item RPC CTest set, all 71 native DiskIO cases with HCA enabled, - and all nine semantic cases. All pass. Full C++/FFI gate evidence is - `.crowdb-runtime/artifacts/unit/cpp-e70af11a38a443f6a7b660245507292a/`; - 146 no-verbs items also pass (one HCA-only item skips), and scoped tree-lint - exits zero with style warnings. All 125 ChunkIO tests pass. The first - full Access gate reaches its 60-second shell bound while compiling test - executables, before running any case; complete `--no-run` separately. The - compiled suite also exceeds the aggregate shell bound across its independent - test programs, with no failed assertion before termination. Run it with - `CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="timeout 60s"`, giving each - existing test program the same 60-second execution bound. Request deadlines - and assertions remain unchanged. Preserve the aggregate-limit log as - `access-suite-shell-limit.log`. The complete per-program-bounded Access - suite passes with zero failures; its complete log is `access.log`. -- [x] **Verify real service flows**: run focused baseline and changed tests, - then real DiskIO plus HTTP/ChunkIO upload/download of objects larger than - 1 MiB on the prepared HCA. Assert direct-payload counters and same data - addresses at producer/SEND and receive/disk handoff, exact bytes and existing - frame integrity/order. Repeat TCP-only and two non-root OCI profiles. Keep - per-case manifests, settings, errors and complete logs. Run required fmt, - clippy/tree-lint and full affected gates before cleanup; then use R241's own - plan for Chunk-KV integration. - Use a service fixture that runs a real Hyper HTTP socket, the production S3 - body-to-ChunkIO transfer function, mirror ChunkIO framing/read verification, - and a separate real DiskIO executable with O_DIRECT/io_uring. Inject only - chunk placement/catalog state, keeping namespace/authentication outside this - datapath case. A `test-util` entry delegates to the production upload function - rather than reimplementing its ownership/pressure loop. Build the normal - routed disk adapter from an already connected semantic client so the fixture - shares production submit/read/error behavior. Use two replicas and an object - over 3 MiB with a non-frame-aligned tail; compare exact download and locations - across strict RDMA and explicit TCP. Assert native direct/copy/fallback and - error counters in machine-readable artifacts. Run the same fixture in two - non-root host-network containers sharing the one configured HCA, recording - this local-loop topology and pinned runtime image and executable identities. - The first real HTTP fixture stalls before any DiskIO RPC. Instrumentation - confirms placement and registered producer allocation succeed, then two - owners fill the 2 MiB HTTP budget without owner handoff. The new test wrapper - omitted `enable_owner_handoff()`, which production PUT/UploadPart already - call. Initialize handoff and deferred MD5 exactly as production does; retain - the original 10-second HTTP deadline and all byte/copy assertions. Remove - temporary per-frame diagnostics. Failed evidence is preserved under - `rdma-datapath/bff1ae9a76714199a7dabe9da47edde8`, - `391b25b30b174bb5bf6b456ac67a2f5a`, and `fc50a99060c84e33a99118fc966c2cf0`. - Upload then completes with actual RDMA writes, but GET fails before issuing a - disk read: the injected catalog seals the chunk without sealing its strips, - so the production reader correctly rejects zero durable strip bytes. Mirror - the production seal handler's bounded strip lengths in the injected catalog; - retain normal reader availability/integrity checks. Failed logs remain in - `ba2b03c512bb4989af8957434422ff1a` and `5f8600b20b364ea5bb2e8ce00d29c3ab`. - Native RDMA and TCP now pass the 3,158,073-byte corpus across two chunks. - Two non-root shared-HCA containers also pass. Registered transport payload - copies and TCP fallback are zero; HTTP's bounded prefetched header/body - prefix copy remains separately reported. OCI's default seccomp denies - io_uring, so that profile uses the existing asynchronous BlockingEngine; - report the actual engine explicitly, while native uses io_uring. - Add a partial-download failure variant: the HTTP fixture publishes one - verified frame, then pauses before further stream delivery. The runner - closes DiskIO and releases that gate. The HTTP body must fail as one object, - with no successful full download or TCP suffix. Preserve exact expected - failure counts separately from verification errors in its result artifact. - The completed five-profile matrix passes with identical SHA256, ETag and - physical/logical location geometry for all four successful profiles. Actual - upload direct payload is 6,319,478 bytes across two replicas; download direct - payload is 3,159,739 bytes. Transport payload copies/fallback/errors are zero. - Final artifacts under `.crowdb-runtime/rdma-check/artifacts/rdma-datapath/`: - native RDMA `dd26085b76ef40f4bb61fe05813c9b17`, native TCP - `73f4fd013ea048338b7d3a18dc7bf53b`, OCI RDMA - `f7c807fcfbaa496ca02c4dd38d37e140`, OCI TCP - `e2de510290114a8ab2e59780235f0110`, OCI peer-close - `24d2205fc7d94bbf92a52df176a2af8b`. OCI uses inspected UID 1000, - host networking and pinned image - `sha256:e3c9c5c6d5fc3424ac3f22cf4cdcc9b1c872f47336484f54962b2541c4ed6cef`. - The failed download returns 3,144,096 verified prefix bytes then one HTTP - body error, with zero verification errors/copy/fallback. Expected subsequent - reader attempts to unavailable replicas are retained in preflight counters; - the semantic client performs no accepted-RPC replay. Both peers use two QPs, - one CQ/poller, `sq_depth = rq_depth = posted_receive_target = 4`, - `cq_depth = 16` (provider 31), 4 MiB send arena, 4,198,400-byte receive arena, - and 1,049,600-byte control/data wire slots. HTTP admission peaks at 2 MiB. - Native disk engine is io_uring with `sq_entries = 32`; OCI uses BlockingEngine - under default seccomp. These are functional tests, not TPS measurements. - Final full C++/FFI gate passes at - `.crowdb-runtime/artifacts/unit/cpp-2032c286bd054ca089f560b5b6fced54/`. - All 278 affected Rust tests pass (three external/owned-fixture cases ignored - by the default suite and exercised separately where applicable). The semantic - read/write/fsync fault tests run on the actual HCA. Workspace all-feature - clippy, rs-lint, Rust/C++ format, scoped tree-lint (zero exit with existing - style warnings) and Python compile checks pass. Final environment-sanitized - OCI fault run is `fef1971b25904a9abff2cec49d6a9bed`, with one expected error - and zero verification errors consistently in both manifest and result. - Remaining work is requirement cleanup and advancing to the existing R241 plan. - -## Phase 1: map existing data flow - -- [x] **Trace upload flow**: identify where HTTP body bytes enter ChunkIO, - become aggregated buffers, are submitted to DiskIO, and are released after - durable completion. Files: `app/crowdb-access-server/`, - `lib/crowdb-chunk-client/`, `lib/crowdb-diskio-client/`, and ChunkIO designs. -- [x] **Trace download flow**: identify how SDK/HTTP consumers receive data, - how read-ahead aggregation chooses message boundaries, and when a caller - releases each buffer. -- [x] **Define transport adapter**: add the smallest Access/DiskIO adapter that - accepts an already formed buffer, actual length, request identity, sequence, - and logical offset. Do not add a second flush timer or aggregation queue. - -## Phase 2: upload to DiskIO - -- [x] **Allocate 1 MiB datapath class**: select the configured 1 MiB data - capacity plus 1 KiB control buffer for S3/HTTP/DiskIO streams; the configured - total wire slot includes both regions and the header. -- [x] **Fill registered send buffer**: make the HTTP/ChunkIO producer write - directly into the send-owned registered buffer and submit its actual length. - The producer must stop when the send/registered budget is exhausted. -- [x] **Receive ownership handoff**: on RDMA receive completion, transfer the - DiskIO buffer to the asynchronous disk writer. Keep request, sequence, - logical offset, and expected length attached to the write. -- [x] **Repost after disk completion**: only reset and repost the receive buffer - after the disk write and integrity result are complete. Propagate disk errors - to the stream and do not silently repost corrupted data. - -## Phase 3: download to SDK/HTTP - -- [x] **Prepost receive buffers**: use the R238 pool and R239 receive window; - publish a `BufferLease` only after header/class/length validation. -- [x] **Preserve caller ownership**: keep the lease caller-owned until explicit - release. Do not let CQ completion or a stream callback recycle it. -- [x] **Implement multi-message assembly**: deliver messages in sequence and - logical offset order, preserving end/error flags and cancellation behavior. -- [x] **Keep TCP parity**: use the same access result and error mapping when the - stream selects TCP before RDMA payload submission. - -## Phase 4: pressure and failure behavior - -- [x] **Disk queue pressure**: stop reposting when the asynchronous disk queue - reaches its byte bound; let receive-window exhaustion backpressure the - sender. -- [x] **SDK pressure**: stop publishing new leases when held bytes reach the - configured limit; resume after release. -- [x] **Partial stream failure**: fail the logical object/request after partial - RDMA transfer and quarantine uncertain buffers; never append TCP bytes. -- [x] **Cancellation**: stop new aggregation submissions, drain known WRs, - preserve in-flight buffer lifetimes, and return only buffers proven safe. - -## Tests - -Unit tests: - -- actual length below 1 MiB does not send padded bytes; -- sequence/offset/end/error parsing and multi-message assembly; -- DiskIO repost is rejected before disk completion; -- SDK lease release is the only return transition for caller-owned buffers; -- no second aggregation or flush timer is invoked by the RDMA adapter. - -Integration tests: - -- HTTP/ChunkIO aggregate enters a registered send buffer directly; -- DiskIO receive completion hands the same buffer to asynchronous disk I/O; -- full disk queue stops repost and resumes after completion; -- all SDK leases held causes bounded producer backpressure; -- TCP and RDMA return identical bytes, lengths, and errors; -- cancellation and partial RDMA failure leave no reusable uncertain buffer. - -E2E tests: - -- single-node HCA local-loop OCI upload into DiskIO and download back to a client; -- object larger than 1 MiB reconstructed from ordered messages; -- TCP-only OCI profile produces the same object and metadata; -- single-node local-loop multi-container profile reports bounded pressure without claiming physical - endpoint isolation. - -## Consolidated files - -- `app/crowdb-access-server/` -- `lib/crowdb-chunk-client/` -- `lib/crowdb-diskio-client/` -- `lib/crowdb-rpc/` -- `app/crowdb-diskio/src/rpc/` and its `tests/` -- `lib/crowdb-chunk-client/src/disk_io/`, `client.rs`, and existing chunk writers -- relevant S3/DiskIO integration tests and OCI E2E runner - -## Verification gates - -```bash -pixi run -- cargo fmt --all -- --check -pixi run -- cargo test -p crowdb-access-server --all-targets -pixi run -- cargo test -p crowdb-diskio-client --all-targets -pixi run -- cargo test -p crowdb-chunk-client --all-targets -pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings -pixi run rs-lint -pixi run tree-lint -``` - -Do not hand off to R242 until both upload and download have explicit lease -ownership tests and the DiskIO receive loop proves repost-after-disk-completion. - -## Provider references - -- [libibverbs receive work requests](https://github.com/linux-rdma/rdma-core/blob/master/libibverbs/man/ibv_post_recv.3) - defines the SGE address/length/lkey array and completion-before-buffer-reuse - requirement. CrowDB additionally waits for the disk/caller lease to end. -- [libibverbs send work requests](https://github.com/linux-rdma/rdma-core/blob/master/libibverbs/man/ibv_post_send.3) - defines scatter/gather submission and its completion ownership. The fixed - 1 KiB control region, total wire admission and layout negotiation are - CrowDB's own protocol contract. diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index c0a29c174..7cf1f1a7b 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -5,8 +5,7 @@ Upstream: [R5](../backlog/R5-rdma-alloc.md), [RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md), and child -backlogs [R240](../backlog/R240-rdma-datapath-stream.md), -[R241](../backlog/R241-rdma-chunk-kv-access.md), and +backlogs [R241](../backlog/R241-rdma-chunk-kv-access.md) and [R242](../backlog/R242-rdma-fat-client-sdk.md). Goal: implement the host-memory RDMA SEND/RECV path end to end, with RDMA as @@ -35,14 +34,23 @@ fallback. The old 1.187M value labeled RDMA was TCP; matching current TCP measures 1.196M TPS. Exact identities, settings and artifact names are retained in [RPC flow analysis](../design/rpc/rpc-flow-analysis.md). -Current work is R240, then R241, using their existing plans. R242 and physical +The HTTP/ChunkIO/DiskIO datapath is accepted at `ab1ffbfb`: immutable registered +HTTP owners, aligned receive-to-disk handoff, read reuse through SEND completion, +caller-held receive credits, retained asynchronous service owners and terminal +accepted-RDMA failures. All 278 affected Rust cases and full C++/FFI gates pass. +A 3,158,073-byte real HTTP corpus crosses two chunks and matches TCP/native/OCI +bytes and metadata geometry. Peer-close fails a partial HTTP download with no +TCP suffix. Exact functional settings, error counts, image and artifact +identities are retained in RPC flow analysis. + +Current work is R241, using its existing plan. R242 and physical multi-node validation remain outside the user's requested continuation. ## Fixed design decisions - Use one common RPC abstraction for TCP and RDMA. - Use RC SEND/RECV. The accepted contiguous RPC layout uses one wire slot - and one SGE per message. R240 adds a negotiated control/data layout with + and one SGE per message. The accepted datapath adds a negotiated control/data layout with separate SGEs for immutable producer payload and per-target control; its total wire admission includes both regions. - Use service discovery for coarse RDMA capability and TCP handshake for the @@ -92,15 +100,14 @@ R242 fat client SDK and FFI lease ## Immediate implementation order -User continuation on 2026-10-22: complete R238 and R239, then continue through -R240 and R241 in this same task. Finish transport/benchmark/OCI acceptance -before advancing; implement R240, then R241, using their existing -[datapath stream plan](plan-rdma-datapath-stream.md) and -[Chunk-KV/access plan](plan-rdma-chunk-kv-access.md), with their own gates. -Do not create duplicate plan files. R242 remains outside this requested continuation. +The foundation, transport and datapath requirements are complete. Continue +through R241 in this same task using its existing +[Chunk-KV/access plan](plan-rdma-chunk-kv-access.md) and gates. Do not create +duplicate plan files. R242 remains outside this requested continuation. -1. Complete R240 using its existing datapath stream plan and service gates. -2. Complete R241 using its existing Chunk-KV/access plan and parity gates. +1. Complete R241 using its existing Chunk-KV/access plan and parity gates. +2. Rebuild matching CLI/server binaries and repeat the 10-second peak controls + after all service-related transport changes, retaining errors and latency. 3. Keep R242 downstream until the user includes that scope. Before modifying each service path, run its focused baseline corpus and record @@ -109,12 +116,9 @@ OCI runtime preparation are already verified. Service acceptance must prove actual RDMA, ownership through disk/caller completion, and bounded pressure; transport echo acceptance alone does not prove those properties. -## Phase 3: R240 and R241 service integration +## Phase 3: Chunk-KV service integration -- [ ] **Execute R240 child plan**: connect ChunkIO-produced 1 MiB buffers to - Access Server/DiskIO, retain DiskIO buffers through disk completion, and test - large multi-message streams. -- [ ] **Execute R241 child plan**: connect Access Server/Chunk-KV with 64 KiB +- [~] **Execute R241 child plan**: connect Access Server/Chunk-KV with 64 KiB metadata and 256 KiB aggregate classes, preserving existing request IDs, ordering, and retry semantics. - [ ] **Run semantic parity**: for the same test corpus, compare TCP and RDMA From ec9152540ae8dd4ef00be8cda5540434045f5acd Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 04:38:45 +0800 Subject: [PATCH 046/258] Specify bounded logical control streams for Chunk-KV access --- doc/backlog/R241-rdma-chunk-kv-access.md | 23 +++++- doc/working/plan-rdma-chunk-kv-access.md | 93 +++++++++++++++++++++++- 2 files changed, 111 insertions(+), 5 deletions(-) diff --git a/doc/backlog/R241-rdma-chunk-kv-access.md b/doc/backlog/R241-rdma-chunk-kv-access.md index 68c98e06e..1e021669d 100644 --- a/doc/backlog/R241-rdma-chunk-kv-access.md +++ b/doc/backlog/R241-rdma-chunk-kv-access.md @@ -19,10 +19,29 @@ without changing access semantics. the access/aggregation layer selects the class for each operation. 3. Represent a result larger than its selected capacity as ordered messages with request ID, stream ID, sequence, logical offset, and end/error flags. + The common RPC transport splits and assembles the already encoded control + message. Chunk-KV keeps its existing serializers and delivers one complete + logical request/response to the existing handler/correlation layer. Use the + negotiated control/data layout: small fragment control accompanies bounded + 64 KiB/256 KiB/1 MiB data containing the encoded logical control bytes. + Negotiate support and the logical-message limit before payload submission. 4. Use the same TCP/RDMA request identity and one-way RDMA-to-TCP fallback. Retry only when the existing operation is idempotent or duplicate-suppressed. -5. Keep aggregation above RDMA. The RDMA layer does not add flush timers, - coalescing, or a second buffer-copy path. +5. Keep application aggregation above RDMA. Do not add an application flush + timer, another aggregate producer, or application payload assembly copy. + Existing contiguous RPC transport coalescing remains bounded by both peers' + slots. Control/data streams send one message per SEND. Reassembly copies + each control fragment once into one bounded logical control owner, which + remains charged until its final native/FFI caller release. +6. Bound logical control bytes independently of registered memory. Configure + a maximum logical message and per-direction send/receive byte budgets; + defaults cover the client's existing 16 MiB response budget. Sender + admission counts the retained original encoding. Receiver admission reserves + the complete logical length before assembly; exhausted admission retains + only the existing receive-window fragments and resumes on caller release. + Never allocate replacement receive storage or return a partial control + message. Disconnect/cancellation/partial-message expiry releases assembly + state while completed caller-owned results retain their admission. ## Dependencies diff --git a/doc/working/plan-rdma-chunk-kv-access.md b/doc/working/plan-rdma-chunk-kv-access.md index 5279bd772..ef74eb3ff 100644 --- a/doc/working/plan-rdma-chunk-kv-access.md +++ b/doc/working/plan-rdma-chunk-kv-access.md @@ -11,16 +11,103 @@ Goal: use the common RDMA stream for Access Server to Chunk-KV operations while preserving existing aggregation, request numbering, generation checks, and TCP semantic parity. +## Execution checkpoint and implementation order + +The datapath is accepted at `ab1ffbfb`; its cleanup is `b38d3d78`. Preserve its +1 MiB direct payload path and the ordinary contiguous RPC benchmark. The +focused Chunk-KV baseline passes all 22 client/CLI cases before these changes. +All 278 affected Access/ChunkIO/DiskIO cases and full native/FFI gates passed +at the preceding service checkpoint. + +Observed contract: + +- `ChunkKvRpcTransport` encodes point/seek/scan/multi-get/batch requests as + FlatBuffer control buffers and calls the existing `RpcClient::call`. Its + connection pool and completion IDs remain the correlation owner. The + server's `ChunkKvRpcService` decodes one complete control, checks the same + RPC ID and submits one encoded response. Large values and aggregates are + inside these control bytes, not the RPC data field. +- The routing/composition layer retains logical mutation identities during + retries and retries only unresolved group operations. Seek/scan retain + partition generation and bounded-page semantics. Do not add a separate + Chunk-KV request table or retry scheduler. +- Native RDMA currently rejects control beyond 1010 bytes. Control/data gives + independently aligned data with exact 64 KiB/256 KiB/1 MiB classes; use this + existing layout for encoded-control fragments. Keep the current direct + path for ordinary small control plus real datapath data. +- Metadata point calls use 64 KiB; grouped/seek/scan calls use 256 KiB. An + explicit class config may select 1 MiB. Each class owns a fixed native + connection group and common RPC completion engine, with no producer thread + per loader. Server default is 256 KiB. QP data capacity is the minimum of + the two advertised capacities and is used for actual RX allocation/posting + as well as SEND admission; expose that effective value in immutable status. + +- [~] **Build bounded common control-stream primitives**: add portable byte + admission, fragment metadata validation and one-message assembly, tested + without verbs. Reserve full logical length, validate request/stream ID, + sequence, exact offset, total, class, generation and final flag. Default + maximum encoded control is 32 MiB, covering existing 16 MiB response limits; + independent send/receive budgets default to 64 MiB each. Reject invalid + arithmetic/limits before allocation. Retain completed reservation until + final caller release. Preserve a failed/blocked fragment's existing owner; + blocked admission must not carve replacement storage. Files: private RPC + control-stream modules, `framing.h` control owner, focused portable tests. +- [ ] **Negotiate and configure common streams**: add a small transport-owned + fragment FlatBuffer in `common_msg.fbs`, preserving the common leading + correlation fields, with reserved message type 4. Bump RDMA protocol to + version four; the existing final 4 handshake bytes advertise maximum logical + control length and participate in the CM capability digest. Legacy decode + remains for explicit pre-payload TCP fallback. Extend native/env/status and + a new versioned C ABI config entry without changing the version-one struct + or existing configure entries. Rust typed config carries max message, + per-direction budget and partial-assembly timeout (default 30 seconds). + `RdmaConfig::for_layout` derives send arena bytes from SQ depth and selected + data capacity, as receive defaults already do; 1 MiB and ordinary defaults + preserve their current values. Files: capability/offer/config/status, FFI. +- [ ] **Integrate CQ-owned fragmentation and assembly**: enqueue one original + logical frame under byte/count admission. The CQ owner walks it in order, + builds only the next bounded fragment, and holds immutable source owners + through all WR completions. Do not materialize an unbounded fragment list. + Receive validates metadata before copying into one logical owner and returns + each consumed RQ credit immediately; result size may exceed the entire RQ + window. On exhausted assembly admission retain at most the existing RQ + frames. Final lease release wakes the owning polling/event worker through + the existing retained operation gate, including safe release after shutdown. + Active partial assemblies expire; consumer-blocked first fragments remain + backpressured without an arbitrary consumer hold deadline. Dispatch only a + complete original Frame. Move quota-owned control storage through C ABI + without another full-message malloc/copy. Files: RDMA send/receive/CQ/CM + pools, common framing/C ABI conversion and owner-lifetime tests. +- [ ] **Wire Chunk-KV classes and typed settings**: add service defaults and + explicit RDMA configuration for metadata/aggregate connection groups, + preserve TCP defaults and current serializers/errors/retries, and wire the + Chunk-KV server plus Access storage construction. Cover point, mutation, + seek, bounded scan, multi-get and batch mutation, including duplicate result + order and stable logical mutation IDs. Files: client transport/config, + server startup/RPC boundary, Access storage config and tests. +- [ ] **Verify real service and clean up**: test malformed sequence/offset, + duplicate/end/generation/class, byte exhaustion, held final leases, + cancellation/close/expiry, and TCP setup fallback. Run the actual Chunk-KV + service on the HCA with metadata and results larger than 256 KiB, then two + non-root host-network containers and a no-device TCP profile. Compare bytes, + typed errors and identity using the same corpus. Rerun the HTTP/DiskIO + acceptance after common transport changes. Finish native/no-verbs/FFI, + workspace format/lint and affected service gates. Rebuild matching release + CLI/fb-server and repeat the 1 s warmup + 10 s peak controls before cleanup. + Record all settings, actual classes/QPs, memory and error categories. Delete + this completed child plan/detail/index entry after permanent docs/evidence + are updated. R242 remains outside the requested scope. + ## Phase 1: access contract mapping -- [ ] **Map request/response types**: identify metadata, point, batch, range, +- [x] **Map request/response types**: identify metadata, point, batch, range, and aggregate operations; record idempotency, request ID, generation, error, and retry behavior. Files: `lib/crowdb-chunk-kv-client/`, Chunk-KV server modules, and Access Server access-core modules. -- [ ] **Map aggregation owner**: identify where Chunk-KV combines responses and +- [x] **Map aggregation owner**: identify where Chunk-KV combines responses and where it decides flush/end boundaries. RDMA must receive the aggregate after that decision, never create another aggregation layer. -- [ ] **Define class selection**: metadata defaults to 64 KiB, aggregated +- [x] **Define class selection**: metadata defaults to 64 KiB, aggregated Chunk-KV results default to 256 KiB, and 1 MiB remains a configured option for larger value/range results. Transport creation receives defaults from config. From 67bb86e1816cffc693df6b280e4918598dbd6a5a Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 04:57:14 +0800 Subject: [PATCH 047/258] Bound logical control assembly and retain admission through final callers --- doc/working/plan-rdma-chunk-kv-access.md | 34 ++- lib/crowdb-rpc/include/crowdb-rpc/framing.h | 6 +- lib/crowdb-rpc/src/c_api.cpp | 44 ---- lib/crowdb-rpc/src/c_api/frame.cpp | 56 +++++ .../transport/rdma/rdma_control_budget.cpp | 91 +++++++ .../transport/rdma/rdma_control_stream.cpp | 111 +++++++++ .../src/transport/rdma/rdma_control_stream.h | 106 ++++++++ lib/crowdb-rpc/tests/c_api_buffer_test.cpp | 26 ++ .../tests/rdma_control_stream_test.cpp | 234 ++++++++++++++++++ 9 files changed, 657 insertions(+), 51 deletions(-) create mode 100644 lib/crowdb-rpc/src/c_api/frame.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_control_budget.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_control_stream.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_control_stream.h create mode 100644 lib/crowdb-rpc/tests/rdma_control_stream_test.cpp diff --git a/doc/working/plan-rdma-chunk-kv-access.md b/doc/working/plan-rdma-chunk-kv-access.md index ef74eb3ff..373de978a 100644 --- a/doc/working/plan-rdma-chunk-kv-access.md +++ b/doc/working/plan-rdma-chunk-kv-access.md @@ -42,7 +42,7 @@ Observed contract: the two advertised capacities and is used for actual RX allocation/posting as well as SEND admission; expose that effective value in immutable status. -- [~] **Build bounded common control-stream primitives**: add portable byte +- [x] **Build bounded common control-stream primitives**: add portable byte admission, fragment metadata validation and one-message assembly, tested without verbs. Reserve full logical length, validate request/stream ID, sequence, exact offset, total, class, generation and final flag. Default @@ -52,18 +52,23 @@ Observed contract: final caller release. Preserve a failed/blocked fragment's existing owner; blocked admission must not carve replacement storage. Files: private RPC control-stream modules, `framing.h` control owner, focused portable tests. -- [ ] **Negotiate and configure common streams**: add a small transport-owned +- [~] **Negotiate and configure common streams**: add a small transport-owned fragment FlatBuffer in `common_msg.fbs`, preserving the common leading correlation fields, with reserved message type 4. Bump RDMA protocol to version four; the existing final 4 handshake bytes advertise maximum logical - control length and participate in the CM capability digest. Legacy decode + control length and participate in the CM capability digest. Extend the CM + offer from 48 to 52 bytes to carry that limit explicitly: the listener does + not retain the client's TCP preflight capability, so its digest alone cannot + provide the client's receive limit for response admission. Validate the + explicit limit against the capability when it is available. Legacy decode remains for explicit pre-payload TCP fallback. Extend native/env/status and a new versioned C ABI config entry without changing the version-one struct or existing configure entries. Rust typed config carries max message, per-direction budget and partial-assembly timeout (default 30 seconds). - `RdmaConfig::for_layout` derives send arena bytes from SQ depth and selected - data capacity, as receive defaults already do; 1 MiB and ordinary defaults - preserve their current values. Files: capability/offer/config/status, FFI. + The Chunk-KV typed class factory derives send arena bytes from selected SQ + depth and data capacity, as receive defaults already do. Preserve native + `RdmaConfig::for_layout`'s existing 64 MiB send arena default for the accepted + 1 MiB datapath and ordinary RPC. Files: capability/offer/config/status, FFI. - [ ] **Integrate CQ-owned fragmentation and assembly**: enqueue one original logical frame under byte/count admission. The CQ owner walks it in order, builds only the next bounded fragment, and holds immutable source owners @@ -185,6 +190,23 @@ E2E tests: ## Verification gates +Portable stream checkpoint: + +- All 11 focused native control-stream/C ABI cases passed, including exact + original allocation transfer through cloned FFI handles and held admission. +- `pixi run test-cpp` passed: 543 tree, 177 RPC, 71 DiskIO and 35 common native + cases, plus both FFI suites. Full logs: + `.crowdb-runtime/artifacts/unit/cpp-4548393f6da54a0e9d3d3ba47a5cc115/`. +- TCP-only/no-verbs build and all 155 CTest cases passed (hardware case skipped). + Changed C++ format and scoped `tree-lint` passed; style warnings remain. +- The first full tree run failed the unchanged + `DoubleBuffer.ConcurrentReadersDuringFrequentFreezeAndDrainNoCorruption` + assertion `bad.load() == false`. The isolated 13-case DoubleBuffer suite + and subsequent complete 543-case tree run passed without source changes. + Preserve the initial failure at + `.crowdb-runtime/artifacts/unit/cpp-f0f7acd09fb541f59d048f81e3923cc2/00-tree.log`; + this checkpoint does not claim to repair that intermittent tree failure. + ```bash pixi run -- cargo fmt --all -- --check pixi run -- cargo test -p crowdb-chunk-kv-client --all-targets diff --git a/lib/crowdb-rpc/include/crowdb-rpc/framing.h b/lib/crowdb-rpc/include/crowdb-rpc/framing.h index dd5a9bd8d..0576f3151 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/framing.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/framing.h @@ -9,6 +9,7 @@ #include #include #include +#include #include namespace crowdb::rpc @@ -59,7 +60,10 @@ struct Frame // msg_size > 0. Common handlers (ping, unknown) use only request_id + // rpc_create_nano and ignore this; service-specific handlers (diskio) // parse the full flatbuffer from here. - std::vector control; + // Optional logical-control admission retained through the final caller. + // Storage is destroyed before admission wakes blocked receivers. + std::shared_ptr control_owner; + std::vector control; ~Frame() { diff --git a/lib/crowdb-rpc/src/c_api.cpp b/lib/crowdb-rpc/src/c_api.cpp index 5c1127abf..530b57b50 100644 --- a/lib/crowdb-rpc/src/c_api.cpp +++ b/lib/crowdb-rpc/src/c_api.cpp @@ -337,50 +337,6 @@ void crowdb_rpc_client_destroy(crowdb_rpc_client_t client) namespace crowdb::rpc { -// Wrap a pool Buffer in a crowdb_rpc_buffer_s handle. The handle holds a -// ref_clone so the frame's release doesn't free the buffer. -static crowdb_rpc_buffer_t wrap_pool_buffer(Buffer *buf) -{ - if (buf == nullptr || buf->len == 0) { - return nullptr; - } - Buffer *clone = buf->ref_clone(); - if (clone == nullptr) { - return nullptr; - } - return new crowdb_rpc_buffer_s{clone}; -} - -void frame_to_c_handles(Frame *frame, crowdb_rpc_buffer_t *out_ctrl, crowdb_rpc_buffer_t *out_data) -{ - *out_ctrl = nullptr; - *out_data = nullptr; - if (frame == nullptr) { - return; - } - // Control: raw bytes from frame->control (flatbuffer). Wrap in a - // malloc'd Buffer with a standalone refcount so release() and - // ref_clone() work normally (no pool recycle on last ref). - if (!frame->control.empty()) { - auto *buf = new Buffer; - buf->data = static_cast(std::malloc(frame->control.size())); - if (buf->data != nullptr) { - std::memcpy(buf->data, frame->control.data(), frame->control.size()); - buf->len = static_cast(frame->control.size()); - buf->capacity = buf->len; - buf->ref = new std::atomic(1); - buf->pool = nullptr; - *out_ctrl = new crowdb_rpc_buffer_s{buf}; - } - else { - delete buf; - } - } - // Data is a pool Buffer — ref_clone so the frame's release doesn't free. - *out_data = wrap_pool_buffer(frame->data_buf); - delete frame; -} - static crowdb_rpc_status rpc_error_to_status(RpcError err) { switch (err) { diff --git a/lib/crowdb-rpc/src/c_api/frame.cpp b/lib/crowdb-rpc/src/c_api/frame.cpp new file mode 100644 index 000000000..1a57bfad8 --- /dev/null +++ b/lib/crowdb-rpc/src/c_api/frame.cpp @@ -0,0 +1,56 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/c_api_internal.h" +#include "crowdb-rpc/framing.h" + +namespace crowdb::rpc +{ +namespace +{ +struct ControlOwner +{ + std::shared_ptr admission; + std::vector bytes; +}; + +crowdb_rpc_buffer_t wrap_control(Frame &frame) +{ + if (frame.control.empty()) + return nullptr; + if (frame.control_owner == nullptr) + return crowdb_rpc_buffer_create(frame.control.data(), static_cast(frame.control.size())); + auto owner = std::make_unique(); + owner->bytes = std::move(frame.control); + owner->admission = std::move(frame.control_owner); + auto handle = crowdb_rpc_buffer_create_external( + owner->bytes.data(), static_cast(owner->bytes.size()), + [](void *context) { delete static_cast(context); }, owner.get()); + if (handle != nullptr) + (void)owner.release(); + return handle; +} + +crowdb_rpc_buffer_t wrap_data(Buffer *buffer) +{ + if (buffer == nullptr || buffer->len == 0) + return nullptr; + auto handle = std::make_unique(); + handle->buf = buffer->ref_clone(); + return handle->buf == nullptr ? nullptr : handle.release(); +} +} // namespace + +void frame_to_c_handles(Frame *frame, crowdb_rpc_buffer_t *out_ctrl, crowdb_rpc_buffer_t *out_data) +{ + *out_ctrl = nullptr; + *out_data = nullptr; + std::unique_ptr incoming(frame); + if (incoming == nullptr) + return; + // Budget-owned controls move their exact storage and admission into the + // external owner. Ordinary controls preserve their existing copied handle. + *out_ctrl = wrap_control(*incoming); + *out_data = wrap_data(incoming->data_buf); +} +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_control_budget.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_control_budget.cpp new file mode 100644 index 000000000..f568a82b3 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_control_budget.cpp @@ -0,0 +1,91 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_control_stream.h" + +#include +#include + +namespace crowdb::rpc +{ +struct ControlStreamBudgetState +{ + uint64_t limit; + std::function wake; + std::atomic used{0}; + std::atomic peak{0}; + std::atomic stalls{0}; +}; + +ControlStreamBudget::ControlStreamBudget(uint64_t limit, std::function wake) + : state_(std::make_shared()) +{ + state_->limit = limit; + state_->wake = std::move(wake); +} + +ControlStreamLease::ControlStreamLease(std::shared_ptr state, uint64_t bytes) + : state_(std::move(state)), + bytes_(bytes) +{ +} + +ControlStreamLease::~ControlStreamLease() +{ + state_->used.fetch_sub(bytes_, std::memory_order_acq_rel); + if (state_->wake) + state_->wake(); +} + +std::shared_ptr ControlStreamBudget::try_reserve(uint64_t bytes) +{ + uint64_t used = state_->used.load(std::memory_order_relaxed); + do { + if (bytes == 0 || bytes > state_->limit || used > state_->limit - bytes) { + state_->stalls.fetch_add(1, std::memory_order_relaxed); + return {}; + } + } while (!state_->used.compare_exchange_weak(used, used + bytes, std::memory_order_acq_rel)); + uint64_t peak = state_->peak.load(std::memory_order_relaxed); + while (peak < used + bytes && !state_->peak.compare_exchange_weak(peak, used + bytes, std::memory_order_relaxed)) { + } + // Build the lease before publishing its shared_ptr. If its control-block + // allocation fails, shared_ptr deletes the lease and returns admission. + ControlStreamLease *lease = nullptr; + try { + lease = new ControlStreamLease(state_, bytes); + } + catch (const std::bad_alloc &) { + state_->used.fetch_sub(bytes, std::memory_order_acq_rel); + if (state_->wake) + state_->wake(); + return {}; + } + try { + return std::shared_ptr(lease); + } + catch (const std::bad_alloc &) { + return {}; + } +} + +uint64_t ControlStreamBudget::used() const +{ + return state_->used.load(std::memory_order_acquire); +} + +uint64_t ControlStreamBudget::high_water() const +{ + return state_->peak.load(std::memory_order_relaxed); +} + +uint64_t ControlStreamBudget::stalls() const +{ + return state_->stalls.load(std::memory_order_relaxed); +} + +uint64_t ControlStreamBudget::limit() const +{ + return state_->limit; +} +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_control_stream.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_control_stream.cpp new file mode 100644 index 000000000..f9656e384 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_control_stream.cpp @@ -0,0 +1,111 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_control_stream.h" + +#include +#include +#include + +namespace crowdb::rpc +{ +ControlStreamReceiver::ControlStreamReceiver(ControlStreamBudget budget, uint32_t capacity, uint64_t generation, + uint32_t max_message_bytes, std::chrono::milliseconds timeout) + : budget_(std::move(budget)), + capacity_(capacity), + generation_(generation), + max_message_bytes_(max_message_bytes), + timeout_(timeout) +{ + BufferCapacityClass type; + uint32_t rounded = 0; + if (!classify_rdma_capacity(capacity, &type, &rounded) || rounded != capacity || generation == 0 || + max_message_bytes == 0 || max_message_bytes > budget_.limit() || timeout.count() <= 0) + throw std::invalid_argument("invalid logical control stream bounds"); +} + +bool ControlStreamReceiver::valid(const ControlFragment &f, size_t bytes) const +{ + if (f.request_id == 0 || f.stream_id == 0 || f.generation != generation_ || f.capacity != capacity_ || + f.total == 0 || f.total > max_message_bytes_ || f.offset > f.total || bytes > f.total - f.offset || + bytes > capacity_ || f.message_type == RPC_CONTROL_FRAGMENT_MSG_TYPE || + (f.message_flags & ~FLAG_ONE_WAY) != 0 || (f.stream_flags & ~uint8_t{7}) != 0) + return false; + const bool start = (f.stream_flags & CONTROL_STREAM_START) != 0; + const bool end = (f.stream_flags & CONTROL_STREAM_END) != 0; + if (start != (f.sequence == 0 && f.offset == 0)) + return false; + if ((f.stream_flags & CONTROL_STREAM_ERROR) != 0) + return end && !start && bytes == 0; + return bytes != 0 && end == (bytes == f.total - f.offset) && (end || bytes == capacity_); +} + +bool ControlStreamReceiver::matches(const ControlFragment &f) const +{ + const auto &a = *assembly_; + const auto &first = a.first; + return f.request_id == first.request_id && f.rpc_create_nano == first.rpc_create_nano && + f.response_create_nano == first.response_create_nano && f.stream_id == first.stream_id && + f.total == first.total && f.message_type == first.message_type && f.message_flags == first.message_flags && + f.sequence == a.next_sequence && f.offset == a.offset; +} + +ControlStreamResult ControlStreamReceiver::accept(const ControlFragment &f, std::span bytes, + Clock::time_point now, std::unique_ptr &completed) +{ + completed.reset(); + if (!valid(f, bytes.size())) + return ControlStreamResult::Invalid; + if (!assembly_) { + if ((f.stream_flags & CONTROL_STREAM_START) == 0 || f.stream_id <= last_stream_id_) + return ControlStreamResult::Invalid; + auto lease = budget_.try_reserve(f.total); + if (lease == nullptr) + return ControlStreamResult::Backpressure; + Assembly next{.first = f, .deadline = now + timeout_, .lease = std::move(lease), .bytes = {}}; + next.bytes.resize(static_cast(f.total)); + assembly_ = std::move(next); + } + if (!matches(f) || now >= assembly_->deadline) + return ControlStreamResult::Invalid; + if ((f.stream_flags & CONTROL_STREAM_ERROR) != 0) { + last_stream_id_ = f.stream_id; + reset(); + return ControlStreamResult::Aborted; + } + auto &a = *assembly_; + std::memcpy(a.bytes.data() + a.offset, bytes.data(), bytes.size()); + a.offset += bytes.size(); + ++a.next_sequence; + if ((f.stream_flags & CONTROL_STREAM_END) == 0) + return ControlStreamResult::Accepted; + auto frame = std::make_unique(); + frame->header.msg_type = f.message_type; + frame->header.msg_size = static_cast(f.total); + frame->header.flags = f.message_flags; + frame->request_id = f.request_id; + frame->rpc_create_nano = f.rpc_create_nano; + frame->response_create_nano = f.response_create_nano; + frame->control = std::move(a.bytes); + frame->control_owner = std::move(a.lease); + completed = std::move(frame); + last_stream_id_ = f.stream_id; + assembly_.reset(); + return ControlStreamResult::Complete; +} + +bool ControlStreamReceiver::active() const +{ + return assembly_.has_value(); +} + +bool ControlStreamReceiver::expired(Clock::time_point now) const +{ + return assembly_ && now >= assembly_->deadline; +} + +void ControlStreamReceiver::reset() +{ + assembly_.reset(); +} +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_control_stream.h b/lib/crowdb-rpc/src/transport/rdma/rdma_control_stream.h new file mode 100644 index 000000000..20a8cb456 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_control_stream.h @@ -0,0 +1,106 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/framing.h" + +#include +#include +#include +#include +#include + +namespace crowdb::rpc +{ +constexpr uint16_t RPC_CONTROL_FRAGMENT_MSG_TYPE = 4; +constexpr uint8_t CONTROL_STREAM_START = 1; +constexpr uint8_t CONTROL_STREAM_END = 2; +constexpr uint8_t CONTROL_STREAM_ERROR = 4; + +struct ControlStreamBudgetState; + +class ControlStreamLease +{ + public: + ~ControlStreamLease(); + ControlStreamLease(const ControlStreamLease &) = delete; + ControlStreamLease &operator=(const ControlStreamLease &) = delete; + + private: + friend class ControlStreamBudget; + ControlStreamLease(std::shared_ptr state, uint64_t bytes); + std::shared_ptr state_; + uint64_t bytes_; +}; + +// Copies share the same admission. The wake callback must not throw and must +// safely handle a lease released after its transport has stopped. +class ControlStreamBudget +{ + public: + explicit ControlStreamBudget(uint64_t limit, std::function wake = {}); + [[nodiscard]] std::shared_ptr try_reserve(uint64_t bytes); + [[nodiscard]] uint64_t used() const; + [[nodiscard]] uint64_t high_water() const; + [[nodiscard]] uint64_t stalls() const; + [[nodiscard]] uint64_t limit() const; + + private: + std::shared_ptr state_; +}; + +struct ControlFragment +{ + uint64_t request_id = 0; + uint64_t rpc_create_nano = 0; + uint64_t response_create_nano = 0; + uint64_t stream_id = 0; + uint64_t generation = 0; + uint64_t sequence = 0; + uint64_t offset = 0; + uint64_t total = 0; + uint32_t capacity = 0; + uint16_t message_type = 0; + uint8_t message_flags = 0; + uint8_t stream_flags = 0; +}; + +enum class ControlStreamResult { Accepted, Complete, Backpressure, Invalid, Aborted }; + +// One QP sends complete logical messages in order, so its CQ owner needs only +// one active assembly. Completed controls retain admission independently. +class ControlStreamReceiver +{ + public: + using Clock = std::chrono::steady_clock; + ControlStreamReceiver(ControlStreamBudget budget, uint32_t capacity, uint64_t generation, + uint32_t max_message_bytes, std::chrono::milliseconds timeout); + ControlStreamResult accept(const ControlFragment &fragment, std::span bytes, Clock::time_point now, + std::unique_ptr &completed); + [[nodiscard]] bool active() const; + [[nodiscard]] bool expired(Clock::time_point now) const; + void reset(); + + private: + struct Assembly + { + ControlFragment first; + uint64_t next_sequence = 0; + uint64_t offset = 0; + Clock::time_point deadline; + std::shared_ptr lease; + std::vector bytes; + }; + + [[nodiscard]] bool valid(const ControlFragment &fragment, size_t bytes) const; + [[nodiscard]] bool matches(const ControlFragment &fragment) const; + ControlStreamBudget budget_; + uint32_t capacity_; + uint64_t generation_; + uint32_t max_message_bytes_; + std::chrono::milliseconds timeout_; + uint64_t last_stream_id_ = 0; + std::optional assembly_; +}; +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/tests/c_api_buffer_test.cpp b/lib/crowdb-rpc/tests/c_api_buffer_test.cpp index 26a1c876c..28a95a90d 100644 --- a/lib/crowdb-rpc/tests/c_api_buffer_test.cpp +++ b/lib/crowdb-rpc/tests/c_api_buffer_test.cpp @@ -1,7 +1,9 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#include "../src/transport/rdma/rdma_control_stream.h" #include "crowdb-rpc/c_api.h" +#include "crowdb-rpc/c_api_internal.h" #include @@ -37,3 +39,27 @@ TEST(CApiBufferTest, ExternalOwnerReleasesOnlyAfterLastIndependentHandle) crowdb_rpc_buffer_release(original); EXPECT_EQ(releases, 1U); } + +TEST(CApiBufferTest, LogicalControlRetainsAdmissionAndOriginalStorageThroughClonedHandles) +{ + crowdb::rpc::ControlStreamBudget budget(32); + auto frame = std::make_unique(); + frame->control.resize(32, 0x5a); + frame->control_owner = budget.try_reserve(frame->control.size()); + auto *original = frame->control.data(); + crowdb_rpc_buffer_t control = nullptr; + crowdb_rpc_buffer_t data = nullptr; + crowdb::rpc::frame_to_c_handles(frame.release(), &control, &data); + ASSERT_NE(control, nullptr); + EXPECT_EQ(data, nullptr); + EXPECT_EQ(crowdb_rpc_buffer_data(control), original); + EXPECT_EQ(crowdb_rpc_buffer_len(control), 32U); + EXPECT_EQ(budget.used(), 32U); + auto clone = crowdb_rpc_buffer_ref(control); + ASSERT_NE(clone, nullptr); + crowdb_rpc_buffer_release(control); + EXPECT_EQ(budget.used(), 32U); + EXPECT_EQ(crowdb_rpc_buffer_data(clone)[31], 0x5a); + crowdb_rpc_buffer_release(clone); + EXPECT_EQ(budget.used(), 0U); +} diff --git a/lib/crowdb-rpc/tests/rdma_control_stream_test.cpp b/lib/crowdb-rpc/tests/rdma_control_stream_test.cpp new file mode 100644 index 000000000..d6a424c63 --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_control_stream_test.cpp @@ -0,0 +1,234 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "../src/transport/rdma/rdma_control_stream.h" + +#include + +#include +#include +#include + +using namespace crowdb::rpc; +using namespace std::chrono_literals; + +namespace +{ +ControlFragment test_fragment(uint64_t total, uint32_t capacity, uint64_t offset = 0, uint64_t stream_id = 1) +{ + return {.request_id = 43, + .rpc_create_nano = 11, + .response_create_nano = 12, + .stream_id = stream_id, + .generation = 7, + .sequence = offset / capacity, + .offset = offset, + .total = total, + .capacity = capacity, + .message_type = 3701, + .stream_flags = static_cast((offset == 0 ? CONTROL_STREAM_START : 0) | + (total - offset <= capacity ? CONTROL_STREAM_END : 0))}; +} + +TEST(ControlStreamTest, ClassesReassembleExactBytesAndRetainAdmissionThroughFinalOwner) +{ + for (const uint32_t capacity : {RDMA_METADATA_CAPACITY, RDMA_AGGREGATE_CAPACITY, RDMA_DATAPATH_CAPACITY}) { + ControlStreamBudget budget(4 << 20); + ControlStreamReceiver receiver(budget, capacity, 7, 4 << 20, 10ms); + std::vector original(2 * capacity + 13); + for (size_t index = 0; index < original.size(); ++index) + original[index] = static_cast(index * 131U); + std::unique_ptr completed; + for (size_t offset = 0; offset < original.size(); offset += capacity) { + const auto part = std::span(original).subspan(offset, std::min(capacity, original.size() - offset)); + const auto expected = + offset + part.size() == original.size() ? ControlStreamResult::Complete : ControlStreamResult::Accepted; + EXPECT_EQ(receiver.accept(test_fragment(original.size(), capacity, offset), part, {}, completed), expected); + EXPECT_EQ(budget.used(), original.size()); + } + ASSERT_NE(completed, nullptr); + EXPECT_EQ(completed->control, original); + EXPECT_EQ(completed->request_id, 43U); + EXPECT_EQ(completed->header.msg_type, 3701U); + EXPECT_EQ(completed->header.msg_size, original.size()); + EXPECT_EQ(completed->header.data_size, 0U); + EXPECT_EQ(completed->rpc_create_nano, 11U); + EXPECT_EQ(completed->response_create_nano, 12U); + EXPECT_FALSE(receiver.active()); + auto retained = completed->control_owner; + completed.reset(); + EXPECT_EQ(budget.used(), original.size()); + retained.reset(); + EXPECT_EQ(budget.used(), 0U); + EXPECT_LE(budget.high_water(), budget.limit()); + } +} + +TEST(ControlStreamTest, HeldCompletedControlBlocksTheNextMessageAndReleaseWakesAdmission) +{ + std::atomic wakes{0}; + ControlStreamBudget budget(32, [&] { wakes.fetch_add(1); }); + ControlStreamReceiver receiver(budget, RDMA_METADATA_CAPACITY, 7, 32, 10ms); + std::array bytes{}; + std::unique_ptr first; + ASSERT_EQ(receiver.accept(test_fragment(bytes.size(), RDMA_METADATA_CAPACITY), bytes, {}, first), + ControlStreamResult::Complete); + std::unique_ptr second; + const auto fragment = test_fragment(bytes.size(), RDMA_METADATA_CAPACITY, 0, 2); + EXPECT_EQ(receiver.accept(fragment, bytes, {}, second), ControlStreamResult::Backpressure); + EXPECT_EQ(second, nullptr); + EXPECT_FALSE(receiver.active()); + EXPECT_EQ(budget.used(), bytes.size()); + EXPECT_EQ(wakes.load(), 0U); + first.reset(); + EXPECT_EQ(wakes.load(), 1U); + EXPECT_EQ(receiver.accept(fragment, bytes, {}, second), ControlStreamResult::Complete); + second.reset(); + EXPECT_EQ(budget.used(), 0U); + EXPECT_EQ(wakes.load(), 2U); +} + +TEST(ControlStreamTest, MetadataDisagreementAndGapsNeverPublishPartialControl) +{ + using Change = std::function; + const std::array changes{[](auto &f) { ++f.request_id; }, + [](auto &f) { ++f.rpc_create_nano; }, + [](auto &f) { ++f.response_create_nano; }, + [](auto &f) { ++f.stream_id; }, + [](auto &f) { ++f.generation; }, + [](auto &f) { ++f.sequence; }, + [](auto &f) { --f.offset; }, + [](auto &f) { ++f.total; }, + [](auto &f) { f.capacity = RDMA_AGGREGATE_CAPACITY; }, + [](auto &f) { ++f.message_type; }, + [](auto &f) { f.message_flags = FLAG_ONE_WAY; }}; + const uint32_t capacity = RDMA_METADATA_CAPACITY; + std::vector bytes(capacity); + for (const auto &change : changes) { + ControlStreamBudget budget(2 * capacity); + ControlStreamReceiver receiver(budget, capacity, 7, 2 * capacity, 10ms); + std::unique_ptr completed; + ASSERT_EQ(receiver.accept(test_fragment(2 * capacity, capacity), bytes, {}, completed), + ControlStreamResult::Accepted); + auto invalid = test_fragment(2 * capacity, capacity, capacity); + change(invalid); + EXPECT_EQ(receiver.accept(invalid, bytes, {}, completed), ControlStreamResult::Invalid); + EXPECT_EQ(completed, nullptr); + EXPECT_EQ(budget.used(), 2 * capacity); + receiver.reset(); + EXPECT_EQ(budget.used(), 0U); + } +} + +TEST(ControlStreamTest, DuplicateCompletedStreamAndDuplicateStartAreRejected) +{ + const uint32_t capacity = RDMA_METADATA_CAPACITY; + ControlStreamBudget budget(2 * capacity); + ControlStreamReceiver receiver(budget, capacity, 7, 2 * capacity, 10ms); + std::vector bytes(capacity); + std::unique_ptr completed; + const auto first = test_fragment(2 * capacity, capacity); + ASSERT_EQ(receiver.accept(first, bytes, {}, completed), ControlStreamResult::Accepted); + EXPECT_EQ(receiver.accept(first, bytes, {}, completed), ControlStreamResult::Invalid); + ASSERT_EQ(receiver.accept(test_fragment(2 * capacity, capacity, capacity), bytes, {}, completed), + ControlStreamResult::Complete); + completed.reset(); + EXPECT_EQ(receiver.accept(first, bytes, {}, completed), ControlStreamResult::Invalid); + EXPECT_EQ(budget.used(), 0U); +} + +TEST(ControlStreamTest, InvalidBoundsFlagsAndLengthsRejectBeforeByteAdmission) +{ + const uint32_t capacity = RDMA_METADATA_CAPACITY; + ControlStreamBudget budget(2 * capacity); + ControlStreamReceiver receiver(budget, capacity, 7, 2 * capacity, 10ms); + const auto first = test_fragment(2 * capacity, capacity); + const std::array, 9> changes{ + [](auto &f) { f.request_id = 0; }, + [](auto &f) { f.stream_id = 0; }, + [](auto &f) { f.total = UINT64_MAX; }, + [](auto &f) { f.offset = UINT64_MAX; }, + [](auto &f) { f.stream_flags |= CONTROL_STREAM_END; }, + [](auto &f) { f.stream_flags |= 8; }, + [](auto &f) { f.message_flags = 0x80; }, + [](auto &f) { f.message_type = RPC_CONTROL_FRAGMENT_MSG_TYPE; }, + [](auto &f) { f.capacity = 128 << 10; }}; + std::vector bytes(capacity); + std::unique_ptr completed; + for (const auto &change : changes) { + auto invalid = first; + change(invalid); + EXPECT_EQ(receiver.accept(invalid, bytes, {}, completed), ControlStreamResult::Invalid); + EXPECT_EQ(budget.used(), 0U); + } + EXPECT_EQ(receiver.accept(first, std::span(bytes).first(capacity - 1), {}, completed), + ControlStreamResult::Invalid); + EXPECT_EQ(receiver.accept(first, {}, {}, completed), ControlStreamResult::Invalid); + EXPECT_EQ(budget.used(), 0U); +} + +TEST(ControlStreamTest, ErrorAndPartialExpiryReturnAdmissionWithoutPublishingBytes) +{ + const uint32_t capacity = RDMA_METADATA_CAPACITY; + ControlStreamBudget budget(2 * capacity); + ControlStreamReceiver receiver(budget, capacity, 7, 2 * capacity, 10ms); + std::vector bytes(capacity); + std::unique_ptr completed; + const auto first = test_fragment(2 * capacity, capacity); + ASSERT_EQ(receiver.accept(first, bytes, {}, completed), ControlStreamResult::Accepted); + auto failure = test_fragment(2 * capacity, capacity, capacity); + failure.stream_flags = CONTROL_STREAM_END | CONTROL_STREAM_ERROR; + EXPECT_EQ(receiver.accept(failure, {}, {}, completed), ControlStreamResult::Aborted); + EXPECT_EQ(budget.used(), 0U); + ASSERT_EQ(receiver.accept(test_fragment(2 * capacity, capacity, 0, 2), bytes, {}, completed), + ControlStreamResult::Accepted); + EXPECT_FALSE(receiver.expired(ControlStreamReceiver::Clock::time_point{} + 9ms)); + EXPECT_TRUE(receiver.expired(ControlStreamReceiver::Clock::time_point{} + 10ms)); + EXPECT_EQ(receiver.accept(test_fragment(2 * capacity, capacity, capacity, 2), bytes, + ControlStreamReceiver::Clock::time_point{} + 10ms, completed), + ControlStreamResult::Invalid); + EXPECT_EQ(completed, nullptr); + receiver.reset(); + EXPECT_EQ(budget.used(), 0U); +} + +TEST(ControlStreamTest, CompletedOwnerOutlivesReceiverAndBudgetFacades) +{ + auto wakes = std::make_shared>(0); + std::unique_ptr completed; + { + ControlStreamBudget budget(32, [wakes] { wakes->fetch_add(1); }); + ControlStreamReceiver receiver(budget, RDMA_METADATA_CAPACITY, 7, 32, 10ms); + std::array bytes{}; + ASSERT_EQ(receiver.accept(test_fragment(bytes.size(), RDMA_METADATA_CAPACITY), bytes, {}, completed), + ControlStreamResult::Complete); + } + EXPECT_EQ(wakes->load(), 0U); + completed.reset(); + EXPECT_EQ(wakes->load(), 1U); +} + +TEST(ControlStreamTest, ConcurrentByteReservationsNeverExceedTheHardBound) +{ + ControlStreamBudget budget(4096); + std::array workers; + std::atomic valid{true}; + for (auto &worker : workers) + worker = std::thread([&] { + for (uint64_t index = 0; index < 2000; ++index) { + auto lease = budget.try_reserve(1 + index % 128); + if (budget.used() > budget.limit()) + valid.store(false); + if (lease == nullptr) + std::this_thread::yield(); + } + }); + for (auto &worker : workers) + worker.join(); + EXPECT_TRUE(valid.load()); + EXPECT_EQ(budget.used(), 0U); + EXPECT_LE(budget.high_water(), budget.limit()); + EXPECT_EQ(budget.try_reserve(UINT64_MAX), nullptr); + EXPECT_EQ(budget.try_reserve(0), nullptr); +} +} // namespace From 6d4f7cabd4f8942cbbc24b0a47e4e867c10a6654 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 05:03:35 +0800 Subject: [PATCH 048/258] Negotiate bounded logical controls and expose directional byte budgets --- doc/working/plan-rdma-chunk-kv-access.md | 23 +++++++- lib/crowdb-protocol/src/fbs/common_msg.fbs | 17 ++++++ lib/crowdb-protocol/src/fbs/msg_type.fbs | 1 + lib/crowdb-rpc/ffi/src/sys.rs | 17 ++++++ lib/crowdb-rpc/ffi/src/transport.rs | 25 ++++++++- .../ffi/tests/transport_config_test.rs | 20 +++++++ lib/crowdb-rpc/include/crowdb-rpc/c_api.h | 17 ++++++ .../transport/rdma/rdma_capability.h | 5 +- .../crowdb-rpc/transport/rdma/rdma_config.h | 4 ++ lib/crowdb-rpc/src/c_api/transport.cpp | 45 ++++++++++++--- .../src/transport/rdma/rdma_capability.cpp | 13 ++++- .../src/transport/rdma/rdma_config.cpp | 10 +++- .../src/transport/rdma/rdma_offer.cpp | 19 ++++--- .../src/transport/rdma/rdma_offer.h | 17 +++--- .../src/transport/rdma/rdma_status.cpp | 4 ++ .../src/transport/rdma/rdma_transport.cpp | 1 + lib/crowdb-rpc/tests/rdma_config_test.cpp | 39 +++++++++++++ lib/crowdb-rpc/tests/rdma_offer_test.cpp | 56 +++++++++++++++---- .../tests/transport_config_test.cpp | 24 ++++++++ tools/rpc-test/src/args.rs | 1 + 20 files changed, 314 insertions(+), 44 deletions(-) diff --git a/doc/working/plan-rdma-chunk-kv-access.md b/doc/working/plan-rdma-chunk-kv-access.md index 373de978a..5308ea77b 100644 --- a/doc/working/plan-rdma-chunk-kv-access.md +++ b/doc/working/plan-rdma-chunk-kv-access.md @@ -52,7 +52,7 @@ Observed contract: final caller release. Preserve a failed/blocked fragment's existing owner; blocked admission must not carve replacement storage. Files: private RPC control-stream modules, `framing.h` control owner, focused portable tests. -- [~] **Negotiate and configure common streams**: add a small transport-owned +- [x] **Negotiate and configure common streams**: add a small transport-owned fragment FlatBuffer in `common_msg.fbs`, preserving the common leading correlation fields, with reserved message type 4. Bump RDMA protocol to version four; the existing final 4 handshake bytes advertise maximum logical @@ -69,7 +69,7 @@ Observed contract: depth and data capacity, as receive defaults already do. Preserve native `RdmaConfig::for_layout`'s existing 64 MiB send arena default for the accepted 1 MiB datapath and ordinary RPC. Files: capability/offer/config/status, FFI. -- [ ] **Integrate CQ-owned fragmentation and assembly**: enqueue one original +- [~] **Integrate CQ-owned fragmentation and assembly**: enqueue one original logical frame under byte/count admission. The CQ owner walks it in order, builds only the next bounded fragment, and holds immutable source owners through all WR completions. Do not materialize an unbounded fragment list. @@ -207,6 +207,25 @@ Portable stream checkpoint: `.crowdb-runtime/artifacts/unit/cpp-f0f7acd09fb541f59d048f81e3923cc2/00-tree.log`; this checkpoint does not claim to repair that intermittent tree failure. +Negotiation checkpoint: + +- Protocol v4 carries maximum logical control bytes in the 160-byte TCP + capability and explicit 52-byte CM offer. Digest/generation/layout/receive + admission validation binds both ends; versions 1/2/3 remain decodable for + pre-payload fallback. Native/env/status and version-three C ABI configuration + expose independent logical send/receive budgets and partial-assembly timeout. + Version-one config layout and previous configure entries remain valid. +- Full RPC binary passed (168 portable tests; 13 hardware cases skipped), + no-verbs build passed all 157 CTest cases with its hardware case skipped, + and protocol/FFI Rust suites passed. Three actual HCA cases passed: direct + payload/lease round trip, layout mismatch and explicit TCP fallback policy. + Native XML is under `.crowdb-runtime/rdma-check/artifacts/r241-control/`. +- CLI's exhaustive typed-config literal now inherits the new stream defaults; + all four coroutine/timed RPC benchmark integration cases passed. Focused + all-feature Rust clippy, workspace `rs-lint`, format and scoped C++ lint + passed. The existing request-count worker fixture requires separate client + and server roles; its role-less invocation failed setup, not a payload test. + ```bash pixi run -- cargo fmt --all -- --check pixi run -- cargo test -p crowdb-chunk-kv-client --all-targets diff --git a/lib/crowdb-protocol/src/fbs/common_msg.fbs b/lib/crowdb-protocol/src/fbs/common_msg.fbs index 17a8f0bc0..59c80eefd 100644 --- a/lib/crowdb-protocol/src/fbs/common_msg.fbs +++ b/lib/crowdb-protocol/src/fbs/common_msg.fbs @@ -26,3 +26,20 @@ table UnknownMessage { rpc_create_nano:uint64; response_create_nano:uint64; } + +// Transport-owned framing for a large encoded control. The data SGE contains +// one bounded part; handlers see only the complete original control message. +table RpcControlFragment { + id:uint64; + rpc_create_nano:uint64; + response_create_nano:uint64; + stream_id:uint64; + generation:uint64; + sequence:uint64; + logical_offset:uint64; + total_length:uint64; + data_capacity:uint32; + original_msg_type:uint16; + original_flags:ubyte; + stream_flags:ubyte; +} diff --git a/lib/crowdb-protocol/src/fbs/msg_type.fbs b/lib/crowdb-protocol/src/fbs/msg_type.fbs index 7fdb23937..24e0ccf87 100644 --- a/lib/crowdb-protocol/src/fbs/msg_type.fbs +++ b/lib/crowdb-protocol/src/fbs/msg_type.fbs @@ -19,6 +19,7 @@ enum FBMsgType : int16 { EUnknownResponse = 1, EConnectionPingRequest = 2, EConnectionPingResponse = 3, + ERpcControlFragment = 4, // diskio service (range 3600s, see diskio.fbs) EDiskWriteRequest = 3600, EDiskWriteResponse = 3601, diff --git a/lib/crowdb-rpc/ffi/src/sys.rs b/lib/crowdb-rpc/ffi/src/sys.rs index 7a0aa9278..16fe19485 100644 --- a/lib/crowdb-rpc/ffi/src/sys.rs +++ b/lib/crowdb-rpc/ffi/src/sys.rs @@ -80,6 +80,16 @@ pub struct CrowdbRpcRdmaConfig { pub gid: *const c_char, } +#[repr(C)] +pub struct CrowdbRpcRdmaStreamConfig { + pub struct_size: u32, + pub version: u32, + pub max_control_message_bytes: u32, + pub control_assembly_timeout_ms: u32, + pub control_send_budget_bytes: u64, + pub control_receive_budget_bytes: u64, +} + #[repr(C)] #[derive(Default, Debug)] pub struct CrowdbRpcClientCounters { @@ -151,6 +161,13 @@ pub type crowdb_rpc_co_on_response = Option< extern "C" { pub fn crowdb_rpc_rdma_config_init(config: *mut CrowdbRpcRdmaConfig); + pub fn crowdb_rpc_rdma_stream_config_init(config: *mut CrowdbRpcRdmaStreamConfig); + pub fn crowdb_rpc_server_configure_rdma_v3( + server: crowdb_rpc_server_t, + config: *const CrowdbRpcRdmaConfig, + wire_layout: u32, + streams: *const CrowdbRpcRdmaStreamConfig, + ) -> crowdb_rpc_status; pub fn crowdb_rpc_server_configure_rdma( server: crowdb_rpc_server_t, config: *const CrowdbRpcRdmaConfig, diff --git a/lib/crowdb-rpc/ffi/src/transport.rs b/lib/crowdb-rpc/ffi/src/transport.rs index 70efdb601..41dcf615b 100644 --- a/lib/crowdb-rpc/ffi/src/transport.rs +++ b/lib/crowdb-rpc/ffi/src/transport.rs @@ -42,6 +42,10 @@ pub struct RdmaConfig { pub send_arena_bytes: u64, pub receive_arena_bytes: u64, pub registered_memory_budget_bytes: u64, + pub max_control_message_bytes: u32, + pub control_assembly_timeout_ms: u32, + pub control_send_budget_bytes: u64, + pub control_receive_budget_bytes: u64, pub device: String, pub source_address: String, pub gid: String, @@ -56,9 +60,19 @@ fn native_defaults() -> sys::CrowdbRpcRdmaConfig { } } +fn native_stream_defaults() -> sys::CrowdbRpcRdmaStreamConfig { + let mut config = MaybeUninit::uninit(); + // SAFETY: initializer fills every field of this versioned C ABI struct. + unsafe { + sys::crowdb_rpc_rdma_stream_config_init(config.as_mut_ptr()); + config.assume_init() + } +} + impl Default for RdmaConfig { fn default() -> Self { let raw = native_defaults(); + let streams = native_stream_defaults(); Self { enabled: raw.enabled != 0, allow_tcp_fallback: raw.allow_tcp_fallback != 0, @@ -81,6 +95,10 @@ impl Default for RdmaConfig { send_arena_bytes: raw.send_arena_bytes, receive_arena_bytes: raw.receive_arena_bytes, registered_memory_budget_bytes: raw.registered_memory_budget_bytes, + max_control_message_bytes: streams.max_control_message_bytes, + control_assembly_timeout_ms: streams.control_assembly_timeout_ms, + control_send_budget_bytes: streams.control_send_budget_bytes, + control_receive_budget_bytes: streams.control_receive_budget_bytes, device: String::new(), source_address: String::new(), gid: String::new(), @@ -121,9 +139,14 @@ impl RpcServer { raw.device = device.as_ptr(); raw.source_address = source.as_ptr(); raw.gid = gid.as_ptr(); + let mut streams = native_stream_defaults(); + streams.max_control_message_bytes = config.max_control_message_bytes; + streams.control_assembly_timeout_ms = config.control_assembly_timeout_ms; + streams.control_send_budget_bytes = config.control_send_budget_bytes; + streams.control_receive_budget_bytes = config.control_receive_budget_bytes; // SAFETY: handle is borrowed, struct and CStrings live through the call. let status = unsafe { - sys::crowdb_rpc_server_configure_rdma_v2(self.handle(), &raw, config.wire_layout as u32) + sys::crowdb_rpc_server_configure_rdma_v3(self.handle(), &raw, config.wire_layout as u32, &streams) }; if status == sys::CROWDB_RPC_OK { Ok(()) diff --git a/lib/crowdb-rpc/ffi/tests/transport_config_test.rs b/lib/crowdb-rpc/ffi/tests/transport_config_test.rs index c1eca2064..081ef46d7 100644 --- a/lib/crowdb-rpc/ffi/tests/transport_config_test.rs +++ b/lib/crowdb-rpc/ffi/tests/transport_config_test.rs @@ -39,6 +39,26 @@ fn invalid_queue_bounds_fail_before_start() { assert!(server.configure_rdma(&config).is_err()); } +#[test] +fn logical_control_limits_are_typed_and_bounded_in_both_directions() { + let server = RpcServer::new(None); + let mut config = RdmaConfig::default(); + assert_eq!(config.max_control_message_bytes, 32 << 20); + assert_eq!(config.control_send_budget_bytes, 64 << 20); + assert_eq!(config.control_receive_budget_bytes, 64 << 20); + config.max_control_message_bytes = 65536; + config.control_send_budget_bytes = 65535; + assert!(server.configure_rdma(&config).is_err()); + config.control_send_budget_bytes = 65536; + config.control_receive_budget_bytes = 65535; + assert!(server.configure_rdma(&config).is_err()); + config.control_receive_budget_bytes = 65536; + config.control_assembly_timeout_ms = 0; + assert!(server.configure_rdma(&config).is_err()); + config.control_assembly_timeout_ms = 125; + server.configure_rdma(&config).unwrap(); +} + #[test] fn service_layout_defaults_validate_capacity_and_preserve_explicit_tcp() { assert!(RpcServer::with_rdma_layout(None, 1, 1, RdmaWireLayout::ControlData, 4096).is_err()); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h index e24e8b8f7..5214b68f5 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h @@ -110,6 +110,23 @@ crowdb_rpc_status crowdb_rpc_server_configure_rdma(crowdb_rpc_server_t server, c // the version-one struct or initializer. wire_layout: 0 contiguous, 1 control/data. crowdb_rpc_status crowdb_rpc_server_configure_rdma_v2(crowdb_rpc_server_t server, const crowdb_rpc_rdma_config_t *config, uint32_t wire_layout); + +// Version-three entry adds bounded logical controls. The original config ABI +// remains unchanged; previous configure entries select the stream defaults. +typedef struct crowdb_rpc_rdma_stream_config +{ + uint32_t struct_size; + uint32_t version; + uint32_t max_control_message_bytes; + uint32_t control_assembly_timeout_ms; + uint64_t control_send_budget_bytes; + uint64_t control_receive_budget_bytes; +} crowdb_rpc_rdma_stream_config_t; + +void crowdb_rpc_rdma_stream_config_init(crowdb_rpc_rdma_stream_config_t *config); +crowdb_rpc_status crowdb_rpc_server_configure_rdma_v3(crowdb_rpc_server_t server, + const crowdb_rpc_rdma_config_t *config, uint32_t wire_layout, + const crowdb_rpc_rdma_stream_config_t *streams); // Owned, versioned UTF-8 JSON snapshots. Release using buffer_release. crowdb_rpc_buffer_t crowdb_rpc_server_status_json(crowdb_rpc_server_t server); crowdb_rpc_buffer_t crowdb_rpc_conn_status_json(crowdb_rpc_conn_t connection); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h index cee92ccfd..3f95f59fb 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h @@ -40,12 +40,13 @@ struct RdmaEndpointCapability uint32_t receive_slot_capacity = 0; uint32_t receive_frame_limit = 1; std::string cm_address{}; - RdmaWireLayout wire_layout = RdmaWireLayout::Contiguous; + RdmaWireLayout wire_layout = RdmaWireLayout::Contiguous; + uint32_t max_control_message_bytes = 0; }; constexpr uint16_t RDMA_HANDSHAKE_MAGIC = 0x5244; constexpr uint16_t RDMA_HANDSHAKE_MSG_TYPE = 0xFFFE; -constexpr uint16_t RDMA_PROTOCOL_VERSION = 3; +constexpr uint16_t RDMA_PROTOCOL_VERSION = 4; constexpr size_t RDMA_LEGACY_HANDSHAKE_SIZE = 128; constexpr size_t RDMA_HANDSHAKE_SIZE = 160; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h index 069da96b8..53007b183 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h @@ -42,6 +42,10 @@ struct RdmaConfig uint32_t aggregation_max_bytes = RDMA_DATAPATH_CAPACITY; uint32_t aggregation_max_frames = 256; uint32_t send_queue_capacity = 4096; + uint32_t max_control_message_bytes = 32U << 20; + uint64_t control_send_budget_bytes = 64ULL << 20; + uint64_t control_receive_budget_bytes = 64ULL << 20; + uint32_t control_assembly_timeout_ms = 30000; [[nodiscard]] bool apply_environment(); [[nodiscard]] bool valid() const; diff --git a/lib/crowdb-rpc/src/c_api/transport.cpp b/lib/crowdb-rpc/src/c_api/transport.cpp index c3919f100..3025ec395 100644 --- a/lib/crowdb-rpc/src/c_api/transport.cpp +++ b/lib/crowdb-rpc/src/c_api/transport.cpp @@ -50,20 +50,49 @@ extern "C" crowdb_rpc_status crowdb_rpc_server_configure_rdma(crowdb_rpc_server_ extern "C" crowdb_rpc_status crowdb_rpc_server_configure_rdma_v2(crowdb_rpc_server_t server, const crowdb_rpc_rdma_config_t *config, uint32_t wire_layout) +{ + crowdb_rpc_rdma_stream_config_t streams; + crowdb_rpc_rdma_stream_config_init(&streams); + return crowdb_rpc_server_configure_rdma_v3(server, config, wire_layout, &streams); +} + +extern "C" void crowdb_rpc_rdma_stream_config_init(crowdb_rpc_rdma_stream_config_t *config) +{ + if (config == nullptr) { + return; + } + const RdmaConfig defaults; + *config = {.struct_size = sizeof(*config), + .version = 1, + .max_control_message_bytes = defaults.max_control_message_bytes, + .control_assembly_timeout_ms = defaults.control_assembly_timeout_ms, + .control_send_budget_bytes = defaults.control_send_budget_bytes, + .control_receive_budget_bytes = defaults.control_receive_budget_bytes}; +} + +extern "C" crowdb_rpc_status crowdb_rpc_server_configure_rdma_v3(crowdb_rpc_server_t server, + const crowdb_rpc_rdma_config_t *config, + uint32_t wire_layout, + const crowdb_rpc_rdma_stream_config_t *streams) { if (server == nullptr || config == nullptr || config->struct_size != sizeof(*config) || config->version != 1 || config->enabled > 1 || config->allow_tcp_fallback > 1 || config->listen_enabled > 1 || - config->aggregation_enabled > 1 || config->port > UINT8_MAX) + config->aggregation_enabled > 1 || config->port > UINT8_MAX || streams == nullptr || + streams->struct_size != sizeof(*streams) || streams->version != 1) return CROWDB_RPC_ERR_INVALID_ARG; try { RdmaConfig native; - native.wire_layout = static_cast(wire_layout); - native.use_environment = false; - native.port = static_cast(config->port); - native.aggregation_enabled = config->aggregation_enabled != 0; - native.device = config->device == nullptr ? "" : config->device; - native.source_address = config->source_address == nullptr ? "" : config->source_address; - native.gid = config->gid == nullptr ? "" : config->gid; + native.wire_layout = static_cast(wire_layout); + native.use_environment = false; + native.port = static_cast(config->port); + native.aggregation_enabled = config->aggregation_enabled != 0; + native.device = config->device == nullptr ? "" : config->device; + native.source_address = config->source_address == nullptr ? "" : config->source_address; + native.gid = config->gid == nullptr ? "" : config->gid; + native.max_control_message_bytes = streams->max_control_message_bytes; + native.control_assembly_timeout_ms = streams->control_assembly_timeout_ms; + native.control_send_budget_bytes = streams->control_send_budget_bytes; + native.control_receive_budget_bytes = streams->control_receive_budget_bytes; #define COPY_VALUE(name) native.name = config->name; CONFIG_NUMBERS(COPY_VALUE) #undef COPY_VALUE diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp index f11509282..f9610d640 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp @@ -64,6 +64,10 @@ std::array serialize_rdma_handshake(const RdmaEndp for (size_t i = 0; i < 4; ++i) bytes[152 + i] = static_cast(layout >> (i * 8)); } + if (capability.protocol_version >= 4) { + for (size_t i = 0; i < 4; ++i) + bytes[156 + i] = static_cast(capability.max_control_message_bytes >> (i * 8)); + } return bytes; } @@ -114,13 +118,17 @@ bool parse_rdma_handshake(std::span bytes, RdmaEndpointCapability uint32_t layout = 0; for (size_t i = 0; i < 4; ++i) { layout |= static_cast(bytes[152 + i]) << (i * 8); - if (bytes[156 + i] != 0) + if (out->protocol_version == 3 && bytes[156 + i] != 0) return false; } if (layout > static_cast(RdmaWireLayout::ControlData)) return false; out->wire_layout = static_cast(layout); } + if (out->protocol_version >= 4) { + for (size_t i = 0; i < 4; ++i) + out->max_control_message_bytes |= static_cast(bytes[156 + i]) << (i * 8); + } return true; } @@ -143,7 +151,8 @@ bool validate_rdma_capability(const RdmaEndpointCapability &capability, uint16_t (expected_protocol < 3 || capability.receive_frame_limit != 1))) return false; } - return capability.data_classes[0] == RDMA_CONTROL_CAPACITY && + return (expected_protocol < 4 || capability.max_control_message_bytes != 0) && + capability.data_classes[0] == RDMA_CONTROL_CAPACITY && capability.data_classes[1] == RDMA_METADATA_CAPACITY && capability.data_classes[2] == RDMA_AGGREGATE_CAPACITY && capability.data_classes[3] == RDMA_DATAPATH_CAPACITY; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp index 1d91284fe..fc616196b 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp @@ -58,7 +58,11 @@ bool RdmaConfig::apply_environment() read("CROWDB_RDMA_AGGREGATION_ENABLED", aggregation_enabled) && read("CROWDB_RDMA_AGGREGATION_MAX_BYTES", aggregation_max_bytes) && read("CROWDB_RDMA_AGGREGATION_MAX_FRAMES", aggregation_max_frames) && - read("CROWDB_RDMA_SEND_QUEUE_CAPACITY", send_queue_capacity); + read("CROWDB_RDMA_SEND_QUEUE_CAPACITY", send_queue_capacity) && + read("CROWDB_RDMA_MAX_CONTROL_MESSAGE_BYTES", max_control_message_bytes) && + read("CROWDB_RDMA_CONTROL_SEND_BUDGET_BYTES", control_send_budget_bytes) && + read("CROWDB_RDMA_CONTROL_RECEIVE_BUDGET_BYTES", control_receive_budget_bytes) && + read("CROWDB_RDMA_CONTROL_ASSEMBLY_TIMEOUT_MS", control_assembly_timeout_ms); } bool RdmaConfig::valid() const @@ -75,7 +79,9 @@ bool RdmaConfig::valid() const min_rnr_timer <= 31 && retry_count <= 7 && rnr_retry_count <= 7 && registration_budget() != 0 && aggregation_max_bytes >= HEADER_SIZE && aggregation_max_bytes <= RDMA_DATAPATH_CAPACITY && aggregation_max_frames > 0 && aggregation_max_frames <= 256 && send_queue_capacity >= 2 && - send_queue_capacity <= (1U << 20) && (send_queue_capacity & (send_queue_capacity - 1)) == 0; + send_queue_capacity <= (1U << 20) && (send_queue_capacity & (send_queue_capacity - 1)) == 0 && + max_control_message_bytes != 0 && max_control_message_bytes <= control_send_budget_bytes && + max_control_message_bytes <= control_receive_budget_bytes && control_assembly_timeout_ms != 0; } uint64_t RdmaConfig::registration_budget() const diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_offer.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_offer.cpp index ff4c4630d..693572f68 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_offer.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_offer.cpp @@ -37,6 +37,7 @@ std::array serialize_rdma_offer(const RdmaEndpointC put(bytes, 24, expected_peer == nullptr ? 0 : expected_peer->generation, 8); put(bytes, 32, rdma_capability_identity(local), 8); put(bytes, 40, expected_peer == nullptr ? 0 : rdma_capability_identity(*expected_peer), 8); + put(bytes, 48, local.max_control_message_bytes, 4); return bytes; } @@ -47,17 +48,18 @@ bool parse_rdma_offer(std::span bytes, RdmaCmOffer *out) get(bytes, 12, 4) > static_cast(RdmaWireLayout::ControlData)) return false; RdmaCmOffer offer; - offer.receive_frame_limit = static_cast(get(bytes, 6, 2)); - offer.receive_slot_capacity = static_cast(get(bytes, 8, 4)); - offer.wire_layout = static_cast(get(bytes, 12, 4)); - offer.generation = get(bytes, 16, 8); - offer.expected_generation = get(bytes, 24, 8); - offer.identity = get(bytes, 32, 8); - offer.expected_identity = get(bytes, 40, 8); + offer.receive_frame_limit = static_cast(get(bytes, 6, 2)); + offer.receive_slot_capacity = static_cast(get(bytes, 8, 4)); + offer.wire_layout = static_cast(get(bytes, 12, 4)); + offer.generation = get(bytes, 16, 8); + offer.expected_generation = get(bytes, 24, 8); + offer.identity = get(bytes, 32, 8); + offer.expected_identity = get(bytes, 40, 8); + offer.max_control_message_bytes = static_cast(get(bytes, 48, 4)); if (!valid_rdma_wire_capacity(offer.receive_slot_capacity, offer.wire_layout) || offer.receive_frame_limit == 0 || offer.receive_frame_limit > 256 || offer.generation == 0 || (offer.wire_layout == RdmaWireLayout::ControlData && offer.receive_frame_limit != 1) || offer.identity == 0 || - (offer.expected_generation == 0) != (offer.expected_identity == 0)) + (offer.expected_generation == 0) != (offer.expected_identity == 0) || offer.max_control_message_bytes == 0) return false; *out = offer; return true; @@ -86,6 +88,7 @@ bool validate_rdma_offer(const RdmaCmOffer &offer, const RdmaEndpointCapability offer.identity == rdma_capability_identity(*expected_peer) && offer.receive_slot_capacity == expected_peer->receive_slot_capacity && offer.wire_layout == expected_peer->wire_layout && + offer.max_control_message_bytes == expected_peer->max_control_message_bytes && offer.receive_frame_limit == expected_peer->receive_frame_limit); } } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_offer.h b/lib/crowdb-rpc/src/transport/rdma/rdma_offer.h index 4389efe9a..f34763cea 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_offer.h +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_offer.h @@ -7,17 +7,18 @@ namespace crowdb::rpc { -constexpr size_t RDMA_CM_OFFER_SIZE = 48; +constexpr size_t RDMA_CM_OFFER_SIZE = 52; struct RdmaCmOffer { - uint16_t receive_frame_limit = 0; - uint32_t receive_slot_capacity = 0; - RdmaWireLayout wire_layout = RdmaWireLayout::Contiguous; - uint64_t generation = 0; - uint64_t expected_generation = 0; - uint64_t identity = 0; - uint64_t expected_identity = 0; + uint16_t receive_frame_limit = 0; + uint32_t receive_slot_capacity = 0; + RdmaWireLayout wire_layout = RdmaWireLayout::Contiguous; + uint64_t generation = 0; + uint64_t expected_generation = 0; + uint64_t identity = 0; + uint64_t expected_identity = 0; + uint32_t max_control_message_bytes = 0; }; std::array serialize_rdma_offer(const RdmaEndpointCapability &local, diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp index 40a3971f1..7ff817faa 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp @@ -41,6 +41,10 @@ std::string RdmaTransport::status_json() const CONFIG_FIELD(send_queue_capacity); CONFIG_FIELD(aggregation_max_bytes); CONFIG_FIELD(aggregation_max_frames); + CONFIG_FIELD(max_control_message_bytes); + CONFIG_FIELD(control_send_budget_bytes); + CONFIG_FIELD(control_receive_budget_bytes); + CONFIG_FIELD(control_assembly_timeout_ms); # undef CONFIG_FIELD # define COUNTER_FIELD(name) status.number(#name, counters.name.load(std::memory_order_relaxed)) COUNTER_FIELD(sends); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index 27bbacd14..20362b05e 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -80,6 +80,7 @@ RdmaTransport::RdmaTransport(RdmaConfig config, BufferPool *application_pool) RDMA_DATAPATH_CAPACITY}; local_capability_.receive_slot_capacity = config_.wire_slot_capacity; local_capability_.wire_layout = config_.wire_layout; + local_capability_.max_control_message_bytes = config_.max_control_message_bytes; local_capability_.receive_frame_limit = config_.wire_layout == RdmaWireLayout::ControlData ? 1 : RDMA_MAX_AGGREGATE_FRAMES; local_capability_.cm_address = config_.source_address; diff --git a/lib/crowdb-rpc/tests/rdma_config_test.cpp b/lib/crowdb-rpc/tests/rdma_config_test.cpp index b4af1139f..cc1ea3361 100644 --- a/lib/crowdb-rpc/tests/rdma_config_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_config_test.cpp @@ -45,6 +45,45 @@ TEST(RdmaConfigTest, EnvironmentRejectsUnknownWireLayout) EXPECT_FALSE(config.apply_environment()); } +TEST(RdmaConfigTest, LogicalControlsRequireNonzeroLimitWithinEachDirectionalBudget) +{ + RdmaConfig config; + EXPECT_EQ(config.max_control_message_bytes, 32U << 20); + EXPECT_EQ(config.control_send_budget_bytes, 64ULL << 20); + EXPECT_EQ(config.control_receive_budget_bytes, 64ULL << 20); + config.max_control_message_bytes = 0; + EXPECT_FALSE(config.valid()); + config.max_control_message_bytes = 32U << 20; + config.control_send_budget_bytes = (32U << 20) - 1; + EXPECT_FALSE(config.valid()); + config.control_send_budget_bytes = 32U << 20; + config.control_receive_budget_bytes = (32U << 20) - 1; + EXPECT_FALSE(config.valid()); + config.control_receive_budget_bytes = 32U << 20; + EXPECT_TRUE(config.valid()); + config.control_assembly_timeout_ms = 0; + EXPECT_FALSE(config.valid()); +} + +TEST(RdmaConfigTest, LogicalControlEnvironmentParses64BitBudgetsWithoutChangingMrAdmission) +{ + const TestEnvironment max("CROWDB_RDMA_MAX_CONTROL_MESSAGE_BYTES", "65536"); + const TestEnvironment send("CROWDB_RDMA_CONTROL_SEND_BUDGET_BYTES", "8589934592"); + const TestEnvironment receive("CROWDB_RDMA_CONTROL_RECEIVE_BUDGET_BYTES", "131072"); + const TestEnvironment timeout("CROWDB_RDMA_CONTROL_ASSEMBLY_TIMEOUT_MS", "125"); + RdmaConfig config; + const auto registered = config.registration_budget(); + ASSERT_TRUE(config.apply_environment()); + EXPECT_TRUE(config.valid()); + EXPECT_EQ(config.max_control_message_bytes, 65536U); + EXPECT_EQ(config.control_send_budget_bytes, 8ULL << 30); + EXPECT_EQ(config.control_receive_budget_bytes, 131072U); + EXPECT_EQ(config.control_assembly_timeout_ms, 125U); + EXPECT_EQ(config.registration_budget(), registered); + const TestEnvironment invalid("CROWDB_RDMA_CONTROL_SEND_BUDGET_BYTES", "-1"); + EXPECT_FALSE(config.apply_environment()); +} + TEST(RdmaConfigTest, PostedReceiveTargetMustFitRqAndRegisteredBudget) { RdmaConfig config; diff --git a/lib/crowdb-rpc/tests/rdma_offer_test.cpp b/lib/crowdb-rpc/tests/rdma_offer_test.cpp index fa0f92666..c42b66566 100644 --- a/lib/crowdb-rpc/tests/rdma_offer_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_offer_test.cpp @@ -23,16 +23,17 @@ namespace RdmaEndpointCapability test_capability(uint64_t generation) { return { - .rdma_available = true, - .protocol_version = RDMA_PROTOCOL_VERSION, - .generation = generation, - .port = 19101, - .device = "test-device", - .gid = "fe80::1", - .data_classes = {1024, 65536, 262144, 1048576}, - .receive_slot_capacity = 1024, - .receive_frame_limit = 1, - .cm_address = "10.200.1.1" + .rdma_available = true, + .protocol_version = RDMA_PROTOCOL_VERSION, + .generation = generation, + .port = 19101, + .device = "test-device", + .gid = "fe80::1", + .data_classes = {1024, 65536, 262144, 1048576}, + .receive_slot_capacity = 1024, + .receive_frame_limit = 1, + .cm_address = "10.200.1.1", + .max_control_message_bytes = 32U << 20 }; } @@ -81,7 +82,7 @@ TEST(RdmaOfferTest, PrivateDataBindsPreflightIdentityAndReceiveAdmission) RdmaCmOffer decoded; const auto wire = serialize_rdma_offer(client, &server); ASSERT_TRUE(parse_rdma_offer(wire, &decoded)); - EXPECT_EQ(wire.size(), 48U); + EXPECT_EQ(wire.size(), 52U); EXPECT_EQ(decoded.receive_slot_capacity, 1024U); EXPECT_TRUE(validate_rdma_offer(decoded, server, &client, true)); auto stale = server; @@ -118,6 +119,7 @@ TEST(RdmaOfferTest, ControlDataLayoutIsBoundByBothCapabilityAndPrivateOffer) invalid[152] = 2; EXPECT_FALSE(parse_rdma_handshake(invalid, &decoded)); invalid = wire; + invalid[2] = 3; invalid[156] = 1; EXPECT_FALSE(parse_rdma_handshake(invalid, &decoded)); client.protocol_version = 2; @@ -128,6 +130,38 @@ TEST(RdmaOfferTest, ControlDataLayoutIsBoundByBothCapabilityAndPrivateOffer) EXPECT_TRUE(validate_rdma_capability(decoded, 2, 11, "test-device")); } +TEST(RdmaOfferTest, LogicalControlLimitIsExplicitAndBoundIntoBothIdentities) +{ + auto client = test_capability(31); + auto server = test_capability(37); + client.max_control_message_bytes = 1U << 20; + server.max_control_message_bytes = 16U << 20; + RdmaEndpointCapability decoded; + ASSERT_TRUE(parse_rdma_handshake(serialize_rdma_handshake(client), &decoded)); + EXPECT_EQ(decoded.max_control_message_bytes, client.max_control_message_bytes); + RdmaCmOffer offer; + ASSERT_TRUE(parse_rdma_offer(serialize_rdma_offer(client, &server), &offer)); + EXPECT_EQ(offer.max_control_message_bytes, client.max_control_message_bytes); + EXPECT_TRUE(validate_rdma_offer(offer, server, &client, true)); + ++client.max_control_message_bytes; + EXPECT_FALSE(validate_rdma_offer(offer, server, &client, true)); + --client.max_control_message_bytes; + ++offer.max_control_message_bytes; + EXPECT_FALSE(validate_rdma_offer(offer, server, &client, true)); + --offer.max_control_message_bytes; + ++server.max_control_message_bytes; + EXPECT_FALSE(validate_rdma_offer(offer, server, &client, true)); + client.max_control_message_bytes = 0; + EXPECT_FALSE(validate_rdma_capability(client, RDMA_PROTOCOL_VERSION, 31, "test-device")); + EXPECT_FALSE(parse_rdma_offer(serialize_rdma_offer(client), &offer)); + for (const uint16_t version : {uint16_t{2}, uint16_t{3}}) { + client.protocol_version = version; + ASSERT_TRUE(parse_rdma_handshake(serialize_rdma_handshake(client), &decoded)); + EXPECT_EQ(decoded.max_control_message_bytes, 0U); + EXPECT_FALSE(validate_rdma_capability(decoded, RDMA_PROTOCOL_VERSION, 31, "test-device")); + } +} + TEST(RdmaOfferTest, MissingPreflightAndMalformedAdmissionFailClosed) { const auto client = test_capability(1); diff --git a/lib/crowdb-rpc/tests/transport_config_test.cpp b/lib/crowdb-rpc/tests/transport_config_test.cpp index 92d959abd..fdafe09d2 100644 --- a/lib/crowdb-rpc/tests/transport_config_test.cpp +++ b/lib/crowdb-rpc/tests/transport_config_test.cpp @@ -57,6 +57,30 @@ TEST(TransportConfigTest, VersionedCAbiRejectsMalformedBoundsBeforePublication) crowdb_rpc_server_destroy(server); } +TEST(TransportConfigTest, VersionedLogicalControlBoundsRejectBeforeConfiguration) +{ + const auto server = crowdb_rpc_server_create(nullptr); + ASSERT_NE(server, nullptr); + crowdb_rpc_rdma_config_t config; + crowdb_rpc_rdma_config_init(&config); + crowdb_rpc_rdma_stream_config_t streams; + crowdb_rpc_rdma_stream_config_init(&streams); + EXPECT_EQ(streams.struct_size, sizeof(streams)); + EXPECT_EQ(streams.version, 1U); + EXPECT_EQ(streams.max_control_message_bytes, 32U << 20); + EXPECT_EQ(crowdb_rpc_server_configure_rdma_v3(server, &config, 0, nullptr), CROWDB_RPC_ERR_INVALID_ARG); + streams.version = 2; + EXPECT_EQ(crowdb_rpc_server_configure_rdma_v3(server, &config, 0, &streams), CROWDB_RPC_ERR_INVALID_ARG); + streams.version = 1; + streams.control_receive_budget_bytes = streams.max_control_message_bytes - 1; + EXPECT_EQ(crowdb_rpc_server_configure_rdma_v3(server, &config, 0, &streams), CROWDB_RPC_ERR_INVALID_ARG); + streams.control_receive_budget_bytes = streams.max_control_message_bytes; + EXPECT_EQ(crowdb_rpc_server_configure_rdma_v3(server, &config, 0, &streams), CROWDB_RPC_OK); + EXPECT_EQ(config.version, 1U); + EXPECT_EQ(config.struct_size, sizeof(config)); + crowdb_rpc_server_destroy(server); +} + TEST(TransportConfigTest, ClientErrorCountersSurviveMetricWindowFlush) { crowdb_rpc_client_counters_t before{}, pending{}, flushed{}; diff --git a/tools/rpc-test/src/args.rs b/tools/rpc-test/src/args.rs index 8c5aa9c0e..e79e78bc7 100644 --- a/tools/rpc-test/src/args.rs +++ b/tools/rpc-test/src/args.rs @@ -176,6 +176,7 @@ impl RpcArgs { aggregation_enabled: q.aggregation_enabled, aggregation_max_bytes: q.aggregation_max_bytes, aggregation_max_frames: q.aggregation_max_frames, + ..RdmaConfig::default() }) } } From 5c3f5c6e60763aa66b9988a577384474387bcb0d Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 05:28:00 +0800 Subject: [PATCH 049/258] Stream large logical controls through bounded CQ-owned RDMA fragments --- doc/working/plan-rdma-chunk-kv-access.md | 46 ++- .../transport/rdma/rdma_transport.h | 95 ++++--- .../src/transport/rdma/rdma_completion.cpp | 3 +- .../src/transport/rdma/rdma_config.cpp | 3 +- .../src/transport/rdma/rdma_connection.cpp | 35 +-- .../src/transport/rdma/rdma_control_codec.cpp | 70 +++++ .../src/transport/rdma/rdma_control_codec.h | 13 + .../src/transport/rdma/rdma_control_send.cpp | 105 +++++++ .../src/transport/rdma/rdma_control_send.h | 19 ++ .../transport/rdma/rdma_control_stream.cpp | 20 +- .../src/transport/rdma/rdma_control_stream.h | 1 + lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp | 9 + lib/crowdb-rpc/src/transport/rdma/rdma_cq.h | 2 + .../transport/rdma/rdma_direct_receive.cpp | 87 ++++-- .../src/transport/rdma/rdma_direct_send.cpp | 19 +- .../src/transport/rdma/rdma_poller.cpp | 60 +++- lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp | 32 ++- .../src/transport/rdma/rdma_receive.cpp | 10 +- .../src/transport/rdma/rdma_send.cpp | 39 ++- .../src/transport/rdma/rdma_status.cpp | 12 + .../src/transport/rdma/rdma_transport.cpp | 11 + .../src/transport/rdma/rdma_work_request.cpp | 29 +- .../tests/common/test_rdma_control.cpp | 93 ++++++ .../tests/common/test_rdma_control.h | 59 ++++ .../tests/common/test_rdma_transport.h | 48 ++++ .../tests/rdma_control_codec_test.cpp | 79 ++++++ .../tests/rdma_control_hardware_test.cpp | 266 ++++++++++++++++++ 27 files changed, 1144 insertions(+), 121 deletions(-) create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_control_codec.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_control_codec.h create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_control_send.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_control_send.h create mode 100644 lib/crowdb-rpc/tests/common/test_rdma_control.cpp create mode 100644 lib/crowdb-rpc/tests/common/test_rdma_control.h create mode 100644 lib/crowdb-rpc/tests/rdma_control_codec_test.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_control_hardware_test.cpp diff --git a/doc/working/plan-rdma-chunk-kv-access.md b/doc/working/plan-rdma-chunk-kv-access.md index 5308ea77b..c486f2abf 100644 --- a/doc/working/plan-rdma-chunk-kv-access.md +++ b/doc/working/plan-rdma-chunk-kv-access.md @@ -69,7 +69,7 @@ Observed contract: depth and data capacity, as receive defaults already do. Preserve native `RdmaConfig::for_layout`'s existing 64 MiB send arena default for the accepted 1 MiB datapath and ordinary RPC. Files: capability/offer/config/status, FFI. -- [~] **Integrate CQ-owned fragmentation and assembly**: enqueue one original +- [x] **Integrate CQ-owned fragmentation and assembly**: enqueue one original logical frame under byte/count admission. The CQ owner walks it in order, builds only the next bounded fragment, and holds immutable source owners through all WR completions. Do not materialize an unbounded fragment list. @@ -83,7 +83,27 @@ Observed contract: complete original Frame. Move quota-owned control storage through C ABI without another full-message malloc/copy. Files: RDMA send/receive/CQ/CM pools, common framing/C ABI conversion and owner-lifetime tests. -- [ ] **Wire Chunk-KV classes and typed settings**: add service defaults and + + Implementation order within this task: + - Add private fragment codec and CQ-only sender cursor. Charge the original + control using a Buffer slice whose retained owner is the byte lease; + rejected submit restores the caller's original buffer. WR records retain + the single original cursor until every posted SEND has completed. + - Set effective data capacity and peer generation/maximum from CM. Active + connections create/prime receive storage after the validated CM reply and + before application publication, so asymmetric classes allocate/post the + actual minimum rather than a larger local default. + - Extend the existing CQ deferred receive vector for byte admission. A + per-QP count preserves FIFO, and a per-pass epoch prevents later fragments + overtaking an admission-blocked first fragment. Existing RQ owners stay + intact; no replacement buffers are carved. + - Check active assembly deadlines under CQ ownership and bound the event + wait to the nearest active deadline. Shared receive budget release wakes + CQ workers through the retained transport operation gate. Keep storage + destruction before returning byte admission. + - Record logical-message, fragment, copied-control, directional admission + and timeout counters separately from direct application payload counters. +- [~] **Wire Chunk-KV classes and typed settings**: add service defaults and explicit RDMA configuration for metadata/aggregate connection groups, preserve TCP defaults and current serializers/errors/retries, and wire the Chunk-KV server plus Access storage construction. Cover point, mutation, @@ -226,6 +246,28 @@ Negotiation checkpoint: passed. The existing request-count worker fixture requires separate client and server roles; its role-less invocation failed setup, not a payload test. +CQ integration checkpoint: + +- The CQ walks one retained source control at a time. Each fragment fits the + actual negotiated 64 KiB/256 KiB/1 MiB data capacity; active CM setup primes + receive storage only after the reply establishes that minimum. Source bytes + stay charged through the final WR, and rejected queue publication restores + the caller's original buffer/reference. +- Six actual HCA cases passed: asymmetric classes and exact round trips larger + than the entire two-slot RQ window; fixed polling/event worker FIFO pressure + with a held completed result and release wakeup; idle partial expiry; + source-owner/queue rejection; peer logical limit; invalid generation and + mismatched encoded-control identity. The 1 MiB case reconstructs 2,097,165 + bytes per direction. A completed control remains readable after transport + destruction, then safely returns admission on final release. +- `pixi run test-cpp` passed at + `.crowdb-runtime/artifacts/unit/cpp-6d344d3e3eaf45ba8d8e1d20a0cd1437/`: + 543 tree, 189 RPC, 71 DiskIO and 35 common CTest cases, plus both FFI suites. + Optional HCA cases are skipped by that default gate and exercised separately. + No-verbs build passed all 159 CTest cases (one hardware skip). Format and + scoped C++ lint passed. Final RPC/XML and six HCA case results are under + `.crowdb-runtime/rdma-check/artifacts/r241-control/`. + ```bash pixi run -- cargo fmt --all -- --check pixi run -- cargo test -p crowdb-chunk-kv-client --all-targets diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index 87eae5306..56c95d17b 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -29,6 +29,9 @@ namespace crowdb::rpc class SocketEngine; struct RdmaDevice; struct RdmaMemoryStats; +struct RdmaCmOffer; +class ControlStreamBudget; +class ControlStreamReceiver; constexpr uint32_t RDMA_WIRE_SLOT_CAPACITY = RDMA_DATAPATH_CAPACITY; @@ -107,6 +110,17 @@ class RdmaTransport : public Transport std::atomic direct_sends{0}; std::atomic direct_payload_bytes{0}; std::atomic copied_payload_bytes{0}; + std::atomic control_fragments_sent{0}; + std::atomic control_fragments_received{0}; + std::atomic copied_control_send_bytes{0}; + std::atomic copied_control_receive_bytes{0}; + std::atomic control_stream_timeouts{0}; + std::atomic control_send_bytes{0}; + std::atomic control_send_bytes_high_water{0}; + std::atomic control_send_stalls{0}; + std::atomic control_receive_bytes{0}; + std::atomic control_receive_bytes_high_water{0}; + std::atomic control_receive_stalls{0}; }; explicit RdmaTransport(RdmaConfig config = {}, BufferPool *application_pool = nullptr); @@ -219,22 +233,25 @@ class RdmaTransport : public Transport enum class WorkKind : uint8_t { Send, Receive }; struct ConnectionPools; + struct ControlSendState; bool create_connection_pools(ConnectionPools &pools); + void configure_peer(ConnectionPools &pools, const RdmaCmOffer &peer); struct WorkRequest { - std::atomic token{0}; - WorkKind kind; - std::shared_ptr conn; - std::atomic connection_identity{nullptr}; - std::shared_ptr pools; - Buffer *slot = nullptr; - Buffer *prefix = nullptr; - OutFrame *frame = nullptr; - uint64_t request_id = 0; - uint64_t stream_id = 0; - uint64_t sequence = 0; - uint64_t logical_offset = 0; + std::atomic token{0}; + WorkKind kind; + std::shared_ptr conn; + std::atomic connection_identity{nullptr}; + std::shared_ptr pools; + Buffer *slot = nullptr; + Buffer *prefix = nullptr; + OutFrame *frame = nullptr; + std::shared_ptr control_send; + uint64_t request_id = 0; + uint64_t stream_id = 0; + uint64_t sequence = 0; + uint64_t logical_offset = 0; }; static constexpr uint64_t WORK_RESERVED = UINT64_MAX; @@ -271,7 +288,11 @@ class RdmaTransport : public Transport uint32_t flush_sends(CompletionQueue &queue, uint32_t budget); uint32_t flush_connection(Connection *connection, ConnectionPools &pools, uint32_t budget); uint32_t flush_direct_connection(Connection *connection, ConnectionPools &pools, uint32_t budget); - bool post_direct(Connection *connection, ConnectionPools &pools, Buffer *slot, Buffer *prefix, OutFrame *frame); + bool post_direct(Connection *connection, ConnectionPools &pools, Buffer *slot, Buffer *prefix, OutFrame *frame, + std::shared_ptr control_send = {}); + bool flush_control_fragment(Connection *connection, ConnectionPools &pools); + Buffer *admit_control_frame(const OutFrame &frame, const ConnectionPools &pools); + void check_control_deadlines(CompletionQueue &queue); bool post_direct_receive(Connection *connection, Buffer *slot); void retire_sends(CompletionQueue &queue, Connection *connection); bool post_aggregate(Connection *connection, ConnectionPools &pools, Buffer *slot, OutFrame *frames, @@ -331,7 +352,9 @@ class RdmaTransport : public Transport bool entered_ = false; }; - std::shared_ptr repost_state_; + std::shared_ptr repost_state_; + std::shared_ptr control_send_budget_; + std::shared_ptr control_receive_budget_; struct ConnectionPools : std::enable_shared_from_this { @@ -341,22 +364,30 @@ class RdmaTransport : public Transport ~ConnectionPools(); - ibv_qp_cap effective_capacity{}; - std::shared_ptr posting_state = std::make_shared(); - std::unique_ptr send; - std::unique_ptr recv; - std::unique_ptr recv_control; - uint32_t cq_index = 0; - uint32_t reserved_capacity = 0; - uint32_t peer_receive_slot_capacity = 0; - uint32_t peer_receive_frame_limit = 1; - uint32_t qp_number = 0; - rdma_cm_id *cm_id = nullptr; // immutable after publication - std::atomic disconnect_requested{false}; - SendQueue send_queue; - std::atomic queued_frames{0}; - OutFrame *pending_frame = nullptr; - uint32_t sends_in_flight = 0; // CQ owner only + ibv_qp_cap effective_capacity{}; + std::shared_ptr posting_state = std::make_shared(); + std::unique_ptr send; + std::unique_ptr recv; + std::unique_ptr recv_control; + uint32_t cq_index = 0; + uint32_t reserved_capacity = 0; + uint32_t peer_receive_slot_capacity = 0; + uint32_t peer_receive_frame_limit = 1; + uint32_t qp_number = 0; + uint32_t effective_data_capacity = 0; + uint32_t peer_max_control_message_bytes = 0; + uint64_t peer_generation = 0; + rdma_cm_id *cm_id = nullptr; // immutable after publication + std::atomic disconnect_requested{false}; + SendQueue send_queue; + std::atomic queued_frames{0}; + OutFrame *pending_frame = nullptr; + uint32_t sends_in_flight = 0; // CQ owner only + uint64_t next_control_stream_id = 1; + std::shared_ptr control_send; + std::shared_ptr control_receive; + uint32_t deferred_receives = 0; + uint64_t deferred_blocked_epoch = 0; }; struct RepostContext @@ -369,8 +400,8 @@ class RdmaTransport : public Transport std::unordered_map> cm_pools_; void release_completed_frame(WorkRequest *work, const struct ibv_wc &wc); - void release_completed_receive(WorkRequest *work, const struct ibv_wc &wc); - void release_completed_direct_receive(WorkRequest *work, const struct ibv_wc &wc); + bool release_completed_receive(WorkRequest *work, const struct ibv_wc &wc); + bool release_completed_direct_receive(WorkRequest *work, const struct ibv_wc &wc); void record_completion_error(const struct ibv_wc &wc, bool expected_flush = false); static void repost_frame_release(void *context, Buffer *slot); bool adopt_cm_context(struct ibv_context *context); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp index 8b4b2618b..41220c757 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp @@ -46,6 +46,7 @@ int RdmaTransport::drain_queue(uint32_t index, uint32_t max_completions, bool wa } if (max_completions == 0) max_completions = std::max(1U, config_.poll_batch); + check_control_deadlines(queue); const uint32_t sent = flush_sends(queue, max_completions); uint32_t drained = process_deferred(queue, max_completions); uint32_t empty_polls = 0; @@ -163,7 +164,7 @@ bool RdmaTransport::prime_receive(const std::shared_ptr &connection) return false; BufferPool *pool = pools->recv.get(); for (uint32_t i = 0; i < depth; ++i) { - Buffer *slot = pool->alloc(config_.data_slot_capacity()); + Buffer *slot = pool->alloc(pools->effective_data_capacity); if (slot == nullptr) { return false; } diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp index fc616196b..7a6d45cfc 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp @@ -80,7 +80,8 @@ bool RdmaConfig::valid() const aggregation_max_bytes >= HEADER_SIZE && aggregation_max_bytes <= RDMA_DATAPATH_CAPACITY && aggregation_max_frames > 0 && aggregation_max_frames <= 256 && send_queue_capacity >= 2 && send_queue_capacity <= (1U << 20) && (send_queue_capacity & (send_queue_capacity - 1)) == 0 && - max_control_message_bytes != 0 && max_control_message_bytes <= control_send_budget_bytes && + max_control_message_bytes != 0 && max_control_message_bytes <= INT32_MAX && + max_control_message_bytes <= control_send_budget_bytes && max_control_message_bytes <= control_receive_budget_bytes && control_assembly_timeout_ms != 0; } diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp index 189e08c06..0110ebea5 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp @@ -155,22 +155,12 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int pools->effective_capacity = qp_attr.cap; pools->cq_index = reservation.queue->index; pools->reserved_capacity = reservation.capacity; - if (!create_connection_pools(*pools)) { - rdma_destroy_qp(id); - rdma_destroy_id(id); - return nullptr; - } - auto connection = std::make_shared(reinterpret_cast(id), addr, application_pool_); + auto connection = std::make_shared(reinterpret_cast(id), addr, application_pool_); set_connection_pools(connection.get(), pools); reservation.committed = true; connection->set_path(ConnectionPath::Rdma); connection->transport_handle = reinterpret_cast(id->qp); install_connection_notifiers(connection, pools->cq_index); - if (!prime_receive(connection)) { - destroy_connection_qp(connection.get(), id); - rdma_destroy_id(id); - return nullptr; - } rdma_conn_param params{}; const auto local = local_capability(); const auto offer = serialize_rdma_offer(local, expected_peer); @@ -210,9 +200,13 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int rdma_destroy_id(id); return nullptr; } - pools->peer_receive_slot_capacity = peer_offer.receive_slot_capacity; - pools->peer_receive_frame_limit = peer_offer.receive_frame_limit; - pools->qp_number = id->qp->qp_num; + configure_peer(*pools, peer_offer); + if (!create_connection_pools(*pools) || !prime_receive(connection)) { + destroy_connection_qp(connection.get(), id); + rdma_destroy_id(id); + return nullptr; + } + pools->qp_number = id->qp->qp_num; ibv_qp_attr receive_attributes{}; receive_attributes.min_rnr_timer = config_.min_rnr_timer; if (ibv_modify_qp(id->qp, &receive_attributes, IBV_QP_MIN_RNR_TIMER) != 0) { @@ -360,13 +354,12 @@ std::shared_ptr RdmaTransport::accept() rdma_destroy_id(id); return nullptr; } - auto pools = std::make_shared(config_.send_queue_capacity); - pools->peer_receive_slot_capacity = peer_offer.receive_slot_capacity; - pools->peer_receive_frame_limit = peer_offer.receive_frame_limit; - pools->qp_number = id->qp->qp_num; - pools->effective_capacity = qp_attr.cap; - pools->cq_index = reservation.queue->index; - pools->reserved_capacity = reservation.capacity; + auto pools = std::make_shared(config_.send_queue_capacity); + configure_peer(*pools, peer_offer); + pools->qp_number = id->qp->qp_num; + pools->effective_capacity = qp_attr.cap; + pools->cq_index = reservation.queue->index; + pools->reserved_capacity = reservation.capacity; if ((config_.wire_layout == RdmaWireLayout::ControlData && (qp_attr.cap.max_send_sge < 2 || qp_attr.cap.max_recv_sge < 2)) || !create_connection_pools(*pools)) { diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_control_codec.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_control_codec.cpp new file mode 100644 index 000000000..47c98d8a0 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_control_codec.cpp @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_control_codec.h" + +#include "common_msg_generated.h" +#include "msg_type_generated.h" + +#include +#include + +namespace crowdb::rpc +{ +static_assert(RPC_CONTROL_FRAGMENT_MSG_TYPE == proto::FBMsgType_ERpcControlFragment); + +bool write_control_fragment(std::span prefix, const ControlFragment &f, uint32_t data_length) +{ + if (prefix.size() != RDMA_CONTROL_CAPACITY || data_length > f.capacity) { + return false; + } + flatbuffers::FlatBufferBuilder builder(256); + builder.Finish(proto::CreateRpcControlFragment(builder, f.request_id, f.rpc_create_nano, f.response_create_nano, + f.stream_id, f.generation, f.sequence, f.offset, f.total, f.capacity, + f.message_type, f.message_flags, f.stream_flags)); + if (builder.GetSize() > RDMA_CONTROL_CAPACITY - HEADER_SIZE) { + return false; + } + std::fill(prefix.begin(), prefix.end(), uint8_t{0}); + Header header; + header.msg_type = RPC_CONTROL_FRAGMENT_MSG_TYPE; + header.msg_size = RDMA_CONTROL_CAPACITY - HEADER_SIZE; + header.data_size = data_length; + header.flags = FLAG_ONE_WAY; + serialize_header(prefix.data(), header); + std::memcpy(prefix.data() + HEADER_SIZE, builder.GetBufferPointer(), builder.GetSize()); + return true; +} + +bool parse_control_fragment(std::span control, ControlFragment &fragment) +{ + if (control.empty() || control.size() > RDMA_CONTROL_CAPACITY - HEADER_SIZE) { + return false; + } + flatbuffers::Verifier verifier(control.data(), control.size()); + if (!verifier.VerifyBuffer()) { + return false; + } + const auto *f = flatbuffers::GetRoot(control.data()); + fragment = {.request_id = f->id(), + .rpc_create_nano = f->rpc_create_nano(), + .response_create_nano = f->response_create_nano(), + .stream_id = f->stream_id(), + .generation = f->generation(), + .sequence = f->sequence(), + .offset = f->logical_offset(), + .total = f->total_length(), + .capacity = f->data_capacity(), + .message_type = f->original_msg_type(), + .message_flags = f->original_flags(), + .stream_flags = f->stream_flags()}; + return true; +} + +bool control_identity_matches(const Frame &frame) +{ + uint64_t id = 0, created = 0, response = 0; + extract_control_fields(frame.control.data(), static_cast(frame.control.size()), id, created, response); + return id == frame.request_id && created == frame.rpc_create_nano && response == frame.response_create_nano; +} +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_control_codec.h b/lib/crowdb-rpc/src/transport/rdma/rdma_control_codec.h new file mode 100644 index 000000000..31b102af6 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_control_codec.h @@ -0,0 +1,13 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "rdma_control_stream.h" + +namespace crowdb::rpc +{ +bool write_control_fragment(std::span prefix, const ControlFragment &fragment, uint32_t data_length); +bool parse_control_fragment(std::span control, ControlFragment &fragment); +bool control_identity_matches(const Frame &frame); +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_control_send.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_control_send.cpp new file mode 100644 index 000000000..3d4a4cb7e --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_control_send.cpp @@ -0,0 +1,105 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_control_send.h" + +#include "rdma_control_codec.h" + +#ifdef CROWDB_RPC_HAVE_RDMA +# include "rdma_aggregate.h" + +# include +# include +# include + +namespace crowdb::rpc +{ +RdmaTransport::ControlSendState::~ControlSendState() +{ + release_rdma_frames(frame); +} + +Buffer *RdmaTransport::admit_control_frame(const OutFrame &frame, const ConnectionPools &pools) +{ + const auto *control = frame.control; + if (control == nullptr || control->data == nullptr || control->view_offset > control->capacity || + control->len > control->capacity - control->view_offset || frame.data_views() != 0 || + frame.header.magic != MAGIC || frame.header.msg_offset != HEADER_SIZE || frame.request_id == 0 || + (frame.header.flags & ~FLAG_ONE_WAY) != 0 || frame.header.msg_type == RDMA_HANDSHAKE_MSG_TYPE || + frame.header.msg_type == RPC_CONTROL_FRAGMENT_MSG_TYPE || + control->len > std::min(config_.max_control_message_bytes, pools.peer_max_control_message_bytes)) { + return nullptr; + } + uint64_t id = 0, created = 0, response = 0; + extract_control_fields(control->payload_data(), control->len, id, created, response); + if (id != frame.request_id) { + return nullptr; + } + auto admission = control_send_budget_->try_reserve(control->len); + if (admission == nullptr) { + return nullptr; + } + auto *view = frame.control->slice(0, control->len); + if (view != nullptr) { + view->pool_owner = std::move(admission); + } + return view; +} + +bool RdmaTransport::flush_control_fragment(Connection *connection, ConnectionPools &pools) +{ + if (pools.control_send == nullptr) { + if (pools.next_control_stream_id == 0) { + connection->close(); + return false; + } + auto state = std::make_shared(); + state->frame = std::exchange(pools.pending_frame, nullptr); + auto &f = state->fragment; + f.request_id = state->frame->request_id; + extract_control_fields(state->frame->control->data, state->frame->control->len, f.request_id, f.rpc_create_nano, + f.response_create_nano); + f.stream_id = pools.next_control_stream_id++; + f.generation = local_capability_.generation; + f.total = state->frame->control->len; + f.capacity = pools.effective_data_capacity; + f.message_type = state->frame->header.msg_type; + f.message_flags = state->frame->header.flags; + pools.control_send = std::move(state); + } + auto state = pools.control_send; + auto &f = state->fragment; + const auto length = static_cast(std::min(f.capacity, f.total - f.offset)); + auto release = [](Buffer *buffer) { buffer->release(); }; + std::unique_ptr prefix(pools.send->alloc(RDMA_CONTROL_CAPACITY), release); + std::unique_ptr data(send_pool_->alloc(length), release); + if (prefix == nullptr || data == nullptr) { + return false; + } + f.stream_flags = static_cast((f.offset == 0 ? CONTROL_STREAM_START : 0) | + (f.offset + length == f.total ? CONTROL_STREAM_END : 0)); + if (!write_control_fragment(std::span(prefix->data, RDMA_CONTROL_CAPACITY), f, length)) { + connection->close(); + return false; + } + prefix->len = RDMA_CONTROL_CAPACITY; + std::memcpy(data->data, state->frame->control->data + f.offset, length); + data->len = length; + data->set_metadata(f.request_id, f.stream_id, f.sequence, f.offset, data->registration_id); + auto *data_slot = data.release(); + auto *control_slot = prefix.release(); + if (!post_direct(connection, pools, data_slot, control_slot, nullptr, state)) { + return false; + } + stats_.control_fragments_sent.fetch_add(1, std::memory_order_relaxed); + stats_.copied_control_send_bytes.fetch_add(length, std::memory_order_relaxed); + f.offset += length; + ++f.sequence; + if (f.offset == f.total) { + pools.control_send.reset(); + pools.queued_frames.fetch_sub(1, std::memory_order_release); + } + return true; +} +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_control_send.h b/lib/crowdb-rpc/src/transport/rdma/rdma_control_send.h new file mode 100644 index 000000000..f0df11816 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_control_send.h @@ -0,0 +1,19 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "rdma_control_stream.h" + +#ifdef CROWDB_RPC_HAVE_RDMA +namespace crowdb::rpc +{ +struct RdmaTransport::ControlSendState +{ + OutFrame *frame = nullptr; + ControlFragment fragment; + ~ControlSendState(); +}; +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_control_stream.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_control_stream.cpp index f9656e384..e0a334143 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_control_stream.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_control_stream.cpp @@ -3,6 +3,8 @@ #include "rdma_control_stream.h" +#include "crowdb-rpc/transport/rdma/rdma_capability.h" + #include #include #include @@ -20,7 +22,8 @@ ControlStreamReceiver::ControlStreamReceiver(ControlStreamBudget budget, uint32_ BufferCapacityClass type; uint32_t rounded = 0; if (!classify_rdma_capacity(capacity, &type, &rounded) || rounded != capacity || generation == 0 || - max_message_bytes == 0 || max_message_bytes > budget_.limit() || timeout.count() <= 0) + max_message_bytes == 0 || max_message_bytes > INT32_MAX || max_message_bytes > budget_.limit() || + timeout.count() <= 0) throw std::invalid_argument("invalid logical control stream bounds"); } @@ -29,7 +32,8 @@ bool ControlStreamReceiver::valid(const ControlFragment &f, size_t bytes) const if (f.request_id == 0 || f.stream_id == 0 || f.generation != generation_ || f.capacity != capacity_ || f.total == 0 || f.total > max_message_bytes_ || f.offset > f.total || bytes > f.total - f.offset || bytes > capacity_ || f.message_type == RPC_CONTROL_FRAGMENT_MSG_TYPE || - (f.message_flags & ~FLAG_ONE_WAY) != 0 || (f.stream_flags & ~uint8_t{7}) != 0) + f.message_type == RDMA_HANDSHAKE_MSG_TYPE || (f.message_flags & ~FLAG_ONE_WAY) != 0 || + (f.stream_flags & ~uint8_t{7}) != 0) return false; const bool start = (f.stream_flags & CONTROL_STREAM_START) != 0; const bool end = (f.stream_flags & CONTROL_STREAM_END) != 0; @@ -42,6 +46,9 @@ bool ControlStreamReceiver::valid(const ControlFragment &f, size_t bytes) const bool ControlStreamReceiver::matches(const ControlFragment &f) const { + if (!assembly_) { + return false; + } const auto &a = *assembly_; const auto &first = a.first; return f.request_id == first.request_id && f.rpc_create_nano == first.rpc_create_nano && @@ -104,6 +111,15 @@ bool ControlStreamReceiver::expired(Clock::time_point now) const return assembly_ && now >= assembly_->deadline; } +int ControlStreamReceiver::wait_ms(Clock::time_point now) const +{ + if (!assembly_) { + return 1000; + } + const auto remaining = std::chrono::ceil(assembly_->deadline - now).count(); + return static_cast(std::clamp(remaining, 1, 1000)); +} + void ControlStreamReceiver::reset() { assembly_.reset(); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_control_stream.h b/lib/crowdb-rpc/src/transport/rdma/rdma_control_stream.h index 20a8cb456..1163fbbc7 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_control_stream.h +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_control_stream.h @@ -80,6 +80,7 @@ class ControlStreamReceiver std::unique_ptr &completed); [[nodiscard]] bool active() const; [[nodiscard]] bool expired(Clock::time_point now) const; + [[nodiscard]] int wait_ms(Clock::time_point now) const; void reset(); private: diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp index 3d20ada86..4b2dd30ef 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp @@ -4,6 +4,7 @@ #include "rdma_cq.h" #include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "rdma_control_stream.h" #include "rdma_device.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -90,6 +91,14 @@ void RdmaTransport::refresh_poll_stats() const std::memory_order_relaxed); stats_.registration_failures.store(memory_stats_->failures.load(std::memory_order_relaxed), std::memory_order_relaxed); + if (control_send_budget_ != nullptr) { + stats_.control_send_bytes.store(control_send_budget_->used(), std::memory_order_relaxed); + stats_.control_send_bytes_high_water.store(control_send_budget_->high_water(), std::memory_order_relaxed); + stats_.control_send_stalls.store(control_send_budget_->stalls(), std::memory_order_relaxed); + stats_.control_receive_bytes.store(control_receive_budget_->used(), std::memory_order_relaxed); + stats_.control_receive_bytes_high_water.store(control_receive_budget_->high_water(), std::memory_order_relaxed); + stats_.control_receive_stalls.store(control_receive_budget_->stalls(), std::memory_order_relaxed); + } } void RdmaTransport::destroy_completion_queues() diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h index e93a07bef..1996f0e07 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h @@ -26,6 +26,8 @@ struct RdmaTransport::CompletionQueue }; std::vector deferred; + uint64_t deferred_epoch = 0; + std::atomic control_wait_ms{1000}; struct ConnectionBinding { diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_direct_receive.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_receive.cpp index 215c90f28..26639f2ab 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_direct_receive.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_receive.cpp @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. #include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "rdma_control_codec.h" #include "rdma_direct_frame.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -10,11 +11,10 @@ namespace crowdb::rpc bool RdmaTransport::post_direct_receive(Connection *connection, Buffer *slot) { Operation operation(repost_state_); - if (!operation || connection == nullptr || slot == nullptr || slot->data == nullptr || - slot->capacity < config_.data_slot_capacity()) + if (!operation || connection == nullptr || slot == nullptr || slot->data == nullptr) return false; auto *pools = pools_for(connection); - if (pools == nullptr || pools->recv_control == nullptr) + if (pools == nullptr || pools->recv_control == nullptr || slot->capacity < pools->effective_data_capacity) return false; Operation posting(pools->posting_state); if (!posting || !connection->is_open() || connection->transport_handle == 0) @@ -44,7 +44,7 @@ bool RdmaTransport::post_direct_receive(Connection *connection, Buffer *slot) } std::array regions{}; regions[0] = {reinterpret_cast(prefix->data), RDMA_CONTROL_CAPACITY, prefix->registration_key}; - regions[1] = {reinterpret_cast(slot->data), config_.data_slot_capacity(), slot->registration_key}; + regions[1] = {reinterpret_cast(slot->data), pools->effective_data_capacity, slot->registration_key}; ibv_recv_wr request{}; request.wr_id = work->token.load(std::memory_order_acquire); request.sg_list = regions.data(); @@ -61,17 +61,44 @@ bool RdmaTransport::post_direct_receive(Connection *connection, Buffer *slot) return true; } -void RdmaTransport::release_completed_direct_receive(WorkRequest *work, const ibv_wc &completion) +bool RdmaTransport::release_completed_direct_receive(WorkRequest *work, const ibv_wc &completion) { - auto *slot = work->slot; - auto *prefix = work->prefix; - Header header; - const bool valid = completion.status == IBV_WC_SUCCESS && work->conn->is_open() && - slot->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight) && - prefix->transition(BufferOwnership::PostedReceive, BufferOwnership::Available) && - parse_rdma_direct_control(std::span(prefix->data, RDMA_CONTROL_CAPACITY), - config_.data_slot_capacity(), completion.byte_len, header) && - slot->transition(BufferOwnership::ReceiveInFlight, BufferOwnership::CallerOwned); + auto *slot = work->slot; + auto *prefix = work->prefix; + Header header; + bool valid = completion.status == IBV_WC_SUCCESS && work->conn->is_open() && + parse_rdma_direct_control(std::span(prefix->data, RDMA_CONTROL_CAPACITY), + work->pools->effective_data_capacity, completion.byte_len, header); + std::unique_ptr logical; + const bool fragment = valid && header.msg_type == RPC_CONTROL_FRAGMENT_MSG_TYPE; + if (fragment) { + ControlFragment metadata; + valid = header.flags == FLAG_ONE_WAY && work->pools->control_receive != nullptr && + parse_control_fragment(std::span(prefix->data + HEADER_SIZE, header.msg_size), metadata); + if (valid) { + ControlStreamResult result = ControlStreamResult::Invalid; + try { + result = work->pools->control_receive->accept(metadata, std::span(slot->data, header.data_size), + ControlStreamReceiver::Clock::now(), logical); + } + catch (const std::bad_alloc &) { + // Bounded admission cannot manufacture replacement memory. + // Allocation failure terminates this logical connection. + result = ControlStreamResult::Invalid; + } + if (result == ControlStreamResult::Backpressure) { + return false; + } + valid = (result == ControlStreamResult::Accepted || result == ControlStreamResult::Complete) && + (logical == nullptr || control_identity_matches(*logical)); + } + } + else if (valid && work->pools->control_receive != nullptr && work->pools->control_receive->active()) { + valid = false; + } + valid = valid && slot->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight) && + prefix->transition(BufferOwnership::PostedReceive, BufferOwnership::Available) && + slot->transition(BufferOwnership::ReceiveInFlight, BufferOwnership::CallerOwned); if (!valid) { if (completion.status != IBV_WC_SUCCESS) record_completion_error(completion, !work->conn->is_open()); @@ -84,28 +111,36 @@ void RdmaTransport::release_completed_direct_receive(WorkRequest *work, const ib connection->close(); slot->release(); prefix->release(); - return; + return true; } auto connection = std::move(work->conn); auto pools = std::move(work->pools); slot->last_release_cb = &RdmaTransport::repost_frame_release; slot->last_release_ctx = new RepostContext{repost_state_, connection, std::move(pools)}; - auto *frame = new Frame; - frame->header = header; - frame->control.assign(prefix->data + HEADER_SIZE, prefix->data + RDMA_CONTROL_CAPACITY); - extract_control_fields(frame->control.data(), header.msg_size, frame->request_id, frame->rpc_create_nano, - frame->response_create_nano); - frame->data_buf = slot->slice(0, header.data_size); + if (!fragment) { + logical = std::make_unique(); + logical->header = header; + logical->control.assign(prefix->data + HEADER_SIZE, prefix->data + RDMA_CONTROL_CAPACITY); + extract_control_fields(logical->control.data(), header.msg_size, logical->request_id, logical->rpc_create_nano, + logical->response_create_nano); + logical->data_buf = slot->slice(0, header.data_size); + } release_work(work); prefix->release(); stats_.receives.fetch_add(1, std::memory_order_relaxed); - stats_.rpc_frames_received.fetch_add(1, std::memory_order_relaxed); + if (fragment) { + stats_.control_fragments_received.fetch_add(1, std::memory_order_relaxed); + stats_.copied_control_receive_bytes.fetch_add(header.data_size, std::memory_order_relaxed); + } + if (logical != nullptr) { + stats_.rpc_frames_received.fetch_add(1, std::memory_order_relaxed); + } stats_.wire_bytes_received.fetch_add(completion.byte_len, std::memory_order_relaxed); - if (connection->is_open()) - connection->on_frame(frame); - else - delete frame; + if (connection->is_open() && logical != nullptr) { + connection->on_frame(logical.release()); + } slot->release(); + return true; } } // namespace crowdb::rpc #endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_direct_send.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_send.cpp index 53f119eee..1f89b0f06 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_direct_send.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_send.cpp @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. #include "rdma_aggregate.h" +#include "rdma_control_send.h" #include "rdma_cq.h" #include "rdma_direct_frame.h" @@ -41,8 +42,16 @@ uint32_t RdmaTransport::flush_direct_connection(Connection *connection, Connecti auto &queue = *completion_queues_[pools.cq_index]; while (sent < budget && connection->is_open() && pools.sends_in_flight < std::min(config_.sq_depth, pools.effective_capacity.max_send_wr)) { - if (pools.pending_frame == nullptr && pools.send_queue.drain(&pools.pending_frame, 1) == 0) + if (pools.control_send == nullptr && pools.pending_frame == nullptr && + pools.send_queue.drain(&pools.pending_frame, 1) == 0) break; + if (pools.control_send != nullptr || pools.pending_frame->control->len > RDMA_CONTROL_CAPACITY - HEADER_SIZE) { + if (!flush_control_fragment(connection, pools)) { + break; + } + ++sent; + continue; + } auto *frame = pools.pending_frame; uint32_t bytes = 0; if (!rdma_direct_frame_size(*frame, limit, bytes)) { @@ -101,7 +110,7 @@ uint32_t RdmaTransport::flush_direct_connection(Connection *connection, Connecti } bool RdmaTransport::post_direct(Connection *connection, ConnectionPools &pools, Buffer *slot, Buffer *prefix, - OutFrame *frame) + OutFrame *frame, std::shared_ptr control_send) { Operation posting(pools.posting_state); WorkRequest *work = nullptr; @@ -110,6 +119,7 @@ bool RdmaTransport::post_direct(Connection *connection, ConnectionPools &pools, (prefix == nullptr || prefix->transition(BufferOwnership::Available, BufferOwnership::SendInFlight))) work = acquire_work(WorkKind::Send, connection, slot, frame, prefix); if (work != nullptr) { + work->control_send = std::move(control_send); auto *control = prefix == nullptr ? slot : prefix; std::array regions{}; regions[0] = {reinterpret_cast(control->data), RDMA_CONTROL_CAPACITY, control->registration_key}; @@ -125,7 +135,10 @@ bool RdmaTransport::post_direct(Connection *connection, ConnectionPools &pools, if (ibv_post_send(reinterpret_cast(connection->transport_handle), &request, &bad) == 0) { ++pools.sends_in_flight; stats_.sends.fetch_add(1, std::memory_order_relaxed); - stats_.rpc_frames_sent.fetch_add(1, std::memory_order_relaxed); + if (work->control_send == nullptr || + work->control_send->fragment.offset + slot->len == work->control_send->fragment.total) { + stats_.rpc_frames_sent.fetch_add(1, std::memory_order_relaxed); + } stats_.wire_bytes_sent.fetch_add(RDMA_CONTROL_CAPACITY + (prefix == nullptr ? 0 : slot->len), std::memory_order_relaxed); return true; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp index 2c7d214a0..036913b3d 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp @@ -1,6 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#include "rdma_control_stream.h" #include "rdma_cq.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -65,7 +66,8 @@ void RdmaTransport::run_poller(CompletionQueue *queue) queue->wake_state.fetch_and(~CompletionQueue::IDLE, std::memory_order_acq_rel); continue; } - const int count = queue->engine->wait(events.data(), events.size(), 1000); + const int count = + queue->engine->wait(events.data(), events.size(), queue->control_wait_ms.load(std::memory_order_relaxed)); if (count > 0) { queue->event_wakeup.store(true, std::memory_order_release); stats_.cq_wakeups.fetch_add(1, std::memory_order_relaxed); @@ -98,18 +100,30 @@ uint32_t RdmaTransport::process_deferred(CompletionQueue &queue, uint32_t max_co { uint32_t completed = 0; size_t retained = 0; + if (++queue.deferred_epoch == 0) { + ++queue.deferred_epoch; + } for (auto pending : queue.deferred) { + auto pools = pending.work->pools; if (completed < max_completions && - (pending.work->conn->has_frame_handler() || !pending.work->conn->is_open())) { + (!pending.work->conn->is_open() || + (pending.work->conn->has_frame_handler() && + (pools == nullptr || pools->deferred_blocked_epoch != queue.deferred_epoch)))) { if (!pending.work->conn->is_open()) pending.completion.status = IBV_WC_WR_FLUSH_ERR; - release_completed_receive(pending.work, pending.completion); - stats_.pending_deferred_receives.fetch_sub(1, std::memory_order_relaxed); - ++completed; - } - else { - queue.deferred[retained++] = pending; + if (release_completed_receive(pending.work, pending.completion)) { + if (pools != nullptr) { + --pools->deferred_receives; + } + stats_.pending_deferred_receives.fetch_sub(1, std::memory_order_relaxed); + ++completed; + continue; + } + if (pools != nullptr) { + pools->deferred_blocked_epoch = queue.deferred_epoch; + } } + queue.deferred[retained++] = pending; } queue.deferred.resize(retained); return completed; @@ -120,8 +134,12 @@ void RdmaTransport::cancel_deferred(CompletionQueue &queue, Connection *connecti size_t retained = 0; for (auto pending : queue.deferred) { if (connection == nullptr || pending.work->conn.get() == connection) { + auto pools = pending.work->pools; pending.completion.status = IBV_WC_WR_FLUSH_ERR; - release_completed_receive(pending.work, pending.completion); + (void)release_completed_receive(pending.work, pending.completion); + if (pools != nullptr) { + --pools->deferred_receives; + } stats_.pending_deferred_receives.fetch_sub(1, std::memory_order_relaxed); } else { @@ -131,6 +149,30 @@ void RdmaTransport::cancel_deferred(CompletionQueue &queue, Connection *connecti queue.deferred.resize(retained); } +void RdmaTransport::check_control_deadlines(CompletionQueue &queue) +{ + if (config_.wire_layout != RdmaWireLayout::ControlData) { + return; + } + int wait_ms = 1000; + const auto now = ControlStreamReceiver::Clock::now(); + for (const auto &binding : queue.connections) { + auto &receiver = binding.pools->control_receive; + if (receiver == nullptr || !binding.connection->is_open()) { + continue; + } + if (receiver->expired(now)) { + stats_.control_stream_timeouts.fetch_add(1, std::memory_order_relaxed); + binding.connection->close(); + receiver->reset(); + } + else { + wait_ms = std::min(wait_ms, receiver->wait_ms(now)); + } + } + queue.control_wait_ms.store(wait_ms, std::memory_order_relaxed); +} + } // namespace crowdb::rpc #endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp index 5ad585809..51e289817 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp @@ -3,7 +3,9 @@ #include "../../status_json.h" #include "crowdb-common/log.h" +#include "rdma_control_stream.h" #include "rdma_cq.h" +#include "rdma_offer.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -14,14 +16,33 @@ namespace crowdb::rpc bool RdmaTransport::create_connection_pools(ConnectionPools &pools) { - const auto control_bytes = config_.receive_control_bytes(); - pools.send = make_pool(config_.send_arena_bytes, config_.sq_depth); - pools.recv = make_pool(config_.receive_arena_bytes - control_bytes, config_.rq_depth); + const auto control_bytes = config_.receive_control_bytes(); + const uint64_t data_bytes = config_.wire_layout == RdmaWireLayout::ControlData + ? uint64_t{config_.posted_receive_target} * pools.effective_data_capacity + : config_.receive_arena_bytes; + pools.send = make_pool(config_.send_arena_bytes, config_.sq_depth); + pools.recv = make_pool(data_bytes, config_.rq_depth); if (control_bytes != 0) pools.recv_control = make_pool(control_bytes, config_.posted_receive_target); return pools.send != nullptr && pools.recv != nullptr && (control_bytes == 0 || pools.recv_control != nullptr); } +void RdmaTransport::configure_peer(ConnectionPools &pools, const RdmaCmOffer &peer) +{ + pools.peer_receive_slot_capacity = peer.receive_slot_capacity; + pools.peer_receive_frame_limit = peer.receive_frame_limit; + pools.peer_generation = peer.generation; + pools.peer_max_control_message_bytes = peer.max_control_message_bytes; + pools.effective_data_capacity = config_.data_slot_capacity(); + if (config_.wire_layout == RdmaWireLayout::ControlData) { + pools.effective_data_capacity = std::min( + pools.effective_data_capacity, rdma_data_slot_capacity(peer.receive_slot_capacity, config_.wire_layout)); + pools.control_receive = std::make_shared( + *control_receive_budget_, pools.effective_data_capacity, peer.generation, config_.max_control_message_bytes, + std::chrono::milliseconds(config_.control_assembly_timeout_ms)); + } +} + RdmaTransport::ConnectionPools *RdmaTransport::pools_for(Connection *connection) { return static_cast(connection->transport_context_.get()); @@ -51,6 +72,11 @@ bool RdmaTransport::publish_cm_connection(rdma_cm_id *id, const std::shared_ptr< info.number("effective_max_recv_sge", pools->effective_capacity.max_recv_sge); info.string("wire_layout", config_.wire_layout == RdmaWireLayout::Contiguous ? "contiguous" : "control_data"); info.number("peer_receive_frame_limit", pools->peer_receive_frame_limit); + info.number("effective_data_capacity", pools->effective_data_capacity); + info.number("peer_generation", pools->peer_generation); + info.number("peer_max_control_message_bytes", pools->peer_max_control_message_bytes); + info.number("effective_max_control_message_bytes", + std::min(config_.max_control_message_bytes, pools->peer_max_control_message_bytes)); connection->transport_info_json_ = info.finish(); std::scoped_lock lock(cm_mu_); if (rdma_migrate_id(id, cm_channel_) != 0) diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp index fac93e3d7..8ee8d4324 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp @@ -8,11 +8,10 @@ namespace crowdb::rpc { -void RdmaTransport::release_completed_receive(WorkRequest *work, const ibv_wc &wc) +bool RdmaTransport::release_completed_receive(WorkRequest *work, const ibv_wc &wc) { if (work->prefix != nullptr) { - release_completed_direct_receive(work, wc); - return; + return release_completed_direct_receive(work, wc); } auto fail = [&] { if (wc.status != IBV_WC_SUCCESS) @@ -29,14 +28,14 @@ void RdmaTransport::release_completed_receive(WorkRequest *work, const ibv_wc &w if (wc.status != IBV_WC_SUCCESS || !work->conn->is_open() || !work->slot->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight)) { fail(); - return; + return true; } RdmaAggregate aggregate; if (wc.byte_len > work->slot->capacity || !parse_rdma_aggregate(std::span(work->slot->data, wc.byte_len), aggregate) || !work->slot->transition(BufferOwnership::ReceiveInFlight, BufferOwnership::CallerOwned)) { fail(); - return; + return true; } auto connection = std::move(work->conn); @@ -67,6 +66,7 @@ void RdmaTransport::release_completed_receive(WorkRequest *work, const ibv_wc &w delete frame; } slot->release(); + return true; } void RdmaTransport::repost_frame_release(void *context, Buffer *slot) diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp index 2380f8478..f89e3a7cb 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. #include "rdma_aggregate.h" +#include "rdma_control_send.h" #include "rdma_cq.h" #include "rdma_direct_frame.h" @@ -25,7 +26,8 @@ bool RdmaTransport::submit(Connection *connection, OutFrame *frame) return false; }; Operation operation(repost_state_); - if (!operation || connection == nullptr || frame == nullptr || !connection->is_open()) + if (!operation || connection == nullptr || frame == nullptr || !connection->is_open() || + frame->header.msg_type == RPC_CONTROL_FRAGMENT_MSG_TYPE) return reject(); auto *pools = pools_for(connection); if (pools == nullptr || pools->send == nullptr || pools->cq_index >= completion_queues_.size()) @@ -37,17 +39,40 @@ bool RdmaTransport::submit(Connection *connection, OutFrame *frame) direct ? std::min(config_.wire_slot_capacity, pools->peer_receive_slot_capacity) : std::min({config_.wire_slot_capacity, pools->peer_receive_slot_capacity, config_.aggregation_max_bytes}); - if (!posting || !connection->is_open() || - !(direct ? rdma_direct_frame_size(*frame, limit, bytes) : rdma_frame_size(*frame, limit, bytes))) + if (!posting || !connection->is_open()) return reject(); + const bool stream = + direct && frame->control != nullptr && frame->control->len > RDMA_CONTROL_CAPACITY - HEADER_SIZE; + Buffer *original_control = frame->control; + if (stream) { + auto *charged = admit_control_frame(*frame, *pools); + if (charged == nullptr) { + return reject(); + } + frame->control = charged; + // The charged slice now owns the original reference before the CQ can + // consume it. Rejected publication restores the caller's ownership. + original_control->release(); + } + else if (!(direct ? rdma_direct_frame_size(*frame, limit, bytes) : rdma_frame_size(*frame, limit, bytes))) { + return reject(); + } + auto restore = [&] { + if (stream) { + auto *charged = frame->control; + frame->control = original_control->ref_clone(); + charged->release(); + } + return reject(); + }; uint32_t queued = pools->queued_frames.load(std::memory_order_relaxed); do { if (queued >= config_.send_queue_capacity) - return reject(); + return restore(); } while (!pools->queued_frames.compare_exchange_weak(queued, queued + 1, std::memory_order_acq_rel)); if (!pools->send_queue.try_push(frame)) { pools->queued_frames.fetch_sub(1, std::memory_order_release); - return reject(); + return restore(); } uint64_t peak = stats_.send_queue_high_water.load(std::memory_order_relaxed); while (peak < queued + 1 && @@ -181,6 +206,10 @@ void RdmaTransport::retire_sends(CompletionQueue &queue, Connection *connection) if (connection != nullptr && entry.connection.get() != connection) return false; auto &pools = *entry.pools; + pools.control_send.reset(); + if (pools.control_receive != nullptr) { + pools.control_receive->reset(); + } release_rdma_frames(std::exchange(pools.pending_frame, nullptr)); OutFrame *frame = nullptr; while (pools.send_queue.drain(&frame, 1) != 0) diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp index 7ff817faa..d4086d9dc 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp @@ -20,6 +20,7 @@ std::string RdmaTransport::status_json() const status.string("cm_address", capability.cm_address); status.number("cm_port", capability.port); status.number("generation", capability.generation); + status.number("protocol_version", capability.protocol_version); status.boolean("aggregation_enabled", config_.aggregation_enabled); status.string("wire_layout", config_.wire_layout == RdmaWireLayout::Contiguous ? "contiguous" : "control_data"); status.number("data_slot_capacity", config_.data_slot_capacity()); @@ -84,6 +85,17 @@ std::string RdmaTransport::status_json() const COUNTER_FIELD(cm_attempts); COUNTER_FIELD(preflight_attempts); COUNTER_FIELD(preflight_failures); + COUNTER_FIELD(control_fragments_sent); + COUNTER_FIELD(control_fragments_received); + COUNTER_FIELD(copied_control_send_bytes); + COUNTER_FIELD(copied_control_receive_bytes); + COUNTER_FIELD(control_stream_timeouts); + COUNTER_FIELD(control_send_bytes); + COUNTER_FIELD(control_send_bytes_high_water); + COUNTER_FIELD(control_send_stalls); + COUNTER_FIELD(control_receive_bytes); + COUNTER_FIELD(control_receive_bytes_high_water); + COUNTER_FIELD(control_receive_stalls); # undef COUNTER_FIELD status.null("quarantined_bytes"); std::string depths = "["; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index 20362b05e..5a53d88b1 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -7,6 +7,7 @@ #include "crowdb-rpc/transport/socket_transport.h" #include "rdma_aggregate.h" #include "rdma_channel.h" +#include "rdma_control_stream.h" #include "rdma_cq.h" #include "rdma_device.h" @@ -57,6 +58,16 @@ RdmaTransport::RdmaTransport(RdmaConfig config, BufferPool *application_pool) CRB_LOG_WARN("rdma configuration: inconsistent queue depths, slot capacity, or arena budget; using TCP"); return; } + control_send_budget_ = std::make_shared(config_.control_send_budget_bytes); + control_receive_budget_ = + std::make_shared(config_.control_receive_budget_bytes, [state = repost_state_] { + Operation operation(state); + if (operation) { + for (uint32_t index = 0; index < state->transport->completion_queues_.size(); ++index) { + state->transport->notify_queue(index); + } + } + }); if (config_.use_environment && config_.device.empty()) { if (const char *requested_device = std::getenv("CROWDB_RDMA_DEVICE"); requested_device != nullptr) config_.device = requested_device; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp index 74d329133..a404421ca 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp @@ -63,6 +63,7 @@ void RdmaTransport::release_work(WorkRequest *work) { work->conn.reset(); work->pools.reset(); + work->control_send.reset(); work->slot = nullptr; work->prefix = nullptr; work->frame = nullptr; @@ -76,14 +77,17 @@ void RdmaTransport::dispatch_completion(const ibv_wc &wc) return; // Error completions do not provide a trustworthy opcode. The retained // work record owns dispatch, even after disconnect or QP destruction. - if (work->kind == WorkKind::Receive && wc.status == IBV_WC_SUCCESS && !work->conn->has_frame_handler()) { + bool deferred = work->kind == WorkKind::Receive && wc.status == IBV_WC_SUCCESS && work->pools != nullptr && + work->pools->deferred_receives != 0; + if (!deferred && work->kind == WorkKind::Receive && wc.status == IBV_WC_SUCCESS && + !work->conn->has_frame_handler()) { RdmaAggregate aggregate; bool valid = false; bool application = false; if (work->prefix != nullptr) { Header header; valid = parse_rdma_direct_control(std::span(work->prefix->data, RDMA_CONTROL_CAPACITY), - config_.data_slot_capacity(), wc.byte_len, header); + work->pools->effective_data_capacity, wc.byte_len, header); application = valid && header.msg_type != RDMA_HANDSHAKE_MSG_TYPE; } else { @@ -92,17 +96,20 @@ void RdmaTransport::dispatch_completion(const ibv_wc &wc) for (uint32_t index = 0; valid && index < aggregate.count; ++index) application |= aggregate.frames[index].header.msg_type != RDMA_HANDSHAKE_MSG_TYPE; } - if (application) { - const uint32_t index = static_cast(wc.wr_id) / config_.cq_depth; - completion_queues_[index]->deferred.push_back({work, wc}); - stats_.pending_deferred_receives.fetch_add(1, std::memory_order_relaxed); - return; - } + deferred = application; } - if (work->kind == WorkKind::Receive) - release_completed_receive(work, wc); - else + if (work->kind != WorkKind::Receive) { release_completed_frame(work, wc); + return; + } + if (deferred || !release_completed_receive(work, wc)) { + const uint32_t index = static_cast(wc.wr_id) / config_.cq_depth; + completion_queues_[index]->deferred.push_back({work, wc}); + if (work->pools != nullptr) { + ++work->pools->deferred_receives; + } + stats_.pending_deferred_receives.fetch_add(1, std::memory_order_relaxed); + } } void RdmaTransport::cancel_work(Connection *connection) diff --git a/lib/crowdb-rpc/tests/common/test_rdma_control.cpp b/lib/crowdb-rpc/tests/common/test_rdma_control.cpp new file mode 100644 index 000000000..1016b6324 --- /dev/null +++ b/lib/crowdb-rpc/tests/common/test_rdma_control.cpp @@ -0,0 +1,93 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "test_rdma_control.h" + +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) +# include "crowdb-rpc/server/message.h" + +# include + +namespace crowdb::rpc +{ +RdmaConfig test_control_config(uint32_t capacity, uint32_t max_control_bytes) +{ + auto config = RdmaConfig::for_layout(RdmaWireLayout::ControlData, capacity); + config.use_environment = false; + config.sq_depth = config.rq_depth = config.posted_receive_target = 2; + config.cq_depth = 4; + config.send_arena_bytes = uint64_t{2} * capacity; + config.receive_arena_bytes = uint64_t{2} * config.wire_slot_capacity; + config.max_control_message_bytes = max_control_bytes; + config.control_send_budget_bytes = uint64_t{3} * max_control_bytes; + config.control_receive_budget_bytes = max_control_bytes; + config.poll_empty_budget = 1; + if (const auto *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR")) { + config.source_address = address; + } + if (const auto *device = std::getenv("CROWDB_RDMA_DEVICE")) { + config.device = device; + } + return config; +} + +OutFrame *test_control_frame(BufferPool &pool, uint64_t id, uint32_t size) +{ + auto *control = pool.alloc(size); + if (control == nullptr) { + return nullptr; + } + for (uint32_t index = 0; index < size; ++index) { + control->data[index] = static_cast(index * 131U + id); + } + auto *identity = build_ping_request(&pool, id, 11); + if (identity == nullptr || size < identity->len) { + if (identity != nullptr) { + identity->release(); + } + control->release(); + return nullptr; + } + std::memcpy(control->data, identity->data, identity->len); + identity->release(); + control->len = size; + return build_out_frame(id, 100, control, nullptr); +} + +OutFrame *test_control_reply(Frame *frame) +{ + auto *control = new Buffer; + control->data = frame->control.data(); + control->len = static_cast(frame->control.size()); + control->capacity = control->len; + control->ref = new std::atomic(1); + control->free_cb = [](void *owner) { delete static_cast(owner); }; + control->free_ctx = frame; + return build_out_frame(frame->request_id, 100, control, nullptr); +} + +std::unique_ptr TestControlPeers::create(const RdmaConfig &client_config, + const RdmaConfig &server_config) +{ + auto pair = std::make_unique(); + pair->server = std::make_unique(server_config, &pair->application); + pair->client = std::make_unique(client_config, &pair->application); + if (!pair->server->listen(server_config.source_address, 19401)) { + return nullptr; + } + std::thread acceptor([&] { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (pair->incoming == nullptr && std::chrono::steady_clock::now() < deadline) { + pair->incoming = pair->server->accept(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + }); + pair->outgoing = pair->client->connect(server_config.source_address, 19401); + acceptor.join(); + if (pair->incoming == nullptr || pair->outgoing == nullptr) { + return nullptr; + } + return pair; +} +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/tests/common/test_rdma_control.h b/lib/crowdb-rpc/tests/common/test_rdma_control.h new file mode 100644 index 000000000..2fe550073 --- /dev/null +++ b/lib/crowdb-rpc/tests/common/test_rdma_control.h @@ -0,0 +1,59 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "test_rdma_transport.h" + +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) +namespace crowdb::rpc +{ +RdmaConfig test_control_config(uint32_t capacity, uint32_t max_control_bytes); +OutFrame *test_control_frame(BufferPool &pool, uint64_t id, uint32_t size); +OutFrame *test_control_reply(Frame *frame); + +struct TestControlPoller +{ + explicit TestControlPoller(RdmaTransport &owner) : transport(&owner) + { + transport->start(); + } + + ~TestControlPoller() + { + stop(); + } + + void stop() + { + if (transport != nullptr) { + transport->stop(); + transport = nullptr; + } + } + + RdmaTransport *transport; +}; + +struct TestControlPeers +{ + SystemBufferPool application; + std::unique_ptr server; + std::unique_ptr client; + std::shared_ptr incoming; + std::shared_ptr outgoing; + + static std::unique_ptr create(const RdmaConfig &client_config, const RdmaConfig &server_config); + + template bool drain_until(Ready ready) + { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!ready() && std::chrono::steady_clock::now() < deadline) { + client->drain_cq(); + server->drain_cq(); + } + return ready(); + } +}; +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/tests/common/test_rdma_transport.h b/lib/crowdb-rpc/tests/common/test_rdma_transport.h index 8e1f28568..9701583ae 100644 --- a/lib/crowdb-rpc/tests/common/test_rdma_transport.h +++ b/lib/crowdb-rpc/tests/common/test_rdma_transport.h @@ -3,6 +3,7 @@ #pragma once +#include "../../src/transport/rdma/rdma_control_codec.h" #include "../../src/transport/rdma/rdma_cq.h" #include "../../src/transport/rdma/rdma_device.h" #include "crowdb-rpc/transport/rdma/rdma_transport.h" @@ -139,6 +140,53 @@ struct TestRdmaTransport ->work_requests[index % transport.config_.cq_depth] .token.load(std::memory_order_acquire) == token; } + + static uint32_t flush_for_tests(RdmaTransport &transport, Connection *connection, uint32_t budget) + { + return transport.flush_connection(connection, *RdmaTransport::pools_for(connection), budget); + } + + static bool post_fragment_for_tests(RdmaTransport &transport, Connection *connection, + const ControlFragment &fragment, std::span bytes) + { + auto *pools = RdmaTransport::pools_for(connection); + auto *prefix = pools->send->alloc(RDMA_CONTROL_CAPACITY); + auto *data = transport.send_pool_->alloc(static_cast(bytes.size())); + if (prefix == nullptr || data == nullptr) { + if (prefix != nullptr) { + prefix->release(); + } + if (data != nullptr) { + data->release(); + } + return false; + } + if (!write_control_fragment(std::span(prefix->data, RDMA_CONTROL_CAPACITY), fragment, bytes.size())) { + prefix->release(); + data->release(); + return false; + } + prefix->len = RDMA_CONTROL_CAPACITY; + std::memcpy(data->data, bytes.data(), bytes.size()); + data->len = static_cast(bytes.size()); + return transport.post_direct(connection, *pools, data, prefix, nullptr); + } + + static std::vector receive_capacities_for_tests(RdmaTransport &transport, Connection *connection) + { + std::vector capacities; + for (const auto &queue : transport.completion_queues_) { + for (uint32_t index = 0; index < transport.config_.cq_depth; ++index) { + const auto &work = queue->work_requests[index]; + const auto token = work.token.load(std::memory_order_acquire); + if (token != 0 && token != RdmaTransport::WORK_RESERVED && + work.kind == RdmaTransport::WorkKind::Receive && work.conn.get() == connection) { + capacities.push_back(work.slot->capacity); + } + } + } + return capacities; + } }; struct TestCompletionQueue diff --git a/lib/crowdb-rpc/tests/rdma_control_codec_test.cpp b/lib/crowdb-rpc/tests/rdma_control_codec_test.cpp new file mode 100644 index 000000000..0cda7c83e --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_control_codec_test.cpp @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "../src/transport/rdma/rdma_control_codec.h" +#include "../src/transport/rdma/rdma_direct_frame.h" +#include "crowdb-rpc/server/message.h" + +#include + +using namespace crowdb::rpc; + +TEST(ControlCodecTest, BoundedFragmentPreservesCommonCorrelationAndAllStreamMetadata) +{ + const ControlFragment source{.request_id = 43, + .rpc_create_nano = 11, + .response_create_nano = 12, + .stream_id = 5, + .generation = 7, + .sequence = 2, + .offset = 131072, + .total = 131089, + .capacity = RDMA_METADATA_CAPACITY, + .message_type = 3701, + .message_flags = FLAG_ONE_WAY, + .stream_flags = CONTROL_STREAM_END}; + std::array prefix{}; + ASSERT_TRUE(write_control_fragment(prefix, source, 17)); + Header header; + ASSERT_TRUE(parse_rdma_direct_control(prefix, source.capacity, prefix.size() + 17, header)); + EXPECT_EQ(header.msg_type, RPC_CONTROL_FRAGMENT_MSG_TYPE); + EXPECT_EQ(header.flags, FLAG_ONE_WAY); + ControlFragment decoded; + const auto control = std::span(prefix).subspan(HEADER_SIZE); + ASSERT_TRUE(parse_control_fragment(control, decoded)); + EXPECT_EQ(decoded.request_id, source.request_id); + EXPECT_EQ(decoded.rpc_create_nano, source.rpc_create_nano); + EXPECT_EQ(decoded.response_create_nano, source.response_create_nano); + EXPECT_EQ(decoded.stream_id, source.stream_id); + EXPECT_EQ(decoded.generation, source.generation); + EXPECT_EQ(decoded.sequence, source.sequence); + EXPECT_EQ(decoded.offset, source.offset); + EXPECT_EQ(decoded.total, source.total); + EXPECT_EQ(decoded.capacity, source.capacity); + EXPECT_EQ(decoded.message_type, source.message_type); + EXPECT_EQ(decoded.message_flags, source.message_flags); + EXPECT_EQ(decoded.stream_flags, source.stream_flags); + uint64_t id = 0, created = 0, response = 0; + extract_control_fields(control.data(), control.size(), id, created, response); + EXPECT_EQ(id, source.request_id); + EXPECT_EQ(created, source.rpc_create_nano); + EXPECT_EQ(response, source.response_create_nano); + EXPECT_FALSE(write_control_fragment(prefix, source, source.capacity + 1)); +} + +TEST(ControlCodecTest, MalformedMetadataAndMismatchedOriginalControlDoNotPassTheBoundary) +{ + std::array prefix{}; + ControlFragment fragment; + EXPECT_FALSE(parse_control_fragment(prefix, fragment)); + for (size_t size = 0; size < 8; ++size) { + EXPECT_FALSE(parse_control_fragment(std::span(prefix).first(size), fragment)); + } + SystemBufferPool pool; + auto *control = build_ping_request(&pool, 43, 11); + ASSERT_NE(control, nullptr); + Frame frame; + frame.control.assign(control->data, control->data + control->len); + frame.request_id = 43; + frame.rpc_create_nano = 11; + EXPECT_TRUE(control_identity_matches(frame)); + ++frame.request_id; + EXPECT_FALSE(control_identity_matches(frame)); + --frame.request_id; + ++frame.response_create_nano; + EXPECT_FALSE(control_identity_matches(frame)); + frame.control.assign(32, 0xFF); + EXPECT_FALSE(control_identity_matches(frame)); + control->release(); +} diff --git a/lib/crowdb-rpc/tests/rdma_control_hardware_test.cpp b/lib/crowdb-rpc/tests/rdma_control_hardware_test.cpp new file mode 100644 index 000000000..fd296ef88 --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_control_hardware_test.cpp @@ -0,0 +1,266 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "../src/transport/rdma/rdma_aggregate.h" +#include "common/test_rdma_control.h" + +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) +# include + +# include + +using namespace crowdb::rpc; +using namespace std::chrono_literals; + +class RdmaControlHardwareTest : public ::testing::Test +{ + protected: + void SetUp() override + { + if (std::getenv("CROWDB_RDMA_E2E") == nullptr || std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR") == nullptr) { + GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; + } + } +}; + +TEST_F(RdmaControlHardwareTest, AsymmetricClassesRoundTripBeyondTheWholeReceiveWindow) +{ + for (const auto capacity : {RDMA_METADATA_CAPACITY, RDMA_AGGREGATE_CAPACITY, RDMA_DATAPATH_CAPACITY}) { + const auto size = (2 * capacity) + 13; + auto client_config = test_control_config(capacity, size); + auto server_config = test_control_config(RDMA_DATAPATH_CAPACITY, size); + auto pair = TestControlPeers::create(client_config, server_config); + ASSERT_NE(pair, nullptr); + for (auto *transport : {pair->client.get(), pair->server.get()}) { + const auto owner = transport == pair->client.get() ? pair->outgoing.get() : pair->incoming.get(); + const auto capacities = TestRdmaTransport::receive_capacities_for_tests(*transport, owner); + ASSERT_EQ(capacities.size(), 2U); + for (const auto actual : capacities) { + EXPECT_EQ(actual, capacity); + } + } + auto *request = test_control_frame(pair->application, 43, size); + ASSERT_NE(request, nullptr); + const std::vector expected(request->control->data, request->control->data + size); + pair->incoming->set_on_frame([&](Frame *frame, Connection *connection) { + EXPECT_EQ(frame->control, expected); + EXPECT_EQ(frame->header.data_size, 0U); + EXPECT_EQ(frame->data_buf, nullptr); + auto *reply = test_control_reply(frame); + if (!pair->server->submit(connection, reply)) { + ADD_FAILURE() << "complete logical response was not admitted"; + release_rdma_frames(reply); + } + }); + std::unique_ptr received; + pair->outgoing->set_on_frame([&](Frame *frame, Connection *) { received.reset(frame); }); + ASSERT_TRUE(pair->client->submit(pair->outgoing.get(), request)); + ASSERT_TRUE(pair->drain_until([&] { return received != nullptr; })); + EXPECT_EQ(received->control, expected); + EXPECT_EQ(received->request_id, 43U); + EXPECT_EQ(pair->client->stats().control_fragments_sent.load(), 3U); + EXPECT_EQ(pair->server->stats().control_fragments_sent.load(), 3U); + EXPECT_EQ(pair->client->stats().rpc_frames_received.load(), 1U); + EXPECT_EQ(pair->server->stats().rpc_frames_received.load(), 1U); + EXPECT_EQ(pair->client->stats().control_receive_bytes.load(), size); + EXPECT_EQ(pair->client->stats().copied_control_receive_bytes.load(), size); + received.reset(); + EXPECT_EQ(pair->client->stats().control_receive_bytes.load(), 0U); + ASSERT_TRUE(pair->drain_until([&] { return pair->server->stats().control_send_bytes.load() == 0; })); + EXPECT_EQ(pair->server->stats().control_receive_bytes.load(), 0U); + for (auto *transport : {pair->client.get(), pair->server.get()}) { + EXPECT_EQ(transport->stats().protocol_errors.load(), 0U); + EXPECT_EQ(transport->stats().cq_errors.load(), 0U); + EXPECT_EQ(transport->stats().receive_post_errors.load(), 0U); + EXPECT_EQ(transport->stats().fallback_count.load(), 0U); + } + } +} + +TEST_F(RdmaControlHardwareTest, HeldFinalOwnerBoundsTheRqAndResumesMessagesInFifoOrder) +{ + const auto size = (2 * RDMA_METADATA_CAPACITY) + 13; + auto config = test_control_config(RDMA_METADATA_CAPACITY, size); + config.control_assembly_timeout_ms = 25; + auto pair = TestControlPeers::create(config, config); + ASSERT_NE(pair, nullptr); + std::array, 3> held{}; + pair->incoming->set_on_frame([&](Frame *frame, Connection *) { + if (frame->request_id >= 1 && frame->request_id <= held.size()) { + held[frame->request_id - 1].store(frame, std::memory_order_release); + } + else { + delete frame; + } + }); + TestControlPoller poller(*pair->server); + auto wait_client = [&](auto ready) { + const auto deadline = std::chrono::steady_clock::now() + 1s; + while (!ready() && std::chrono::steady_clock::now() < deadline) { + pair->client->drain_cq(); + std::this_thread::yield(); + } + return ready(); + }; + ASSERT_TRUE(pair->client->submit(pair->outgoing.get(), test_control_frame(pair->application, 1, size))); + ASSERT_TRUE(wait_client([&] { return held[0].load(std::memory_order_acquire) != nullptr; })); + ASSERT_TRUE(wait_client([&] { return pair->client->stats().control_send_bytes.load() == 0; })); + for (uint64_t id : {2, 3}) { + ASSERT_TRUE(pair->client->submit(pair->outgoing.get(), test_control_frame(pair->application, id, size))); + } + ASSERT_TRUE(wait_client([&] { return pair->server->stats().pending_deferred_receives.load() == 2; })); + const auto registered = pair->server->stats().registered_bytes.load(); + const auto posted = pair->server->stats().posted_receives.load(); + std::this_thread::sleep_for(50ms); + EXPECT_TRUE(pair->incoming->is_open()); + EXPECT_EQ(held[1].load(std::memory_order_acquire), nullptr); + EXPECT_EQ(pair->server->stats().control_stream_timeouts.load(), 0U); + EXPECT_EQ(pair->server->stats().control_receive_bytes.load(), size); + EXPECT_EQ(pair->server->stats().registered_bytes.load(), registered); + EXPECT_EQ(pair->server->stats().posted_receives.load(), posted); + EXPECT_GT(pair->server->stats().control_receive_stalls.load(), 0U); + const auto wakeups = pair->server->stats().cq_wakeups.load(); + delete held[0].exchange(nullptr, std::memory_order_acq_rel); + ASSERT_TRUE(wait_client([&] { return held[1].load(std::memory_order_acquire) != nullptr; })); + EXPECT_EQ(held[1].load()->request_id, 2U); + EXPECT_GT(pair->server->stats().cq_wakeups.load(), wakeups); + delete held[1].exchange(nullptr, std::memory_order_acq_rel); + ASSERT_TRUE(wait_client([&] { return held[2].load(std::memory_order_acquire) != nullptr; })); + EXPECT_EQ(held[2].load()->request_id, 3U); + poller.stop(); + std::unique_ptr final_owner(held[2].exchange(nullptr)); + EXPECT_EQ(pair->server->stats().control_receive_bytes_high_water.load(), size); + EXPECT_EQ(pair->server->stats().control_receive_bytes.load(), size); + EXPECT_EQ(pair->server->stats().protocol_errors.load(), 0U); + EXPECT_EQ(pair->server->stats().cq_errors.load(), 0U); + EXPECT_EQ(pair->server->stats().poller_count.load(), 1U); + pair.reset(); + EXPECT_EQ(final_owner->control.size(), size); + EXPECT_EQ(final_owner->request_id, 3U); + final_owner.reset(); +} + +TEST_F(RdmaControlHardwareTest, PartialAssemblyExpiresAndNeverPublishesAnApplicationFrame) +{ + const auto size = (2 * RDMA_METADATA_CAPACITY) + 13; + auto config = test_control_config(RDMA_METADATA_CAPACITY, size); + config.control_assembly_timeout_ms = 10; + auto pair = TestControlPeers::create(config, config); + ASSERT_NE(pair, nullptr); + uint32_t published = 0; + pair->incoming->set_on_frame([&](Frame *frame, Connection *) { + ++published; + delete frame; + }); + ASSERT_TRUE(pair->client->submit(pair->outgoing.get(), test_control_frame(pair->application, 1, size))); + ASSERT_EQ(TestRdmaTransport::flush_for_tests(*pair->client, pair->outgoing.get(), 1), 1U); + TestControlPoller poller(*pair->server); + const auto deadline = std::chrono::steady_clock::now() + 1s; + while (pair->server->stats().control_fragments_received.load() == 0 && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + ASSERT_EQ(pair->server->stats().control_fragments_received.load(), 1U); + EXPECT_EQ(pair->server->stats().control_receive_bytes.load(), size); + while (pair->incoming->is_open() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + poller.stop(); + EXPECT_FALSE(pair->incoming->is_open()); + EXPECT_EQ(published, 0U); + EXPECT_EQ(pair->server->stats().control_receive_bytes.load(), 0U); + EXPECT_EQ(pair->server->stats().control_stream_timeouts.load(), 1U); + EXPECT_EQ(pair->server->stats().fallback_count.load(), 0U); + pair->client->shutdown(); + EXPECT_EQ(pair->client->stats().control_send_bytes.load(), 0U); +} + +TEST_F(RdmaControlHardwareTest, QueueRejectionRestoresCallerAndInFlightSourceRetainsItsByteLease) +{ + const auto size = RDMA_METADATA_CAPACITY + 13; + auto config = test_control_config(RDMA_METADATA_CAPACITY, size); + config.send_queue_capacity = 2; + auto pair = TestControlPeers::create(config, config); + ASSERT_NE(pair, nullptr); + pair->incoming->set_on_frame([](Frame *frame, Connection *) { delete frame; }); + for (uint64_t id : {1, 2}) { + ASSERT_TRUE(pair->client->submit(pair->outgoing.get(), test_control_frame(pair->application, id, size))); + } + auto *rejected = test_control_frame(pair->application, 3, size); + auto *original = rejected->control; + EXPECT_FALSE(pair->client->submit(pair->outgoing.get(), rejected)); + EXPECT_EQ(rejected->control, original); + EXPECT_EQ(original->ref->load(), 1); + release_rdma_frames(rejected); + EXPECT_EQ(pair->client->stats().control_send_bytes.load(), uint64_t{2} * size); + ASSERT_EQ(TestRdmaTransport::flush_for_tests(*pair->client, pair->outgoing.get(), 2), 2U); + EXPECT_EQ(pair->client->stats().control_send_bytes.load(), uint64_t{2} * size); + EXPECT_EQ(pair->client->stats().rpc_frames_sent.load(), 1U); + ASSERT_TRUE(pair->drain_until([&] { return pair->server->stats().rpc_frames_received.load() == 2; })); + ASSERT_TRUE(pair->drain_until([&] { return pair->client->stats().control_send_bytes.load() == 0; })); + EXPECT_EQ(pair->client->stats().send_errors.load(), 0U); +} + +TEST_F(RdmaControlHardwareTest, PeerLogicalLimitRejectsBeforePublishingOrChargingBytes) +{ + const auto size = RDMA_METADATA_CAPACITY + 13; + const auto server_config = test_control_config(RDMA_METADATA_CAPACITY, size); + const auto client_config = test_control_config(RDMA_METADATA_CAPACITY, 2 * size); + auto pair = TestControlPeers::create(client_config, server_config); + ASSERT_NE(pair, nullptr); + auto *request = test_control_frame(pair->application, 43, size + 1); + auto *original = request->control; + EXPECT_FALSE(pair->client->submit(pair->outgoing.get(), request)); + EXPECT_EQ(request->control, original); + EXPECT_EQ(pair->client->stats().control_send_bytes.load(), 0U); + EXPECT_EQ(pair->client->stats().send_queue_high_water.load(), 0U); + EXPECT_EQ(pair->client->stats().sends.load(), 0U); + EXPECT_EQ(pair->client->stats().submit_rejects.load(), 1U); + release_rdma_frames(request); +} + +TEST_F(RdmaControlHardwareTest, MalformedWireIdentityFailsTheWholeLogicalControl) +{ + const auto size = RDMA_METADATA_CAPACITY + 13; + const auto config = test_control_config(RDMA_METADATA_CAPACITY, size); + for (const bool wrong_generation : {true, false}) { + auto pair = TestControlPeers::create(config, config); + ASSERT_NE(pair, nullptr); + uint32_t published = 0; + pair->incoming->set_on_frame([&](Frame *frame, Connection *) { + ++published; + delete frame; + }); + auto *original = test_control_frame(pair->application, 44, size); + ControlFragment f{.request_id = 43, + .rpc_create_nano = 11, + .stream_id = 1, + .generation = pair->client->local_capability().generation + (wrong_generation ? 1 : 0), + .total = size, + .capacity = RDMA_METADATA_CAPACITY, + .message_type = 100, + .stream_flags = CONTROL_STREAM_START}; + const auto bytes = std::span(original->control->data, size); + ASSERT_TRUE(TestRdmaTransport::post_fragment_for_tests(*pair->client, pair->outgoing.get(), f, + bytes.first(RDMA_METADATA_CAPACITY))); + ASSERT_TRUE(pair->drain_until([&] { + return pair->server->stats().control_fragments_received.load() != 0 || !pair->incoming->is_open(); + })); + if (!wrong_generation) { + f.sequence = 1; + f.offset = RDMA_METADATA_CAPACITY; + f.stream_flags = CONTROL_STREAM_END; + ASSERT_TRUE(TestRdmaTransport::post_fragment_for_tests(*pair->client, pair->outgoing.get(), f, + bytes.subspan(RDMA_METADATA_CAPACITY))); + ASSERT_TRUE(pair->drain_until([&] { return !pair->incoming->is_open(); })); + } + pair->server->drain_cq(); + EXPECT_EQ(published, 0U); + EXPECT_FALSE(pair->incoming->is_open()); + EXPECT_EQ(pair->server->stats().protocol_errors.load(), 1U); + EXPECT_EQ(pair->server->stats().control_receive_bytes.load(), 0U); + EXPECT_EQ(pair->server->stats().fallback_count.load(), 0U); + release_rdma_frames(original); + } +} +#endif From ce95310da4562db5c62be8f15dff4238a4cda586 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 06:06:33 +0800 Subject: [PATCH 050/258] Wire bounded Chunk-KV RPC classes through Access services --- Cargo.lock | 2 + app/crowdb-access-server/src/config.rs | 6 + app/crowdb-access-server/src/dataset.rs | 7 +- .../src/iceberg/runtime.rs | 10 +- app/crowdb-access-server/src/main.rs | 18 +- app/crowdb-access-server/tests/config_test.rs | 24 ++ app/crowdb-chunk-kv-server/src/config.rs | 11 + app/crowdb-chunk-kv-server/src/main.rs | 12 +- app/crowdb-chunk-kv-server/src/rpc.rs | 95 +++----- .../src/rpc/response.rs | 40 ++++ .../src/rpc/transport.rs | 30 +++ .../tests/config_test.rs | 31 +++ doc/working/plan-rdma-chunk-kv-access.md | 76 ++++++- lib/crowdb-access-iceberg/src/storage.rs | 31 ++- lib/crowdb-access-s3/src/storage.rs | 29 ++- lib/crowdb-chunk-kv-client/Cargo.toml | 1 + lib/crowdb-chunk-kv-client/src/lib.rs | 2 +- lib/crowdb-chunk-kv-client/src/transport.rs | 132 +---------- .../src/transport/config.rs | 38 ++++ .../src/transport/endpoint.rs | 27 +++ .../src/transport/rpc.rs | 207 ++++++++++++++++++ .../src/transport/status.rs | 71 ++++++ .../tests/transport_class_test.rs | 128 +++++++++++ lib/crowdb-rpc/ffi/Cargo.toml | 1 + lib/crowdb-rpc/ffi/src/server.rs | 7 + lib/crowdb-rpc/ffi/src/sys.rs | 1 + lib/crowdb-rpc/ffi/src/transport.rs | 37 +++- .../ffi/tests/transport_config_test.rs | 31 +++ lib/crowdb-rpc/include/crowdb-rpc/c_api.h | 2 + .../include/crowdb-rpc/c_api_internal.h | 8 +- lib/crowdb-rpc/include/crowdb-rpc/framing.h | 5 +- lib/crowdb-rpc/src/c_api.cpp | 1 + lib/crowdb-rpc/src/c_api/frame.cpp | 47 ++++ lib/crowdb-rpc/src/c_api/handlers.cpp | 43 ---- .../tests/c_api_handler_lifetime_test.cpp | 40 ++++ 35 files changed, 985 insertions(+), 266 deletions(-) create mode 100644 app/crowdb-chunk-kv-server/src/rpc/response.rs create mode 100644 app/crowdb-chunk-kv-server/src/rpc/transport.rs create mode 100644 lib/crowdb-chunk-kv-client/src/transport/config.rs create mode 100644 lib/crowdb-chunk-kv-client/src/transport/endpoint.rs create mode 100644 lib/crowdb-chunk-kv-client/src/transport/rpc.rs create mode 100644 lib/crowdb-chunk-kv-client/src/transport/status.rs create mode 100644 lib/crowdb-chunk-kv-client/tests/transport_class_test.rs create mode 100644 lib/crowdb-rpc/tests/c_api_handler_lifetime_test.cpp diff --git a/Cargo.lock b/Cargo.lock index d2f722c24..bd30d2e87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -824,6 +824,7 @@ dependencies = [ "crowdb-rpc-ffi", "futures", "rand 0.8.6", + "serde", "serde_json", "thiserror 2.0.18", "tokio", @@ -1249,6 +1250,7 @@ dependencies = [ "crowdb-rpc-ffi", "ctor", "flatbuffers", + "serde", "serde_json", "tokio", ] diff --git a/app/crowdb-access-server/src/config.rs b/app/crowdb-access-server/src/config.rs index f17c5f381..c37c7bd35 100644 --- a/app/crowdb-access-server/src/config.rs +++ b/app/crowdb-access-server/src/config.rs @@ -58,6 +58,7 @@ pub struct CommonConfig { pub management_seeds: Vec, pub diskio_connections_per_endpoint: usize, pub diskio_rpc_workers: u32, + pub chunk_kv_rpc: crowdb_chunk_kv_client::ChunkKvRpcTransportConfig, } impl Default for CommonConfig { @@ -66,6 +67,7 @@ impl Default for CommonConfig { management_seeds: Vec::new(), diskio_connections_per_endpoint: 2, diskio_rpc_workers: 2, + chunk_kv_rpc: crowdb_chunk_kv_client::ChunkKvRpcTransportConfig::default(), } } } @@ -311,6 +313,10 @@ pub struct IcebergGcConfig { impl BaseConfig for AccessConfig { #[allow(clippy::too_many_lines)] fn validate(&self) -> Result<(), String> { + self.common + .chunk_kv_rpc + .validate() + .map_err(|error| error.to_string())?; if self.common.diskio_connections_per_endpoint == 0 || self.common.diskio_rpc_workers == 0 { return Err("common DiskIO connections and RPC workers must be nonzero".into()); } diff --git a/app/crowdb-access-server/src/dataset.rs b/app/crowdb-access-server/src/dataset.rs index a2523639f..b5b015a67 100644 --- a/app/crowdb-access-server/src/dataset.rs +++ b/app/crowdb-access-server/src/dataset.rs @@ -162,7 +162,12 @@ pub async fn run_with_shutdown( let control = Arc::new(CrowdbKvClient::new(KvConfig::new(seeds.clone()))); let source = Arc::new(Group0ChunkKvRangeCatalogSource::from_shared(Arc::clone(&control))); let chunk_config = ChunkKvConfig::default(); - let transport = Arc::new(ChunkKvRpcTransport::new(chunk_config.max_owner_connections, 1, 2)); + let transport = Arc::new(ChunkKvRpcTransport::with_config( + chunk_config.max_owner_connections, + 1, + 2, + &access.common.chunk_kv_rpc, + )?); let client = Arc::new(ChunkKvClient::new(chunk_config, source, transport)?); client.refresh_catalog().await?; let store = Arc::new(ChunkKvDatasetStore::new(client)); diff --git a/app/crowdb-access-server/src/iceberg/runtime.rs b/app/crowdb-access-server/src/iceberg/runtime.rs index 60195ebdb..c1f52a75d 100644 --- a/app/crowdb-access-server/src/iceberg/runtime.rs +++ b/app/crowdb-access-server/src/iceberg/runtime.rs @@ -6,7 +6,9 @@ use crowdb_access_iceberg::catalog::{ RoutedCatalogStore, }; use crowdb_access_iceberg::operation::{ManagementAction, ManagementRequest, RequestIdentity}; -use crowdb_access_iceberg::storage::{connect, foreground_blocks, IcebergLargeWriteSettings}; +use crowdb_access_iceberg::storage::{ + connect_with_chunk_kv_config, foreground_blocks, IcebergLargeWriteSettings, +}; use crowdb_access_iceberg::wire::BearerAuthenticator; use crowdb_access_s3::native_buffer::NativeBodyAllocator; use crowdb_chunk_client::ChunkIoClient; @@ -93,12 +95,13 @@ pub async fn run_with_shutdown( if arguments.len() > 7 { return Err("too many Iceberg command arguments".into()); } - let (repository, store, chunks) = connect( + let (repository, store, chunks) = connect_with_chunk_kv_config( config.management_seeds.clone(), access_config.read.policy(), access_config.iceberg_small_write().policy(), access_config.common.diskio_connections_per_endpoint, access_config.common.diskio_rpc_workers, + &access_config.common.chunk_kv_rpc, ) .await?; let result = if arguments.is_empty() || arguments == ["serve"] { @@ -245,12 +248,13 @@ async fn connect_gc_pool( if !enabled { return Ok((store, None)); } - let (_, gc_store, gc_chunks) = connect( + let (_, gc_store, gc_chunks) = connect_with_chunk_kv_config( management_seeds, access_config.read.policy(), access_config.iceberg_small_write().policy(), access_config.common.diskio_connections_per_endpoint, access_config.common.diskio_rpc_workers, + &access_config.common.chunk_kv_rpc, ) .await?; Ok((gc_store, Some(gc_chunks))) diff --git a/app/crowdb-access-server/src/main.rs b/app/crowdb-access-server/src/main.rs index 16d7a735b..35f61da20 100644 --- a/app/crowdb-access-server/src/main.rs +++ b/app/crowdb-access-server/src/main.rs @@ -165,11 +165,7 @@ async fn run_s3( .clone() .or_else(|| std::env::var("CROWDB_S3_LISTEN").ok()) { - let management_seeds = if access_config.common.management_seeds.is_empty() { - management_seeds()? - } else { - access_config.common.management_seeds.clone() - }; + let management_seeds = configured_management_seeds(access_config)?; let tenant_name = match &access_config.s3.tenant { Some(name) => name.clone(), None => required_env("CROWDB_S3_TENANT")?, @@ -179,12 +175,13 @@ async fn run_s3( let credential_cipher = Arc::new(CredentialCipher::new(&master_key)); let continuation_key = credential_cipher.continuation_key().to_vec(); let write_policies = s3_write_policies(access_config)?; - let storage = S3StorageClients::connect_with_read_policy( + let storage = S3StorageClients::connect_with_chunk_kv_config( management_seeds, access_config.common.diskio_connections_per_endpoint, access_config.common.diskio_rpc_workers, write_policies.small, access_config.read.policy(), + &access_config.common.chunk_kv_rpc, ) .await?; let chunks = Arc::clone(&storage.chunks); @@ -258,6 +255,15 @@ async fn run_s3( Ok(()) } +#[cfg(feature = "s3")] +fn configured_management_seeds(config: &AccessConfig) -> Result, Box> { + if config.common.management_seeds.is_empty() { + management_seeds() + } else { + Ok(config.common.management_seeds.clone()) + } +} + #[cfg(feature = "s3")] async fn bind_health_listener(access_config: &AccessConfig) -> std::io::Result { let address = access_config diff --git a/app/crowdb-access-server/tests/config_test.rs b/app/crowdb-access-server/tests/config_test.rs index 07c72b2db..5cd0a826b 100644 --- a/app/crowdb-access-server/tests/config_test.rs +++ b/app/crowdb-access-server/tests/config_test.rs @@ -6,6 +6,30 @@ use std::path::Path; use crowdb_access_server::config::{load_args, AccessConfig, SmallWriteConfig}; use crowdb_common::config::{load_from_file, BaseConfig}; +#[test] +fn all_access_frontends_share_the_serialized_chunk_kv_rpc_class_settings() { + let mut config = AccessConfig::default(); + let native = crowdb_rpc_ffi::RdmaConfig { + enabled: true, + ..crowdb_rpc_ffi::RdmaConfig::default() + }; + config.common.chunk_kv_rpc.metadata_rdma = + Some(native.clone().with_control_data_capacity(65_536).unwrap()); + config.common.chunk_kv_rpc.aggregate_rdma = Some(native.with_control_data_capacity(262_144).unwrap()); + config.validate().unwrap(); + let encoded = serde_json::to_vec(&config).unwrap(); + let decoded: AccessConfig = serde_json::from_slice(&encoded).unwrap(); + assert_eq!(decoded.common.chunk_kv_rpc, config.common.chunk_kv_rpc); + config + .common + .chunk_kv_rpc + .metadata_rdma + .as_mut() + .unwrap() + .wire_layout = crowdb_rpc_ffi::RdmaWireLayout::Contiguous; + assert!(config.validate().unwrap_err().contains("metadata_rdma")); +} + #[test] fn tracked_access_configs_load_and_set_bounded_read_resources() { let root = Path::new(env!("CARGO_MANIFEST_DIR")); diff --git a/app/crowdb-chunk-kv-server/src/config.rs b/app/crowdb-chunk-kv-server/src/config.rs index ae19dd380..ad44e2e38 100644 --- a/app/crowdb-chunk-kv-server/src/config.rs +++ b/app/crowdb-chunk-kv-server/src/config.rs @@ -39,6 +39,7 @@ pub struct ChunkKvServerConfig { pub max_split_catchup_lag_records: u64, pub shutdown_drain_timeout_ms: u64, pub rpc_workers: u32, + pub rpc_rdma: Option, pub storage: StorageConfig, pub bootstrap_partition: Option, pub monitor: DomainMonitorDescriptor, @@ -60,6 +61,7 @@ impl Default for ChunkKvServerConfig { max_split_catchup_lag_records: 1_024, shutdown_drain_timeout_ms: 30_000, rpc_workers: 2, + rpc_rdma: None, storage: StorageConfig::default(), bootstrap_partition: None, monitor: default_monitor(), @@ -130,6 +132,15 @@ impl ChunkKvServerConfig { )); } self.storage.validate()?; + if self + .rpc_rdma + .as_ref() + .is_some_and(|config| config.enabled && config.control_data_capacity().is_none()) + { + return Err(ConfigError::Invalid( + "rpc_rdma requires control_data with a supported data capacity".into(), + )); + } if let Some(bootstrap) = &self.bootstrap_partition { bootstrap.validate()?; } diff --git a/app/crowdb-chunk-kv-server/src/main.rs b/app/crowdb-chunk-kv-server/src/main.rs index bfed44c12..a8f2a3648 100644 --- a/app/crowdb-chunk-kv-server/src/main.rs +++ b/app/crowdb-chunk-kv-server/src/main.rs @@ -506,11 +506,13 @@ async fn main() -> std::process::ExitCode { } }); - let rpc_server = Arc::new(crowdb_rpc_ffi::RpcServer::with_engines( - None, - 1, - config.rpc_workers, - )); + let rpc_server = match ChunkKvRpcService::create_server(config.rpc_workers, config.rpc_rdma.as_ref()) { + Ok(server) => server, + Err(error) => { + error!(%error, "data RPC transport configuration failed"); + return std::process::ExitCode::FAILURE; + } + }; if let Err(error) = rpc_server.listen(&rpc_addr.ip().to_string(), i32::from(rpc_addr.port())) { error!(%rpc_addr, %error, "data RPC bind failed"); return std::process::ExitCode::FAILURE; diff --git a/app/crowdb-chunk-kv-server/src/rpc.rs b/app/crowdb-chunk-kv-server/src/rpc.rs index 8fab5ade7..da3f7ef17 100644 --- a/app/crowdb-chunk-kv-server/src/rpc.rs +++ b/app/crowdb-chunk-kv-server/src/rpc.rs @@ -18,13 +18,18 @@ use crowdb_protocol::chunk_kv_group_wire::{ use crowdb_protocol::chunk_kv_ordered_wire::{ decode_bounded_scan_request, decode_scan_request, decode_seek_request, }; -use crowdb_protocol::chunk_kv_wire::{decode_point_request, encode_point_response}; +use crowdb_protocol::chunk_kv_wire::decode_point_request; use crowdb_protocol::fb::FBMsgType; use crowdb_rpc_ffi::{Buffer, RpcServer, ServerRequest}; use tokio::runtime::Handle; use crate::ChunkKvService; +mod response; +mod transport; + +use response::{submit_group_response, submit_response}; + pub struct ChunkKvRpcService { service: Arc, runtime: Handle, @@ -74,7 +79,6 @@ impl ChunkKvRpcService { let server = Arc::clone(server); self.runtime.spawn(async move { let rpc_request_id = request.request_id; - let rpc_create_nano = request.rpc_create_nano; let response = match decode_point_request(request.control()) { Ok(envelope) if envelope.rpc_request_id == rpc_request_id => { service @@ -84,13 +88,7 @@ impl ChunkKvRpcService { Ok(_) => invalid_response("RPC frame and control request IDs differ"), Err(error) => invalid_response(&error.to_string()), }; - submit_response( - &server, - request.conn_handle, - rpc_request_id, - rpc_create_nano, - &response, - ); + submit_response(&server, &request, &response); }); } @@ -99,7 +97,6 @@ impl ChunkKvRpcService { let server = Arc::clone(server); self.runtime.spawn(async move { let rpc_request_id = request.request_id; - let rpc_create_nano = request.rpc_create_nano; let response = match decode_seek_request(request.control()) { Ok(envelope) if envelope.rpc_request_id == rpc_request_id => { service @@ -109,13 +106,7 @@ impl ChunkKvRpcService { Ok(_) => invalid_response("RPC frame and control request IDs differ"), Err(error) => invalid_response(&error.to_string()), }; - submit_response( - &server, - request.conn_handle, - rpc_request_id, - rpc_create_nano, - &response, - ); + submit_response(&server, &request, &response); }); } @@ -124,7 +115,6 @@ impl ChunkKvRpcService { let server = Arc::clone(server); self.runtime.spawn(async move { let rpc_request_id = request.request_id; - let rpc_create_nano = request.rpc_create_nano; let response = match decode_scan_request(request.control()) { Ok(envelope) if envelope.rpc_request_id == rpc_request_id => { service @@ -134,13 +124,7 @@ impl ChunkKvRpcService { Ok(_) => invalid_response("RPC frame and control request IDs differ"), Err(error) => invalid_response(&error.to_string()), }; - submit_response( - &server, - request.conn_handle, - rpc_request_id, - rpc_create_nano, - &response, - ); + submit_response(&server, &request, &response); }); } @@ -149,7 +133,6 @@ impl ChunkKvRpcService { let server = Arc::clone(server); self.runtime.spawn(async move { let id = request.request_id; - let created = request.rpc_create_nano; let response = match decode_bounded_scan_request(request.control()) { Ok((envelope, budget)) if envelope.rpc_request_id == id => { service @@ -159,7 +142,7 @@ impl ChunkKvRpcService { Ok(_) => invalid_response("RPC frame and control request IDs differ"), Err(error) => invalid_response(&error.to_string()), }; - submit_response(&server, request.conn_handle, id, created, &response); + submit_response(&server, &request, &response); }); } @@ -178,14 +161,17 @@ impl ChunkKvRpcService { Ok(_) => invalid_multi_get("RPC frame and control request IDs differ"), Err(error) => invalid_multi_get(&error.to_string()), }; - if let Ok(bytes) = encode_multi_get_response(id, create_nano, &response) { - submit_group_response( + match encode_multi_get_response(id, create_nano, &response) { + Ok(bytes) => submit_group_response( &server, - request.conn_handle, - id, + &request, Buffer::from_vec(bytes), FBMsgType::EChunkKvMultiGetResponse.0 as u16, - ); + ), + Err(error) => { + tracing::error!(request_id = id, %error, "Chunk-KV multi-get response encoding failed; closing connection"); + request.close_connection(); + } } }); } @@ -205,14 +191,17 @@ impl ChunkKvRpcService { Ok(_) => invalid_batch("RPC frame and control request IDs differ"), Err(error) => invalid_batch(&error.to_string()), }; - if let Ok(bytes) = encode_batch_mutation_response(id, create_nano, &response) { - submit_group_response( + match encode_batch_mutation_response(id, create_nano, &response) { + Ok(bytes) => submit_group_response( &server, - request.conn_handle, - id, + &request, Buffer::from_vec(bytes), FBMsgType::EChunkKvBatchMutationResponse.0 as u16, - ); + ), + Err(error) => { + tracing::error!(request_id = id, %error, "Chunk-KV batch response encoding failed; closing connection"); + request.close_connection(); + } } }); } @@ -256,38 +245,6 @@ fn invalid_response(message: &str) -> ChunkKvResponse { } } -fn submit_response( - server: &RpcServer, - conn_handle: *mut std::ffi::c_void, - rpc_request_id: u64, - rpc_create_nano: u64, - response: &ChunkKvResponse, -) { - let (bytes, offset) = encode_point_response(rpc_request_id, rpc_create_nano, response); - let buffer = Buffer::from_vec_offset(bytes, offset); - unsafe { - let _ = server.submit_response_buffer( - conn_handle, - buffer, - None, - FBMsgType::EChunkKvPointResponse.0 as u16, - rpc_request_id, - ); - } -} - -fn submit_group_response( - server: &RpcServer, - conn_handle: *mut std::ffi::c_void, - rpc_request_id: u64, - buffer: Buffer, - message_type: u16, -) { - unsafe { - let _ = server.submit_response_buffer(conn_handle, buffer, None, message_type, rpc_request_id); - } -} - fn wall_time_ms() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/app/crowdb-chunk-kv-server/src/rpc/response.rs b/app/crowdb-chunk-kv-server/src/rpc/response.rs new file mode 100644 index 000000000..1d6ed7fbc --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/rpc/response.rs @@ -0,0 +1,40 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::chunk_kv::ChunkKvResponse; +use crowdb_protocol::chunk_kv_wire::encode_point_response; +use crowdb_protocol::fb::FBMsgType; +use crowdb_rpc_ffi::{Buffer, RpcServer, ServerRequest}; + +pub(super) fn submit_response(server: &RpcServer, request: &ServerRequest, response: &ChunkKvResponse) { + let (bytes, offset) = encode_point_response(request.request_id, request.rpc_create_nano, response); + submit_group_response( + server, + request, + Buffer::from_vec_offset(bytes, offset), + FBMsgType::EChunkKvPointResponse.0 as u16, + ); +} + +pub(super) fn submit_group_response( + server: &RpcServer, + request: &ServerRequest, + buffer: Buffer, + message_type: u16, +) { + // SAFETY: request owns the native Frame and its retained connection; + // the borrowed server and response owner live through this submission. + let result = unsafe { + server.submit_response_buffer( + request.conn_handle, + buffer, + None, + message_type, + request.request_id, + ) + }; + if let Err(error) = result { + tracing::error!(request_id = request.request_id, %error, "Chunk-KV response submission failed; closing connection"); + request.close_connection(); + } +} diff --git a/app/crowdb-chunk-kv-server/src/rpc/transport.rs b/app/crowdb-chunk-kv-server/src/rpc/transport.rs new file mode 100644 index 000000000..056ee2f7b --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/rpc/transport.rs @@ -0,0 +1,30 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_rpc_ffi::{RdmaConfig, RdmaWireLayout, RpcError, RpcServer}; + +use super::ChunkKvRpcService; + +impl ChunkKvRpcService { + /// Create the service's aggregate-capacity listener before binding it. + /// + /// # Errors + /// Rejects invalid classes/queues or unavailable explicitly enabled RDMA. + pub fn create_server(workers: u32, rdma: Option<&RdmaConfig>) -> Result, RpcError> { + let server = if let Some(rdma) = rdma { + if rdma.enabled && rdma.control_data_capacity().is_none() { + return Err(RpcError::InvalidArg); + } + let server = RpcServer::with_engines(None, 1, workers.max(1)); + let mut native = rdma.clone(); + native.listen_enabled = native.enabled; + server.configure_rdma(&native)?; + server + } else { + RpcServer::with_rdma_layout(None, 1, workers.max(1), RdmaWireLayout::ControlData, 262_144)? + }; + Ok(Arc::new(server)) + } +} diff --git a/app/crowdb-chunk-kv-server/tests/config_test.rs b/app/crowdb-chunk-kv-server/tests/config_test.rs index 03292e9d7..b39d5906f 100644 --- a/app/crowdb-chunk-kv-server/tests/config_test.rs +++ b/app/crowdb-chunk-kv-server/tests/config_test.rs @@ -3,6 +3,37 @@ use crowdb_chunk_kv_server::{BalanceConfig, ChunkKvServerConfig, ConfigError}; +#[test] +fn rpc_rdma_settings_round_trip_and_require_a_stream_layout() { + let mut config = ChunkKvServerConfig { + instance_id: 1, + ..ChunkKvServerConfig::default() + }; + config.rpc_rdma = Some(crowdb_rpc_ffi::RdmaConfig { + enabled: true, + ..crowdb_rpc_ffi::RdmaConfig::default() + }); + assert!(config.validate().is_err()); + config.rpc_rdma = Some( + config + .rpc_rdma + .take() + .unwrap() + .with_control_data_capacity(262_144) + .unwrap(), + ); + config.validate().unwrap(); + let encoded = toml::to_string(&config).unwrap(); + assert_eq!(toml::from_str::(&encoded).unwrap(), config); + let server = crowdb_chunk_kv_server::ChunkKvRpcService::create_server( + 1, + Some(&crowdb_rpc_ffi::RdmaConfig::default()), + ) + .unwrap(); + let status: serde_json::Value = serde_json::from_slice(&server.transport_status_json().unwrap()).unwrap(); + assert!(status["rdma"].is_null()); +} + #[test] fn defaults_close_the_documented_timing_contract() { let mut config = ChunkKvServerConfig { diff --git a/doc/working/plan-rdma-chunk-kv-access.md b/doc/working/plan-rdma-chunk-kv-access.md index c486f2abf..db413e9ee 100644 --- a/doc/working/plan-rdma-chunk-kv-access.md +++ b/doc/working/plan-rdma-chunk-kv-access.md @@ -103,14 +103,44 @@ Observed contract: destruction before returning byte admission. - Record logical-message, fragment, copied-control, directional admission and timeout counters separately from direct application payload counters. -- [~] **Wire Chunk-KV classes and typed settings**: add service defaults and +- [x] **Wire Chunk-KV classes and typed settings**: add service defaults and explicit RDMA configuration for metadata/aggregate connection groups, preserve TCP defaults and current serializers/errors/retries, and wire the Chunk-KV server plus Access storage construction. Cover point, mutation, seek, bounded scan, multi-get and batch mutation, including duplicate result order and stable logical mutation IDs. Files: client transport/config, server startup/RPC boundary, Access storage config and tests. -- [ ] **Verify real service and clean up**: test malformed sequence/offset, + + Implementation order within this task: + - Serialize the existing Rust `RdmaConfig` for cold service configuration; + preserve its native defaults. Add an explicit + `with_control_data_capacity` helper that derives wire capacity and SQ/RQ + arena bytes from the selected data class and configured depths. + - Add `ChunkKvRpcTransportConfig` with optional `metadata_rdma` and + `aggregate_rdma` overrides. Missing overrides preserve environment policy + and select native 64 KiB/256 KiB service defaults. Explicit enabled + overrides require control/data and one of 64 KiB/256 KiB/1 MiB. Each + group retains its own existing connection index and one fixed transport; + the transport shares one `RpcClient`, reaper and request-ID generator. + - Split the current client transport runtime and endpoint parsing into + private child modules before extending the 337-line facade. Add read-only + class/endpoint status for actual path, QP and provider capacities. + - Configure the Chunk-KV server with a 256 KiB control/data service default + and optional explicit `rpc_rdma`. Server construction owns listener + policy; client groups are always non-listeners. Wire Access common + `chunk_kv_rpc` settings through S3, Iceberg and Dataset storage construction, + preserving existing convenience constructors with default configuration. + - Pin the native connection in Frames transferred to asynchronous FFI + handlers. A completed logical control has no RQ data lease to retain that + connection; the existing raw handler pointer otherwise outlives CM teardown. + Add safe `ServerRequest::close_connection` backed by that retained owner. + Close and report rejected Chunk-KV response submissions instead of silently + dropping a response. Cover handler hold/peer close/final release. + - Preserve the current serializers, stable logical mutation IDs, exact pool + generation invalidation and retry/error behavior. Cover class selection, + serialized settings, invalid bounds and existing client/server RPC cases + before the real service corpus. +- [~] **Verify real service and clean up**: test malformed sequence/offset, duplicate/end/generation/class, byte exhaustion, held final leases, cancellation/close/expiry, and TCP setup fallback. Run the actual Chunk-KV service on the HCA with metadata and results larger than 256 KiB, then two @@ -123,6 +153,22 @@ Observed contract: this completed child plan/detail/index entry after permanent docs/evidence are updated. R242 remains outside the requested scope. + Implementation order within this task: + - Run production `ChunkKvRpcService`, `CrowdbPartitionTree` and partition WAL + logic in a separate process. Inject only catalog/serving authority and the + memory ChunkStream backing; label that scope explicitly. Use the same + point/conditional/seek/scan/multi-get/batch corpus and stable mutation IDs + for strict RDMA, explicit TCP and pre-payload setup fallback. + - Reuse the canonical benchmark `Runtime` for native and two non-root OCI + processes. Record exact binaries, source, image, UID, actual connections, + classes, queues, copied-control bytes, directional admission and errors. + - Add a `test-util`-only native fragment gate. After the normal real-service + corpus, pause the response after exactly one posted control fragment and + close the server. Verify one fragment arrived, no logical result was + published, the caller fails promptly and TCP fallback remains zero. + - Repeat HTTP/DiskIO semantic profiles and matching 10-second CLI throughput + controls after the common RPC changes, then update permanent evidence. + ## Phase 1: access contract mapping - [x] **Map request/response types**: identify metadata, point, batch, range, @@ -268,6 +314,32 @@ CQ integration checkpoint: scoped C++ lint passed. Final RPC/XML and six HCA case results are under `.crowdb-runtime/rdma-check/artifacts/r241-control/`. +Chunk-KV class wiring checkpoint: + +- All 1,244 client/server and Access/S3/Iceberg/Dataset target cases passed + (three optional cases ignored), including + serialized class settings, point/scan pool separation, one global RPC ID + sequence and existing routing/retry/duplicate-result semantics. Logs: + `/tmp/r241-class-rust-tests.log`. +- Full C++/FFI gate passed at + `.crowdb-runtime/artifacts/unit/cpp-c2122fc1d6a14e7e9beb7959fe2ad245/`: + 543 tree, 190 RPC, 71 DiskIO and 35 common native cases plus both FFI suites. + No-verbs build passed 160 CTest cases (one hardware skip). The added C ABI + handler lifetime case proves a control-only Frame pins its connection until + final release; all eight Rust transport config cases passed. +- Workspace all-feature clippy passed. The first `rs-lint` invocation expired + while waiting for another Cargo build lock; rerun it without competing Cargo + work passed. Format and scoped C++ lint passed (existing style warnings). + All six real HCA control cases passed again at `r241-control/class-wiring.xml`. +- The first complete native gate failed the unchanged + `SqFullBackpressureTest.UringEngineTinySqManyWrites`: five callbacks returned + `-ENOMEM` at `sq_full_test.cpp:240`. Existing `DiskIOUring::submit_lockfree` + returns that error when its 1000 yield attempts cannot obtain an SQ slot. + The exact case, its 14-case suite and subsequent full native gate passed + without source changes. Preserve initial evidence at + `.crowdb-runtime/artifacts/unit/cpp-bdbe5707db274b8aa625247ce0eb00eb/`; + this task does not claim to repair that existing scheduling sensitivity. + ```bash pixi run -- cargo fmt --all -- --check pixi run -- cargo test -p crowdb-chunk-kv-client --all-targets diff --git a/lib/crowdb-access-iceberg/src/storage.rs b/lib/crowdb-access-iceberg/src/storage.rs index 24edddf67..9edec2727 100644 --- a/lib/crowdb-access-iceberg/src/storage.rs +++ b/lib/crowdb-access-iceberg/src/storage.rs @@ -11,7 +11,8 @@ use crowdb_chunk_client::{ FramedWriteBuffer, IoError, LargeWritePolicy, SmallWritePolicy, }; use crowdb_chunk_kv_client::{ - ChunkKvClient, ChunkKvRpcTransport, ClientConfig, Group0ChunkKvRangeCatalogSource, + ChunkKvClient, ChunkKvRpcTransport, ChunkKvRpcTransportConfig, ClientConfig, + Group0ChunkKvRangeCatalogSource, }; use crowdb_common::ec::EcScheme; use crowdb_kv_client::{ClientConfig as KvConfig, CrowdbKvClient}; @@ -228,21 +229,45 @@ impl IcebergLargeWriteSettings { /// # Errors /// Returns a discovery, catalog, or chunk-client configuration failure. pub async fn connect( + seeds: Vec, + read_policy: ChunkReadPolicy, + small_write: SmallWritePolicy, + diskio_connections_per_endpoint: usize, + diskio_rpc_workers: u32, +) -> Result<(Arc, Arc, ChunkIoClient), IcebergStorageError> { + connect_with_chunk_kv_config( + seeds, + read_policy, + small_write, + diskio_connections_per_endpoint, + diskio_rpc_workers, + &ChunkKvRpcTransportConfig::default(), + ) + .await +} + +/// Connects metadata through the configured fixed RPC data classes. +/// +/// # Errors +/// Returns a transport, discovery or storage configuration failure. +pub async fn connect_with_chunk_kv_config( seeds: Vec, read_policy: ChunkReadPolicy, mut small_write: SmallWritePolicy, diskio_connections_per_endpoint: usize, diskio_rpc_workers: u32, + chunk_kv_rpc: &ChunkKvRpcTransportConfig, ) -> Result<(Arc, Arc, ChunkIoClient), IcebergStorageError> { small_write.chunk_type = ChunkType::IcebergTable; let control = Arc::new(CrowdbKvClient::new(KvConfig::new(seeds.clone()))); let client_config = ClientConfig::default(); let source = Arc::new(Group0ChunkKvRangeCatalogSource::from_shared(Arc::clone(&control))); - let transport = Arc::new(ChunkKvRpcTransport::new( + let transport = Arc::new(ChunkKvRpcTransport::with_config( client_config.max_owner_connections, 1, 2, - )); + chunk_kv_rpc, + )?); let client = Arc::new(ChunkKvClient::new(client_config, source, transport)?); client.refresh_catalog().await?; let store = Arc::new(RoutedCatalogStore::new(client)); diff --git a/lib/crowdb-access-s3/src/storage.rs b/lib/crowdb-access-s3/src/storage.rs index 40fe5fadf..b878fd003 100644 --- a/lib/crowdb-access-s3/src/storage.rs +++ b/lib/crowdb-access-s3/src/storage.rs @@ -11,7 +11,8 @@ use crowdb_chunk_client::{ SmallWritePolicy, }; use crowdb_chunk_kv_client::{ - ChunkKvClient, ChunkKvRpcTransport, ClientConfig as ChunkKvConfig, Group0ChunkKvRangeCatalogSource, + ChunkKvClient, ChunkKvRpcTransport, ChunkKvRpcTransportConfig, ClientConfig as ChunkKvConfig, + Group0ChunkKvRangeCatalogSource, }; use crowdb_common::ec::EcScheme; use crowdb_kv_client::{ClientConfig as KvConfig, CrowdbKvClient}; @@ -156,17 +157,41 @@ impl S3StorageClients { /// # Errors /// Returns an error when the management or `DiskIO` connection cannot be established. pub async fn connect_with_read_policy( + management_seeds: Vec, + diskio_connections_per_endpoint: usize, + diskio_rpc_workers: u32, + small_write: SmallWritePolicy, + read_policy: ChunkReadPolicy, + ) -> Result { + Self::connect_with_chunk_kv_config( + management_seeds, + diskio_connections_per_endpoint, + diskio_rpc_workers, + small_write, + read_policy, + &ChunkKvRpcTransportConfig::default(), + ) + .await + } + + /// # Errors + /// Returns before readiness on invalid transport settings or discovery failure. + pub async fn connect_with_chunk_kv_config( management_seeds: Vec, diskio_connections_per_endpoint: usize, diskio_rpc_workers: u32, mut small_write: SmallWritePolicy, read_policy: ChunkReadPolicy, + chunk_kv_rpc: &ChunkKvRpcTransportConfig, ) -> Result { small_write.chunk_type = ChunkType::S3; let kv = Arc::new(CrowdbKvClient::new(KvConfig::new(management_seeds.clone()))); let config = ChunkKvConfig::default(); let catalog = Arc::new(Group0ChunkKvRangeCatalogSource::from_shared(Arc::clone(&kv))); - let transport = Arc::new(ChunkKvRpcTransport::new(config.max_owner_connections, 1, 2)); + let transport = Arc::new( + ChunkKvRpcTransport::with_config(config.max_owner_connections, 1, 2, chunk_kv_rpc) + .map_err(|error| StorageConnectError::ChunkKv(error.to_string()))?, + ); let metadata = ChunkKvClient::new(config, catalog, transport) .map_err(|error| StorageConnectError::ChunkKv(error.to_string()))?; metadata diff --git a/lib/crowdb-chunk-kv-client/Cargo.toml b/lib/crowdb-chunk-kv-client/Cargo.toml index b659eb1b3..24a2e2aec 100644 --- a/lib/crowdb-chunk-kv-client/Cargo.toml +++ b/lib/crowdb-chunk-kv-client/Cargo.toml @@ -21,6 +21,7 @@ clap = { version = "4", features = ["derive"] } futures = "0.3" rand = "0.8" serde_json = "1" +serde = { version = "1", features = ["derive"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } diff --git a/lib/crowdb-chunk-kv-client/src/lib.rs b/lib/crowdb-chunk-kv-client/src/lib.rs index c75704887..c4646710f 100644 --- a/lib/crowdb-chunk-kv-client/src/lib.rs +++ b/lib/crowdb-chunk-kv-client/src/lib.rs @@ -20,4 +20,4 @@ pub use compose::{ }; pub use config::ClientConfig; pub use error::{ClientError, Result}; -pub use transport::{ChunkKvRpcTransport, ChunkKvTransport}; +pub use transport::{ChunkKvRpcTransport, ChunkKvRpcTransportConfig, ChunkKvTransport}; diff --git a/lib/crowdb-chunk-kv-client/src/transport.rs b/lib/crowdb-chunk-kv-client/src/transport.rs index 680ea25a0..0fac0966e 100644 --- a/lib/crowdb-chunk-kv-client/src/transport.rs +++ b/lib/crowdb-chunk-kv-client/src/transport.rs @@ -1,9 +1,6 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; - use async_trait::async_trait; use crowdb_protocol::chunk_kv::{ BatchMutationRequest, BatchMutationResponse, ChunkKvResponse, MultiGetRequest, MultiGetResponse, @@ -16,11 +13,9 @@ use crowdb_protocol::chunk_kv_group_wire::{ use crowdb_protocol::chunk_kv_ordered_wire::{ encode_bounded_scan_request, encode_scan_request, encode_seek_request, }; -use crowdb_protocol::chunk_kv_wire::{decode_point_response, encode_point_request, ChunkKvWireError}; +use crowdb_protocol::chunk_kv_wire::{encode_point_request, ChunkKvWireError}; use crowdb_protocol::fb::FBMsgType; -use crowdb_rpc_ffi::{ - Buffer, ConnectionPoolError, ConnectionPoolIndex, RpcClient, RpcError, RpcServer, SelectedConnection, -}; +use crowdb_rpc_ffi::Buffer; use crate::{ClientError, Result}; @@ -90,101 +85,13 @@ pub trait ChunkKvTransport: Send + Sync { } } -/// Bounded production crowdb-rpc connection pool for direct owner calls. -pub struct ChunkKvRpcTransport { - server: Arc, - rpc: RpcClient, - connections: ConnectionPoolIndex, - next_rpc_request_id: AtomicU64, -} - -impl ChunkKvRpcTransport { - #[must_use] - pub fn new(max_owners: usize, pool_size: usize, workers: u32) -> Self { - let server = Arc::new(RpcServer::with_engines(None, 1, workers.max(1))); - server.start(); - server.register_conn_count_gauge("chunk_kv.client.connections"); - let rpc = RpcClient::new(); - rpc.set_completion_pool_size(1024); - rpc.start_reaper(30_000_000_000, 500_000_000); - Self { - server, - rpc, - connections: ConnectionPoolIndex::new(pool_size, Some(max_owners)), - next_rpc_request_id: AtomicU64::new(1), - } - } - - fn next_id(&self) -> Result { - let id = self.next_rpc_request_id.fetch_add(1, Ordering::Relaxed); - if id == 0 { - Err(ClientError::Transport("RPC request identity exhausted".into())) - } else { - Ok(id) - } - } - - fn connection(&self, endpoint: &str) -> Result { - let endpoint = normalize_endpoint(endpoint); - let (host, port) = parse_endpoint(&endpoint)?; - self.connections - .get_or_try_install(&endpoint, || { - let connection = self.server.connect(&host, port).map_err(|error| { - ClientError::Transport(format!("connect to {host}:{port}: {error:?}")) - })?; - self.rpc.attach(&connection); - Ok(connection) - }) - .map_err(|error| match error { - ConnectionPoolError::Closed => { - ClientError::Transport("connection closed during acquisition".into()) - } - ConnectionPoolError::Connect(error) => error, - ConnectionPoolError::Capacity { .. } => { - ClientError::Transport("owner connection limit reached".into()) - } - }) - } - - async fn call_control( - &self, - endpoint: &str, - request_id: u64, - control: Buffer, - message_type: u16, - ) -> Result { - let connection = self.connection(endpoint)?; - let normalized = normalize_endpoint(endpoint); - let map_error = |error: RpcError| { - if error.is_retryable() { - self.connections.invalidate(&normalized, connection.generation()); - } - ClientError::Transport(error.to_string()) - }; - let response = self - .rpc - .call(&self.server, &connection, request_id, control, None, message_type) - .map_err(&map_error)? - .await - .map_err(map_error)?; - response - .control - .ok_or_else(|| ClientError::Transport("RPC response omitted its control buffer".into())) - } +mod config; +mod endpoint; +mod rpc; +mod status; - async fn call_point_response( - &self, - endpoint: &str, - request_id: u64, - control: Buffer, - message_type: u16, - ) -> Result { - let response = self - .call_control(endpoint, request_id, control, message_type) - .await?; - decode_point_response(response.bytes()).map_err(|error| wire_error(&error)) - } -} +pub use config::ChunkKvRpcTransportConfig; +pub use rpc::ChunkKvRpcTransport; #[async_trait] impl ChunkKvTransport for ChunkKvRpcTransport { @@ -299,29 +206,6 @@ impl ChunkKvTransport for ChunkKvRpcTransport { } } -fn normalize_endpoint(endpoint: &str) -> String { - let endpoint = endpoint - .strip_prefix("http://") - .or_else(|| endpoint.strip_prefix("https://")) - .unwrap_or(endpoint); - endpoint.replacen("0.0.0.0:", "127.0.0.1:", 1) -} - -fn parse_endpoint(endpoint: &str) -> Result<(String, i32)> { - let (host, port) = endpoint - .rsplit_once(':') - .ok_or_else(|| ClientError::Transport(format!("invalid owner endpoint: {endpoint}")))?; - let port = port - .parse::() - .map_err(|_| ClientError::Transport(format!("invalid owner endpoint port: {endpoint}")))?; - if host.is_empty() || !(1..=u16::MAX.into()).contains(&port) { - return Err(ClientError::Transport(format!( - "invalid owner endpoint: {endpoint}" - ))); - } - Ok((host.into(), port)) -} - fn wall_time_ns() -> u64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/lib/crowdb-chunk-kv-client/src/transport/config.rs b/lib/crowdb-chunk-kv-client/src/transport/config.rs new file mode 100644 index 000000000..55c90c109 --- /dev/null +++ b/lib/crowdb-chunk-kv-client/src/transport/config.rs @@ -0,0 +1,38 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_rpc_ffi::RdmaConfig; +use serde::{Deserialize, Serialize}; + +use crate::{ClientError, Result}; + +/// Fixed metadata and aggregate connection groups sharing one RPC caller. +/// Missing overrides retain environment policy with 64 KiB/256 KiB defaults. +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct ChunkKvRpcTransportConfig { + pub metadata_rdma: Option, + pub aggregate_rdma: Option, +} + +impl ChunkKvRpcTransportConfig { + /// Validate the layouts required by encoded-control streams. + /// + /// # Errors + /// Enabled RDMA groups require a supported control/data class. + pub fn validate(&self) -> Result<()> { + for (name, config) in [ + ("metadata_rdma", &self.metadata_rdma), + ("aggregate_rdma", &self.aggregate_rdma), + ] { + if let Some(config) = config { + if config.enabled && config.control_data_capacity().is_none() { + return Err(ClientError::InvalidRequest(format!( + "{name} requires control_data with 64 KiB, 256 KiB or 1 MiB data capacity" + ))); + } + } + } + Ok(()) + } +} diff --git a/lib/crowdb-chunk-kv-client/src/transport/endpoint.rs b/lib/crowdb-chunk-kv-client/src/transport/endpoint.rs new file mode 100644 index 000000000..cbe5dde39 --- /dev/null +++ b/lib/crowdb-chunk-kv-client/src/transport/endpoint.rs @@ -0,0 +1,27 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::{ClientError, Result}; + +pub(super) fn normalize_endpoint(endpoint: &str) -> String { + let endpoint = endpoint + .strip_prefix("http://") + .or_else(|| endpoint.strip_prefix("https://")) + .unwrap_or(endpoint); + endpoint.replacen("0.0.0.0:", "127.0.0.1:", 1) +} + +pub(super) fn parse_endpoint(endpoint: &str) -> Result<(String, i32)> { + let (host, port) = endpoint + .rsplit_once(':') + .ok_or_else(|| ClientError::Transport(format!("invalid owner endpoint: {endpoint}")))?; + let port = port + .parse::() + .map_err(|_| ClientError::Transport(format!("invalid owner endpoint port: {endpoint}")))?; + if host.is_empty() || !(1..=u16::MAX.into()).contains(&port) { + return Err(ClientError::Transport(format!( + "invalid owner endpoint: {endpoint}" + ))); + } + Ok((host.into(), port)) +} diff --git a/lib/crowdb-chunk-kv-client/src/transport/rpc.rs b/lib/crowdb-chunk-kv-client/src/transport/rpc.rs new file mode 100644 index 000000000..4e9df2880 --- /dev/null +++ b/lib/crowdb-chunk-kv-client/src/transport/rpc.rs @@ -0,0 +1,207 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use crowdb_protocol::chunk_kv::ChunkKvResponse; +use crowdb_protocol::chunk_kv_wire::decode_point_response; +use crowdb_protocol::fb::FBMsgType; +use crowdb_rpc_ffi::{ + Buffer, ConnectionPoolError, ConnectionPoolIndex, RdmaConfig, RdmaWireLayout, RpcClient, RpcError, + RpcServer, SelectedConnection, +}; + +use super::config::ChunkKvRpcTransportConfig; +use super::endpoint::{normalize_endpoint, parse_endpoint}; +use crate::{ClientError, Result}; + +pub(super) struct RpcGroup { + pub(super) server: Arc, + pub(super) connections: ConnectionPoolIndex, + pub(super) data_capacity: Option, +} + +impl RpcGroup { + fn new( + max_owners: usize, + pool_size: usize, + workers: u32, + capacity: u32, + config: Option<&RdmaConfig>, + ) -> Result { + let server = if let Some(config) = config { + let server = RpcServer::with_engines(None, 1, workers.max(1)); + let mut native = config.clone(); + native.listen_enabled = false; + server.configure_rdma(&native).map_err(rpc_error)?; + server + } else { + RpcServer::with_rdma_layout(None, 1, workers.max(1), RdmaWireLayout::ControlData, capacity) + .map_err(rpc_error)? + }; + server.start(); + Ok(Self { + server: Arc::new(server), + connections: ConnectionPoolIndex::new(pool_size, Some(max_owners)), + data_capacity: config.map_or(Some(capacity), RdmaConfig::control_data_capacity), + }) + } + + fn connection(&self, endpoint: &str, rpc: &RpcClient) -> Result { + let (host, port) = parse_endpoint(endpoint)?; + self.connections + .get_or_try_install(endpoint, || { + let connection = self.server.connect(&host, port).map_err(|error| { + ClientError::Transport(format!("connect to {host}:{port}: {error:?}")) + })?; + rpc.attach(&connection); + Ok(connection) + }) + .map_err(|error| match error { + ConnectionPoolError::Closed => { + ClientError::Transport("connection closed during acquisition".into()) + } + ConnectionPoolError::Connect(error) => error, + ConnectionPoolError::Capacity { .. } => { + ClientError::Transport("owner connection limit reached".into()) + } + }) + } +} + +/// Fixed data-class transports sharing the existing RPC completion engine. +pub struct ChunkKvRpcTransport { + pub(super) metadata: RpcGroup, + pub(super) aggregate: RpcGroup, + pub(super) rpc: RpcClient, + next_rpc_request_id: AtomicU64, +} + +impl ChunkKvRpcTransport { + /// Create the fixed service layouts with environment transport policy. + /// + /// # Panics + /// Panics if the native engine cannot create the fixed service layouts. + /// `with_config` returns initialization failures to callers that need them. + #[must_use] + pub fn new(max_owners: usize, pool_size: usize, workers: u32) -> Self { + Self::with_config( + max_owners, + pool_size, + workers, + &ChunkKvRpcTransportConfig::default(), + ) + .expect("fixed Chunk-KV service data classes are valid") + } + + /// Build fixed class groups before publishing the client. + /// + /// # Errors + /// Rejects invalid class/queue/budget settings or unavailable explicit RDMA. + pub fn with_config( + max_owners: usize, + pool_size: usize, + workers: u32, + config: &ChunkKvRpcTransportConfig, + ) -> Result { + config.validate()?; + let metadata = RpcGroup::new( + max_owners, + pool_size, + workers, + 65_536, + config.metadata_rdma.as_ref(), + )?; + let aggregate = RpcGroup::new( + max_owners, + pool_size, + workers, + 262_144, + config.aggregate_rdma.as_ref(), + )?; + metadata + .server + .register_conn_count_gauge("chunk_kv.client.metadata.connections"); + aggregate + .server + .register_conn_count_gauge("chunk_kv.client.aggregate.connections"); + let rpc = RpcClient::new(); + rpc.set_completion_pool_size(1024); + rpc.start_reaper(30_000_000_000, 500_000_000); + Ok(Self { + metadata, + aggregate, + rpc, + next_rpc_request_id: AtomicU64::new(1), + }) + } + + pub(super) fn next_id(&self) -> Result { + let id = self.next_rpc_request_id.fetch_add(1, Ordering::Relaxed); + if id == 0 { + Err(ClientError::Transport("RPC request identity exhausted".into())) + } else { + Ok(id) + } + } + + fn group(&self, message_type: u16) -> &RpcGroup { + if message_type == FBMsgType::EChunkKvPointRequest.0 as u16 { + &self.metadata + } else { + &self.aggregate + } + } + + pub(super) async fn call_control( + &self, + endpoint: &str, + request_id: u64, + control: Buffer, + message_type: u16, + ) -> Result { + let group = self.group(message_type); + let normalized = normalize_endpoint(endpoint); + let connection = group.connection(&normalized, &self.rpc)?; + let map_error = |error: RpcError| { + if error.is_retryable() { + group.connections.invalidate(&normalized, connection.generation()); + } + rpc_error(error) + }; + let response = self + .rpc + .call( + &group.server, + &connection, + request_id, + control, + None, + message_type, + ) + .map_err(&map_error)? + .await + .map_err(map_error)?; + response + .control + .ok_or_else(|| ClientError::Transport("RPC response omitted its control buffer".into())) + } + + pub(super) async fn call_point_response( + &self, + endpoint: &str, + request_id: u64, + control: Buffer, + message_type: u16, + ) -> Result { + let response = self + .call_control(endpoint, request_id, control, message_type) + .await?; + decode_point_response(response.bytes()).map_err(|error| super::wire_error(&error)) + } +} + +fn rpc_error(error: RpcError) -> ClientError { + ClientError::Transport(error.to_string()) +} diff --git a/lib/crowdb-chunk-kv-client/src/transport/status.rs b/lib/crowdb-chunk-kv-client/src/transport/status.rs new file mode 100644 index 000000000..93533d41b --- /dev/null +++ b/lib/crowdb-chunk-kv-client/src/transport/status.rs @@ -0,0 +1,71 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use serde_json::{json, Value}; + +use super::endpoint::normalize_endpoint; +use super::rpc::RpcGroup; +use super::ChunkKvRpcTransport; +use crate::{ClientError, Result}; + +impl ChunkKvRpcTransport { + /// Owned class settings, native transport/error counters and pool counts. + /// + /// # Errors + /// Returns a native snapshot or JSON encoding failure. + pub fn transport_status_json(&self) -> Result> { + let counters = self.rpc.counters(); + serde_json::to_vec(&json!({ + "schema_version": 1, + "metadata": group_status(&self.metadata)?, + "aggregate": group_status(&self.aggregate)?, + "rpc_counters": { + "submit_fail": counters.submit_fail, "submit_retry": counters.submit_retry, + "resp_missed": counters.resp_missed, "reaped": counters.reaped, + }, + })) + .map_err(|error| ClientError::Transport(error.to_string())) + } + + /// Snapshot the first existing connection in each endpoint class group. + /// This diagnostic never opens a connection or changes the pool cursor. + /// + /// # Errors + /// Returns a native snapshot or JSON encoding failure. + pub fn connection_status_json(&self, endpoint: &str) -> Result> { + let endpoint = normalize_endpoint(endpoint); + serde_json::to_vec(&json!({ + "schema_version": 1, + "metadata": connection_status(&self.metadata, &endpoint)?, + "aggregate": connection_status(&self.aggregate, &endpoint)?, + })) + .map_err(|error| ClientError::Transport(error.to_string())) + } +} + +fn group_status(group: &RpcGroup) -> Result { + let native = group + .server + .transport_status_json() + .map_err(|error| ClientError::Transport(error.to_string()))?; + let native: Value = + serde_json::from_slice(&native).map_err(|error| ClientError::Transport(error.to_string()))?; + Ok( + json!({"data_capacity": group.data_capacity, "pool_size": group.connections.pool_size(), + "endpoint_count": group.connections.len(), "connection_count": group.connections.connection_count(), + "transport": native}), + ) +} + +fn connection_status(group: &RpcGroup, endpoint: &str) -> Result { + let Some(connection) = group.connections.get_first(endpoint) else { + return Ok(Value::Null); + }; + let native = connection + .transport_status_json() + .map_err(|error| ClientError::Transport(error.to_string()))?; + let mut native: Value = + serde_json::from_slice(&native).map_err(|error| ClientError::Transport(error.to_string()))?; + native["pool_generation"] = json!(connection.generation()); + Ok(native) +} diff --git a/lib/crowdb-chunk-kv-client/tests/transport_class_test.rs b/lib/crowdb-chunk-kv-client/tests/transport_class_test.rs new file mode 100644 index 000000000..a5a8ca72e --- /dev/null +++ b/lib/crowdb-chunk-kv-client/tests/transport_class_test.rs @@ -0,0 +1,128 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use crowdb_chunk_kv_client::{ChunkKvRpcTransport, ChunkKvRpcTransportConfig, ChunkKvTransport}; +use crowdb_protocol::chunk_kv::{ + ChunkKvResponse, ChunkKvRpcErrorCode, ClientRequestId, Id128, PointOperation, PointRequest, + RequestRouting, RpcFailure, ScanDirection, ScanRequest, +}; +use crowdb_protocol::chunk_kv_wire::encode_point_response; +use crowdb_protocol::fb::FBMsgType; +use crowdb_rpc_ffi::{RdmaConfig, RpcServer}; +use serde_json::Value; + +#[test] +fn explicit_class_overrides_validate_before_connect_and_keep_service_defaults() { + let transport = ChunkKvRpcTransport::new(2, 1, 1); + let status: Value = serde_json::from_slice(&transport.transport_status_json().unwrap()).unwrap(); + assert_eq!(status["metadata"]["data_capacity"], 65_536); + assert_eq!(status["aggregate"]["data_capacity"], 262_144); + let invalid = ChunkKvRpcTransportConfig { + metadata_rdma: Some(RdmaConfig { + enabled: true, + ..RdmaConfig::default() + }), + ..ChunkKvRpcTransportConfig::default() + }; + assert!(invalid.validate().is_err()); + assert!(ChunkKvRpcTransport::with_config(2, 1, 1, &invalid).is_err()); + let aggregate = RdmaConfig::default() + .with_control_data_capacity(1_048_576) + .unwrap(); + let config = ChunkKvRpcTransportConfig { + metadata_rdma: None, + aggregate_rdma: Some(aggregate), + }; + let encoded = serde_json::to_vec(&config).unwrap(); + assert_eq!( + serde_json::from_slice::(&encoded).unwrap(), + config + ); + let transport = ChunkKvRpcTransport::with_config(2, 1, 1, &config).unwrap(); + let status: Value = serde_json::from_slice(&transport.transport_status_json().unwrap()).unwrap(); + assert_eq!(status["aggregate"]["data_capacity"], 1_048_576); + assert!(status["aggregate"]["transport"]["rdma"].is_null()); +} + +#[tokio::test] +async fn point_and_scan_use_distinct_pools_and_one_correlated_rpc_identity_sequence() { + let server = RpcServer::new(None); + server.configure_rdma(&RdmaConfig::default()).unwrap(); + server.listen("127.0.0.1", 0).unwrap(); + let observed = Arc::new(AtomicU64::new(0)); + for kind in [FBMsgType::EChunkKvPointRequest, FBMsgType::EChunkKvScanRequest] { + let observed = observed.clone(); + server.register_reply_for_tests(u16::try_from(kind.0).unwrap(), move |request| { + observed.fetch_or(1 << request.request_id, Ordering::Relaxed); + let response = ChunkKvResponse { + map_revision: 4, + journal_position: None, + result: Err(RpcFailure { + code: ChunkKvRpcErrorCode::NotMyRange, + message: "test owner".into(), + retry_after_ms: None, + latest_map_revision: None, + owner_hint: None, + }), + }; + let (bytes, offset) = + encode_point_response(request.request_id, request.rpc_create_nano, &response); + (bytes[offset..].to_vec(), Vec::new()) + }); + } + server.start(); + let config = ChunkKvRpcTransportConfig { + metadata_rdma: Some(RdmaConfig::default().with_control_data_capacity(65_536).unwrap()), + aggregate_rdma: Some(RdmaConfig::default().with_control_data_capacity(262_144).unwrap()), + }; + let transport = ChunkKvRpcTransport::with_config(2, 1, 1, &config).unwrap(); + let endpoint = format!("http://0.0.0.0:{}", server.port()); + let routing = RequestRouting { + request_id: ClientRequestId { + client_instance_id: Id128 { high: 1, low: 2 }, + client_sequence: 3, + }, + map_revision: 4, + partition_id: Id128 { high: 5, low: 6 }, + owner_epoch: 7, + min_journal_position: None, + deadline_ms: None, + }; + let point = PointRequest { + routing: routing.clone(), + operation: PointOperation::Get { + key: b"object".to_vec(), + }, + }; + let scan = ScanRequest { + routing, + start: Some(Vec::new()), + end: None, + direction: ScanDirection::Forward, + limit: 10, + continuation: None, + }; + let (point, scan) = tokio::time::timeout(std::time::Duration::from_secs(5), async { + tokio::join!( + transport.point(&endpoint, &point), + transport.scan(&endpoint, &scan) + ) + }) + .await + .expect("class RPC responses did not complete"); + for response in [point.unwrap(), scan.unwrap()] { + assert_eq!(response.result.unwrap_err().code, ChunkKvRpcErrorCode::NotMyRange); + } + assert_eq!(observed.load(Ordering::Relaxed), (1 << 1) | (1 << 2)); + let status: Value = serde_json::from_slice(&transport.transport_status_json().unwrap()).unwrap(); + assert_eq!(status["metadata"]["connection_count"], 1); + assert_eq!(status["aggregate"]["connection_count"], 1); + let connections: Value = + serde_json::from_slice(&transport.connection_status_json(&endpoint).unwrap()).unwrap(); + assert_eq!(connections["metadata"]["path"], "tcp"); + assert_eq!(connections["aggregate"]["path"], "tcp"); + server.stop(); +} diff --git a/lib/crowdb-rpc/ffi/Cargo.toml b/lib/crowdb-rpc/ffi/Cargo.toml index c9c895a37..12d9052ef 100644 --- a/lib/crowdb-rpc/ffi/Cargo.toml +++ b/lib/crowdb-rpc/ffi/Cargo.toml @@ -23,6 +23,7 @@ arc-swap = "1.9" crowdb-common = { path = "../../crowdb-common/rust" } tokio = { version = "1.40", features = ["rt", "rt-multi-thread", "macros", "sync"] } bytes = "1" +serde = { version = "1", features = ["derive"] } ctor = { version = "1.0.13", optional = true } [dev-dependencies] diff --git a/lib/crowdb-rpc/ffi/src/server.rs b/lib/crowdb-rpc/ffi/src/server.rs index a2cd0bf6c..5856fb889 100644 --- a/lib/crowdb-rpc/ffi/src/server.rs +++ b/lib/crowdb-rpc/ffi/src/server.rs @@ -537,6 +537,13 @@ pub struct ServerRequest { } impl ServerRequest { + /// Fail this logical connection when a response cannot be submitted. + /// The native Frame retains the connection through this request's release. + pub fn close_connection(&self) { + // SAFETY: this wrapper owns the handler Frame for the duration of self. + unsafe { crate::sys::crowdb_rpc_frame_close_connection(self.frame_handle) }; + } + /// Borrow the control (flatbuffer) bytes. Valid as long as `self` /// is alive. #[must_use] diff --git a/lib/crowdb-rpc/ffi/src/sys.rs b/lib/crowdb-rpc/ffi/src/sys.rs index 16fe19485..c9a5b8c83 100644 --- a/lib/crowdb-rpc/ffi/src/sys.rs +++ b/lib/crowdb-rpc/ffi/src/sys.rs @@ -347,6 +347,7 @@ extern "C" { pub fn crowdb_rpc_server_register_echo_handler(server: crowdb_rpc_server_t, msg_type: u16); pub fn crowdb_rpc_frame_release(frame_handle: *mut c_void); + pub fn crowdb_rpc_frame_close_connection(frame_handle: *mut c_void); pub fn crowdb_rpc_server_register_handler( server: crowdb_rpc_server_t, diff --git a/lib/crowdb-rpc/ffi/src/transport.rs b/lib/crowdb-rpc/ffi/src/transport.rs index 41dcf615b..85934279b 100644 --- a/lib/crowdb-rpc/ffi/src/transport.rs +++ b/lib/crowdb-rpc/ffi/src/transport.rs @@ -6,10 +6,13 @@ use std::ffi::CString; use std::mem::MaybeUninit; +use serde::{Deserialize, Serialize}; + use crate::{sys, Buffer, Connection, RpcError, RpcServer}; /// Layout selected during TCP preflight and bound into the CM offer. -#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)] +#[derive(Debug, Default, Clone, Copy, Eq, PartialEq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] #[repr(u32)] pub enum RdmaWireLayout { #[default] @@ -18,7 +21,8 @@ pub enum RdmaWireLayout { } /// Explicit configuration overrides native environment defaults before listen. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq, PartialEq, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] #[allow(clippy::struct_excessive_bools)] pub struct RdmaConfig { pub enabled: bool, @@ -106,6 +110,35 @@ impl Default for RdmaConfig { } } +impl RdmaConfig { + /// Select a data class and derive arenas from the configured SQ/RQ window. + /// Existing `Default` and native service defaults remain unchanged. + /// + /// # Errors + /// Rejects classes outside 64 KiB/256 KiB/1 MiB and transport aggregation. + pub fn with_control_data_capacity(mut self, capacity: u32) -> Result { + if !matches!(capacity, 65_536 | 262_144 | 1_048_576) || self.aggregation_enabled { + return Err(RpcError::InvalidArg); + } + self.wire_layout = RdmaWireLayout::ControlData; + self.wire_slot_capacity = capacity + 1024; + self.send_arena_bytes = u64::from(self.sq_depth) * u64::from(capacity); + self.receive_arena_bytes = u64::from(self.posted_receive_target) * u64::from(self.wire_slot_capacity); + Ok(self) + } + + /// The selected control/data class, if it is a supported service class. + #[must_use] + pub fn control_data_capacity(&self) -> Option { + if self.wire_layout != RdmaWireLayout::ControlData { + return None; + } + self.wire_slot_capacity + .checked_sub(1024) + .filter(|capacity| matches!(capacity, 65_536 | 262_144 | 1_048_576)) + } +} + impl RpcServer { /// Configure before listen/start; strings are copied by the native layer. /// diff --git a/lib/crowdb-rpc/ffi/tests/transport_config_test.rs b/lib/crowdb-rpc/ffi/tests/transport_config_test.rs index 081ef46d7..6b13dfdfe 100644 --- a/lib/crowdb-rpc/ffi/tests/transport_config_test.rs +++ b/lib/crowdb-rpc/ffi/tests/transport_config_test.rs @@ -59,6 +59,37 @@ fn logical_control_limits_are_typed_and_bounded_in_both_directions() { server.configure_rdma(&config).unwrap(); } +#[test] +fn serialized_control_data_settings_derive_the_selected_sq_and_rq_windows() { + for capacity in [65_536, 262_144, 1_048_576] { + let config = RdmaConfig { + sq_depth: 8, + rq_depth: 4, + posted_receive_target: 4, + cq_depth: 16, + ..RdmaConfig::default() + } + .with_control_data_capacity(capacity) + .unwrap(); + assert_eq!(config.control_data_capacity(), Some(capacity)); + assert_eq!(config.wire_slot_capacity, capacity + 1024); + assert_eq!(config.send_arena_bytes, u64::from(capacity) * 8); + assert_eq!(config.receive_arena_bytes, u64::from(capacity + 1024) * 4); + let encoded = serde_json::to_value(&config).unwrap(); + assert_eq!(encoded["wire_layout"], "control_data"); + assert_eq!(serde_json::from_value::(encoded).unwrap(), config); + RpcServer::new(None).configure_rdma(&config).unwrap(); + } + assert!(RdmaConfig::default().with_control_data_capacity(4096).is_err()); + assert!(RdmaConfig { + aggregation_enabled: true, + ..RdmaConfig::default() + } + .with_control_data_capacity(65_536) + .is_err()); + assert!(serde_json::from_str::(r#"{"receive_queue_length": 1024}"#).is_err()); +} + #[test] fn service_layout_defaults_validate_capacity_and_preserve_explicit_tcp() { assert!(RpcServer::with_rdma_layout(None, 1, 1, RdmaWireLayout::ControlData, 4096).is_err()); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h index 5214b68f5..c39798644 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h @@ -359,6 +359,8 @@ typedef void (*crowdb_rpc_handler_release_fn)(void *user_data); // Release a frame_handle previously passed to a crowdb_rpc_handler_fn. // Must be called exactly once per frame_handle. Passing nullptr is a no-op. void crowdb_rpc_frame_release(void *frame_handle); +// Close the retained connection of a live handler Frame without releasing it. +void crowdb_rpc_frame_close_connection(void *frame_handle); // Register a custom dispatch callback for the given msg_type. The // callback is invoked for every incoming frame with that msg_type. This diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api_internal.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api_internal.h index dc6d1b8c0..07e1bb952 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api_internal.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api_internal.h @@ -43,12 +43,12 @@ namespace crowdb::rpc { // Internal helpers shared between RpcClient (slab path) and the C API -// adapter. Implemented in c_api.cpp where the crowdb_rpc_buffer_s concrete -// struct is defined. +// adapter. Implemented in the private C ABI frame module. // Convert a response Frame into C ABI buffer handles. Takes ownership of // the frame (deletes it). The data buffer is ref_clone'd so the handle -// owns its own reference. Control is nullptr (fields extracted during parse). +// owns its own reference. Budget-owned controls move their exact storage and +// admission; ordinary controls preserve their copied response handle. // On error (frame == nullptr), both handles are set to nullptr. void frame_to_c_handles(Frame *frame, crowdb_rpc_buffer_t *out_ctrl, crowdb_rpc_buffer_t *out_data); @@ -58,7 +58,7 @@ void frame_to_c_handles(Frame *frame, crowdb_rpc_buffer_t *out_ctrl, crowdb_rpc_ void invoke_c_complete(crowdb_rpc_on_complete cb, void *user_data, uint64_t request_id, Frame *frame, RpcError err); // Shared handler trampoline: extracts request fields from the frame, -// invokes the C dispatch callback, and deletes the frame. Used by both +// invokes the C dispatch callback, and transfers the retained frame. Used by both // server-side and client-side handler dispatch. The callback submits the // response later via crowdb_rpc_server_submit_response (async pattern). void invoke_c_handler(crowdb_rpc_handler_fn callback, void *user_data, Frame *request, Connection *conn); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/framing.h b/lib/crowdb-rpc/include/crowdb-rpc/framing.h index 0576f3151..d85ce5155 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/framing.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/framing.h @@ -14,6 +14,7 @@ namespace crowdb::rpc { +class Connection; // ── Wire format (14-byte header) ────────────────────────────────── // @@ -63,7 +64,9 @@ struct Frame // Optional logical-control admission retained through the final caller. // Storage is destroyed before admission wakes blocked receivers. std::shared_ptr control_owner; - std::vector control; + // Asynchronous FFI handlers retain their connection through Frame release. + std::shared_ptr connection_owner; + std::vector control; ~Frame() { diff --git a/lib/crowdb-rpc/src/c_api.cpp b/lib/crowdb-rpc/src/c_api.cpp index 530b57b50..5738d113f 100644 --- a/lib/crowdb-rpc/src/c_api.cpp +++ b/lib/crowdb-rpc/src/c_api.cpp @@ -672,6 +672,7 @@ void crowdb_rpc_server_register_echo_handler(crowdb_rpc_server_t server, uint16_ } + // ── Logging ─────────────────────────────────────────────────────── void crowdb_rpc_init_logging(const char *log_dir, const char *level, size_t max_file_mb, size_t max_files, diff --git a/lib/crowdb-rpc/src/c_api/frame.cpp b/lib/crowdb-rpc/src/c_api/frame.cpp index 1a57bfad8..0e4acda9d 100644 --- a/lib/crowdb-rpc/src/c_api/frame.cpp +++ b/lib/crowdb-rpc/src/c_api/frame.cpp @@ -53,4 +53,51 @@ void frame_to_c_handles(Frame *frame, crowdb_rpc_buffer_t *out_ctrl, crowdb_rpc_ *out_ctrl = wrap_control(*incoming); *out_data = wrap_data(incoming->data_buf); } + +void invoke_c_handler(crowdb_rpc_handler_fn callback, void *user_data, Frame *request, Connection *conn) +{ + if (callback == nullptr || request == nullptr) { + delete request; + return; + } + request->connection_owner = conn == nullptr ? nullptr : conn->weak_from_this().lock(); + uint64_t req_id = request->request_id; + uint64_t create_nano = request->rpc_create_nano; + uint16_t msg_type = request->header.msg_type; + const uint8_t *ctrl_ptr = request->control.empty() ? nullptr : request->control.data(); + auto ctrl_len = static_cast(request->control.size()); + const uint8_t *data_ptr = nullptr; + uint32_t data_len = 0; + if (request->data_buf != nullptr && request->data_buf->len > 0) { + data_ptr = request->data_buf->payload_data(); + data_len = request->data_buf->len; + } + void *conn_handle = static_cast(conn); + void *frame_handle = static_cast(request); + + // Transfer frame ownership to the callback. The callback must call + // crowdb_rpc_frame_release(frame_handle) when done. + callback(req_id, create_nano, msg_type, ctrl_ptr, ctrl_len, data_ptr, data_len, conn_handle, frame_handle, + user_data); +} + } // namespace crowdb::rpc + +extern "C" void crowdb_rpc_frame_release(void *frame_handle) +{ + delete static_cast(frame_handle); +} + +extern "C" void crowdb_rpc_frame_close_connection(void *frame_handle) +{ + try { + if (frame_handle != nullptr) { + auto connection = static_cast(frame_handle)->connection_owner; + if (connection != nullptr) { + connection->close(); + } + } + } + catch (...) { + } +} diff --git a/lib/crowdb-rpc/src/c_api/handlers.cpp b/lib/crowdb-rpc/src/c_api/handlers.cpp index 41f189078..0cc1432b9 100644 --- a/lib/crowdb-rpc/src/c_api/handlers.cpp +++ b/lib/crowdb-rpc/src/c_api/handlers.cpp @@ -17,49 +17,6 @@ static std::shared_ptr own_handler_context(void *user_data, crowdb_rpc_han // ── Custom handler dispatch ────────── -// Shared handler trampoline: extracts request fields from the frame, -// invokes the C dispatch callback, and transfers frame ownership to the -// callback. Used by both server-side (c_handler_trampoline) and client-side -// (RpcClient::dispatch_request) handler dispatch. The callback submits the -// response later via crowdb_rpc_server_submit_response (async pattern). -// -// The callback OWNS the frame via frame_handle — it must call -// crowdb_rpc_frame_release(frame_handle) when done. The dispatch layer does -// NOT delete the frame. -void crowdb::rpc::invoke_c_handler(crowdb_rpc_handler_fn callback, void *user_data, Frame *request, Connection *conn) -{ - if (callback == nullptr || request == nullptr) { - delete request; - return; - } - uint64_t req_id = request->request_id; - uint64_t create_nano = request->rpc_create_nano; - uint16_t msg_type = request->header.msg_type; - const uint8_t *ctrl_ptr = request->control.empty() ? nullptr : request->control.data(); - auto ctrl_len = static_cast(request->control.size()); - const uint8_t *data_ptr = nullptr; - uint32_t data_len = 0; - if (request->data_buf != nullptr && request->data_buf->len > 0) { - data_ptr = request->data_buf->payload_data(); - data_len = request->data_buf->len; - } - void *conn_handle = static_cast(conn); - void *frame_handle = static_cast(request); - - // Transfer frame ownership to the callback. The callback must call - // crowdb_rpc_frame_release(frame_handle) when done. - callback(req_id, create_nano, msg_type, ctrl_ptr, ctrl_len, data_ptr, data_len, conn_handle, frame_handle, - user_data); -} - -// Release a frame_handle passed to a crowdb_rpc_handler_fn callback. -void crowdb_rpc_frame_release(void *frame_handle) -{ - if (frame_handle != nullptr) { - delete static_cast(frame_handle); - } -} - void crowdb_rpc_server_register_handler(crowdb_rpc_server_t server, uint16_t msg_type, crowdb_rpc_handler_fn callback, void *user_data, crowdb_rpc_handler_release_fn release) { diff --git a/lib/crowdb-rpc/tests/c_api_handler_lifetime_test.cpp b/lib/crowdb-rpc/tests/c_api_handler_lifetime_test.cpp new file mode 100644 index 000000000..ff926d844 --- /dev/null +++ b/lib/crowdb-rpc/tests/c_api_handler_lifetime_test.cpp @@ -0,0 +1,40 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/c_api_internal.h" + +#include + +using namespace crowdb::rpc; + +TEST(CApiHandlerLifetimeTest, ControlOnlyRequestPinsConnectionUntilFinalFrameRelease) +{ + SystemBufferPool pool; + auto connection = std::make_shared(1, "handler", &pool); + std::weak_ptr lifetime = connection; + uint32_t closes = 0; + connection->set_on_close([&](Connection *) { ++closes; }); + + struct TestRequest + { + void *frame = nullptr; + void *connection = nullptr; + } held; + + const auto callback = [](uint64_t, uint64_t, uint16_t, const uint8_t *, uint32_t, const uint8_t *, uint32_t, + void *conn, void *frame, void *context) { + auto &request = *static_cast(context); + request.frame = frame; + request.connection = conn; + }; + invoke_c_handler(callback, &held, new Frame, connection.get()); + ASSERT_NE(held.frame, nullptr); + EXPECT_EQ(held.connection, connection.get()); + connection->close(); + connection.reset(); + EXPECT_FALSE(lifetime.expired()); + crowdb_rpc_frame_close_connection(held.frame); + EXPECT_EQ(closes, 1U); + crowdb_rpc_frame_release(held.frame); + EXPECT_TRUE(lifetime.expired()); +} From 057d26bfc4e07cd4b25d7b61abe08715f2403bec Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 06:22:55 +0800 Subject: [PATCH 051/258] Verify large Chunk-KV controls and keep staging slots reusable --- Cargo.lock | 2 + app/crowdb-chunk-kv-server/Cargo.toml | 2 + .../tests/common/rdma_chunk_kv.rs | 205 +++++++++++ .../tests/common/rdma_chunk_kv/corpus.rs | 324 ++++++++++++++++++ .../tests/common/rdma_chunk_kv/service.rs | 155 +++++++++ .../tests/rdma_rpc_test.rs | 17 + doc/design/rpc/design-crowdb-rpc-rdma.md | 115 ++++++- doc/working/plan-rdma-chunk-kv-access.md | 53 ++- lib/crowdb-rpc/ffi/build.rs | 3 + lib/crowdb-rpc/ffi/src/sys.rs | 5 + lib/crowdb-rpc/ffi/src/test_support.rs | 8 + lib/crowdb-rpc/include/crowdb-rpc/c_api.h | 4 + .../transport/rdma/rdma_transport.h | 8 + .../crowdb-rpc/transport/socket_transport.h | 9 + lib/crowdb-rpc/src/c_api/transport.cpp | 13 + .../src/transport/rdma/rdma_control_send.cpp | 11 +- .../tests/rdma_control_hardware_test.cpp | 31 ++ tools/rdma/run-chunk-kv.py | 168 +++++++++ 18 files changed, 1108 insertions(+), 25 deletions(-) create mode 100644 app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv.rs create mode 100644 app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv/corpus.rs create mode 100644 app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv/service.rs create mode 100644 app/crowdb-chunk-kv-server/tests/rdma_rpc_test.rs create mode 100644 tools/rdma/run-chunk-kv.py diff --git a/Cargo.lock b/Cargo.lock index bd30d2e87..ef1b7a699 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -841,6 +841,7 @@ dependencies = [ "clap", "crowdb-chunk-client", "crowdb-chunk-kv", + "crowdb-chunk-kv-client", "crowdb-chunk-kv-server", "crowdb-chunk-stream", "crowdb-common", @@ -852,6 +853,7 @@ dependencies = [ "crowdb-tree-ffi", "serde", "serde_json", + "sha2", "thiserror 2.0.18", "tokio", "toml", diff --git a/app/crowdb-chunk-kv-server/Cargo.toml b/app/crowdb-chunk-kv-server/Cargo.toml index d441698d8..706a65d1b 100644 --- a/app/crowdb-chunk-kv-server/Cargo.toml +++ b/app/crowdb-chunk-kv-server/Cargo.toml @@ -36,6 +36,8 @@ tracing = { workspace = true } [dev-dependencies] crowdb-test-harness = { path = "../../lib/crowdb-test-harness" } +crowdb-chunk-kv-client = { path = "../../lib/crowdb-chunk-kv-client" } +sha2 = "0.10" crowdb-kv = { path = "../../lib/crowdb-kv" } crowdb-chunk-kv-server = { path = ".", features = ["test-util"] } crowdb-chunk-kv = { path = "../../lib/crowdb-chunk-kv", features = ["test-util"] } diff --git a/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv.rs b/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv.rs new file mode 100644 index 000000000..4ed16f1c5 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv.rs @@ -0,0 +1,205 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use crowdb_chunk_kv_client::{ChunkKvRpcTransport, ChunkKvRpcTransportConfig, ChunkKvTransport}; +use crowdb_chunk_kv_server::ChunkKvRpcService; +use crowdb_rpc_ffi::{RdmaConfig, RpcServer}; +use serde_json::{json, Value}; + +#[path = "rdma_chunk_kv/corpus.rs"] +mod corpus; +#[path = "rdma_chunk_kv/service.rs"] +mod service; + +pub(super) struct TestChunkKvProfile { + endpoint: String, + output: PathBuf, + profile: String, +} + +impl TestChunkKvProfile { + pub(super) fn load() -> Self { + Self { + endpoint: std::env::var("CROWDB_RDMA_CHUNK_KV_ENDPOINT").unwrap(), + output: PathBuf::from(std::env::var("CROWDB_RDMA_CHUNK_KV_OUTPUT").unwrap()), + profile: std::env::var("CROWDB_RDMA_CHUNK_KV_PROFILE").unwrap(), + } + } + + fn config(&self, capacity: u32, server: bool) -> RdmaConfig { + RdmaConfig { + enabled: self.profile != "tcp" && !(server && self.profile == "fallback"), + allow_tcp_fallback: self.profile == "fallback", + listen_enabled: server, + cq_count: 1, + cq_depth: 16, + sq_depth: 2, + rq_depth: 2, + posted_receive_target: 2, + device: std::env::var("CROWDB_RDMA_DEVICE").unwrap_or_default(), + source_address: self.endpoint.rsplit_once(':').unwrap().0.into(), + ..RdmaConfig::default() + } + .with_control_data_capacity(capacity) + .unwrap() + } + + fn write(&self, name: &str, value: &Value) { + let path = self.output.join(name); + let temporary = path.with_extension("tmp"); + std::fs::write(&temporary, serde_json::to_vec_pretty(value).unwrap()).unwrap(); + std::fs::rename(temporary, path).unwrap(); + } + + fn status(server: &RpcServer) -> Value { + serde_json::from_slice(&server.transport_status_json().unwrap()).unwrap() + } + + pub(super) async fn server(&self) { + let service = service::create(&self.endpoint).await; + let handlers = Arc::new(ChunkKvRpcService::new( + service.clone(), + tokio::runtime::Handle::current(), + )); + let rpc = ChunkKvRpcService::create_server(1, Some(&self.config(262_144, true))).unwrap(); + let (host, port) = self.endpoint.rsplit_once(':').unwrap(); + rpc.listen(host, port.parse().unwrap()).unwrap(); + handlers.register_handlers(&rpc); + rpc.start(); + self.write("ready.json", &Self::status(&rpc)); + let mut tick = tokio::time::interval(Duration::from_millis(5)); + let mut lease_sequence = 1; + let mut next_renewal = tokio::time::Instant::now() + Duration::from_secs(2); + let mut fragment_limit = None; + while !self.output.join("stop").exists() { + tick.tick().await; + if tokio::time::Instant::now() >= next_renewal { + lease_sequence += 1; + service::renew(&service, lease_sequence); + next_renewal = tokio::time::Instant::now() + Duration::from_secs(2); + self.write("server-progress.json", &Self::status(&rpc)); + } + if fragment_limit.is_none() && self.output.join("arm-fault").exists() { + let before = Self::status(&rpc)["rdma"]["control_fragments_sent"] + .as_u64() + .unwrap(); + assert!(rpc.limit_control_fragments_for_tests(before + 1)); + fragment_limit = Some(before + 1); + self.write("fault-armed.json", &json!({"fragment_limit": before + 1})); + } + if let Some(limit) = fragment_limit { + let status = Self::status(&rpc); + if status["rdma"]["control_fragments_sent"].as_u64().unwrap() == limit { + self.write("fault-ready.json", &status); + } + } + } + self.write("server-final.json", &Self::status(&rpc)); + rpc.stop(); + } + + pub(super) async fn client(&self) { + let config = ChunkKvRpcTransportConfig { + metadata_rdma: Some(self.config(65_536, false)), + aggregate_rdma: Some(self.config(262_144, false)), + }; + let transport = ChunkKvRpcTransport::with_config(2, 1, 1, &config).unwrap(); + let mut result = match tokio::time::timeout( + Duration::from_secs(10), + corpus::run(&transport, &self.endpoint), + ) + .await + { + Ok(result) => result, + Err(error) => { + self.write("client-timeout.json", &json!({ + "transport": serde_json::from_slice::(&transport.transport_status_json().unwrap()).unwrap(), + "connections": serde_json::from_slice::(&transport.connection_status_json(&self.endpoint).unwrap()).unwrap(), + })); + panic!("real Chunk-KV corpus deadline: {error}"); + } + }; + let connections: Value = + serde_json::from_slice(&transport.connection_status_json(&self.endpoint).unwrap()).unwrap(); + let expected_path = if self.profile == "tcp" || self.profile == "fallback" { + "tcp" + } else { + "rdma" + }; + assert_eq!(connections["metadata"]["path"], expected_path); + assert_eq!(connections["aggregate"]["path"], expected_path); + result["connections"] = connections; + result["settings"] = serde_json::to_value(config).unwrap(); + result["expected_transport_errors"] = json!(0); + if self.profile == "disconnect" { + self.partial_response(&transport, &mut result).await; + } + result["transport"] = serde_json::from_slice(&transport.transport_status_json().unwrap()).unwrap(); + result["effective_uid"] = json!(std::fs::read_to_string("/proc/self/status") + .unwrap() + .lines() + .find(|line| line.starts_with("Uid:")) + .unwrap() + .split_whitespace() + .nth(2) + .unwrap()); + result["profile"] = json!(self.profile); + self.write("client.json", &result); + } + + async fn partial_response(&self, transport: &ChunkKvRpcTransport, result: &mut Value) { + std::fs::write(self.output.join("arm-fault"), "arm\n").unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + while !self.output.join("fault-armed.json").exists() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("server fault gate"); + let before: Value = serde_json::from_slice(&transport.transport_status_json().unwrap()).unwrap(); + let error = tokio::time::timeout( + Duration::from_secs(5), + self.wait_partial_close(transport, &before), + ) + .await + .expect("partial RDMA close did not complete") + .expect_err("incomplete logical response was published"); + let after: Value = serde_json::from_slice(&transport.transport_status_json().unwrap()).unwrap(); + let field = "control_fragments_received"; + let received = after["aggregate"]["transport"]["rdma"][field].as_u64().unwrap() + - before["aggregate"]["transport"]["rdma"][field].as_u64().unwrap(); + assert_eq!(received, 1, "fault must follow one actual response fragment"); + assert_eq!(after["aggregate"]["transport"]["rdma"]["fallback_count"], 0); + result["expected_transport_errors"] = json!(1); + result["partial_response"] = + json!({"received_fragments": received, "error": error.to_string(), "published_results": 0}); + } + + async fn wait_partial_close( + &self, + transport: &ChunkKvRpcTransport, + before: &Value, + ) -> crowdb_chunk_kv_client::Result { + let scan = corpus::scan(); + let response = transport.scan(&self.endpoint, &scan); + tokio::pin!(response); + let mut tick = tokio::time::interval(Duration::from_millis(5)); + loop { + tokio::select! { + result = &mut response => return result, + _ = tick.tick() => { + let current: Value = serde_json::from_slice(&transport.transport_status_json().unwrap()).unwrap(); + let received = current["aggregate"]["transport"]["rdma"]["control_fragments_received"].as_u64().unwrap() + - before["aggregate"]["transport"]["rdma"]["control_fragments_received"].as_u64().unwrap(); + if received == 1 { + self.write("client-partial-ready.json", ¤t); + } + } + } + } + } +} diff --git a/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv/corpus.rs b/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv/corpus.rs new file mode 100644 index 000000000..9275e2e1e --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv/corpus.rs @@ -0,0 +1,324 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunk_kv_client::{ChunkKvRpcTransport, ChunkKvTransport}; +use crowdb_protocol::chunk_kv::{ + BatchMutationItem, BatchMutationRequest, ChunkKvResponse, ChunkKvRpcErrorCode, ClientRequestId, Id128, + MultiGetRequest, OperationResult, PartitionRouting, PointOperation, PointRequest, RequestRouting, + RpcCompareCondition, RpcValue, ScanDirection, ScanRequest, SeekKind, SeekRequest, +}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; + +use super::service::{EPOCH, PARTITION}; + +pub(super) fn routing(sequence: u64) -> RequestRouting { + RequestRouting { + request_id: ClientRequestId { + client_instance_id: Id128 { high: 20, low: 21 }, + client_sequence: sequence, + }, + map_revision: 1, + partition_id: PARTITION, + owner_epoch: EPOCH, + min_journal_position: None, + deadline_ms: None, + } +} + +fn records() -> Vec<(Vec, Vec)> { + [ + 63 * 1024, + 65_553, + 255 * 1024, + 262_175, + 768 * 1024, + 1_048_583, + 300 * 1024, + 600 * 1024, + ] + .into_iter() + .enumerate() + .map(|(index, size)| { + let key = format!("k{index:02}").into_bytes(); + let value = (0..size) + .map(|offset| u8::try_from(offset % 251).unwrap() ^ u8::try_from(index).unwrap()) + .collect(); + (key, value) + }) + .collect() +} + +fn values(response: ChunkKvResponse) -> (Vec, Option) { + match response.result.unwrap() { + OperationResult::Scan { items, continuation } => (items, continuation), + other => panic!("expected scan response: {other:?}"), + } +} + +pub(super) fn scan() -> ScanRequest { + ScanRequest { + routing: routing(300), + start: None, + end: None, + direction: ScanDirection::Forward, + limit: 100, + continuation: None, + } +} + +pub(super) async fn run(transport: &ChunkKvRpcTransport, endpoint: &str) -> Value { + let records = records(); + let results = populate(transport, endpoint, &records).await; + check_conditions(transport, endpoint, &records[0]).await; + let mut expected = Vec::new(); + for (index, (key, value)) in records.iter().enumerate() { + let response = transport + .point( + endpoint, + &PointRequest { + routing: routing(50 + u64::try_from(index).unwrap()), + operation: PointOperation::Get { key: key.clone() }, + }, + ) + .await + .unwrap(); + match response.result.unwrap() { + OperationResult::Value(Some(found)) => { + assert_eq!(&found.key, key); + assert_eq!(&found.value, value); + expected.push(found); + } + other => panic!("expected stored value: {other:?}"), + } + } + check_groups(transport, endpoint, &expected).await; + let error = transport + .point( + endpoint, + &PointRequest { + routing: RequestRouting { + deadline_ms: Some(1), + ..routing(200) + }, + operation: PointOperation::Get { + key: records[0].0.clone(), + }, + }, + ) + .await + .unwrap() + .result + .unwrap_err(); + assert_eq!(error.code, ChunkKvRpcErrorCode::RequestExpired); + let mut stale = scan(); + stale.continuation = Some(crowdb_protocol::chunk_kv::ScanContinuation { + direction: ScanDirection::Forward, + last_key: b"k00".to_vec(), + partition_id: PARTITION, + owner_epoch: EPOCH + 1, + map_revision: 1, + }); + let stale = transport.scan(endpoint, &stale).await.unwrap().result; + assert!(stale.is_err(), "stale continuation must not produce a scan"); + assert_eq!(stale.unwrap_err().code, ChunkKvRpcErrorCode::RefreshRequired); + json!({ + "records": records.len(), "payload_bytes": records.iter().map(|(_, v)| v.len()).sum::(), + "values_sha256": format!("{:x}", Sha256::digest(serde_json::to_vec(&expected).unwrap())), + "mutation_results_sha256": format!("{:x}", Sha256::digest(serde_json::to_vec(&results).unwrap())), + "expected_typed_errors": 2, "typed_errors": [error.code, ChunkKvRpcErrorCode::RefreshRequired], "logical_mutation_sequences": (1..=8).collect::>(), + "errors": 0, "verification_errors": 0, + }) +} + +async fn populate( + transport: &ChunkKvRpcTransport, + endpoint: &str, + records: &[(Vec, Vec)], +) -> Vec { + let mut results = Vec::new(); + for (index, (key, value)) in records.iter().take(6).enumerate() { + let request = PointRequest { + routing: routing(u64::try_from(index).unwrap() + 1), + operation: PointOperation::Put { + key: key.clone(), + value: value.clone(), + }, + }; + let result = transport.point(endpoint, &request).await.unwrap(); + assert!(matches!( + result.result, + Ok(OperationResult::Mutation { applied: true, .. }) + )); + assert_eq!( + transport.point(endpoint, &request).await.unwrap(), + result, + "stable mutation identity must deduplicate" + ); + results.push(serde_json::to_value(result).unwrap()); + } + let batch = BatchMutationRequest { + routing: PartitionRouting { + map_revision: 1, + partition_id: PARTITION, + owner_epoch: EPOCH, + deadline_ms: None, + }, + operations: records + .iter() + .enumerate() + .skip(6) + .map(|(index, (key, value))| BatchMutationItem { + request_id: routing(u64::try_from(index).unwrap() + 1).request_id, + operation: PointOperation::Put { + key: key.clone(), + value: value.clone(), + }, + }) + .collect(), + }; + let result = transport.batch_mutate(endpoint, &batch).await.unwrap(); + assert_eq!(transport.batch_mutate(endpoint, &batch).await.unwrap(), result); + for (item, request) in result.result.as_ref().unwrap().iter().zip(&batch.operations) { + assert_eq!(item.request_id, request.request_id); + assert!(matches!( + item.result, + Ok(OperationResult::Mutation { applied: true, .. }) + )); + } + results.push(serde_json::to_value(result).unwrap()); + results +} + +async fn check_conditions(transport: &ChunkKvRpcTransport, endpoint: &str, record: &(Vec, Vec)) { + let response = transport + .point( + endpoint, + &PointRequest { + routing: routing(100), + operation: PointOperation::PutIfAbsent { + key: record.0.clone(), + value: b"wrong".to_vec(), + }, + }, + ) + .await + .unwrap(); + assert!( + matches!(response.result, Ok(OperationResult::Mutation { applied: false, observed: Some(ref v), .. }) if v.value == record.1) + ); + let response = transport + .point( + endpoint, + &PointRequest { + routing: routing(101), + operation: PointOperation::CompareExchange { + key: record.0.clone(), + condition: RpcCompareCondition::Value(b"wrong".to_vec()), + value: b"wrong".to_vec(), + }, + }, + ) + .await + .unwrap(); + assert!(matches!( + response.result, + Ok(OperationResult::Mutation { applied: false, .. }) + )); + for (sequence, operation) in [ + ( + 102, + PointOperation::Put { + key: b"temp".to_vec(), + value: b"value".to_vec(), + }, + ), + ( + 103, + PointOperation::ConditionalDelete { + key: b"temp".to_vec(), + condition: RpcCompareCondition::Value(b"value".to_vec()), + }, + ), + ( + 104, + PointOperation::Delete { + key: b"missing".to_vec(), + }, + ), + ] { + let response = transport + .point( + endpoint, + &PointRequest { + routing: routing(sequence), + operation, + }, + ) + .await + .unwrap(); + assert!(matches!(response.result, Ok(OperationResult::Mutation { .. }))); + } +} + +async fn check_groups(transport: &ChunkKvRpcTransport, endpoint: &str, expected: &[RpcValue]) { + let indices = [7, 0, 7, 2]; + let request = MultiGetRequest { + routing: routing(210), + keys: indices.iter().map(|&i| expected[i].key.clone()).collect(), + }; + let result = transport + .multi_get(endpoint, &request) + .await + .unwrap() + .result + .unwrap(); + assert_eq!( + result, + indices + .iter() + .map(|&i| Some(expected[i].clone())) + .collect::>() + ); + let seek = SeekRequest { + routing: routing(211), + key: b"k03".to_vec(), + kind: SeekKind::Ceiling, + }; + assert_eq!( + transport.seek(endpoint, &seek).await.unwrap().result.unwrap(), + OperationResult::Value(Some(expected[3].clone())) + ); + let (items, continuation) = values(transport.scan(endpoint, &scan()).await.unwrap()); + assert_eq!(items, expected); + assert!(continuation.is_none()); + let mut reverse = scan(); + reverse.direction = ScanDirection::Reverse; + assert_eq!( + values(transport.scan(endpoint, &reverse).await.unwrap()).0, + expected.iter().rev().cloned().collect::>() + ); + let mut page = scan(); + page.limit = 2; + let mut paged = Vec::new(); + loop { + let (items, continuation) = values(transport.scan(endpoint, &page).await.unwrap()); + assert!(items.len() <= 2); + paged.extend(items); + if continuation.is_none() { + break; + } + page.continuation = continuation; + } + assert_eq!(paged, expected); + let (bounded, continuation) = values( + transport + .scan_bounded(endpoint, &scan(), 600 * 1024) + .await + .unwrap(), + ); + assert!(!bounded.is_empty()); + assert!(bounded.len() < expected.len()); + assert!(continuation.is_some()); + assert_eq!(bounded, expected[..bounded.len()]); +} diff --git a/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv/service.rs b/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv/service.rs new file mode 100644 index 000000000..ec0a0ffff --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv/service.rs @@ -0,0 +1,155 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crowdb_chunk_kv::{ + CrowdbPartitionTree, Partition, PartitionConfig, PartitionId, PartitionJournal, PartitionRange, + PartitionTree, StreamPartitionJournal, +}; +use crowdb_chunk_kv_server::ChunkKvService; +use crowdb_chunk_stream::memory::MemoryStreamStore; +use crowdb_chunk_stream::{ + ChunkStream, StreamBinding, StreamBindingState, StreamChunkStore, StreamConfig, StreamMetadataStore, + StreamName, StreamRegistry, +}; +use crowdb_protocol::chunk_kv::{ + ChunkKvRangeBalancePolicy, ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, + ChunkKvRangeCatalogPageRef, ChunkKvRangeCatalogPartitionState, DomainFailurePolicy, + DomainMonitorDescriptor, Id128, KeyRange, OwnerDescriptor, PartitionArtifact, ServingAssignment, + ServingGrant, +}; + +pub(super) const PARTITION: Id128 = Id128 { high: 1, low: 2 }; +pub(super) const EPOCH: u64 = 4; + +pub(super) async fn create(endpoint: &str) -> Arc { + let stream_name = StreamName { high: 30, low: 31 }; + let store = Arc::new(MemoryStreamStore::new(32 << 20)); + let stream = ChunkStream::create( + StreamBinding { + purpose: crowdb_protocol::chunk_stream::StreamPurpose::Wal, + stream_name, + metadata_group_id: 7, + binding_generation: 1, + state: StreamBindingState::Active, + owner_kind: Some("chunk-kv-partition".into()), + }, + EPOCH, + StreamConfig::default(), + store.clone() as Arc, + store.clone() as Arc, + store as Arc, + ) + .await + .unwrap(); + let tree: Arc = Arc::new( + CrowdbPartitionTree::open( + 1, + &crowdb_tree_ffi::Config { + buffer_pool_bytes: 64 << 20, + ..Default::default() + }, + ) + .unwrap(), + ); + let journal: Arc = + Arc::new(StreamPartitionJournal::new(stream, stream_name).unwrap()); + let partition = Partition::open( + PartitionId { high: 1, low: 2 }, + PartitionRange { + start: Some(Vec::new()), + end: None, + }, + EPOCH, + PartitionConfig::default(), + tree, + journal, + ) + .unwrap(); + let service = Arc::new(ChunkKvService::new(7, 8).unwrap()); + let (head, page) = catalog(endpoint, stream_name); + service + .install_catalog_and_reconcile(&head, &[page], &[partition]) + .unwrap(); + renew(&service, 1); + service +} + +fn catalog(endpoint: &str, stream_name: StreamName) -> (ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage) { + let mut page = ChunkKvRangeCatalogPage { + generation: 1, + page_index: 0, + entries: vec![ChunkKvRangeCatalogEntry { + partition_id: PARTITION, + range: KeyRange { + start: Vec::new(), + end: None, + }, + owner: OwnerDescriptor { + instance_id: 7, + rpc_endpoint: endpoint.into(), + }, + owner_epoch: EPOCH, + state: ChunkKvRangeCatalogPartitionState::Serving, + artifact: PartitionArtifact { + tree_id: 1, + stream_name, + tail_overlay: None, + }, + transition_id: None, + }], + checksum: [0; 32], + }; + page.seal().unwrap(); + let mut head = ChunkKvRangeCatalogHead { + generation: 1, + previous_generation: None, + pages: vec![ChunkKvRangeCatalogPageRef { + page_generation: 1, + page_index: 0, + first_key: Vec::new(), + page_checksum: page.checksum, + }], + checksum: [0; 32], + }; + head.seal().unwrap(); + (head, page) +} + +pub(super) fn renew(service: &ChunkKvService, sequence: u64) { + let policy = DomainMonitorDescriptor { + domain: "chunk-kv".into(), + service_registry_name: "chunk-kv".into(), + driver_version: 1, + capability_version: 1, + heartbeat_interval_ms: 2_000, + suspect_after_ms: 6_000, + dead_after_ms: 10_000, + lease_duration_ms: 12_000, + max_clock_skew_ms: 1_000, + self_fence_margin_ms: 1_000, + failure_policy: DomainFailurePolicy::AutomaticSharedStorage, + balance_policy: "count-first-v1".into(), + chunk_kv_range_balance: Some(ChunkKvRangeBalancePolicy::default()), + }; + let now = u64::try_from(SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis()).unwrap(); + let mut grant = ServingGrant { + instance_id: 7, + lease_sequence: sequence, + catalog_generation: 1, + issued_at_ms: now, + expires_at_ms: now + policy.lease_duration_ms, + assignments: vec![ServingAssignment { + partition_id: PARTITION, + owner_epoch: EPOCH, + }], + assignment_digest: [0; 32], + }; + grant.seal(); + service + .authority() + .install(grant, &policy, now, service.monotonic_ms()) + .unwrap(); +} diff --git a/app/crowdb-chunk-kv-server/tests/rdma_rpc_test.rs b/app/crowdb-chunk-kv-server/tests/rdma_rpc_test.rs new file mode 100644 index 000000000..7d3c06fd4 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/rdma_rpc_test.rs @@ -0,0 +1,17 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/rdma_chunk_kv.rs"] +mod fixture; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "run through tools/rdma/run-chunk-kv.py with explicit transport and topology"] +async fn real_chunk_kv_service_preserves_large_controls_and_typed_results() { + crowdb_rpc_ffi::init_test_logging(); + let profile = fixture::TestChunkKvProfile::load(); + match std::env::var("CROWDB_RDMA_CHUNK_KV_ROLE").unwrap().as_str() { + "server" => profile.server().await, + "client" => profile.client().await, + other => panic!("invalid Chunk-KV test role: {other}"), + } +} diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index 264c53ee9..0de939732 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -49,13 +49,15 @@ version, and supported data classes. The client verifies that the generation and peer identity are current before creating the RDMA connection. `SocketTransport` performs this control exchange on a temporary nonblocking -TCP socket with one deadline. The 160-byte version 3 capability advertises -receive slot capacity, the maximum number of frames accepted in one SEND and -`wire_layout`. Older capability versions can be decoded for pre-payload TCP +TCP socket with one deadline. The 160-byte version 4 capability advertises +receive slot capacity, the maximum number of frames accepted in one SEND, +`wire_layout` and `max_control_message_bytes`. Older capability versions can be decoded for pre-payload TCP fallback; an RDMA connection requires the current protocol and matching layout. -A 48-byte CM offer +A 52-byte CM offer echoes the expected peer generation and identity digest, and declares the -sender's generation, digest, layout and receive admission. Both CM sides validate it +sender's generation, digest, layout, receive admission and maximum logical +control length. The explicit limit is bound into the digest and validated +against TCP capability when available. Both CM sides validate it before publishing the logical connection. The digest detects inconsistent or stale discovery; it does not authenticate a peer. Low-level CM diagnostics may omit the TCP expectation but still negotiate receive admission. @@ -91,10 +93,9 @@ The common header carries the control and payload lengths. The complete wire length must fit `min(aggregation_max_bytes, wire_slot_capacity, peer_receive_slot_capacity)`, including every frame's header and control. `aggregation_max_frames` and the peer's advertised frame limit also bound each -SEND; disabled aggregation sends one frame. Oversized individual frames fail -admission without fragmentation or fallback. A logical result larger than the -selected capacity is an ordered sequence identified by request ID, stream ID, -sequence, logical offset, and START/DATA/END/ERROR flags. +SEND; disabled aggregation sends one frame. Oversized individual frames in contiguous layout fail +admission without fragmentation or fallback. Control/data layout also carries +bounded streams of encoded logical control bytes. In the default `Contiguous` layout, each SEND uses one SGE pointing at the slot and the actual wire length. Each RECV is posted with one SGE pointing at a complete receive slot and its full @@ -127,6 +128,55 @@ uses a bounded copy into the device arena. Per-CQ `direct_payload_bytes` and using this layout. A failed SEND quarantines the canonical source allocation, including when the WR retains a nested view. +### 3.1. Encoded logical control streams + +A control larger than the 1010-byte FlatBuffer prefix uses transport message +`ERpcControlFragment`. Each fragment has the common correlation fields plus +stream ID, transport generation, sequence, exact logical offset and total, +negotiated data capacity, original message type/flags and START/END/ERROR flags. +Its data SGE contains the next encoded-control bytes. The original application +message has no separate data payload; ordinary small control plus direct data +keeps its existing path. + +The sender admits one original encoding under `control_send_budget_bytes` and +the logical queue count. Its CQ owner walks one cursor, stages only the next +fragment and completes that cursor before taking the next logical frame on the +QP. Every posted WR retains the original source until its own completion. +Control staging descriptors use the configured `data_slot_capacity()` even +when the negotiated capacity or final fragment length is smaller. This keeps +the fixed SQ descriptor window reusable across mixed sizes and peer classes. +SEND carries only the actual fragment length. Rejected queue publication +restores the caller's original control ownership. + +QP data capacity is `min(local_data_capacity, peer_data_capacity)`. Active CM +setup allocates and primes receives after validating the peer reply; passive +setup already has the offer. Both actual RX allocation/posting and SEND +validation use this minimum, which immutable connection status exposes as +`effective_data_capacity`. + +The receiver reserves the complete logical length under +`control_receive_budget_bytes` before assembly. One active message per QP +validates generation, identity, sequence, offset, total, negotiated capacity +and final flag. It copies each fragment once into one logical control owner +and releases the consumed RQ slot immediately. A result may exceed the entire +RQ window. Only a complete validated original Frame reaches the existing +application handler or RPC completion table. + +Admission exhaustion retains only existing RQ owners in CQ FIFO order. It does +not carve replacement buffers. Completed controls retain admission through the +final native/FFI caller release; C ABI conversion moves their storage without a +second full-message copy. Storage is destroyed before admission wakes CQ +workers through the retained operation gate. Active partial messages expire +under `control_assembly_timeout_ms`; a first fragment waiting for consumer byte +admission has no arbitrary consumer hold deadline. Close, cancellation and +expiry discard incomplete assembly, while completed caller-owned results stay +valid after transport shutdown. + +Logical RPC counts, fragment counts and `copied_control_send_bytes` / +`copied_control_receive_bytes` are distinct from direct application payload +copy counters. Directional admission reports current bytes, high-water and +stalls. Ordinary contiguous traffic does not scan control-stream deadlines. + Each QP has a bounded `SendQueue` in `ConnectionPools`. Producers validate and transfer immutable frame ownership through its MPSC admission; the CQ owner alone drains it, carries one next-slot frame and posts SENDs within effective @@ -344,9 +394,33 @@ SEND sources and eventually stops new request receives. Eligible aligned O_DIRECT reads use that same allocation; alignment edges use explicit staging. TCP uses the ordinary application pool and the same disk result mapping. -For Access Server and Chunk-KV, the existing access and aggregation layer -selects the 64 KiB or 256 KiB class. RDMA transports each aggregate with the -same request, sequence, and response semantics as TCP. +For Access Server and Chunk-KV, `ChunkKvRpcTransport` has fixed metadata and +aggregate connection groups. Point calls select 64 KiB; seek, bounded scan, +scan, multi-get and batch mutation select 256 KiB. An explicit override can +select 1 MiB. The server defaults to 256 KiB and negotiates the smaller class +for each QP. These are encoded FlatBuffer control bytes, including values and +aggregated results. Common control streaming reconstructs one existing message; +it adds no Chunk-KV aggregation or correlation layer. + +Both groups share one `RpcClient`, completion reaper and global RPC ID +sequence, while retaining separate connection indexes and generation-based +invalidation. Existing logical mutation identities, duplicate suppression, +result order, typed errors and eligible retries remain in the application +layer. Transport generation validates fragments independently of the catalog +and partition owner epochs. + +Access common `chunk_kv_rpc` settings provide `metadata_rdma` and +`aggregate_rdma` overrides consistently to S3, Iceberg, Iceberg GC and Dataset. +Missing overrides keep environment policy and fixed service classes. Server +`rpc_rdma` config owns listener policy. Explicit disabled RDMA selects TCP; +`allow_tcp_fallback` controls failed setup before payload. Client class groups +always disable listener creation. + +An asynchronous C ABI handler's Frame retains its connection until final +request release. This owner is also required for assembled controls, which no +longer hold an RQ data lease. `ServerRequest::close_connection` closes that +retained connection when response encoding or submission fails, waking the +client instead of silently discarding its response. The fat client uses the access core directly. Its caller owns each completed host buffer lease until release. GPU support initially consists of an explicit @@ -369,6 +443,10 @@ cq_depth = 8192 poll_batch = 256 verbs_poll_batch = 32 poll_empty_budget = 256 +max_control_message_bytes = 33554432 +control_send_budget_bytes = 67108864 +control_receive_budget_bytes = 67108864 +control_assembly_timeout_ms = 30000 ``` CQ depth is per CQ; arena budgets and SQ/RQ depths are per QP. For example, @@ -379,7 +457,13 @@ benchmarks select 1024-byte slots and matching arena budgets explicitly. The versioned C ABI and Rust `RdmaConfig` expose cold configuration. `crowdb_rpc_server_configure_rdma_v2` accepts `wire_layout` separately from the unchanged version-one config structure; the original entry selects contiguous -layout. `CROWDB_RDMA_WIRE_LAYOUT` uses 0 for contiguous and 1 for control/data. +layout. `crowdb_rpc_server_configure_rdma_v3` adds a separately versioned +logical-stream config while preserving the version-one structure and previous +entry points. Rust `RdmaConfig::with_control_data_capacity` derives wire and +SQ/RQ arena capacities from an explicit service data class. Logical message +limits must be nonzero, fit FlatBuffer length bounds and fit each directional +logical byte budget; these budgets are independent of registered MR bytes. +`CROWDB_RDMA_WIRE_LAYOUT` uses 0 for contiguous and 1 for control/data. Owned status JSON contains requested settings, actual QP/CQ/provider capacities, transport errors, registration budgets and process resource counters. Connection @@ -426,6 +510,11 @@ slot class only when the complete wire message fits that class. dispatch occurs in the polling worker. - **I10:** Unsupported RDMA leaves the TCP service usable and does not require RDMA libraries or devices at runtime. +- **I11:** An encoded logical control publishes exactly once after complete + validation. Its admission remains charged until final caller release; + incomplete controls never reach application dispatch. +- **I12:** Fragment staging uses the configured data class, so smaller cached + allocations cannot consume every descriptor needed by a later larger stream. ## 10. Scope boundary diff --git a/doc/working/plan-rdma-chunk-kv-access.md b/doc/working/plan-rdma-chunk-kv-access.md index db413e9ee..a10676d2c 100644 --- a/doc/working/plan-rdma-chunk-kv-access.md +++ b/doc/working/plan-rdma-chunk-kv-access.md @@ -169,6 +169,37 @@ Observed contract: - Repeat HTTP/DiskIO semantic profiles and matching 10-second CLI throughput controls after the common RPC changes, then update permanent evidence. +Real service checkpoint: + +- Production RPC/service/native tree and partition WAL logic pass an eight-value + 3,409,975-byte corpus in native and two non-root OCI processes. Only catalog, + serving authority and memory ChunkStream backing are injected. Metadata and + aggregate groups negotiate 64 KiB and 256 KiB with SQ/RQ/post target two. + Both value and mutation-result digests match strict RDMA, explicit no-device + TCP and pre-payload fallback; duplicate mutation IDs and multi-get order match. +- Expired request and stale scan-continuation errors remain typed and equal. + Native and OCI failure profiles receive exactly one response fragment before + server close, publish no incomplete logical result and return connection + closed within the existing deadline. Fallback stays zero after payload. +- The first real RDMA corpus exposed a fixed-descriptor staging defect: cached + smaller allocations could occupy every SQ descriptor and prevent a later + larger fragment from progressing. The dedicated HCA mixed-size case failed + before the fix and passes after control staging allocates the configured + data class while SEND retains the actual length. All seven HCA control cases + pass. Before/after evidence: `r241-control/mixed-sizes-before.log`, + `mixed-sizes-after.log`, and `mixed-sizes.xml` under the isolated artifacts. +- Full C++/FFI gate passes after this fix at + `.crowdb-runtime/artifacts/unit/cpp-a4be5996987e48c68bf6fa53219fde7a/`: + 543 tree, 191 RPC, 71 DiskIO and 35 common native cases plus both FFI suites. + Workspace all-feature clippy, scoped C++ lint and formatting pass. + The refreshed no-verbs build passes 160 cases. All 13 selected real HCA + transport/control cases and both direct DiskIO HCA cases also pass. + All 1,244 affected Rust cases pass again (four optional cases ignored), + workspace `rs-lint` passes, and all 13 canonical runner contract cases pass. +- Keep this task active until the clean-source semantic rerun, HTTP/DiskIO + regression, matching release CLI 10-second controls and permanent evidence + are complete. + ## Phase 1: access contract mapping - [x] **Map request/response types**: identify metadata, point, batch, range, @@ -184,37 +215,37 @@ Observed contract: ## Phase 2: common client/server wiring -- [ ] **Add RDMA transport choice**: make the Chunk-KV client use the R239 +- [x] **Add RDMA transport choice**: make the Chunk-KV client use the R239 logical connection interface and preserve TCP as fallback. Do not duplicate request correlation in the Chunk-KV layer. -- [ ] **Advertise selected class**: include class and capacity in stream setup +- [x] **Advertise selected class**: include class and capacity in stream setup or request metadata, validate the peer's negotiated class, and reject a payload larger than that class before publishing it. -- [ ] **Preserve request identity**: carry request ID, stream ID, sequence, +- [x] **Preserve request identity**: carry request ID, stream ID, sequence, logical offset, generation, and operation type through RDMA completion and response assembly. -- [ ] **Preserve access errors**: map transport, timeout, cancellation, +- [x] **Preserve access errors**: map transport, timeout, cancellation, generation, integrity, and Chunk-KV errors identically for TCP and RDMA. ## Phase 3: multi-message response assembly -- [ ] **Split aggregate responses**: when an existing aggregate exceeds 64 KiB +- [x] **Split aggregate responses**: when an existing aggregate exceeds 64 KiB or 256 KiB, emit ordered messages without changing its logical result. -- [ ] **Reassemble on client**: validate sequence, offset, total/end flags, and +- [x] **Reassemble on client**: validate sequence, offset, total/end flags, and duplicate request identity; reject gaps, overlaps, stale generations, and wrong classes. -- [ ] **Bound assembly memory**: use R238 classes and caller/consumer release +- [x] **Bound assembly memory**: use R238 classes and caller/consumer release to keep aggregate response memory within the configured budget. -- [ ] **Retry safely**: use shared request IDs and only retry operations whose +- [x] **Retry safely**: use shared request IDs and only retry operations whose current semantics provide idempotency or duplicate suppression. ## Phase 4: fallback and compatibility -- [ ] **Pre-payload fallback**: when RDMA setup fails, retry the same eligible +- [x] **Pre-payload fallback**: when RDMA setup fails, retry the same eligible request on TCP with the same request ID and generation. -- [ ] **Partial response failure**: fail the whole logical response after an +- [x] **Partial response failure**: fail the whole logical response after an uncertain RDMA message; never append a TCP suffix or publish incomplete data. -- [ ] **Config compatibility**: preserve TCP defaults and allow RDMA classes, +- [x] **Config compatibility**: preserve TCP defaults and allow RDMA classes, endpoint selection, and enablement to be configured when creating the transport. diff --git a/lib/crowdb-rpc/ffi/build.rs b/lib/crowdb-rpc/ffi/build.rs index 538debc67..987685552 100644 --- a/lib/crowdb-rpc/ffi/build.rs +++ b/lib/crowdb-rpc/ffi/build.rs @@ -123,6 +123,9 @@ fn main() -> Result<(), Box> { if have_spdlog { build.define("CROWDB_HAVE_SPDLOG", "1"); } + if std::env::var_os("CARGO_FEATURE_TEST_UTIL").is_some() { + build.define("CROWDB_RPC_TEST_UTIL", "1"); + } if have_rdma { build.define("CROWDB_RPC_HAVE_RDMA", "1"); if let Some(prefix) = &conda_prefix { diff --git a/lib/crowdb-rpc/ffi/src/sys.rs b/lib/crowdb-rpc/ffi/src/sys.rs index c9a5b8c83..c72dd640c 100644 --- a/lib/crowdb-rpc/ffi/src/sys.rs +++ b/lib/crowdb-rpc/ffi/src/sys.rs @@ -348,6 +348,11 @@ extern "C" { pub fn crowdb_rpc_frame_release(frame_handle: *mut c_void); pub fn crowdb_rpc_frame_close_connection(frame_handle: *mut c_void); + #[cfg(feature = "test-util")] + pub fn crowdb_rpc_server_limit_control_fragments_for_tests( + server: crowdb_rpc_server_t, + limit: u64, + ) -> c_int; pub fn crowdb_rpc_server_register_handler( server: crowdb_rpc_server_t, diff --git a/lib/crowdb-rpc/ffi/src/test_support.rs b/lib/crowdb-rpc/ffi/src/test_support.rs index 32e8d2710..64050d6fc 100644 --- a/lib/crowdb-rpc/ffi/src/test_support.rs +++ b/lib/crowdb-rpc/ffi/src/test_support.rs @@ -6,6 +6,14 @@ use crate::{sys, RpcServer, ServerRequest}; impl RpcServer { + /// Pause encoded-control SENDs after the absolute fragment counter limit. + /// Used to fail a real service response at a deterministic partial boundary. + #[must_use] + pub fn limit_control_fragments_for_tests(&self, limit: u64) -> bool { + // SAFETY: self retains the native server; the hook changes one atomic. + unsafe { sys::crowdb_rpc_server_limit_control_fragments_for_tests(self.handle(), limit) != 0 } + } + /// Register a synchronous reply fixture, keeping raw handles in the FFI. pub fn register_reply_for_tests(&self, msg_type: u16, reply: F) where diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h index c39798644..94c91936d 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h @@ -361,6 +361,10 @@ typedef void (*crowdb_rpc_handler_release_fn)(void *user_data); void crowdb_rpc_frame_release(void *frame_handle); // Close the retained connection of a live handler Frame without releasing it. void crowdb_rpc_frame_close_connection(void *frame_handle); +#ifdef CROWDB_RPC_TEST_UTIL +// Deterministic service failure after a bounded number of control fragments. +int crowdb_rpc_server_limit_control_fragments_for_tests(crowdb_rpc_server_t server, uint64_t limit); +#endif // Register a custom dispatch callback for the given msg_type. The // callback is invoked for every incoming frame with that msg_type. This diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index 56c95d17b..a08771f79 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -195,9 +195,17 @@ class RdmaTransport : public Transport return capability_available_.load(std::memory_order_acquire); } +# ifdef CROWDB_RPC_TEST_UTIL + void limit_control_fragments_for_tests(uint64_t limit) + { + control_fragment_limit_for_tests_.store(limit, std::memory_order_release); + } +# endif + private: # ifdef CROWDB_RPC_TEST_UTIL friend struct TestRdmaTransport; + std::atomic control_fragment_limit_for_tests_{UINT64_MAX}; # endif RdmaConfig config_; RdmaEndpointCapability local_capability_; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h index fac9fac9f..234836369 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h @@ -539,6 +539,15 @@ class SocketTransport : public Transport return rdma_transport_ == nullptr ? nullptr : &rdma_transport_->stats(); } + bool limit_control_fragments_for_tests(uint64_t limit) + { + if (rdma_transport_ == nullptr) { + return false; + } + rdma_transport_->limit_control_fragments_for_tests(limit); + return true; + } + #endif private: diff --git a/lib/crowdb-rpc/src/c_api/transport.cpp b/lib/crowdb-rpc/src/c_api/transport.cpp index 3025ec395..b984e7df1 100644 --- a/lib/crowdb-rpc/src/c_api/transport.cpp +++ b/lib/crowdb-rpc/src/c_api/transport.cpp @@ -188,3 +188,16 @@ extern "C" void crowdb_rpc_server_register_status_handler(crowdb_rpc_server_t se catch (...) { } } + +#ifdef CROWDB_RPC_TEST_UTIL +extern "C" int crowdb_rpc_server_limit_control_fragments_for_tests(crowdb_rpc_server_t server, uint64_t limit) +{ +# ifdef CROWDB_RPC_HAVE_RDMA + return server != nullptr && server->server->transport()->limit_control_fragments_for_tests(limit); +# else + (void)server; + (void)limit; + return 0; +# endif +} +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_control_send.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_control_send.cpp index 3d4a4cb7e..4cb408a1e 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_control_send.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_control_send.cpp @@ -48,6 +48,12 @@ Buffer *RdmaTransport::admit_control_frame(const OutFrame &frame, const Connecti bool RdmaTransport::flush_control_fragment(Connection *connection, ConnectionPools &pools) { +# ifdef CROWDB_RPC_TEST_UTIL + if (stats_.control_fragments_sent.load(std::memory_order_relaxed) >= + control_fragment_limit_for_tests_.load(std::memory_order_acquire)) { + return false; + } +# endif if (pools.control_send == nullptr) { if (pools.next_control_stream_id == 0) { connection->close(); @@ -72,7 +78,10 @@ bool RdmaTransport::flush_control_fragment(Connection *connection, ConnectionPoo const auto length = static_cast(std::min(f.capacity, f.total - f.offset)); auto release = [](Buffer *buffer) { buffer->release(); }; std::unique_ptr prefix(pools.send->alloc(RDMA_CONTROL_CAPACITY), release); - std::unique_ptr data(send_pool_->alloc(length), release); + // Keep every staging descriptor in the configured data class. Smaller + // cached allocations cannot grow and would strand a later larger stream + // after the fixed SQ descriptor window has been carved. + std::unique_ptr data(send_pool_->alloc(config_.data_slot_capacity()), release); if (prefix == nullptr || data == nullptr) { return false; } diff --git a/lib/crowdb-rpc/tests/rdma_control_hardware_test.cpp b/lib/crowdb-rpc/tests/rdma_control_hardware_test.cpp index fd296ef88..311f9925f 100644 --- a/lib/crowdb-rpc/tests/rdma_control_hardware_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_control_hardware_test.cpp @@ -264,3 +264,34 @@ TEST_F(RdmaControlHardwareTest, MalformedWireIdentityFailsTheWholeLogicalControl } } #endif + +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) +TEST_F(RdmaControlHardwareTest, MixedControlSizesDoNotStrandTheFixedSendWindow) +{ + constexpr auto capacity = RDMA_AGGREGATE_CAPACITY; + auto config = test_control_config(capacity, capacity + 13); + auto pair = TestControlPeers::create(config, config); + ASSERT_NE(pair, nullptr); + uint32_t received = 0; + pair->incoming->set_on_frame([&](Frame *frame, Connection *connection) { + auto *reply = test_control_reply(frame); + if (!pair->server->submit(connection, reply)) { + ADD_FAILURE() << "logical response admission failed"; + release_rdma_frames(reply); + } + }); + pair->outgoing->set_on_frame([&](Frame *frame, Connection *) { + ++received; + delete frame; + }); + for (uint64_t id : {1, 2}) { + ASSERT_TRUE(pair->client->submit(pair->outgoing.get(), test_control_frame(pair->application, id, 4096))); + } + ASSERT_TRUE(pair->drain_until([&] { return received == 2; })); + ASSERT_TRUE(pair->client->submit(pair->outgoing.get(), test_control_frame(pair->application, 3, capacity + 13))); + EXPECT_TRUE(pair->drain_until([&] { return received == 3; })) + << "cached smaller allocations must not strand the configured SQ data slots"; + EXPECT_EQ(pair->client->stats().fallback_count.load(), 0U); + EXPECT_EQ(pair->server->stats().fallback_count.load(), 0U); +} +#endif diff --git a/tools/rdma/run-chunk-kv.py b/tools/rdma/run-chunk-kv.py new file mode 100644 index 000000000..97b53c193 --- /dev/null +++ b/tools/rdma/run-chunk-kv.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Real Chunk-KV RPC/native-tree corpus through the shared native/OCI launcher.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import socket +import subprocess +import sys +import time +import uuid + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tools/benchmark")) +from rpc_bench_process import Runtime, start # noqa: E402 + +CASE = "real_chunk_kv_service_preserves_large_controls_and_typed_results" + + +def build(pixi, output): + command = [pixi, "run", "--", "cargo", "test", "-j32", "-p", "crowdb-chunk-kv-server", + "--test", "rdma_rpc_test", "--no-run", "--message-format=json"] + result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True) + (output / "build.jsonl").write_text(result.stdout) + print(result.stderr, end="", flush=True) + executable = None + for line in result.stdout.splitlines(): + item = json.loads(line) + if item.get("reason") == "compiler-message": + print(item["message"].get("rendered", ""), end="", flush=True) + if item.get("target", {}).get("name") == "rdma_rpc_test" and item.get("executable"): + executable = Path(item["executable"]) + result.check_returncode() + if executable is None: + raise RuntimeError("Cargo did not identify the Chunk-KV fixture executable") + return executable + + +def wait_json(path, process, timeout=10): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError(f"process exited before {path.name}: {process.returncode}") + try: + return json.loads(path.read_text()) + except (FileNotFoundError, json.JSONDecodeError): + time.sleep(0.01) + raise RuntimeError(f"deadline waiting for {path}") + + +def check(result, server, profile): + if result["errors"] or result["verification_errors"] or result["payload_bytes"] <= 3 << 20: + raise RuntimeError("invalid corpus or unexpected errors") + if result["expected_typed_errors"] != 2 or result["expected_transport_errors"] != int(profile == "disconnect"): + raise RuntimeError("incorrect expected error counts") + snapshots = [result["transport"][group]["transport"] for group in ("metadata", "aggregate")] + if profile == "tcp": + if any(item["rdma"] is not None for item in [server, *snapshots]): + raise RuntimeError("explicit TCP unexpectedly attempted RDMA") + elif profile == "fallback": + if any(item["rdma"]["fallback_count"] != 1 for item in snapshots): + raise RuntimeError("each fixed class must fall back exactly once before payload") + else: + qps = [] + for group, capacity in (("metadata", 65536), ("aggregate", 262144)): + info = result["connections"][group]["info"] + if info["effective_data_capacity"] != capacity or info["effective_sq_depth"] < 2 or info["effective_rq_depth"] < 2: + raise RuntimeError("actual QP does not match the selected class or queue window") + qps.append(info["qpair_number"]) + if len(set(qps)) != 2: + raise RuntimeError("metadata and aggregate groups must use separate QPs") + for item in [server, *snapshots]: + rdma = item["rdma"] + for field in ("fallback_count", "protocol_errors", "receive_post_errors", "control_stream_timeouts"): + if rdma[field] != 0: + raise RuntimeError(f"unexpected RDMA {field}={rdma[field]}") + if profile != "disconnect" and (rdma["cq_errors"] or rdma["send_errors"]): + raise RuntimeError("unexpected CQ/SEND errors") + if rdma["control_fragments_received"] == 0 or rdma["control_fragments_sent"] == 0: + raise RuntimeError("large controls did not traverse RDMA fragments") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile", choices=("rdma", "tcp", "fallback", "disconnect"), required=True) + parser.add_argument("--topology", choices=("native", "containers"), default="native") + parser.add_argument("--address", default="10.200.1.1") + parser.add_argument("--device", default="rocep35s0f0") + parser.add_argument("--image", default="crowdb-node:dev") + args = parser.parse_args() + pixi = shutil.which("pixi") + if not pixi: + raise RuntimeError("pixi is required") + output = (Path(os.environ.get("CROWDB_RUNTIME_ROOT", ROOT / ".crowdb-runtime")) + / "artifacts/rdma-chunk-kv" / uuid.uuid4().hex).resolve() + output.mkdir(parents=True) + binary = build(pixi, output) + if args.topology == "containers": + args.image = subprocess.check_output([pixi, "run", "--", "docker", "image", "inspect", + args.image, "--format", "{{.Id}}"], text=True).strip() + address = "127.0.0.1" if args.profile == "tcp" else args.address + with socket.socket() as reservation: + reservation.bind((address, 0)) + port = reservation.getsockname()[1] + env = {key: value for key, value in os.environ.items() if not key.startswith("CROWDB_RDMA_")} + env.update(CROWDB_RUNTIME_ROOT=str(output / "runtime"), CROWDB_RDMA_DEVICE=args.device, + CROWDB_RDMA_CHUNK_KV_ENDPOINT=f"{address}:{port}", CROWDB_RDMA_CHUNK_KV_OUTPUT=str(output), + CROWDB_RDMA_CHUNK_KV_PROFILE=args.profile) + topology = "containers-tcp" if args.topology == "containers" and args.profile == "tcp" else args.topology + runtime = Runtime(ROOT, output, pixi, topology, args.image) + manifest = {"profile": args.profile, "topology": args.topology, "physical_hosts": 1, + "image": args.image if args.topology == "containers" else None, + "source": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip(), + "source_dirty": bool(subprocess.check_output(["git", "status", "--porcelain"], cwd=ROOT, text=True)), + "binary_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), + "scope": "production RPC/service/native tree/WAL; injected catalog, serving authority and memory ChunkStream backing", + "errors": None, "expected_transport_errors": int(args.profile == "disconnect"), "verification_errors": None} + processes = {} + try: + for side in ("server", "client"): + side_env = dict(env, CROWDB_RDMA_CHUNK_KV_ROLE=side) + with (output / f"{side}.log").open("w") as log: + processes[side] = start(runtime.command([binary, "--ignored", "--exact", CASE, "--nocapture"], + side_env, side), side_env, log, ROOT) + if side == "server": + manifest["startup"] = wait_json(output / "ready.json", processes[side]) + if args.topology == "containers": + info = json.loads(subprocess.check_output([pixi, "run", "--", "docker", "inspect", + runtime.names[side]], text=True))[0] + manifest["server_container"] = {"image": info["Image"], "user": info["Config"]["User"], + "network": info["HostConfig"]["NetworkMode"], "devices": info["HostConfig"]["Devices"]} + if info["Config"]["User"].split(":")[0] in ("", "0", "root"): + raise RuntimeError("server container must be non-root") + if args.profile == "disconnect": + manifest["partial_response_gate"] = wait_json(output / "fault-ready.json", processes["client"]) + manifest["partial_receive_gate"] = wait_json(output / "client-partial-ready.json", processes["client"]) + (output / "stop").write_text("disconnect after one RDMA response fragment\n") + if processes["client"].wait(timeout=30) != 0: + raise RuntimeError("Chunk-KV client failed") + (output / "stop").write_text("corpus complete\n") + if processes["server"].wait(timeout=10) != 0: + raise RuntimeError("Chunk-KV server failed") + result = json.loads((output / "client.json").read_text()) + server = json.loads((output / "server-final.json").read_text()) + check(result, server, args.profile) + if args.topology == "containers" and result["effective_uid"] == "0": + raise RuntimeError("client container must be non-root") + result["server_transport"] = server + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n") + manifest.update(errors=0, verification_errors=0) + except Exception as error: + manifest.update(verification_errors=1, failure=str(error)) + raise + finally: + for side, process in processes.items(): + runtime.stop(process, side) + print((output / f"{side}.log").read_text(), end="", flush=True) + (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + print(f"Artifacts: {output}", flush=True) + + +if __name__ == "__main__": + main() From 7eb24dc57c411a81d1f00455e2a2605a1e7ed4ff Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 06:35:06 +0800 Subject: [PATCH 052/258] Record measured benchmark environments and throughput in runner scripts --- tools/benchmark/bench-rdma-rpc-regression.sh | 38 ++++++++++++++++++++ tools/benchmark/bench-rpc-regression.sh | 24 +++++++++++++ 2 files changed, 62 insertions(+) diff --git a/tools/benchmark/bench-rdma-rpc-regression.sh b/tools/benchmark/bench-rdma-rpc-regression.sh index b9ed4d33a..cad692351 100755 --- a/tools/benchmark/bench-rdma-rpc-regression.sh +++ b/tools/benchmark/bench-rdma-rpc-regression.sh @@ -2,6 +2,44 @@ # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. # Shared coroutine workload, structured readiness and canonical result files. +# +# Measured 2026-10-23, source 018da2fc, matching release CLI/server, protocol v4. +# Environment: AMD Ryzen 9 5950X, 16 cores / 32 threads, about 62 GiB RAM. +# HCA: rocep35s0f0 port 1, 100 Gb/s, PCIe 8 GT/s x4; one physical host. +# raw-server = two native host processes. OCI = two non-root UID 1000 +# host-network containers sharing this HCA, with independent QPs and memlock +# unlimited. Image: sha256:e3c9c5c6d5fc3424ac3f22cf4cdcc9b1c872f47336484f54962b2541c4ed6cef. +# CPU affinity/frequency were not pinned. Native builds used -j32. +# Workload: coroutine, 128-byte payload, one in-flight RPC per loader, +# 1 s warmup + 10 s validated completion window, drain counted separately. +# Peak rows select median TPS of three repeats; latency is from that same row. +# +# topology loaders QPs CQ/pollers socket_workers aggregation TPS avg_us p90_us p99_us errors fallback +# raw-server 2048 16 16 1 on 2,901,185.6 755.61 802.82 954.37 0 0 +# OCI 2048 16 16 1 on 3,073,435.4 725.12 741.38 946.18 0 0 +# Raw TPS repeats: 2,911,356.0 / 2,814,582.2 / 2,901,185.6. +# OCI TPS repeats: 3,073,435.4 / 3,067,371.3 / 3,121,091.5. +# +# Progression, raw-server (one 10 s trial each): +# loaders QPs CQ/pollers aggregation TPS avg_us p90_us p99_us errors +# 1 1 1 off 57,620.7 18.57 16.00 30.08 0 +# 512 8 8 on 2,223,708.2 249.36 243.71 284.67 0 +# +# Per peer: io_engines=1, io_workers=1; per QP: sq_depth=rq_depth= +# posted_receive_target=1024, send_queue_capacity=4096 logical RPCs. +# cq_depth=2048 requested per CQ (provider 4095); wire_slot_capacity=1024 B; +# send_arena_bytes=receive_arena_bytes=1 MiB per QP; registered total 34 MiB +# per peer, including shared pools. poll_batch/verbs_poll_batch/ +# poll_empty_budget=256/32/256. aggregation_max_bytes=1024, +# aggregation_max_frames=256; complete frames only, bounded by both slots. +# +# Startup-tail caveat: observed cold first responses reached about 1.5-1.8 s; +# the single-loader trial had zero warmup completions and max_ns=1460998030. +# Completion-window latency includes requests submitted during warmup. P90 is +# not maximum latency; retain this snapshot during the startup investigation. +# Original canonical artifacts: .crowdb-runtime/rdma-check/artifacts/bench/ +# accepted-018da2fc-rdma-native/ and accepted-018da2fc-rdma-oci/. +# set -Eeuo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/../.." exec pixi run -- python tools/benchmark/rpc_regression.py --transport rdma "$@" diff --git a/tools/benchmark/bench-rpc-regression.sh b/tools/benchmark/bench-rpc-regression.sh index 338793230..ad2e7d11e 100755 --- a/tools/benchmark/bench-rpc-regression.sh +++ b/tools/benchmark/bench-rpc-regression.sh @@ -2,6 +2,30 @@ # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. # Shared coroutine workload, structured readiness and canonical result files. +# +# Measured 2026-10-23, source 018da2fc, matching release CLI/server, protocol v4. +# Environment: AMD Ryzen 9 5950X, 16 cores / 32 threads, about 62 GiB RAM. +# HCA: rocep35s0f0 port 1, 100 Gb/s, PCIe 8 GT/s x4; one physical host. +# raw-server = two native host processes. OCI = two non-root UID 1000 +# host-network containers sharing this HCA, with independent QPs and memlock +# unlimited. Image: sha256:e3c9c5c6d5fc3424ac3f22cf4cdcc9b1c872f47336484f54962b2541c4ed6cef. +# CPU affinity/frequency were not pinned. Native builds used -j32. +# Workload: coroutine, 128-byte payload, one in-flight RPC per loader, +# 1 s warmup + 10 s validated completion window, drain counted separately. +# Peak rows select median TPS of three repeats; latency is from that same row. +# +# topology Nagle loaders connections socket_workers TPS avg_us p90_us p99_us errors fallback +# raw-server off 1000 32 16 1,206,800.2 827.23 1654.78 6815.74 0 0 +# raw-server on 1000 32 16 2,197,134.7 453.80 839.68 1515.52 0 0 +# OCI -- -- -- -- not measured in this performance snapshot +# Raw Nagle-off TPS repeats: 1,206,800.2 / 1,199,487.9 / 1,214,054.5. +# Raw Nagle-on TPS repeats: 2,197,134.7 / 2,186,038.9 / 2,198,891.2. +# Per peer: io_engines=1, io_workers=16, send_queue_capacity=4096. +# RDMA is explicitly disabled; CQ/SQ/RQ/registered-slot settings are n/a. +# The no-device OCI TCP functional corpus passed; no OCI TPS is inferred from it. +# Original canonical artifacts: .crowdb-runtime/rdma-check/artifacts/bench/ +# accepted-018da2fc-tcp-off/ and accepted-018da2fc-tcp-on/. +# set -Eeuo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/../.." exec pixi run -- python tools/benchmark/rpc_regression.py --transport tcp "$@" From 100639b7b616d20703a0d3f0b804887c0592f64c Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 06:52:33 +0800 Subject: [PATCH 053/258] Prime contiguous receive slots before active RDMA setup --- doc/design/rpc/design-crowdb-rpc-rdma.md | 12 +- doc/working/plan-rdma-chunk-kv-access.md | 29 +- .../transport/rdma/rdma_transport.h | 8 +- .../src/transport/rdma/rdma_active.cpp | 249 ++++++++++++++++++ .../src/transport/rdma/rdma_connection.cpp | 204 -------------- lib/crowdb-rpc/tests/rdma_startup_test.cpp | 89 +++++++ 6 files changed, 379 insertions(+), 212 deletions(-) create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_active.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_startup_test.cpp diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index 0de939732..0c1e0b029 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -148,11 +148,13 @@ the fixed SQ descriptor window reusable across mixed sizes and peer classes. SEND carries only the actual fragment length. Rejected queue publication restores the caller's original control ownership. -QP data capacity is `min(local_data_capacity, peer_data_capacity)`. Active CM -setup allocates and primes receives after validating the peer reply; passive -setup already has the offer. Both actual RX allocation/posting and SEND -validation use this minimum, which immutable connection status exposes as -`effective_data_capacity`. +Control/data QP data capacity is `min(local_data_capacity, peer_data_capacity)`. +Active CM setup allocates and primes those receives after validating the peer +reply; passive setup already has the offer. Both actual RX allocation/posting +and SEND validation use this minimum, which immutable connection status exposes +as `effective_data_capacity`. Contiguous slots retain their fixed local wire +capacity and are posted before active CM connect, so an immediate peer response +finds receive credits without entering an RNR retry cycle. The receiver reserves the complete logical length under `control_receive_budget_bytes` before assembly. One active message per QP diff --git a/doc/working/plan-rdma-chunk-kv-access.md b/doc/working/plan-rdma-chunk-kv-access.md index a10676d2c..771f60f35 100644 --- a/doc/working/plan-rdma-chunk-kv-access.md +++ b/doc/working/plan-rdma-chunk-kv-access.md @@ -90,7 +90,7 @@ Observed contract: rejected submit restores the caller's original buffer. WR records retain the single original cursor until every posted SEND has completed. - Set effective data capacity and peer generation/maximum from CM. Active - connections create/prime receive storage after the validated CM reply and + control/data connections create/prime receive storage after the validated CM reply and before application publication, so asymmetric classes allocate/post the actual minimum rather than a larger local default. - Extend the existing CQ deferred receive vector for byte admission. A @@ -199,6 +199,33 @@ Real service checkpoint: - Keep this task active until the clean-source semantic rerun, HTTP/DiskIO regression, matching release CLI 10-second controls and permanent evidence are complete. +- The clean v4 timed controls expose a startup tail: the single-loader run has + zero warmup completions and a 1.46-second first response; high-load rows also + contain roughly 1.8-second outliers. Preserve `accepted-018da2fc-*` artifacts. + Check whether moving contiguous RX priming after CM setup caused this change; + only control/data needs the peer's negotiated data class before allocation. + Keep the one-second warmup and ten-second window unchanged. Repeat controls + after diagnosis, and embed the final environment/config/results directly in + both benchmark wrapper scripts as the user requested. +- The controlled contiguous-only fix restores RX priming before active CM + connect. The unchanged single-loader 1 s warmup + 10 s probe now completes + 60,804 warmup RPCs, measures 60,814.3 TPS, average 15,935.74 ns, P90 16,063 ns, + P99 18,687 ns and maximum 90,601 ns with zero errors/fallback. Control/data + still allocates the negotiated minimum after the validated reply. Active + route/QP/CM stages now live in a private implementation with bounded helpers. +- The dedicated HCA first-response regression and all 13 existing selected + transport/control cases pass. Full C++/FFI gate passes at + `.crowdb-runtime/artifacts/unit/cpp-b493f2503a414232a8c41bc2a13824d6/`: + 543 tree, 192 RPC, 71 DiskIO and 35 common native cases plus both FFI suites. + The refreshed no-verbs build passes all 160 CTest cases (one hardware skip). + Changed C++ format/tree-lint, workspace Rust format, shell syntax and all + 13 runner contract cases pass. Logs remain at `/tmp/r241-startup-*.log`. + Repeat clean-source timed raw-server/OCI controls next; include OCI TCP TPS. +- Earlier build/artifact directories were removed during this session outside + this task's commands. Previous canonical artifact IDs remain historical; + surviving `/tmp/r241-accepted-*.log`, `/tmp/r241-datapath-*.log` and gate logs + retain the completed semantic checks. Keep fresh perf artifacts under + `/tmp/crowdb-rdma-probe/artifacts/` as well as the repository runtime root. ## Phase 1: access contract mapping diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index a08771f79..7edcb471b 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -242,8 +242,12 @@ class RdmaTransport : public Transport enum class WorkKind : uint8_t { Send, Receive }; struct ConnectionPools; struct ControlSendState; - bool create_connection_pools(ConnectionPools &pools); - void configure_peer(ConnectionPools &pools, const RdmaCmOffer &peer); + bool create_connection_pools(ConnectionPools &pools); + void configure_peer(ConnectionPools &pools, const RdmaCmOffer &peer); + std::shared_ptr create_active_connection(rdma_cm_id *id, const std::string &addr, + std::shared_ptr &pools); + bool establish_active_connection(rdma_cm_id *id, rdma_event_channel *channel, const std::string &addr, + const RdmaEndpointCapability *expected_peer, RdmaCmOffer &peer_offer); struct WorkRequest { diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_active.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_active.cpp new file mode 100644 index 000000000..2f0b3a5de --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_active.cpp @@ -0,0 +1,249 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-common/log.h" +#include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "rdma_channel.h" +#include "rdma_cq.h" +#include "rdma_offer.h" + +#ifdef CROWDB_RPC_HAVE_RDMA + +# include + +# include +# include +# include + +namespace crowdb::rpc +{ + +namespace +{ +bool resolve_active_route(rdma_cm_id *id, rdma_event_channel *channel, const std::string &addr, int port, + const std::string &source_address_text) +{ + sockaddr_in destination{}; + destination.sin_family = AF_INET; + destination.sin_port = htons(static_cast(port)); + if (::inet_pton(AF_INET, addr.c_str(), &destination.sin_addr) != 1) { + CRB_LOG_WARN("rdma connect: invalid destination {}", addr); + return false; + } + sockaddr_in source{}; + sockaddr *source_address = nullptr; + if (const char *source_env = source_address_text.c_str(); *source_env != '\0') { + source.sin_family = AF_INET; + if (::inet_pton(AF_INET, source_env, &source.sin_addr) != 1) { + CRB_LOG_WARN("rdma connect: invalid source address {}", source_env); + return false; + } + source_address = reinterpret_cast(&source); + if (rdma_bind_addr(id, source_address) != 0) { + + CRB_LOG_WARN("rdma connect: bind source failed source={} errno={}", source_env, errno); + return false; + } + } + // When a source address was explicitly bound above, librdmacm expects a + // null source here; passing it twice can fail immediately with EINVAL. + if (rdma_resolve_addr(id, nullptr, reinterpret_cast(&destination), 2000) != 0) { + + CRB_LOG_WARN("rdma connect: resolve_addr failed addr={} port={} errno={}", addr, port, errno); + return false; + } + rdma_cm_event *event = nullptr; + if (!rdma_detail::wait_cm_event(channel, &event, 3000) || event == nullptr || + event->event != RDMA_CM_EVENT_ADDR_RESOLVED) { + + CRB_LOG_WARN("rdma connect: ADDR_RESOLVED missing addr={} event={} errno={}", addr, + event == nullptr ? -1 : event->event, errno); + if (event != nullptr) + rdma_ack_cm_event(event); + return false; + } + rdma_ack_cm_event(event); + event = nullptr; + if (rdma_resolve_route(id, 2000) != 0) { + CRB_LOG_WARN("rdma connect: resolve_route failed addr={} errno={}", addr, errno); + return false; + } + if (!rdma_detail::wait_cm_event(channel, &event, 3000) || event == nullptr || + event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { + + CRB_LOG_WARN("rdma connect: ROUTE_RESOLVED missing addr={} event={} errno={}", addr, + event == nullptr ? -1 : event->event, errno); + if (event != nullptr) + rdma_ack_cm_event(event); + return false; + } + rdma_ack_cm_event(event); + event = nullptr; + return true; +} +} // namespace + +std::shared_ptr RdmaTransport::create_active_connection(rdma_cm_id *id, const std::string &addr, + std::shared_ptr &pools) +{ + if (!adopt_cm_context(id->verbs)) { + return nullptr; + } + + ibv_device_attr device_attributes{}; + if (ibv_query_device(id->verbs, &device_attributes) != 0) { + return nullptr; + } + const uint64_t required_capacity = static_cast(config_.sq_depth) + config_.posted_receive_target; + CqReservation reservation(required_capacity > UINT32_MAX ? nullptr + : reserve_cq(static_cast(required_capacity)), + static_cast(required_capacity)); + if (reservation.queue == nullptr) { + return nullptr; + } + ibv_qp_init_attr qp_attr{}; + qp_attr.send_cq = reservation.queue->cq; + qp_attr.recv_cq = reservation.queue->cq; + qp_attr.qp_type = IBV_QPT_RC; + qp_attr.cap.max_send_wr = std::min(config_.sq_depth, static_cast(device_attributes.max_qp_wr)); + qp_attr.cap.max_recv_wr = std::min(config_.rq_depth, static_cast(device_attributes.max_qp_wr)); + qp_attr.cap.max_send_sge = std::min(config_.qp_max_send_sge, static_cast(device_attributes.max_sge)); + qp_attr.cap.max_recv_sge = std::min(config_.qp_max_recv_sge, static_cast(device_attributes.max_sge)); + qp_attr.cap.max_inline_data = 0; + qp_attr.sq_sig_all = 1; + qp_attr.qp_context = id; + + if (rdma_create_qp(id, pd_, &qp_attr) != 0) { + + return nullptr; + } + if (config_.wire_layout == RdmaWireLayout::ControlData && + (qp_attr.cap.max_send_sge < 2 || qp_attr.cap.max_recv_sge < 2)) { + rdma_destroy_qp(id); + return nullptr; + } + pools = std::make_shared(config_.send_queue_capacity); + pools->effective_capacity = qp_attr.cap; + pools->cq_index = reservation.queue->index; + pools->reserved_capacity = reservation.capacity; + auto connection = std::make_shared(reinterpret_cast(id), addr, application_pool_); + set_connection_pools(connection.get(), pools); + reservation.committed = true; + connection->set_path(ConnectionPath::Rdma); + connection->transport_handle = reinterpret_cast(id->qp); + install_connection_notifiers(connection, pools->cq_index); + // Contiguous RQ slots have a fixed local wire capacity. Prepost before CM + // connect so an immediate response cannot enter an RNR retry cycle. + if (config_.wire_layout == RdmaWireLayout::Contiguous) { + pools->effective_data_capacity = config_.data_slot_capacity(); + if (!create_connection_pools(*pools) || !prime_receive(connection)) { + destroy_connection_qp(connection.get(), id); + return nullptr; + } + } + return connection; +} + +bool RdmaTransport::establish_active_connection(rdma_cm_id *id, rdma_event_channel *channel, const std::string &addr, + const RdmaEndpointCapability *expected_peer, RdmaCmOffer &peer_offer) +{ + rdma_cm_event *event = nullptr; + rdma_conn_param params{}; + const auto local = local_capability(); + const auto offer = serialize_rdma_offer(local, expected_peer); + params.private_data = offer.data(); + params.private_data_len = static_cast(offer.size()); + params.initiator_depth = 1; + params.responder_resources = 1; + params.retry_count = config_.retry_count; + params.rnr_retry_count = config_.rnr_retry_count; + if (rdma_connect(id, ¶ms) != 0) { + + CRB_LOG_WARN("rdma connect: rdma_connect failed addr={} errno={}", addr, errno); + return false; + } + if (!rdma_detail::wait_cm_event(channel, &event, 5000) || event == nullptr || + event->event != RDMA_CM_EVENT_ESTABLISHED) { + + CRB_LOG_WARN("rdma connect: ESTABLISHED missing addr={} event={} status={} errno={}", addr, + event == nullptr ? -1 : event->event, event == nullptr ? 0 : event->status, errno); + if (event != nullptr) + rdma_ack_cm_event(event); + return false; + } + const bool valid_peer = event->param.conn.private_data != nullptr && + parse_rdma_offer(std::span(static_cast(event->param.conn.private_data), + event->param.conn.private_data_len), + &peer_offer) && + validate_rdma_offer(peer_offer, local, expected_peer, true); + rdma_ack_cm_event(event); + if (!valid_peer) { + return false; + } + return true; +} + +std::shared_ptr RdmaTransport::connect(const std::string &addr, int port, + const RdmaEndpointCapability *expected_peer) +{ + if (!ready() || addr.empty() || port <= 0 || port > 65535 || + (config_.require_tcp_preflight && expected_peer == nullptr)) { + + CRB_LOG_WARN("rdma connect: transport unavailable addr={} port={}", addr, port); + return nullptr; + } + stats_.cm_attempts.fetch_add(1, std::memory_order_relaxed); + { + std::scoped_lock lock(cm_mu_); + if (cm_channel_ == nullptr) + cm_channel_ = rdma_create_event_channel(); + if (cm_channel_ == nullptr) + return nullptr; + rdma_detail::make_nonblocking(cm_channel_->fd); + } + // Active setup has a private channel. The epoll worker may drain existing + // connections without consuming this attempt's address/route events. + auto setup_channel = std::unique_ptr( + rdma_create_event_channel(), &rdma_destroy_event_channel); + if (setup_channel == nullptr) + return nullptr; + rdma_detail::make_nonblocking(setup_channel->fd); + std::shared_ptr connection; + auto destroy_pending = [this, &connection](rdma_cm_id *id) { + if (connection != nullptr) + destroy_connection_qp(connection.get(), id); + rdma_destroy_id(id); + }; + rdma_cm_id *id = nullptr; + if (rdma_create_id(setup_channel.get(), &id, this, RDMA_PS_TCP) != 0) + return nullptr; + auto pending = std::unique_ptr(id, destroy_pending); + if (!resolve_active_route(id, setup_channel.get(), addr, port, config_.source_address)) + return nullptr; + std::shared_ptr pools; + connection = create_active_connection(id, addr, pools); + if (connection == nullptr) + return nullptr; + RdmaCmOffer peer_offer; + if (!establish_active_connection(id, setup_channel.get(), addr, expected_peer, peer_offer)) + return nullptr; + configure_peer(*pools, peer_offer); + if (pools->recv == nullptr && (!create_connection_pools(*pools) || !prime_receive(connection))) { + return nullptr; + } + pools->qp_number = id->qp->qp_num; + ibv_qp_attr receive_attributes{}; + receive_attributes.min_rnr_timer = config_.min_rnr_timer; + if (ibv_modify_qp(id->qp, &receive_attributes, IBV_QP_MIN_RNR_TIMER) != 0) { + return nullptr; + } + if (!publish_cm_connection(id, connection, std::move(pools))) { + return nullptr; + } + pending.release(); + return connection; +} + +} // namespace crowdb::rpc + +#endif // CROWDB_RPC_HAVE_RDMA diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp index 0110ebea5..bd0b23a51 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp @@ -18,210 +18,6 @@ namespace crowdb::rpc { -std::shared_ptr RdmaTransport::connect(const std::string &addr, int port, - const RdmaEndpointCapability *expected_peer) -{ - if (!ready() || addr.empty() || port <= 0 || port > 65535 || - (config_.require_tcp_preflight && expected_peer == nullptr)) { - - CRB_LOG_WARN("rdma connect: transport unavailable addr={} port={}", addr, port); - return nullptr; - } - stats_.cm_attempts.fetch_add(1, std::memory_order_relaxed); - { - std::scoped_lock lock(cm_mu_); - if (cm_channel_ == nullptr) - cm_channel_ = rdma_create_event_channel(); - if (cm_channel_ == nullptr) - return nullptr; - rdma_detail::make_nonblocking(cm_channel_->fd); - } - // Active setup has a private channel. The epoll worker may drain existing - // connections without consuming this attempt's address/route events. - auto setup_channel = std::unique_ptr( - rdma_create_event_channel(), &rdma_destroy_event_channel); - if (setup_channel == nullptr) - return nullptr; - rdma_detail::make_nonblocking(setup_channel->fd); - rdma_cm_id *id = nullptr; - if (rdma_create_id(setup_channel.get(), &id, this, RDMA_PS_TCP) != 0) - return nullptr; - sockaddr_in destination{}; - destination.sin_family = AF_INET; - destination.sin_port = htons(static_cast(port)); - if (::inet_pton(AF_INET, addr.c_str(), &destination.sin_addr) != 1) { - CRB_LOG_WARN("rdma connect: invalid destination {}", addr); - rdma_destroy_id(id); - return nullptr; - } - sockaddr_in source{}; - sockaddr *source_address = nullptr; - if (const char *source_env = config_.source_address.c_str(); *source_env != '\0') { - source.sin_family = AF_INET; - if (::inet_pton(AF_INET, source_env, &source.sin_addr) != 1) { - CRB_LOG_WARN("rdma connect: invalid source address {}", source_env); - rdma_destroy_id(id); - return nullptr; - } - source_address = reinterpret_cast(&source); - if (rdma_bind_addr(id, source_address) != 0) { - - CRB_LOG_WARN("rdma connect: bind source failed source={} errno={}", source_env, errno); - rdma_destroy_id(id); - return nullptr; - } - } - // When a source address was explicitly bound above, librdmacm expects a - // null source here; passing it twice can fail immediately with EINVAL. - if (rdma_resolve_addr(id, nullptr, reinterpret_cast(&destination), 2000) != 0) { - - CRB_LOG_WARN("rdma connect: resolve_addr failed addr={} port={} errno={}", addr, port, errno); - rdma_destroy_id(id); - return nullptr; - } - rdma_cm_event *event = nullptr; - if (!rdma_detail::wait_cm_event(setup_channel.get(), &event, 3000) || event == nullptr || - event->event != RDMA_CM_EVENT_ADDR_RESOLVED) { - - CRB_LOG_WARN("rdma connect: ADDR_RESOLVED missing addr={} event={} errno={}", addr, - event == nullptr ? -1 : event->event, errno); - if (event != nullptr) - rdma_ack_cm_event(event); - rdma_destroy_id(id); - return nullptr; - } - rdma_ack_cm_event(event); - event = nullptr; - if (rdma_resolve_route(id, 2000) != 0) { - CRB_LOG_WARN("rdma connect: resolve_route failed addr={} errno={}", addr, errno); - rdma_destroy_id(id); - return nullptr; - } - if (!rdma_detail::wait_cm_event(setup_channel.get(), &event, 3000) || event == nullptr || - event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { - - CRB_LOG_WARN("rdma connect: ROUTE_RESOLVED missing addr={} event={} errno={}", addr, - event == nullptr ? -1 : event->event, errno); - if (event != nullptr) - rdma_ack_cm_event(event); - rdma_destroy_id(id); - return nullptr; - } - rdma_ack_cm_event(event); - event = nullptr; - - if (!adopt_cm_context(id->verbs)) { - rdma_destroy_id(id); - return nullptr; - } - - ibv_device_attr device_attributes{}; - if (ibv_query_device(id->verbs, &device_attributes) != 0) { - rdma_destroy_id(id); - return nullptr; - } - const uint64_t required_capacity = static_cast(config_.sq_depth) + config_.posted_receive_target; - CqReservation reservation(required_capacity > UINT32_MAX ? nullptr - : reserve_cq(static_cast(required_capacity)), - static_cast(required_capacity)); - if (reservation.queue == nullptr) { - rdma_destroy_id(id); - return nullptr; - } - ibv_qp_init_attr qp_attr{}; - qp_attr.send_cq = reservation.queue->cq; - qp_attr.recv_cq = reservation.queue->cq; - qp_attr.qp_type = IBV_QPT_RC; - qp_attr.cap.max_send_wr = std::min(config_.sq_depth, static_cast(device_attributes.max_qp_wr)); - qp_attr.cap.max_recv_wr = std::min(config_.rq_depth, static_cast(device_attributes.max_qp_wr)); - qp_attr.cap.max_send_sge = std::min(config_.qp_max_send_sge, static_cast(device_attributes.max_sge)); - qp_attr.cap.max_recv_sge = std::min(config_.qp_max_recv_sge, static_cast(device_attributes.max_sge)); - qp_attr.cap.max_inline_data = 0; - qp_attr.sq_sig_all = 1; - qp_attr.qp_context = id; - - if (rdma_create_qp(id, pd_, &qp_attr) != 0) { - - rdma_destroy_id(id); - return nullptr; - } - if (config_.wire_layout == RdmaWireLayout::ControlData && - (qp_attr.cap.max_send_sge < 2 || qp_attr.cap.max_recv_sge < 2)) { - rdma_destroy_qp(id); - rdma_destroy_id(id); - return nullptr; - } - auto pools = std::make_shared(config_.send_queue_capacity); - pools->effective_capacity = qp_attr.cap; - pools->cq_index = reservation.queue->index; - pools->reserved_capacity = reservation.capacity; - auto connection = std::make_shared(reinterpret_cast(id), addr, application_pool_); - set_connection_pools(connection.get(), pools); - reservation.committed = true; - connection->set_path(ConnectionPath::Rdma); - connection->transport_handle = reinterpret_cast(id->qp); - install_connection_notifiers(connection, pools->cq_index); - rdma_conn_param params{}; - const auto local = local_capability(); - const auto offer = serialize_rdma_offer(local, expected_peer); - params.private_data = offer.data(); - params.private_data_len = static_cast(offer.size()); - params.initiator_depth = 1; - params.responder_resources = 1; - params.retry_count = config_.retry_count; - params.rnr_retry_count = config_.rnr_retry_count; - if (rdma_connect(id, ¶ms) != 0) { - - CRB_LOG_WARN("rdma connect: rdma_connect failed addr={} errno={}", addr, errno); - destroy_connection_qp(connection.get(), id); - rdma_destroy_id(id); - return nullptr; - } - if (!rdma_detail::wait_cm_event(setup_channel.get(), &event, 5000) || event == nullptr || - event->event != RDMA_CM_EVENT_ESTABLISHED) { - - CRB_LOG_WARN("rdma connect: ESTABLISHED missing addr={} event={} status={} errno={}", addr, - event == nullptr ? -1 : event->event, event == nullptr ? 0 : event->status, errno); - if (event != nullptr) - rdma_ack_cm_event(event); - destroy_connection_qp(connection.get(), id); - rdma_destroy_id(id); - return nullptr; - } - RdmaCmOffer peer_offer; - const bool valid_peer = event->param.conn.private_data != nullptr && - parse_rdma_offer(std::span(static_cast(event->param.conn.private_data), - event->param.conn.private_data_len), - &peer_offer) && - validate_rdma_offer(peer_offer, local, expected_peer, true); - rdma_ack_cm_event(event); - if (!valid_peer) { - destroy_connection_qp(connection.get(), id); - rdma_destroy_id(id); - return nullptr; - } - configure_peer(*pools, peer_offer); - if (!create_connection_pools(*pools) || !prime_receive(connection)) { - destroy_connection_qp(connection.get(), id); - rdma_destroy_id(id); - return nullptr; - } - pools->qp_number = id->qp->qp_num; - ibv_qp_attr receive_attributes{}; - receive_attributes.min_rnr_timer = config_.min_rnr_timer; - if (ibv_modify_qp(id->qp, &receive_attributes, IBV_QP_MIN_RNR_TIMER) != 0) { - destroy_connection_qp(connection.get(), id); - rdma_destroy_id(id); - return nullptr; - } - if (!publish_cm_connection(id, connection, std::move(pools))) { - destroy_connection_qp(connection.get(), id); - rdma_destroy_id(id); - return nullptr; - } - return connection; -} - bool RdmaTransport::listen(const std::string &addr, int port) { if (addr.empty() || port <= 0 || port > 65535 || listen_id_ != nullptr) { diff --git a/lib/crowdb-rpc/tests/rdma_startup_test.cpp b/lib/crowdb-rpc/tests/rdma_startup_test.cpp new file mode 100644 index 000000000..2e77b5d2a --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_startup_test.cpp @@ -0,0 +1,89 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/client/client.h" +#include "crowdb-rpc/server/handler.h" +#include "crowdb-rpc/server/message.h" +#include "crowdb-rpc/server/server.h" +#include "crowdb-rpc/transport/socket_transport.h" + +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) +# include + +# include +# include +# include +# include +# include +# include + +using namespace crowdb::rpc; + +namespace +{ +struct TestFirstResponse +{ + std::atomic completed{false}; + bool matches = false; +}; + +void complete_first_response(uint64_t id, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, + crowdb_rpc_status status, void *context) +{ + auto &result = *static_cast(context); + result.matches = status == CROWDB_RPC_OK && id == 42 && data != nullptr && crowdb_rpc_buffer_len(data) == 128 && + std::all_of(crowdb_rpc_buffer_data(data), crowdb_rpc_buffer_data(data) + 128, + [](uint8_t byte) { return byte == 0xab; }); + crowdb_rpc_buffer_release(control); + crowdb_rpc_buffer_release(data); + result.completed.store(true, std::memory_order_release); +} +} // namespace + +TEST(RdmaHardwareTest, ContiguousFirstResponseCompletesWithinTheWarmupWindow) +{ + const auto *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) + GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; + RdmaConfig config; + config.use_environment = false; + config.wire_layout = RdmaWireLayout::Contiguous; + config.source_address = address; + if (const auto *device = std::getenv("CROWDB_RDMA_DEVICE")) + config.device = device; + config.wire_slot_capacity = 1024; + config.sq_depth = config.rq_depth = config.posted_receive_target = 1024; + config.cq_depth = 2048; + config.send_arena_bytes = config.receive_arena_bytes = 1024 * 1024; + TestFirstResponse result; + RpcClient caller; + RpcServer server; + ASSERT_TRUE(server.configure_rdma(config, true, false, true)); + ASSERT_TRUE(server.listen(address, 0)); + server.register_handler(100, handle_echo); + server.start(); + SocketTransport client(1, 1); + ASSERT_TRUE(client.configure_rdma(config, true, false, false)); + client.start(); + auto connection = client.connect(address, server.listen_port()); + ASSERT_NE(connection, nullptr); + ASSERT_EQ(connection->path(), ConnectionPath::Rdma); + caller.attach(connection.get()); + auto *data = client.pool()->alloc(128); + ASSERT_NE(data, nullptr); + std::memset(data->data, 0xab, 128); + data->len = 128; + ASSERT_TRUE(caller.send(&client, connection.get(), 42, build_ping_request(client.pool(), 42, 0), data, 100, + complete_first_response, &result)); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(1); + while (!result.completed.load(std::memory_order_acquire) && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + const bool completed_in_time = result.completed.load(std::memory_order_acquire); + client.stop(); + server.stop(); + caller.fail_all(nullptr, RpcError::ConnectionClosed); + EXPECT_TRUE(completed_in_time) << "first RPC exceeded the unchanged one-second benchmark warmup"; + EXPECT_TRUE(result.matches); + EXPECT_EQ(client.rdma_stats_for_tests()->fallback_count.load(), 0U); +} +#endif From 52dc630e1c91d7e19f2033bccf78ec8faa649c55 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 07:06:48 +0800 Subject: [PATCH 054/258] Record final native and OCI throughput with service acceptance evidence --- doc/design/rpc/rpc-flow-analysis.md | 375 ++++++++++++++----- doc/working/plan-rdma-chunk-kv-access.md | 19 + tools/benchmark/bench-rdma-rpc-regression.sh | 57 +-- tools/benchmark/bench-rpc-regression.sh | 39 +- 4 files changed, 355 insertions(+), 135 deletions(-) diff --git a/doc/design/rpc/rpc-flow-analysis.md b/doc/design/rpc/rpc-flow-analysis.md index 4a6a75a48..31445a51f 100644 --- a/doc/design/rpc/rpc-flow-analysis.md +++ b/doc/design/rpc/rpc-flow-analysis.md @@ -3,31 +3,32 @@ # RPC Flow Analysis -End-to-end analysis of the standalone RPC echo benchmark. It measures -crowdb-rpc transport only: epoll/kqueue, framing, request correlation, and the -request/response callback. There is no KV or storage layer. +The standalone RPC echo benchmark measures crowdb-rpc transport: +epoll/kqueue, framing, request correlation and request/response completion. +Echo throughput excludes KV/storage work. Separate functional corpora below +verify actual service and HTTP/DiskIO behavior. ## 1. Flow ```text -Rust benchmark worker - -> build flatbuffer control and pooled data buffers +crowdb-cli bench rpc / bench rdma + -> co_spawn native coroutine loaders, one in-flight request each + -> Rust callback builds flatbuffer control and pooled data buffers -> RpcClient::send(..., callback) claim a preallocated completion slab slot - enqueue the frame and writev on the caller's I/O worker - -> standalone server readable event - read into a 256KiB worker buffer - parse all frames, copy the request into an echo response - enqueue the response and batch writev per connection - -> client readable event - parse response and claim the slab slot DONE - invoke bench_on_complete inline on the I/O worker + submit through the selected TCP or RDMA transport, suspend coroutine + -> standalone server receives complete request frames + validate control, copy request data into an echo response + submit through the same selected transport + -> client completion owner parses response and claims the slab slot DONE + resume coroutine and validate response through the Rust callback + record owner-sharded counters and nanosecond histogram release buffers and submit the next request - -> worker unparks after its in-flight requests drain + -> co_spawn returns after all coroutines and active callbacks drain ``` -The callback mode avoids a tokio scheduler round trip. The tokio mode uses -`RpcClient::call`, an oneshot channel, and a task wake for each response. Both +The native coroutine mode avoids a tokio scheduler round trip. The tokio mode +uses `RpcClient::call`, a oneshot channel, and a task wake for each response. Both models share `RpcClient`, `Connection`, `BufferPool`, and `RpcServer`. The TCP hot path uses connection pointers in epoll/kqueue user data, per-worker @@ -58,8 +59,8 @@ fail the runner. RDMA remote request totals must equal accepted submissions. Hardware: Ryzen 9 5950X, 16 physical/32 logical CPUs, approximately 62 GiB RAM, `rocep35s0f0` port 1, 100 Gb/s link, PCIe 8 GT/s x4. Both peers share this -physical host/HCA. Native means two processes; OCI means two host-network -containers using independent QPs, UID 1000, required Infiniband device mappings +physical host/HCA. raw-server means two native host processes; OCI means two +host-network containers using independent QPs, UID 1000, Infiniband device mappings and unlimited memlock. CPU affinity/frequency are not pinned. This topology measures local HCA traversal rather than physical-node isolation. @@ -72,50 +73,119 @@ nanosecond buckets, exact count/sum/extrema, nearest-rank upper bounds and explicit zero/overflow counts. Resource and transport counter deltas include warmup and drain, as declared in each row. -The following rows select the median TPS of three peak repeats and retain -latency from that same trial. All use 128-byte payloads, zero errors and valid -reports. RDMA rows use clean source `55ffee38`; the Nagle-enabled TCP control -is retained from the preceding runner revision with its own binary identity. - -| Path / topology | Loaders | Connections / QPs | CQ / pollers per peer | Socket workers per peer | Aggregation / Nagle | RPC TPS | Avg us | P90 us | P99 us | Errors | Valid | -|--------------------------|---------|-------------------|-----------------------|-------------------------|---------------------|-----------|--------|----------|----------|--------|-------| -| RDMA native | 2048 | 16 | 16 | 1 | aggregation on | 2,903,006 | 704.47 | 819.20 | 987.14 | 0 | true | -| RDMA two OCI containers | 2048 | 16 | 16 | 1 | aggregation on | 2,954,105 | 692.26 | 851.97 | 1015.81 | 0 | true | -| TCP native | 1000 | 32 | n/a | 16 | Nagle off | 1,196,115 | 834.50 | 1679.36 | 6815.74 | 0 | true | -| TCP native | 1000 | 32 | n/a | 16 | Nagle on | 2,114,935 | 471.48 | 872.45 | 1597.44 | 0 | true | - -RDMA queue settings for both topology rows: - -| Code variable | Requested value | Observed / scope | -|------------------------------------|------------------------------|------------------------------------------| -| `io_engines`, `io_workers` | 1, 1 | per peer | -| `cq_count`, `poller_count` | 16, 16 | one fixed thread per CQ per peer | -| `sq_depth`, `rq_depth` | 1024, 1024 | per QP; provider values in canonical row | -| `posted_receive_target` | 1024 | per QP | -| `cq_depth` | 2048 | per CQ; provider returns 4095 | -| `wire_slot_capacity` | 1024 bytes | all headers/control/data included | -| `send_arena_bytes` | 1 MiB | per QP | -| `receive_arena_bytes` | 1 MiB | per QP | -| `send_queue_capacity` | 4096 | per QP | -| `poll_batch`, `verbs_poll_batch` | 256, 32 | per owner | -| `poll_empty_budget` | 256 | per owner | -| `aggregation_max_bytes` | 1024 | also bounded by peer receive slot | -| `aggregation_max_frames` | 256 | complete frames only, no flush timer | -| `registered_bytes` | derived budget | 34 MiB per peer at 16 QPs | -| `rnr_nak_retry_err` | cumulative provider counter | 53 before/after; delta 0 | -| CPU migrations/quarantined bytes | unavailable aggregate | null, never fabricated zero | - -CPU user/system time, context switches, affinity, SEND counts/rate, -frames per SEND, effective capacities and all error categories are present -in the wide canonical tables alongside the raw before/after peer snapshots. -The primary artifacts under `.crowdb-runtime/rdma-check/artifacts/bench/` are -`rdma-rpc-20261022T170138.574260Z` (native), -`rdma-rpc-20261022T170225.046025Z` (OCI), -`tcp-rpc-20261022T170313.500461Z` (TCP Nagle off) and -`tcp-rpc-20261022T162548.233330Z` (TCP Nagle on). -RDMA repeat TPS values are 2,892,325 / 2,907,907 / 2,903,006 native and -2,896,746 / 2,954,105 / 3,002,761 OCI. The OCI median is 1.76% higher; -the observed variation does not establish a container overhead or benefit. +All six peak rows below use clean source `944274af` and matching release +binaries. They select median TPS from three independent repeats, retaining +latency from that same trial. All 26 final trials have valid reports, zero +unexpected errors and zero fallback. TCP OCI has no RDMA device mappings. +These are closed-loop coroutine workloads, not fixed-rate open-loop SLO tests. +Peak rows use different tuned concurrency; they do not compare latency at +equal offered load. + +| Path / topology | Loaders | Connections / QPs | CQ / pollers | Socket workers | Aggregation / Nagle | RPC TPS | Avg us | P90 us | P99 us | Max ms | Errors | Fallback | +|-----------------|---------|-------------------|--------------|----------------|---------------------|-------------|--------|---------|---------|--------|--------|----------| +| RDMA raw-server | 2048 | 16 | 16 | 1 | aggregation on | 3,026,938.6 | 675.55 | 823.29 | 1097.73 | 5.788 | 0 | 0 | +| RDMA OCI | 2048 | 16 | 16 | 1 | aggregation on | 3,236,760.8 | 631.64 | 704.51 | 892.93 | 3.559 | 0 | 0 | +| TCP raw-server | 1000 | 32 | n/a | 16 | Nagle off | 1,203,469.3 | 829.48 | 1589.25 | 7143.42 | 22.432 | 0 | 0 | +| TCP raw-server | 1000 | 32 | n/a | 16 | Nagle on | 2,167,732.3 | 459.98 | 851.97 | 1548.29 | 5.356 | 0 | 0 | +| TCP OCI | 1000 | 32 | n/a | 16 | Nagle off | 1,094,684.9 | 912.03 | 1441.79 | 9109.50 | 38.539 | 0 | 0 | +| TCP OCI | 1000 | 32 | n/a | 16 | Nagle on | 2,112,923.9 | 471.92 | 888.83 | 1671.17 | 6.659 | 0 | 0 | + +The worst TCP maximum across the three repeats is 24.996 ms raw-server/Nagle +off, 7.170 ms raw-server/Nagle on, 38.539 ms OCI/Nagle off and 7.063 ms +OCI/Nagle on. P90 does not describe this tail. RDMA's worst repeated maximum +is 5.788 ms raw-server and 3.559 ms OCI. All timeout, payload mismatch, +duplicate completion, CQ, protocol and report error checks remain enabled. + +Single-loader latency and high-load throughput are separate controls: + +| RDMA raw-server loaders | QPs | CQ / pollers | Aggregation | RPC TPS | Avg us | P90 us | P99 us | Max ms | Errors | +|-------------------------|-----|--------------|-------------|-------------|--------|--------|--------|--------|--------| +| 1 | 1 | 1 | off | 60,964.4 | 15.88 | 15.94 | 21.25 | 0.361 | 0 | +| 512 | 8 | 8 | on | 2,290,244.8 | 222.58 | 257.02 | 299.01 | 0.656 | 0 | + +At 2048 outstanding RPCs, roughly 3M TPS implies average RTT around 0.67 ms; +this queueing cost must not be presented as the single-loader 16 us latency. +The contiguous layout posts fixed local receive slots before active CM +connect. Moving that step after CM had produced zero warmup completions and a +1.46-second cold response. The corrected single-loader trial completes 57,184 +warmup RPCs and has a 0.361 ms maximum, with unchanged one-second warmup, +ten-second measurement and five-second request deadline. Control/data still +allocates its actual negotiated minimum after validating the CM reply. + +RDMA queue settings for both peak topology rows: + +| Code variable | Requested value | Observed / scope | +|----------------------------------|-----------------------------|------------------------------------------| +| `io_engines`, `io_workers` | 1, 1 | per peer | +| `cq_count`, `poller_count` | 16, 16 | one fixed thread per CQ per peer | +| `sq_depth`, `rq_depth` | 1024, 1024 | per QP; provider values in canonical row | +| `posted_receive_target` | 1024 | per QP | +| `cq_depth` | 2048 | per CQ; provider returns 4095 | +| `wire_slot_capacity` | 1024 bytes | all headers/control/data included | +| `send_arena_bytes` | 1 MiB | per QP | +| `receive_arena_bytes` | 1 MiB | per QP | +| `send_queue_capacity` | 4096 | per QP | +| `poll_batch`, `verbs_poll_batch` | 256, 32 | per owner | +| `poll_empty_budget` | 256 | per owner | +| `aggregation_max_bytes` | 1024 | also bounded by peer receive slot | +| `aggregation_max_frames` | 256 | complete frames only, no flush timer | +| `registered_bytes` | derived budget | 34 MiB per peer at 16 QPs | +| `rnr_nak_retry_err` | cumulative provider counter | 1789 before/after; delta 0 in raw median | +| CPU migrations/quarantined bytes | unavailable aggregate | null, never fabricated zero | + +`connection_count` counts RC QPs; `qpair_number` in connection status is a +provider QP identifier. CQ count and polling thread count match one-for-one. +TCP uses `io_engines = 1`, `io_workers = 16`, 32 connections and +`send_queue_capacity = 4096`; RDMA is explicitly disabled there. CQ/SQ/RQ and +registered slots are not TCP settings. + +CPU user/system time, context switches, affinity, SEND counts/rate, frames per +SEND, actual capacities and every error category remain in the canonical wide +CSV/TSV alongside raw before/after snapshots. Counter deltas include warmup +and drain. Native peers load the pixi runtime libraries; OCI preloads its +system verbs libraries and configured provider. CPU affinity/frequency are not +pinned. OCI RDMA is 6.93% higher than raw-server; OCI TCP is 9.04% lower with +Nagle off and 2.53% lower with Nagle on. These observations do not isolate a +container effect from library/provider and scheduling differences. The shared +workload, settings, deadlines and report validation are identical. + +Canonical evidence is retained in both the repository runtime directory +`.crowdb-runtime/rdma-check/artifacts/bench/` and +`/tmp/crowdb-rdma-probe/artifacts/`. Profile directory names are: + +- `accepted-944274af-rdma-native-final` and `accepted-944274af-rdma-oci-final`; +- `accepted-944274af-tcp-native-off-final` and `accepted-944274af-tcp-native-on-final`; +- `accepted-944274af-tcp-oci-off-final` and `accepted-944274af-tcp-oci-on-final`. + +RDMA raw-server repeat TPS: 3,036,947.8 / 3,026,938.6 / 3,026,730.4. + +RDMA OCI repeat TPS: 3,236,760.8 / 3,228,268.4 / 3,251,586.3. + +TCP raw-server Nagle off repeat TPS: 1,207,836.4 / 1,203,469.3 / 1,202,207.6. + +TCP raw-server Nagle on repeat TPS: 2,177,406.0 / 2,165,395.3 / 2,167,732.3. + +TCP OCI Nagle off repeat TPS: 1,092,294.1 / 1,100,987.3 / 1,094,684.9. + +TCP OCI Nagle on repeat TPS: 2,116,257.1 / 2,112,923.9 / 2,100,983.2. + +Server SHA256 is +`227fe3de6d7b337855d99f7e76ac48a4d348e0a13c700b347e654f9c64c5c55b`; +CLI SHA256 is +`ee349771016cf06012406806040266947e83b26ec50d3f12eea5f993306a26b0`. +The pinned OCI image is +`sha256:e3c9c5c6d5fc3424ac3f22cf4cdcc9b1c872f47336484f54962b2541c4ed6cef`. +Both wrapper scripts embed the environment, parameters, repeats and results. + +The earlier transport-tuning source `55ffee38` measured 2.903M TPS native and +2.954M OCI; matching TCP Nagle off measured 1.196M TPS. The pre-fix v4 source +`018da2fc` measured 2.901M/3.073M RDMA native/OCI with 1.5–1.8-second cold +outliers. Those controls exposed the contiguous receive-priming regression; +low P90 alone had hidden it. Earlier canonical directories were removed during +this session; their IDs below are historical evidence identifiers rather than +currently available downloads. Surviving acceptance/gate logs are retained +under `/tmp/r241-*.log`; current perf and gate artifacts are also backed up in +the repository runtime directory. The staged sweep retained 24 rows and 203,195,362 measured completions at `rdma-rpc-20261022T160711.769136Z`, then nine higher-load rows at @@ -126,7 +196,7 @@ counts 2/4/8 produce about 880k/1.48M/2.21M TPS. At higher load, 16 CQs reach approximately 2.9M; 32 CQs per peer fall to 2.02M and P99 4.00 ms because both peers together have 64 polling threads on 32 logical CPUs. RQ 4096 and larger 64/256 KiB slots do not improve matching controls. -Polling-budget and slot controls are retained at +Polling-budget and slot controls were recorded as `rdma-rpc-20261022T162311.140989Z`; this is the best repeatable tested configuration, not proof of the HCA's hardware maximum. @@ -140,22 +210,39 @@ has average/P90 latency about 44 ms, so small-message latency must not be presented as the latency of every size/load case. The no-device TCP OCI corpus at `tcp-rpc-20261022T163555.799432Z` also passes with zero RDMA attempts. -Historical audit: the 1,186,876 TPS value once labeled RDMA came from TCP -socket reads/writev in -`.crowdb-runtime/artifacts/bench/rdma-rpc-20261018-220617/server/metrics.log`. -Its old TSV records error rows despite a successful script exit. The new -matching TCP Nagle-off median is 1,196,115 TPS (+0.78%), explaining that -baseline without attributing it to RDMA. The older 2,213,182 Nagle-on TCP -reference is 4.4% above the new 2,114,935 control, but timing, validation and -histogram contracts changed; it is not a controlled transport regression. - -The accepted transport source passes full C++/FFI verification at -`.crowdb-runtime/artifacts/unit/cpp-93edc5b4128b456696da0095db768d20/`, all -137 no-verbs CTest cases, four CLI benchmark cases, thirteen runner contract -cases, formatting and workspace lint. Separate real-HCA cases cover held -receive/MR lifetime, registration exhaustion, TCP-only peers, expired CM -endpoints and peer/server close after payload. Close tests allow fallback but -require exactly one failed completion, retained RDMA path and no TCP payload. +Historical audit: the 1,186,876 TPS value once labeled RDMA used TCP socket +reads/writev, and its TSV contained error rows despite successful script exit. +The current matching Nagle-off TCP median is 1,203,469 TPS (+1.40%). It is +not an RDMA regression. The previous fixed-request echo did not test large +encoded Chunk-KV controls, mixed fragment sizes, asynchronous handler lifetime +or interrupted logical results. Current runners require strict path identity, +zero errors and fallback, matching remote submissions, complete reports and +successful process exits. Echo throughput and the service corpora below prove +different properties and share the same native/two-container process launcher. + +Final transport verification passes 543 tree, 192 RPC, 71 DiskIO and 35 common +native cases plus both FFI suites at +`.crowdb-runtime/artifacts/unit/cpp-b493f2503a414232a8c41bc2a13824d6/`. +All 160 no-verbs CTest cases pass (one hardware case skipped), and all 14 +selected real-HCA cases pass, including first-response latency, asymmetric +classes, mixed-size staging, held results, bounded admission, expiry and +peer-close behavior. Changed C++ format/tree-lint, Rust format and thirteen +runner contract cases pass. Current gate logs and HCA XML are backed up at +`.crowdb-runtime/rdma-check/artifacts/verification-944274af/`. +The unchanged Rust service integration passed 1,244 affected cases (four +optional cases ignored), workspace all-feature clippy and `rs-lint` at clean +source `018da2fc`; the final native setup correction preserves those service +control/data semantics. Earlier four CLI timed/coroutine contract cases pass. + +Two existing intermittent failures were retained rather than repaired here: +`DoubleBuffer.ConcurrentReadersDuringFrequentFreezeAndDrainNoCorruption` +failed `bad.load() == false`, then its thirteen-case suite and full tree gate +passed without source changes. `SqFullBackpressureTest.UringEngineTinySqManyWrites` +returned five `-ENOMEM` callbacks when the existing 1000 yield attempts could +not obtain an SQ slot; the exact case, fourteen-case suite and full gate then +passed unchanged. Initial gate IDs are `cpp-f0f7acd09fb541f59d048f81e3923cc2` +and `cpp-bdbe5707db274b8aa625247ce0eb00eb`. Complete surviving failure logs +include `/tmp/r241-class-cpp.log`; this work does not claim either fix. The sections below retain earlier 20-second TCP references. `Nagle` means TCP coalescing is enabled; old histogram precision and measurement scopes differ. @@ -172,13 +259,13 @@ location geometry. TPS and latency percentiles are not measured by this functional fixture; the RPC throughput runner above remains the common load measurement. -| Topology | Path | Disk engine | Downloaded bytes | Upload direct bytes | Download direct bytes | Transport copy bytes | TCP fallback | Expected errors | Verification errors | -|--------------------------|------|-----------------|------------------|---------------------|-----------------------|----------------------|--------------|-----------------|---------------------| -| Two native processes | RDMA | io_uring | 3,158,073 | 6,319,478 | 3,159,739 | 0 | 0 | 0 | 0 | -| Two native processes | TCP | io_uring | 3,158,073 | n/a | n/a | n/a | n/a | 0 | 0 | -| Two non-root containers | RDMA | BlockingEngine | 3,158,073 | 6,319,478 | 3,159,739 | 0 | 0 | 0 | 0 | -| Two non-root containers | TCP | BlockingEngine | 3,158,073 | n/a | n/a | n/a | n/a | 0 | 0 | -| Containers / peer close | RDMA | BlockingEngine | 3,144,096 prefix | 6,319,478 | 3,145,728 | 0 | 0 | 1 | 0 | +| Topology | Path | Disk engine | Downloaded bytes | Upload direct bytes | Download direct bytes | Transport copy bytes | TCP fallback | Expected errors | Verification errors | +|-------------------------|------|----------------|------------------|---------------------|-----------------------|----------------------|--------------|-----------------|---------------------| +| Two native processes | RDMA | io_uring | 3,158,073 | 6,319,478 | 3,159,739 | 0 | 0 | 0 | 0 | +| Two native processes | TCP | io_uring | 3,158,073 | n/a | n/a | n/a | n/a | 0 | 0 | +| Two non-root containers | RDMA | BlockingEngine | 3,158,073 | 6,319,478 | 3,159,739 | 0 | 0 | 0 | 0 | +| Two non-root containers | TCP | BlockingEngine | 3,158,073 | n/a | n/a | n/a | n/a | 0 | 0 | +| Containers / peer close | RDMA | BlockingEngine | 3,144,096 prefix | 6,319,478 | 3,145,728 | 0 | 0 | 1 | 0 | The failure case gates HTTP delivery after one verified frame, closes DiskIO, then resumes delivery. The body terminates with an error and never publishes a @@ -201,16 +288,114 @@ engine selection is independent of the RDMA/TCP path. Both containers use UID 1000, host networking and the same host/HCA, with independent QPs. The pinned runtime image is `sha256:e3c9c5c6d5fc3424ac3f22cf4cdcc9b1c872f47336484f54962b2541c4ed6cef`. -All five profiles use clean source `ab1ffbfb`. Manifests retain binary hashes, +All five profiles also pass the clean `018da2fc` common-control regression. +Manifests retain binary hashes, source/dirty state, actual server identity, -engine, settings, complete logs and raw counters. Artifacts under -`.crowdb-runtime/rdma-check/artifacts/rdma-datapath/` are: - -- `7852ce8aef7a49e6804b1f25241f8424`: native RDMA; -- `9235eeb5ad6f407dae9a8fab190d0f73`: native TCP; -- `5268f02d8ffb4e458113d39e6a5f9dd2`: shared-HCA OCI RDMA; -- `a09e1892158d4b1aa2499c39d926f3fb`: OCI TCP without RDMA devices; -- `227f2ddb147a4fedb5fff32ffe187eda`: OCI partial-download peer close. +engine, settings, complete logs and raw counters. Final regression artifact +IDs below are historical after runtime cleanup; complete role/runner logs +remain under `/tmp/r241-datapath-*.log`: + +- `4228dda544c44d5c84f7f8ad4e215521`: native RDMA; +- `2d6bfc8b120b49648a7cd467cdfb8ae4`: native TCP; +- `6307140655914a9f99ffc95e7cf1a1a4`: shared-HCA OCI RDMA; +- `da8e41b8e25340058c67f057894b1732`: OCI TCP without RDMA devices; +- `e15280fc8ca649c5a9ccf77cf05f0ac3`: OCI partial-download peer close. + +### Chunk-KV / Access functional acceptance — 2026-10-23 + +`tools/rdma/run-chunk-kv.py` uses the same native/two-container `Runtime` as +the throughput runners. Each profile runs production `ChunkKvRpcService`, +`CrowdbPartitionTree` and partition WAL logic in a separate server process. +Catalog, serving authority and memory ChunkStream backing are injected; this +corpus does not start a complete Group-0/namespace/authentication deployment. +Access configuration and construction are covered by the affected Rust suites. + +Eight values total 3,409,975 bytes: 63 KiB, 65,553 B, 255 KiB, 262,175 B, +768 KiB, 1,048,583 B, 300 KiB and 600 KiB. The corpus verifies exact point +put/get, conditional mutation, delete, seek, forward/reverse/paged scan, +600 KiB bounded scan, multi-get duplicate order and a two-mutation 900 KiB +batch. Stable mutation IDs 1–8 replay without duplicate effects. Expired +requests return `RequestExpired`; stale scan continuations return +`RefreshRequired`. A point route hint keeps the existing current-catalog +behavior. TCP/RDMA results and typed errors match. + +| Topology | Profile | Typed errors expected | Transport errors expected | Setup fallback | Unexpected errors | Verification errors | +|------------|-----------------|-----------------------|---------------------------|----------------|-------------------|---------------------| +| raw-server | strict RDMA | 2 | 0 | 0 | 0 | 0 | +| raw-server | explicit TCP | 2 | 0 | 0 | 0 | 0 | +| OCI | strict RDMA | 2 | 0 | 0 | 0 | 0 | +| OCI | no-device TCP | 2 | 0 | 0 | 0 | 0 | +| OCI | setup fallback | 2 | 0 | 2 | 0 | 0 | +| raw-server | RDMA peer close | 2 | 1 | 0 | 0 | 0 | +| OCI | RDMA peer close | 2 | 1 | 0 | 0 | 0 | + +Value SHA256: +`4a9ba75a81f4b56d08eb8a9de0ddc1feee810c932c26f40bf35a2765b324e3af`. +Mutation-result SHA256: +`4b9633bb71ba81353fb3fa5f88be2dd369b9b8a1e2a3a34278cdcdd94be7486b`. +The no-device TCP profile makes no RDMA attempt. Setup fallback occurs once +per metadata/aggregate group before any RDMA application payload, preserving +the corpus, IDs and digests. + +Both peer-close profiles first finish the complete corpus, then pause the +server after one actual response fragment. The client observes exactly one +fragment before server termination, publishes zero incomplete logical results +and returns connection-closed within the existing five-second deadline. +There is no TCP replay/suffix. The fragment gate exists only in `test-util`. + +Client metadata and aggregate groups have separate fixed transports and QPs +but share one `RpcClient`, reaper and global request ID sequence. Metadata +uses 64 KiB data slots; aggregate operations use 256 KiB. The server uses a +256 KiB local class and negotiates actual 64 KiB/256 KiB receives per QP. +`control_slot_capacity = 1024`, `sq_depth = rq_depth = posted_receive_target = 2`, +requested `cq_depth = 16` (provider 31). Each client group has one CQ/poller; +the server has one CQ/poller for both QPs. Control/data transport aggregation +is disabled. One socket worker serves each peer. + +| Owner | QPs | CQ / pollers | Actual data capacity | Registered peak B | Logical send peak B | Logical receive peak B | +|------------------|-----|--------------|----------------------|-------------------|---------------------|------------------------| +| Client metadata | 1 | 1 | 64 KiB | 526,336 | 1,048,736 | 1,048,688 | +| Client aggregate | 1 | 1 | 256 KiB | 2,099,200 | 3,282,896 | 5,536,656 | +| Server | 2 | 1 | 64 / 256 KiB | 2,756,608 | 5,536,656 | 3,282,896 | + +Metadata wire/send/receive arena bytes are 66,560 / 131,072 / 133,120; +aggregate values are 263,168 / 524,288 / 526,336. Logical controls have a +32 MiB maximum and independent 64 MiB send/receive admission budgets. Results +larger than the whole two-slot RQ window complete while those slots are +reposted as fragments are consumed. Completed logical controls retain their +byte admission through final native/FFI caller release and remain readable +after transport destruction. + +Metadata client sends/receives 84/59 control fragments and copies +4,978,576 / 3,540,120 encoded control bytes. Aggregate client sends/receives +26/69 and copies 6,565,792 / 16,422,136 bytes. The server sends/receives +128/110 and copies 19,962,256 / 11,544,368 bytes. Encoded control assembly +copies each fragment once; these counters are distinct from direct data +payload copies in the HTTP/DiskIO corpus. CQ/protocol/post/SEND errors and +unexpected fallback remain zero in strict normal profiles. + +The first mixed-size real service corpus exposed cached smaller staging +allocations occupying every SQ descriptor. A later larger fragment then +could not progress despite available arena bytes. Control staging now keeps +descriptors at the configured data class while SEND uses actual length. +The dedicated mixed-size HCA case failed before the fix and passed afterward; +admission limits and timeouts were unchanged. This traffic is absent from +the small contiguous echo, explaining why that older benchmark could pass. + +All seven profiles pass clean source `018da2fc`, with the same pinned OCI +image, UID 1000 and host/HCA topology as the throughput runs. Historical +artifact IDs are `1b0ca70d7c624f91a043da5a4eb28355` (native RDMA), +`e05741caef6545489d1cf68346c09ab2` (native TCP), +`6a005f6dafde4ad6b384ba42ce457bc6` (OCI RDMA), +`168119889e2b4d308ceedb8d31aa3368` (OCI TCP), +`c2f6e04af419473786c5bc2de4964632` (OCI fallback), +`bacc9abc534a4e79bbcc4f56d9800c46` (native peer close) and +`74401eb8000c42fb91721ade5697e81a` (OCI peer close). Complete surviving logs +are `/tmp/r241-accepted-*.log`; the measured corpus is functional acceptance, +not a Chunk-KV TPS claim. +Surviving service, gate and mixed-size failure logs are also backed up under +`service-018da2fc-logs/` in both current artifact roots. + ### Linux — 2026-08-27 diff --git a/doc/working/plan-rdma-chunk-kv-access.md b/doc/working/plan-rdma-chunk-kv-access.md index 771f60f35..55b407955 100644 --- a/doc/working/plan-rdma-chunk-kv-access.md +++ b/doc/working/plan-rdma-chunk-kv-access.md @@ -409,3 +409,22 @@ pixi run tree-lint Do not hand off to R242 until Chunk-KV response assembly and TCP/RDMA semantic parity pass for metadata and multi-message aggregate responses. + +Final clean-source checkpoint: + +- All 26 final 1 s warmup + 10 s trials pass source `944274af` with matching + release binaries, valid reports, zero errors and zero fallback. RDMA peak + median is 3,026,938.6 TPS raw-server and 3,236,760.8 TPS OCI. TCP Nagle off/on + medians are 1,203,469.3 / 2,167,732.3 raw-server and + 1,094,684.9 / 2,112,923.9 OCI. Single-loader RDMA measures 60,964.4 TPS, + average 15.88 us, P90 15.94 us and maximum 0.361 ms. Artifacts and current + gate logs are backed up in both runtime and `/tmp/crowdb-rdma-probe/`. +- Both requested benchmark scripts now embed AMD Ryzen 9 5950X / 16-core / + 32-thread hardware, raw-server/two non-root OCI topology, exact source/image, + queue/slot/CQ/worker/aggregation settings, repeats, latency tails and errors. + Permanent RPC flow analysis records service corpus/digests, memory/copy/error + counters, the five HTTP/DiskIO reruns, old TCP mislabeling, the mixed-size + staging and cold-response fixes, and known unchanged intermittent failures. +- Implementation and acceptance are complete. Remaining work is final document + consistency/link verification and removal of this completed child plan, its + detail/index entry, and completed tasks in the persistent umbrella plan. diff --git a/tools/benchmark/bench-rdma-rpc-regression.sh b/tools/benchmark/bench-rdma-rpc-regression.sh index cad692351..e67f47841 100755 --- a/tools/benchmark/bench-rdma-rpc-regression.sh +++ b/tools/benchmark/bench-rdma-rpc-regression.sh @@ -3,42 +3,47 @@ # Licensed under the Apache License, Version 2.0. # Shared coroutine workload, structured readiness and canonical result files. # -# Measured 2026-10-23, source 018da2fc, matching release CLI/server, protocol v4. +# Measured 2026-10-23, clean source 944274af, matching release CLI/server, v4. # Environment: AMD Ryzen 9 5950X, 16 cores / 32 threads, about 62 GiB RAM. # HCA: rocep35s0f0 port 1, 100 Gb/s, PCIe 8 GT/s x4; one physical host. # raw-server = two native host processes. OCI = two non-root UID 1000 -# host-network containers sharing this HCA, with independent QPs and memlock -# unlimited. Image: sha256:e3c9c5c6d5fc3424ac3f22cf4cdcc9b1c872f47336484f54962b2541c4ed6cef. +# host-network containers; RDMA shares this HCA with independent QPs and +# unlimited memlock. TCP OCI has no RDMA device mappings. Runtime image: +# sha256:e3c9c5c6d5fc3424ac3f22cf4cdcc9b1c872f47336484f54962b2541c4ed6cef. # CPU affinity/frequency were not pinned. Native builds used -j32. # Workload: coroutine, 128-byte payload, one in-flight RPC per loader, -# 1 s warmup + 10 s validated completion window, drain counted separately. -# Peak rows select median TPS of three repeats; latency is from that same row. +# 1 s warmup + 10 s validated completion window; drain counted separately. +# request_timeout_secs=5. Peak rows select median TPS of three repeats; +# avg/P90/P99/max are from that same trial. All 26 final trials have valid +# reports, errors_total=0, report_errors=0 and fallback_count=0. # -# topology loaders QPs CQ/pollers socket_workers aggregation TPS avg_us p90_us p99_us errors fallback -# raw-server 2048 16 16 1 on 2,901,185.6 755.61 802.82 954.37 0 0 -# OCI 2048 16 16 1 on 3,073,435.4 725.12 741.38 946.18 0 0 -# Raw TPS repeats: 2,911,356.0 / 2,814,582.2 / 2,901,185.6. -# OCI TPS repeats: 3,073,435.4 / 3,067,371.3 / 3,121,091.5. +# topology loaders QPs CQ/pollers socket_workers aggregation TPS avg_us p90_us p99_us max_ms errors fallback +# raw-server 2048 16 16 1 on 3,026,938.6 675.55 823.29 1097.73 5.788 0 0 +# OCI 2048 16 16 1 on 3,236,760.8 631.64 704.51 892.93 3.559 0 0 +# Raw TPS repeats: 3,036,947.8 / 3,026,938.6 / 3,026,730.4. +# OCI TPS repeats: 3,236,760.8 / 3,228,268.4 / 3,251,586.3. # # Progression, raw-server (one 10 s trial each): -# loaders QPs CQ/pollers aggregation TPS avg_us p90_us p99_us errors -# 1 1 1 off 57,620.7 18.57 16.00 30.08 0 -# 512 8 8 on 2,223,708.2 249.36 243.71 284.67 0 +# loaders QPs CQ/pollers aggregation TPS avg_us p90_us p99_us max_ms errors +# 1 1 1 off 60,964.4 15.88 15.94 21.25 0.361 0 +# 512 8 8 on 2,290,244.8 222.58 257.02 299.01 0.656 0 # -# Per peer: io_engines=1, io_workers=1; per QP: sq_depth=rq_depth= -# posted_receive_target=1024, send_queue_capacity=4096 logical RPCs. -# cq_depth=2048 requested per CQ (provider 4095); wire_slot_capacity=1024 B; -# send_arena_bytes=receive_arena_bytes=1 MiB per QP; registered total 34 MiB -# per peer, including shared pools. poll_batch/verbs_poll_batch/ -# poll_empty_budget=256/32/256. aggregation_max_bytes=1024, +# Per peer: io_engines=1, io_workers=1; connection_count=16 counts RC QPs. +# qpair_number in connection status is the provider QP ID, not a count. +# Per QP: sq_depth=rq_depth=posted_receive_target=1024, +# send_queue_capacity=4096 logical RPCs. cq_depth=2048 per CQ (provider 4095), +# wire_slot_capacity=1024 B, send_arena_bytes=receive_arena_bytes=1 MiB; +# registered peak=34 MiB per peer including shared pools. poll_batch/ +# verbs_poll_batch/poll_empty_budget=256/32/256. aggregation_max_bytes=1024, # aggregation_max_frames=256; complete frames only, bounded by both slots. -# -# Startup-tail caveat: observed cold first responses reached about 1.5-1.8 s; -# the single-loader trial had zero warmup completions and max_ns=1460998030. -# Completion-window latency includes requests submitted during warmup. P90 is -# not maximum latency; retain this snapshot during the startup investigation. -# Original canonical artifacts: .crowdb-runtime/rdma-check/artifacts/bench/ -# accepted-018da2fc-rdma-native/ and accepted-018da2fc-rdma-oci/. +# Strict RDMA: TCP fallback disabled. Contiguous RQ is primed before CM; +# single-loader warmup completed 57,184 RPCs, removing the previous 1.46 s +# cold-response tail without changing warmup/deadlines. High-load latency +# includes queueing; P90 is not maximum latency. This is the highest repeated +# tested configuration, not a measured hardware ceiling. +# Canonical JSON/CSV/TSV, full counters, errors and binary hashes: +# .crowdb-runtime/rdma-check/artifacts/bench/ +# accepted-944274af-rdma-native-final/ and accepted-944274af-rdma-oci-final/. # set -Eeuo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/../.." diff --git a/tools/benchmark/bench-rpc-regression.sh b/tools/benchmark/bench-rpc-regression.sh index ad2e7d11e..4fed38858 100755 --- a/tools/benchmark/bench-rpc-regression.sh +++ b/tools/benchmark/bench-rpc-regression.sh @@ -3,28 +3,39 @@ # Licensed under the Apache License, Version 2.0. # Shared coroutine workload, structured readiness and canonical result files. # -# Measured 2026-10-23, source 018da2fc, matching release CLI/server, protocol v4. +# Measured 2026-10-23, clean source 944274af, matching release CLI/server, v4. # Environment: AMD Ryzen 9 5950X, 16 cores / 32 threads, about 62 GiB RAM. # HCA: rocep35s0f0 port 1, 100 Gb/s, PCIe 8 GT/s x4; one physical host. # raw-server = two native host processes. OCI = two non-root UID 1000 -# host-network containers sharing this HCA, with independent QPs and memlock -# unlimited. Image: sha256:e3c9c5c6d5fc3424ac3f22cf4cdcc9b1c872f47336484f54962b2541c4ed6cef. +# host-network containers; RDMA shares this HCA with independent QPs and +# unlimited memlock. TCP OCI has no RDMA device mappings. Runtime image: +# sha256:e3c9c5c6d5fc3424ac3f22cf4cdcc9b1c872f47336484f54962b2541c4ed6cef. # CPU affinity/frequency were not pinned. Native builds used -j32. # Workload: coroutine, 128-byte payload, one in-flight RPC per loader, -# 1 s warmup + 10 s validated completion window, drain counted separately. -# Peak rows select median TPS of three repeats; latency is from that same row. +# 1 s warmup + 10 s validated completion window; drain counted separately. +# request_timeout_secs=5. Peak rows select median TPS of three repeats; +# avg/P90/P99/max are from that same trial. All 26 final trials have valid +# reports, errors_total=0, report_errors=0 and fallback_count=0. # -# topology Nagle loaders connections socket_workers TPS avg_us p90_us p99_us errors fallback -# raw-server off 1000 32 16 1,206,800.2 827.23 1654.78 6815.74 0 0 -# raw-server on 1000 32 16 2,197,134.7 453.80 839.68 1515.52 0 0 -# OCI -- -- -- -- not measured in this performance snapshot -# Raw Nagle-off TPS repeats: 1,206,800.2 / 1,199,487.9 / 1,214,054.5. -# Raw Nagle-on TPS repeats: 2,197,134.7 / 2,186,038.9 / 2,198,891.2. +# topology Nagle loaders connections socket_workers TPS avg_us p90_us p99_us max_ms errors fallback +# raw-server off 1000 32 16 1,203,469.3 829.48 1589.25 7143.42 22.432 0 0 +# raw-server on 1000 32 16 2,167,732.3 459.98 851.97 1548.29 5.356 0 0 +# OCI off 1000 32 16 1,094,684.9 912.03 1441.79 9109.50 38.539 0 0 +# OCI on 1000 32 16 2,112,923.9 471.92 888.83 1671.17 6.659 0 0 +# Raw Nagle-off TPS repeats: 1,207,836.4 / 1,203,469.3 / 1,202,207.6. +# Raw Nagle-off worst max over those repeats: 24.996 ms. +# Raw Nagle-on TPS repeats: 2,177,406.0 / 2,165,395.3 / 2,167,732.3. +# Raw Nagle-on worst max over those repeats: 7.170 ms. +# OCI Nagle-off TPS repeats: 1,092,294.1 / 1,100,987.3 / 1,094,684.9. +# OCI Nagle-off worst max over those repeats: 38.539 ms. +# OCI Nagle-on TPS repeats: 2,116,257.1 / 2,112,923.9 / 2,100,983.2. +# OCI Nagle-on worst max over those repeats: 7.063 ms. # Per peer: io_engines=1, io_workers=16, send_queue_capacity=4096. # RDMA is explicitly disabled; CQ/SQ/RQ/registered-slot settings are n/a. -# The no-device OCI TCP functional corpus passed; no OCI TPS is inferred from it. -# Original canonical artifacts: .crowdb-runtime/rdma-check/artifacts/bench/ -# accepted-018da2fc-tcp-off/ and accepted-018da2fc-tcp-on/. +# Canonical JSON/CSV/TSV, full counters, errors and binary hashes: +# .crowdb-runtime/rdma-check/artifacts/bench/ +# accepted-944274af-tcp-native-off-final/, accepted-944274af-tcp-native-on-final/, +# accepted-944274af-tcp-oci-off-final/ and accepted-944274af-tcp-oci-on-final/. # set -Eeuo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/../.." From d4479a47ebf3497e05e752b3a26ba495bca5413a Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 07:09:17 +0800 Subject: [PATCH 055/258] Retire completed Chunk-KV work and preserve downstream SDK scope --- doc/backlog/R241-rdma-chunk-kv-access.md | 74 ---- doc/backlog/R242-rdma-fat-client-sdk.md | 3 +- doc/backlog/R5-rdma-alloc.md | 14 +- doc/backlog/backlog.md | 9 +- doc/working/plan-rdma-chunk-kv-access.md | 430 ----------------------- doc/working/plan-rdma-fat-client-sdk.md | 2 +- doc/working/plan-rdma.md | 157 ++++----- 7 files changed, 87 insertions(+), 602 deletions(-) delete mode 100644 doc/backlog/R241-rdma-chunk-kv-access.md delete mode 100644 doc/working/plan-rdma-chunk-kv-access.md diff --git a/doc/backlog/R241-rdma-chunk-kv-access.md b/doc/backlog/R241-rdma-chunk-kv-access.md deleted file mode 100644 index 1e021669d..000000000 --- a/doc/backlog/R241-rdma-chunk-kv-access.md +++ /dev/null @@ -1,74 +0,0 @@ - - - -### R241: Access server — RDMA Chunk-KV integration - -## Problem - -Access Server and Chunk-KV need RDMA for access operations, but their message -sizes differ from the S3/DiskIO datapath. Metadata is usually small, while an -aggregated result may be larger and must be returned as multiple messages -without changing access semantics. - -## Solution - -1. Route Access Server to Chunk-KV through R239 while retaining the existing - access planner, request correlation, generation checks, and error mapping. -2. Configure 64 KiB data capacity for metadata and 256 KiB for aggregated - Chunk-KV results. The transport is created with these defaults from config; - the access/aggregation layer selects the class for each operation. -3. Represent a result larger than its selected capacity as ordered messages - with request ID, stream ID, sequence, logical offset, and end/error flags. - The common RPC transport splits and assembles the already encoded control - message. Chunk-KV keeps its existing serializers and delivers one complete - logical request/response to the existing handler/correlation layer. Use the - negotiated control/data layout: small fragment control accompanies bounded - 64 KiB/256 KiB/1 MiB data containing the encoded logical control bytes. - Negotiate support and the logical-message limit before payload submission. -4. Use the same TCP/RDMA request identity and one-way RDMA-to-TCP fallback. - Retry only when the existing operation is idempotent or duplicate-suppressed. -5. Keep application aggregation above RDMA. Do not add an application flush - timer, another aggregate producer, or application payload assembly copy. - Existing contiguous RPC transport coalescing remains bounded by both peers' - slots. Control/data streams send one message per SEND. Reassembly copies - each control fragment once into one bounded logical control owner, which - remains charged until its final native/FFI caller release. -6. Bound logical control bytes independently of registered memory. Configure - a maximum logical message and per-direction send/receive byte budgets; - defaults cover the client's existing 16 MiB response budget. Sender - admission counts the retained original encoding. Receiver admission reserves - the complete logical length before assembly; exhausted admission retains - only the existing receive-window fragments and resumes on caller release. - Never allocate replacement receive storage or return a partial control - message. Disconnect/cancellation/partial-message expiry releases assembly - state while completed caller-owned results retain their admission. - -## Dependencies - -- Accepted [registered buffers and RDMA RPC](../design/rpc/design-crowdb-rpc-rdma.md). -- Existing Chunk-KV client/server and Access Server access semantics. -- Existing ChunkIO aggregation and request-number system. - -## Acceptance - -- Given metadata smaller than 64 KiB, when Access Server selects RDMA, the - result matches TCP and uses the metadata class. Invariant: semantic parity. - Integration test. -- Given an aggregated Chunk-KV response larger than 256 KiB, when it is - returned, ordered messages reconstruct the exact response. Invariant: - bounded message capacity and ordering. Integration test. -- Given an RDMA setup failure before payload, when an idempotent access request - is retried, TCP returns one response with the same request identity. - Invariant: shared correlation and fallback. Integration test. -- Given a partial RDMA response, when the connection fails, the access result - is failed and no TCP suffix is accepted. Invariant: no mixed transport - result. Integration test. - -Run: - -```bash -pixi run -- cargo fmt --all -- --check -pixi run -- cargo test -p crowdb-chunk-kv-client --all-targets -pixi run -- cargo test -p crowdb-access-server --all-targets -pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings -``` diff --git a/doc/backlog/R242-rdma-fat-client-sdk.md b/doc/backlog/R242-rdma-fat-client-sdk.md index 0c84551fd..98d307510 100644 --- a/doc/backlog/R242-rdma-fat-client-sdk.md +++ b/doc/backlog/R242-rdma-fat-client-sdk.md @@ -31,7 +31,8 @@ recycling memory while the caller still reads it. ## Dependencies -- Accepted registered buffers/RDMA RPC and R241. +- Accepted registered buffers/RDMA RPC and + [Chunk-KV/access integration](../design/rpc/design-crowdb-rpc-rdma.md#7-service-interactions). - Existing Access Server access-core and FFI conventions. - R220 remains a separate Dataset SDK plan; this requirement supplies the transport and lease contract it may reuse. diff --git a/doc/backlog/R5-rdma-alloc.md b/doc/backlog/R5-rdma-alloc.md index 238a1f209..a2f4e04e8 100644 --- a/doc/backlog/R5-rdma-alloc.md +++ b/doc/backlog/R5-rdma-alloc.md @@ -5,9 +5,9 @@ ## Status -The registered host-buffer foundation, common RDMA RPC transport and -HTTP/ChunkIO/DiskIO datapath are accepted. Remaining child requirements R241 -and R242 retain the Chunk-KV and fat-client goals. +The registered host-buffer foundation, common RDMA RPC transport, +HTTP/ChunkIO/DiskIO datapath and Chunk-KV/access integration are accepted. +Remaining child requirement R242 retains the fat-client goal. ## Problem @@ -27,9 +27,9 @@ defined in [`design-crowdb-rpc-rdma.md`](../design/rpc/design-crowdb-rpc-rdma.md 3. Reuse the accepted [HTTP/ChunkIO/DiskIO stream](../design/rpc/design-crowdb-rpc-rdma.md#7-service-interactions), preserving existing aggregation, aligned 1 MiB receive data, disk completion ownership, caller leases and partial-result failure without a TCP suffix. -4. Implement Access Server to Chunk-KV integration in - [R241](R241-rdma-chunk-kv-access.md), using 64 KiB metadata and 256 KiB - aggregated result messages. +4. Reuse the accepted [Chunk-KV/access stream](../design/rpc/design-crowdb-rpc-rdma.md#7-service-interactions), + using 64 KiB metadata and 256 KiB aggregate classes, bounded logical control + assembly, shared request identity and final-caller byte admission ownership. 5. Implement the native fat client and FFI lease API in [R242](R242-rdma-fat-client-sdk.md), with host memory first and explicit host-to-GPU copy for the initial GPU simulation. @@ -76,7 +76,7 @@ Run: ```bash pixi run -- cargo fmt --all -- --check -pixi run -- cargo test -p crowdb-rpc --all-targets +pixi run clean-env && pixi run test-cpp pixi run -- cargo test -p crowdb-access-server --all-targets pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings ``` diff --git a/doc/backlog/backlog.md b/doc/backlog/backlog.md index 49cd489c5..d67c2fcc2 100644 --- a/doc/backlog/backlog.md +++ b/doc/backlog/backlog.md @@ -32,13 +32,9 @@ live in the linked document; implementation follows - **[R170](R170-s3-cuobject-rdma.md)** — optional cuObject RDMA data plane. **Area:** access server / S3 / DiskIO / RDMA. **Complexity:** High. **Status:** Deferred until the basic TCP S3 path is stable and measured. -- **[R241](R241-rdma-chunk-kv-access.md)** — RDMA Access Server to Chunk-KV - transport with bounded metadata and aggregate messages. **Area:** access - server / Chunk-KV / RDMA. **Complexity:** High. **Dependencies:** accepted - registered buffers and RDMA RPC. - **[R242](R242-rdma-fat-client-sdk.md)** — native fat-client access SDK and FFI buffer leases. **Area:** access server / SDK / RDMA. **Complexity:** High. - **Dependencies:** accepted RDMA RPC and R241. + **Dependencies:** accepted RDMA RPC and Chunk-KV/access integration. - **[R185](R185-access-iceberg-cache-invalidation.md)** — bounded Iceberg cache and invalidation. **Area:** access server / Iceberg / Group 0 / Chunk-KV. **Complexity:** Medium. **Status:** Deferred pending focused cache measurements. @@ -105,7 +101,8 @@ live in the linked document; implementation follows **Area:** crowdb-tree engine. **Complexity:** Medium. - **[R5](R5-rdma-alloc.md)** — RDMA host-buffer RPC and data transport umbrella. **Area:** RPC / access server / Chunk-KV / DiskIO / SDK / RDMA. - **Complexity:** High. **Status:** Foundation/transport/datapath accepted; R241–R242 remain. + **Complexity:** High. **Status:** Foundation/transport/datapath/Chunk-KV + accepted; R242 remains. - **[R60](R60-tree-scan-sibling-leaf-readahead.md)** — sibling-leaf readahead on cold scans. **Area:** crowdb-tree / scan / DiskIO. **Complexity:** Medium. **Status:** Deferred pending cold file/block-backed measurements. diff --git a/doc/working/plan-rdma-chunk-kv-access.md b/doc/working/plan-rdma-chunk-kv-access.md deleted file mode 100644 index 55b407955..000000000 --- a/doc/working/plan-rdma-chunk-kv-access.md +++ /dev/null @@ -1,430 +0,0 @@ - - - -# RDMA Chunk-KV access Plan - -Upstream: [R241](../backlog/R241-rdma-chunk-kv-access.md), -[root RDMA plan](plan-rdma.md), [registered buffers](../design/rpc/design-crowdb-rpc-rdma.md#4-registered-arenas-and-slot-ownership), -and [RDMA RPC transport](../design/rpc/design-crowdb-rpc-rdma.md). - -Goal: use the common RDMA stream for Access Server to Chunk-KV operations while -preserving existing aggregation, request numbering, generation checks, and TCP -semantic parity. - -## Execution checkpoint and implementation order - -The datapath is accepted at `ab1ffbfb`; its cleanup is `b38d3d78`. Preserve its -1 MiB direct payload path and the ordinary contiguous RPC benchmark. The -focused Chunk-KV baseline passes all 22 client/CLI cases before these changes. -All 278 affected Access/ChunkIO/DiskIO cases and full native/FFI gates passed -at the preceding service checkpoint. - -Observed contract: - -- `ChunkKvRpcTransport` encodes point/seek/scan/multi-get/batch requests as - FlatBuffer control buffers and calls the existing `RpcClient::call`. Its - connection pool and completion IDs remain the correlation owner. The - server's `ChunkKvRpcService` decodes one complete control, checks the same - RPC ID and submits one encoded response. Large values and aggregates are - inside these control bytes, not the RPC data field. -- The routing/composition layer retains logical mutation identities during - retries and retries only unresolved group operations. Seek/scan retain - partition generation and bounded-page semantics. Do not add a separate - Chunk-KV request table or retry scheduler. -- Native RDMA currently rejects control beyond 1010 bytes. Control/data gives - independently aligned data with exact 64 KiB/256 KiB/1 MiB classes; use this - existing layout for encoded-control fragments. Keep the current direct - path for ordinary small control plus real datapath data. -- Metadata point calls use 64 KiB; grouped/seek/scan calls use 256 KiB. An - explicit class config may select 1 MiB. Each class owns a fixed native - connection group and common RPC completion engine, with no producer thread - per loader. Server default is 256 KiB. QP data capacity is the minimum of - the two advertised capacities and is used for actual RX allocation/posting - as well as SEND admission; expose that effective value in immutable status. - -- [x] **Build bounded common control-stream primitives**: add portable byte - admission, fragment metadata validation and one-message assembly, tested - without verbs. Reserve full logical length, validate request/stream ID, - sequence, exact offset, total, class, generation and final flag. Default - maximum encoded control is 32 MiB, covering existing 16 MiB response limits; - independent send/receive budgets default to 64 MiB each. Reject invalid - arithmetic/limits before allocation. Retain completed reservation until - final caller release. Preserve a failed/blocked fragment's existing owner; - blocked admission must not carve replacement storage. Files: private RPC - control-stream modules, `framing.h` control owner, focused portable tests. -- [x] **Negotiate and configure common streams**: add a small transport-owned - fragment FlatBuffer in `common_msg.fbs`, preserving the common leading - correlation fields, with reserved message type 4. Bump RDMA protocol to - version four; the existing final 4 handshake bytes advertise maximum logical - control length and participate in the CM capability digest. Extend the CM - offer from 48 to 52 bytes to carry that limit explicitly: the listener does - not retain the client's TCP preflight capability, so its digest alone cannot - provide the client's receive limit for response admission. Validate the - explicit limit against the capability when it is available. Legacy decode - remains for explicit pre-payload TCP fallback. Extend native/env/status and - a new versioned C ABI config entry without changing the version-one struct - or existing configure entries. Rust typed config carries max message, - per-direction budget and partial-assembly timeout (default 30 seconds). - The Chunk-KV typed class factory derives send arena bytes from selected SQ - depth and data capacity, as receive defaults already do. Preserve native - `RdmaConfig::for_layout`'s existing 64 MiB send arena default for the accepted - 1 MiB datapath and ordinary RPC. Files: capability/offer/config/status, FFI. -- [x] **Integrate CQ-owned fragmentation and assembly**: enqueue one original - logical frame under byte/count admission. The CQ owner walks it in order, - builds only the next bounded fragment, and holds immutable source owners - through all WR completions. Do not materialize an unbounded fragment list. - Receive validates metadata before copying into one logical owner and returns - each consumed RQ credit immediately; result size may exceed the entire RQ - window. On exhausted assembly admission retain at most the existing RQ - frames. Final lease release wakes the owning polling/event worker through - the existing retained operation gate, including safe release after shutdown. - Active partial assemblies expire; consumer-blocked first fragments remain - backpressured without an arbitrary consumer hold deadline. Dispatch only a - complete original Frame. Move quota-owned control storage through C ABI - without another full-message malloc/copy. Files: RDMA send/receive/CQ/CM - pools, common framing/C ABI conversion and owner-lifetime tests. - - Implementation order within this task: - - Add private fragment codec and CQ-only sender cursor. Charge the original - control using a Buffer slice whose retained owner is the byte lease; - rejected submit restores the caller's original buffer. WR records retain - the single original cursor until every posted SEND has completed. - - Set effective data capacity and peer generation/maximum from CM. Active - control/data connections create/prime receive storage after the validated CM reply and - before application publication, so asymmetric classes allocate/post the - actual minimum rather than a larger local default. - - Extend the existing CQ deferred receive vector for byte admission. A - per-QP count preserves FIFO, and a per-pass epoch prevents later fragments - overtaking an admission-blocked first fragment. Existing RQ owners stay - intact; no replacement buffers are carved. - - Check active assembly deadlines under CQ ownership and bound the event - wait to the nearest active deadline. Shared receive budget release wakes - CQ workers through the retained transport operation gate. Keep storage - destruction before returning byte admission. - - Record logical-message, fragment, copied-control, directional admission - and timeout counters separately from direct application payload counters. -- [x] **Wire Chunk-KV classes and typed settings**: add service defaults and - explicit RDMA configuration for metadata/aggregate connection groups, - preserve TCP defaults and current serializers/errors/retries, and wire the - Chunk-KV server plus Access storage construction. Cover point, mutation, - seek, bounded scan, multi-get and batch mutation, including duplicate result - order and stable logical mutation IDs. Files: client transport/config, - server startup/RPC boundary, Access storage config and tests. - - Implementation order within this task: - - Serialize the existing Rust `RdmaConfig` for cold service configuration; - preserve its native defaults. Add an explicit - `with_control_data_capacity` helper that derives wire capacity and SQ/RQ - arena bytes from the selected data class and configured depths. - - Add `ChunkKvRpcTransportConfig` with optional `metadata_rdma` and - `aggregate_rdma` overrides. Missing overrides preserve environment policy - and select native 64 KiB/256 KiB service defaults. Explicit enabled - overrides require control/data and one of 64 KiB/256 KiB/1 MiB. Each - group retains its own existing connection index and one fixed transport; - the transport shares one `RpcClient`, reaper and request-ID generator. - - Split the current client transport runtime and endpoint parsing into - private child modules before extending the 337-line facade. Add read-only - class/endpoint status for actual path, QP and provider capacities. - - Configure the Chunk-KV server with a 256 KiB control/data service default - and optional explicit `rpc_rdma`. Server construction owns listener - policy; client groups are always non-listeners. Wire Access common - `chunk_kv_rpc` settings through S3, Iceberg and Dataset storage construction, - preserving existing convenience constructors with default configuration. - - Pin the native connection in Frames transferred to asynchronous FFI - handlers. A completed logical control has no RQ data lease to retain that - connection; the existing raw handler pointer otherwise outlives CM teardown. - Add safe `ServerRequest::close_connection` backed by that retained owner. - Close and report rejected Chunk-KV response submissions instead of silently - dropping a response. Cover handler hold/peer close/final release. - - Preserve the current serializers, stable logical mutation IDs, exact pool - generation invalidation and retry/error behavior. Cover class selection, - serialized settings, invalid bounds and existing client/server RPC cases - before the real service corpus. -- [~] **Verify real service and clean up**: test malformed sequence/offset, - duplicate/end/generation/class, byte exhaustion, held final leases, - cancellation/close/expiry, and TCP setup fallback. Run the actual Chunk-KV - service on the HCA with metadata and results larger than 256 KiB, then two - non-root host-network containers and a no-device TCP profile. Compare bytes, - typed errors and identity using the same corpus. Rerun the HTTP/DiskIO - acceptance after common transport changes. Finish native/no-verbs/FFI, - workspace format/lint and affected service gates. Rebuild matching release - CLI/fb-server and repeat the 1 s warmup + 10 s peak controls before cleanup. - Record all settings, actual classes/QPs, memory and error categories. Delete - this completed child plan/detail/index entry after permanent docs/evidence - are updated. R242 remains outside the requested scope. - - Implementation order within this task: - - Run production `ChunkKvRpcService`, `CrowdbPartitionTree` and partition WAL - logic in a separate process. Inject only catalog/serving authority and the - memory ChunkStream backing; label that scope explicitly. Use the same - point/conditional/seek/scan/multi-get/batch corpus and stable mutation IDs - for strict RDMA, explicit TCP and pre-payload setup fallback. - - Reuse the canonical benchmark `Runtime` for native and two non-root OCI - processes. Record exact binaries, source, image, UID, actual connections, - classes, queues, copied-control bytes, directional admission and errors. - - Add a `test-util`-only native fragment gate. After the normal real-service - corpus, pause the response after exactly one posted control fragment and - close the server. Verify one fragment arrived, no logical result was - published, the caller fails promptly and TCP fallback remains zero. - - Repeat HTTP/DiskIO semantic profiles and matching 10-second CLI throughput - controls after the common RPC changes, then update permanent evidence. - -Real service checkpoint: - -- Production RPC/service/native tree and partition WAL logic pass an eight-value - 3,409,975-byte corpus in native and two non-root OCI processes. Only catalog, - serving authority and memory ChunkStream backing are injected. Metadata and - aggregate groups negotiate 64 KiB and 256 KiB with SQ/RQ/post target two. - Both value and mutation-result digests match strict RDMA, explicit no-device - TCP and pre-payload fallback; duplicate mutation IDs and multi-get order match. -- Expired request and stale scan-continuation errors remain typed and equal. - Native and OCI failure profiles receive exactly one response fragment before - server close, publish no incomplete logical result and return connection - closed within the existing deadline. Fallback stays zero after payload. -- The first real RDMA corpus exposed a fixed-descriptor staging defect: cached - smaller allocations could occupy every SQ descriptor and prevent a later - larger fragment from progressing. The dedicated HCA mixed-size case failed - before the fix and passes after control staging allocates the configured - data class while SEND retains the actual length. All seven HCA control cases - pass. Before/after evidence: `r241-control/mixed-sizes-before.log`, - `mixed-sizes-after.log`, and `mixed-sizes.xml` under the isolated artifacts. -- Full C++/FFI gate passes after this fix at - `.crowdb-runtime/artifacts/unit/cpp-a4be5996987e48c68bf6fa53219fde7a/`: - 543 tree, 191 RPC, 71 DiskIO and 35 common native cases plus both FFI suites. - Workspace all-feature clippy, scoped C++ lint and formatting pass. - The refreshed no-verbs build passes 160 cases. All 13 selected real HCA - transport/control cases and both direct DiskIO HCA cases also pass. - All 1,244 affected Rust cases pass again (four optional cases ignored), - workspace `rs-lint` passes, and all 13 canonical runner contract cases pass. -- Keep this task active until the clean-source semantic rerun, HTTP/DiskIO - regression, matching release CLI 10-second controls and permanent evidence - are complete. -- The clean v4 timed controls expose a startup tail: the single-loader run has - zero warmup completions and a 1.46-second first response; high-load rows also - contain roughly 1.8-second outliers. Preserve `accepted-018da2fc-*` artifacts. - Check whether moving contiguous RX priming after CM setup caused this change; - only control/data needs the peer's negotiated data class before allocation. - Keep the one-second warmup and ten-second window unchanged. Repeat controls - after diagnosis, and embed the final environment/config/results directly in - both benchmark wrapper scripts as the user requested. -- The controlled contiguous-only fix restores RX priming before active CM - connect. The unchanged single-loader 1 s warmup + 10 s probe now completes - 60,804 warmup RPCs, measures 60,814.3 TPS, average 15,935.74 ns, P90 16,063 ns, - P99 18,687 ns and maximum 90,601 ns with zero errors/fallback. Control/data - still allocates the negotiated minimum after the validated reply. Active - route/QP/CM stages now live in a private implementation with bounded helpers. -- The dedicated HCA first-response regression and all 13 existing selected - transport/control cases pass. Full C++/FFI gate passes at - `.crowdb-runtime/artifacts/unit/cpp-b493f2503a414232a8c41bc2a13824d6/`: - 543 tree, 192 RPC, 71 DiskIO and 35 common native cases plus both FFI suites. - The refreshed no-verbs build passes all 160 CTest cases (one hardware skip). - Changed C++ format/tree-lint, workspace Rust format, shell syntax and all - 13 runner contract cases pass. Logs remain at `/tmp/r241-startup-*.log`. - Repeat clean-source timed raw-server/OCI controls next; include OCI TCP TPS. -- Earlier build/artifact directories were removed during this session outside - this task's commands. Previous canonical artifact IDs remain historical; - surviving `/tmp/r241-accepted-*.log`, `/tmp/r241-datapath-*.log` and gate logs - retain the completed semantic checks. Keep fresh perf artifacts under - `/tmp/crowdb-rdma-probe/artifacts/` as well as the repository runtime root. - -## Phase 1: access contract mapping - -- [x] **Map request/response types**: identify metadata, point, batch, range, - and aggregate operations; record idempotency, request ID, generation, error, - and retry behavior. Files: `lib/crowdb-chunk-kv-client/`, Chunk-KV server - modules, and Access Server access-core modules. -- [x] **Map aggregation owner**: identify where Chunk-KV combines responses and - where it decides flush/end boundaries. RDMA must receive the aggregate after - that decision, never create another aggregation layer. -- [x] **Define class selection**: metadata defaults to 64 KiB, aggregated - Chunk-KV results default to 256 KiB, and 1 MiB remains a configured option for - larger value/range results. Transport creation receives defaults from config. - -## Phase 2: common client/server wiring - -- [x] **Add RDMA transport choice**: make the Chunk-KV client use the R239 - logical connection interface and preserve TCP as fallback. Do not duplicate - request correlation in the Chunk-KV layer. -- [x] **Advertise selected class**: include class and capacity in stream setup - or request metadata, validate the peer's negotiated class, and reject a - payload larger than that class before publishing it. -- [x] **Preserve request identity**: carry request ID, stream ID, sequence, - logical offset, generation, and operation type through RDMA completion and - response assembly. -- [x] **Preserve access errors**: map transport, timeout, cancellation, - generation, integrity, and Chunk-KV errors identically for TCP and RDMA. - -## Phase 3: multi-message response assembly - -- [x] **Split aggregate responses**: when an existing aggregate exceeds 64 KiB - or 256 KiB, emit ordered messages without changing its logical result. -- [x] **Reassemble on client**: validate sequence, offset, total/end flags, and - duplicate request identity; reject gaps, overlaps, stale generations, and - wrong classes. -- [x] **Bound assembly memory**: use R238 classes and caller/consumer release - to keep aggregate response memory within the configured budget. -- [x] **Retry safely**: use shared request IDs and only retry operations whose - current semantics provide idempotency or duplicate suppression. - -## Phase 4: fallback and compatibility - -- [x] **Pre-payload fallback**: when RDMA setup fails, retry the same eligible - request on TCP with the same request ID and generation. -- [x] **Partial response failure**: fail the whole logical response after an - uncertain RDMA message; never append a TCP suffix or publish incomplete data. -- [x] **Config compatibility**: preserve TCP defaults and allow RDMA classes, - endpoint selection, and enablement to be configured when creating the - transport. - -## Tests - -Unit tests: - -- class selection for metadata, aggregate, and large range operations; -- split and reassembly of responses at 64 KiB and 256 KiB boundaries; -- sequence gaps, overlap, duplicate, stale generation, and wrong-class errors; -- request identity and idempotency/retry decisions; -- TCP/RDMA error mapping parity. - -Integration tests: - -- metadata point operation over RDMA matches TCP; -- aggregate result larger than 256 KiB is reconstructed exactly; -- aggregate producer remains the only flush/aggregation owner; -- pre-payload RDMA failure retries eligible requests on TCP; -- partial RDMA response fails without accepting a TCP suffix; -- bounded consumer release applies backpressure to Chunk-KV response production. - -E2E tests: - -- single-node HCA local-loop OCI Access Server to Chunk-KV operations; -- TCP-only OCI profile with no RDMA attempts; -- single-node local-loop multi-container profile with multiple Chunk-KV connections; -- mixed metadata, point, batch, and range corpus compared byte-for-byte across - TCP and RDMA. - -## Consolidated files - -- `lib/crowdb-chunk-kv-client/src/` -- `lib/crowdb-chunk-kv-client/tests/` -- Chunk-KV server RPC/service modules -- Access Server access-core and configuration modules -- `lib/crowdb-rpc/` shared frame and transport modules -- OCI E2E test configuration - -## Verification gates - -Portable stream checkpoint: - -- All 11 focused native control-stream/C ABI cases passed, including exact - original allocation transfer through cloned FFI handles and held admission. -- `pixi run test-cpp` passed: 543 tree, 177 RPC, 71 DiskIO and 35 common native - cases, plus both FFI suites. Full logs: - `.crowdb-runtime/artifacts/unit/cpp-4548393f6da54a0e9d3d3ba47a5cc115/`. -- TCP-only/no-verbs build and all 155 CTest cases passed (hardware case skipped). - Changed C++ format and scoped `tree-lint` passed; style warnings remain. -- The first full tree run failed the unchanged - `DoubleBuffer.ConcurrentReadersDuringFrequentFreezeAndDrainNoCorruption` - assertion `bad.load() == false`. The isolated 13-case DoubleBuffer suite - and subsequent complete 543-case tree run passed without source changes. - Preserve the initial failure at - `.crowdb-runtime/artifacts/unit/cpp-f0f7acd09fb541f59d048f81e3923cc2/00-tree.log`; - this checkpoint does not claim to repair that intermittent tree failure. - -Negotiation checkpoint: - -- Protocol v4 carries maximum logical control bytes in the 160-byte TCP - capability and explicit 52-byte CM offer. Digest/generation/layout/receive - admission validation binds both ends; versions 1/2/3 remain decodable for - pre-payload fallback. Native/env/status and version-three C ABI configuration - expose independent logical send/receive budgets and partial-assembly timeout. - Version-one config layout and previous configure entries remain valid. -- Full RPC binary passed (168 portable tests; 13 hardware cases skipped), - no-verbs build passed all 157 CTest cases with its hardware case skipped, - and protocol/FFI Rust suites passed. Three actual HCA cases passed: direct - payload/lease round trip, layout mismatch and explicit TCP fallback policy. - Native XML is under `.crowdb-runtime/rdma-check/artifacts/r241-control/`. -- CLI's exhaustive typed-config literal now inherits the new stream defaults; - all four coroutine/timed RPC benchmark integration cases passed. Focused - all-feature Rust clippy, workspace `rs-lint`, format and scoped C++ lint - passed. The existing request-count worker fixture requires separate client - and server roles; its role-less invocation failed setup, not a payload test. - -CQ integration checkpoint: - -- The CQ walks one retained source control at a time. Each fragment fits the - actual negotiated 64 KiB/256 KiB/1 MiB data capacity; active CM setup primes - receive storage only after the reply establishes that minimum. Source bytes - stay charged through the final WR, and rejected queue publication restores - the caller's original buffer/reference. -- Six actual HCA cases passed: asymmetric classes and exact round trips larger - than the entire two-slot RQ window; fixed polling/event worker FIFO pressure - with a held completed result and release wakeup; idle partial expiry; - source-owner/queue rejection; peer logical limit; invalid generation and - mismatched encoded-control identity. The 1 MiB case reconstructs 2,097,165 - bytes per direction. A completed control remains readable after transport - destruction, then safely returns admission on final release. -- `pixi run test-cpp` passed at - `.crowdb-runtime/artifacts/unit/cpp-6d344d3e3eaf45ba8d8e1d20a0cd1437/`: - 543 tree, 189 RPC, 71 DiskIO and 35 common CTest cases, plus both FFI suites. - Optional HCA cases are skipped by that default gate and exercised separately. - No-verbs build passed all 159 CTest cases (one hardware skip). Format and - scoped C++ lint passed. Final RPC/XML and six HCA case results are under - `.crowdb-runtime/rdma-check/artifacts/r241-control/`. - -Chunk-KV class wiring checkpoint: - -- All 1,244 client/server and Access/S3/Iceberg/Dataset target cases passed - (three optional cases ignored), including - serialized class settings, point/scan pool separation, one global RPC ID - sequence and existing routing/retry/duplicate-result semantics. Logs: - `/tmp/r241-class-rust-tests.log`. -- Full C++/FFI gate passed at - `.crowdb-runtime/artifacts/unit/cpp-c2122fc1d6a14e7e9beb7959fe2ad245/`: - 543 tree, 190 RPC, 71 DiskIO and 35 common native cases plus both FFI suites. - No-verbs build passed 160 CTest cases (one hardware skip). The added C ABI - handler lifetime case proves a control-only Frame pins its connection until - final release; all eight Rust transport config cases passed. -- Workspace all-feature clippy passed. The first `rs-lint` invocation expired - while waiting for another Cargo build lock; rerun it without competing Cargo - work passed. Format and scoped C++ lint passed (existing style warnings). - All six real HCA control cases passed again at `r241-control/class-wiring.xml`. -- The first complete native gate failed the unchanged - `SqFullBackpressureTest.UringEngineTinySqManyWrites`: five callbacks returned - `-ENOMEM` at `sq_full_test.cpp:240`. Existing `DiskIOUring::submit_lockfree` - returns that error when its 1000 yield attempts cannot obtain an SQ slot. - The exact case, its 14-case suite and subsequent full native gate passed - without source changes. Preserve initial evidence at - `.crowdb-runtime/artifacts/unit/cpp-bdbe5707db274b8aa625247ce0eb00eb/`; - this task does not claim to repair that existing scheduling sensitivity. - -```bash -pixi run -- cargo fmt --all -- --check -pixi run -- cargo test -p crowdb-chunk-kv-client --all-targets -pixi run -- cargo test -p crowdb-access-server --all-targets -pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings -pixi run rs-lint -pixi run tree-lint -``` - -Do not hand off to R242 until Chunk-KV response assembly and TCP/RDMA semantic -parity pass for metadata and multi-message aggregate responses. - -Final clean-source checkpoint: - -- All 26 final 1 s warmup + 10 s trials pass source `944274af` with matching - release binaries, valid reports, zero errors and zero fallback. RDMA peak - median is 3,026,938.6 TPS raw-server and 3,236,760.8 TPS OCI. TCP Nagle off/on - medians are 1,203,469.3 / 2,167,732.3 raw-server and - 1,094,684.9 / 2,112,923.9 OCI. Single-loader RDMA measures 60,964.4 TPS, - average 15.88 us, P90 15.94 us and maximum 0.361 ms. Artifacts and current - gate logs are backed up in both runtime and `/tmp/crowdb-rdma-probe/`. -- Both requested benchmark scripts now embed AMD Ryzen 9 5950X / 16-core / - 32-thread hardware, raw-server/two non-root OCI topology, exact source/image, - queue/slot/CQ/worker/aggregation settings, repeats, latency tails and errors. - Permanent RPC flow analysis records service corpus/digests, memory/copy/error - counters, the five HTTP/DiskIO reruns, old TCP mislabeling, the mixed-size - staging and cold-response fixes, and known unchanged intermittent failures. -- Implementation and acceptance are complete. Remaining work is final document - consistency/link verification and removal of this completed child plan, its - detail/index entry, and completed tasks in the persistent umbrella plan. diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index 1b7ff4f9b..bc2e22c3e 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -6,7 +6,7 @@ Upstream: [R242](../backlog/R242-rdma-fat-client-sdk.md), [root RDMA plan](plan-rdma.md), [registered buffers](../design/rpc/design-crowdb-rpc-rdma.md#4-registered-arenas-and-slot-ownership), [RDMA RPC transport](../design/rpc/design-crowdb-rpc-rdma.md), and -[R241](../backlog/R241-rdma-chunk-kv-access.md). +[accepted Chunk-KV/access integration](../design/rpc/design-crowdb-rpc-rdma.md#7-service-interactions). Goal: expose Access Server semantics through a native fat-client SDK that can consume the same RDMA stream directly, with caller-owned host buffer leases diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index 7cf1f1a7b..1f2d2f3f7 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -4,9 +4,8 @@ # CROWDB RDMA implementation Plan Upstream: [R5](../backlog/R5-rdma-alloc.md), -[RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md), and child -backlogs [R241](../backlog/R241-rdma-chunk-kv-access.md) and -[R242](../backlog/R242-rdma-fat-client-sdk.md). +[RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md), and the remaining +child backlog [R242](../backlog/R242-rdma-fat-client-sdk.md). Goal: implement the host-memory RDMA SEND/RECV path end to end, with RDMA as the preferred connection path, TCP as the one-way fallback, shared aggregation @@ -25,26 +24,43 @@ current endpoint and receive admission. QPs have one fixed polling/event owner per CQ; bounded aggregation preserves complete frames. Strict RDMA CLI tests share the coroutine workload and canonical results with TCP. -The final transport source `55ffee38` passes full C++/FFI gates, 137 no-verbs -cases, CLI/runner contract tests, real HCA lifetime/budget/fallback/disconnect -cases, formatting and workspace lint. Native and two non-root containers on -one shared HCA pass repeated 1 s warmup + 10 s measurement profiles. Median -RDMA throughput is 2.903M TPS native and 2.954M TPS in OCI, with zero errors or -fallback. The old 1.187M value labeled RDMA was TCP; matching current TCP -measures 1.196M TPS. Exact identities, settings and artifact names are retained -in [RPC flow analysis](../design/rpc/rpc-flow-analysis.md). - -The HTTP/ChunkIO/DiskIO datapath is accepted at `ab1ffbfb`: immutable registered -HTTP owners, aligned receive-to-disk handoff, read reuse through SEND completion, -caller-held receive credits, retained asynchronous service owners and terminal -accepted-RDMA failures. All 278 affected Rust cases and full C++/FFI gates pass. -A 3,158,073-byte real HTTP corpus crosses two chunks and matches TCP/native/OCI -bytes and metadata geometry. Peer-close fails a partial HTTP download with no -TCP suffix. Exact functional settings, error counts, image and artifact -identities are retained in RPC flow analysis. - -Current work is R241, using its existing plan. R242 and physical -multi-node validation remain outside the user's requested continuation. +The final transport source `944274af` passes 543 tree, 192 RPC, 71 DiskIO and +35 common native cases plus both FFI suites, 160 no-verbs CTest cases, fourteen +real-HCA cases, formatting and scoped C++ lint. The unchanged Rust integration +passes 1,244 affected cases, workspace all-feature clippy and `rs-lint` at +`018da2fc`. Native and two non-root containers on one shared HCA pass all 26 +final 1 s warmup + 10 s measurement trials. Median RDMA throughput is 3.027M +TPS raw-server and 3.237M OCI, with zero errors/fallback. Matching TCP medians +are 1.203M/2.168M raw-server and 1.095M/2.113M OCI with Nagle off/on. Exact +settings, identities, latency tails and the old TCP mislabeling are retained +in [RPC flow analysis](../design/rpc/rpc-flow-analysis.md) and both benchmark +wrapper scripts. + +The HTTP/ChunkIO/DiskIO datapath is accepted: immutable registered HTTP owners, +aligned receive-to-disk handoff, read reuse through SEND completion, caller-held +receive credits, retained asynchronous service owners and terminal accepted-RDMA +failures. A 3,158,073-byte HTTP corpus crosses two chunks and matches +TCP/native/OCI bytes and geometry. All five functional profiles also pass the +common-control regression at `018da2fc`; peer-close fails a partial download +without a TCP suffix. The fixture injects placement/catalog state and records +its actual io_uring/OCI BlockingEngine selection. + +Chunk-KV/access integration is accepted with distinct 64 KiB metadata and +256 KiB aggregate connection groups sharing request correlation. Protocol v4 +supports bounded logical controls beyond one RQ window, independent byte +budgets, negotiated capacity and final-caller admission ownership. The real +Chunk-KV service/tree/WAL corpus verifies eight values totalling 3,409,975 bytes, +typed errors, mutation-ID replay and exact TCP/RDMA equality in seven native/OCI +profiles. Partial-fragment peer close publishes no incomplete result or TCP +suffix. Catalog/serving authority/memory ChunkStream backing are injected; +Access construction is covered by the affected Rust suites. Formal service +contracts and complete acceptance evidence are linked from the RDMA design +and RPC flow analysis. + +The requested foundation, transport, datapath and Chunk-KV continuation is +complete. Only the fat-client SDK child remains. R242 and physical multi-node +validation are outside the completed continuation; no SDK implementation is +started by this plan update. ## Fixed design decisions @@ -61,8 +77,8 @@ multi-node validation remain outside the user's requested continuation. every SEND fits both the local send slot and peer receive slot, including headers, and retains each RPC's identity and completion ownership. - Assign each QP to one CQ and each CQ to one fixed polling/event worker. - Implement this together with `crowdb-cli bench rdma` and the existing RPC - regression runner; report QP count, slot size, aggregation and errors. + Use `crowdb-cli bench rdma` and the shared RPC regression runner to report + QP count, slot size, aggregation and errors. - Use 64 KiB metadata, 256 KiB aggregated Chunk-KV, and 1 MiB datapath capacities, configured as transport defaults. - Use posted receives as implicit data credit; do not add an extra credit @@ -82,57 +98,30 @@ multi-node validation remain outside the user's requested continuation. - Physical multi-node testing is deferred to [the separate follow-up plan](plan-rdma-multi-node.md) and is not part of the current backlog scope. -## Dependency graph +## Remaining dependency order ```text -R238 registered buffers - | - v -R239 common RDMA RPC transport - / \ - v v -R240 R241 -datapath Chunk-KV/access - \ / - v -R242 fat client SDK and FFI lease +Accepted registered host buffers and common RPC + | +Accepted HTTP/ChunkIO/DiskIO and Chunk-KV/access + | + v + R242 fat client SDK and FFI lease ``` -## Immediate implementation order +The remaining child uses the existing [fat-client SDK plan](plan-rdma-fat-client-sdk.md) +when the user includes that scope. Do not create a duplicate plan or repeat the +completed service integration. The shared benchmark/functional `Runtime` +already launches native, no-device TCP OCI and shared-HCA OCI profiles. -The foundation, transport and datapath requirements are complete. Continue -through R241 in this same task using its existing -[Chunk-KV/access plan](plan-rdma-chunk-kv-access.md) and gates. Do not create -duplicate plan files. R242 remains outside this requested continuation. - -1. Complete R241 using its existing Chunk-KV/access plan and parity gates. -2. Rebuild matching CLI/server binaries and repeat the 10-second peak controls - after all service-related transport changes, retaining errors and latency. -3. Keep R242 downstream until the user includes that scope. - -Before modifying each service path, run its focused baseline corpus and record -it in that child's plan. Transport baseline, hardware topology, queue names and -OCI runtime preparation are already verified. Service acceptance must prove -actual RDMA, ownership through disk/caller completion, and bounded pressure; -transport echo acceptance alone does not prove those properties. - -## Phase 3: Chunk-KV service integration - -- [~] **Execute R241 child plan**: connect Access Server/Chunk-KV with 64 KiB - metadata and 256 KiB aggregate classes, preserving existing request IDs, - ordering, and retry semantics. -- [ ] **Run semantic parity**: for the same test corpus, compare TCP and RDMA - results byte-for-byte and compare errors, sequence, generation, and - cancellation behavior. - -Exit condition: S3/DiskIO and Chunk-KV paths use RDMA without a second -aggregation layer, without an intermediate CPU payload copy, and with TCP -fallback before payload submission. +Before changing the SDK frontend, run its focused baseline corpus, retain the +accepted transport/service settings and compare exact result bytes and errors. +Transport echo acceptance alone does not prove SDK lease semantics. ## Phase 4: R242 fat client SDK - [ ] **Execute R242 child plan**: split access core from HTTP frontend, expose - FFI-safe host leases, and connect SDK operations to R241/R240 transports. + FFI-safe host leases, and reuse the accepted Chunk-KV/ChunkIO transports. - [ ] **Host-to-GPU simulation**: hold each lease until the explicit host-to-GPU copy completes, then release it; no GPU pointer is exposed by the base API. - [ ] **SDK pressure tests**: hold all returned leases, verify bounded admission, @@ -141,22 +130,24 @@ fallback before payload submission. Exit condition: native SDK and HTTP frontend return equivalent access results, and the caller-controlled lease lifetime is visible and testable through FFI. -## Phase 5: OCI E2E matrix +## Phase 5: SDK OCI E2E matrix + +- [ ] **SDK TCP-only profile**: extend the shared launcher with the SDK corpus, + omit RDMA devices and require zero RDMA attempts with exact frontend parity. +- [ ] **SDK shared-HCA profile**: run the SDK corpus in two non-root host-network + containers, requiring independent QPs, retained caller leases, bounded + pressure and setup-only TCP fallback. +- [ ] **SDK topology selection**: reuse existing runtime/readiness/report + contracts for the SDK fixture when each profile's prerequisites are present. -- [ ] **TCP-only profile**: run OCI containers without RDMA devices and assert - service discovery skips RDMA entirely. -- [ ] **Single-node multi-container profile**: run `setup-rdma.sh local`, start - multiple containers on one physical host, share the configured HCA, and - assert independent QPs, request correlation, bounded buffers, and TCP - fallback. -- [ ] **Automated local topology selection**: make the test runner execute the - TCP-only and single-node local-loop profiles when their prerequisites are - present. +Exit condition: SDK profiles report topology, actual path/classes/QPs, fallback, +held lease pressure, typed errors and exact frontend result equality. Existing +HTTP/DiskIO and Chunk-KV native/OCI corpora remain regression controls. -Exit condition: every profile reports its topology, RDMA capability, selected -device, selected path, fallback count, buffer pressure, and final result. +## Invariants to retain in SDK work -## Cross-phase invariants +The accepted transport/service tests already cover these properties. Reapply +them to the new SDK frontend and its caller-controlled leases: - [ ] **No mixed stream**: never append TCP bytes to a partially completed RDMA stream. @@ -181,7 +172,7 @@ Unit and integration gates: ```bash pixi run -- cargo fmt --all -- --check -pixi run -- cargo test -p crowdb-rpc --all-targets +pixi run clean-env && pixi run test-cpp pixi run -- cargo test -p crowdb-rpc-ffi --all-targets pixi run -- cargo test -p crowdb-chunk-kv-client --all-targets pixi run -- cargo test -p crowdb-access-server --all-targets @@ -207,5 +198,5 @@ follow-up plan. Namespace profiles are not part of acceptance. Each child plan is marked `[x]` only after its own acceptance cases and gates pass. If a child discovers a protocol ambiguity, stop that child, record the question here, and resolve it before changing the formal design. If the host -cannot provide RDMA, do not block R238/R239 mock work or TCP fallback tests; -mark only hardware E2E as pending. +cannot provide RDMA, continue portable SDK/TCP checks and mark only hardware +E2E as pending. From a219551d7ff5cf24479da0ec69d026b74aa261ca Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 09:02:25 +0800 Subject: [PATCH 056/258] Plan RDMA send optimization and trusted native SDK --- doc/backlog/R220-dataset-fat-client-plan.md | 146 +++++---- doc/backlog/R242-rdma-fat-client-sdk.md | 190 ++++++++--- doc/backlog/backlog.md | 15 +- doc/working/plan-rdma-fat-client-sdk.md | 333 ++++++++++++++----- doc/working/plan-rdma-mixed-size-send.md | 336 ++++++++++++++++++++ doc/working/plan-rdma.md | 18 +- 6 files changed, 853 insertions(+), 185 deletions(-) create mode 100644 doc/working/plan-rdma-mixed-size-send.md diff --git a/doc/backlog/R220-dataset-fat-client-plan.md b/doc/backlog/R220-dataset-fat-client-plan.md index 5e56d3f17..87e1b2533 100644 --- a/doc/backlog/R220-dataset-fat-client-plan.md +++ b/doc/backlog/R220-dataset-fat-client-plan.md @@ -3,68 +3,102 @@ ### R220: Dataset — Fat-client direct access plan -Status: Deferred until dedicated CPU HTTP workload measurements establish -latency, throughput, memory and digest costs. The public Rust HTTP SDK and -Python wrapper are available under `sdk/` as independent packages; functional -container acceptance is complete. Performance measurements remain future work. +Status: Deferred until the shared runtime and first S3 frontend in +[R242](R242-rdma-fat-client-sdk.md) are accepted and dedicated CPU HTTP +measurements establish latency, throughput, memory, and digest costs. The +public Rust HTTP SDK and Python wrapper are independently packaged under +`sdk/`; functional container acceptance is complete. These Dataset follow-up +gates do not defer R242's shared foundation or S3 delivery. + +## Problem + +Trusted internal data-processing and AI systems need Dataset semantics over +direct Chunk-KV and ChunkIO/DiskIO access, with exposed storage topology for +scheduling. Routing freshness, retries, buffer lifetime, backpressure, and +worker partitioning must remain correct without duplicating authority logic. +Performance claims need an HTTP workload baseline. -Problem: The SDK fat client can avoid HTTP routing and payload relay by connecting -directly to internal Chunk-KV and DiskIO, but it also takes responsibility for -routing freshness, retries, buffer lifetime, backpressure, and worker -partitioning. Implementing it before the HTTP baseline would make optimization -claims hard to measure and could duplicate authority logic. The root contract is [Dataset design](../design/access-server/dataset/design-crowdb-access-dataset.md). -Solution: +## Solution + +1. Attach Dataset after S3 to R242's native runtime in the main workspace. + Reuse topology, transports, admission, and buffer leases; do not introduce + another runtime or make the independent HTTP SDK depend on storage FFI. + The fat client shares Access Server's internal trust domain, takes explicit + Dataset identity/context, and needs neither SigV4 verification, the S3 + user-secret database, nor external-user capability issuance. +2. Reuse `crowdb-access-dataset` authority, sessions, composition, Split, and + upload logic. Preserve Snapshot/Read Plan binding, selection, worker + partitioning, integrity, cleanup, and active-read protection. Topology is + a supported scheduling surface, not placement authority. +3. Preserve explicit acknowledgement and resume: fetching data, releasing a + buffer, and acknowledging processing are separate events. Native delivery + uses the shared owner-backed streams and bounded prefetch. Cancellation + stops producers; caller-held leases remain valid and charged until release. +4. Compare native Chunk-KV and chunk-range reads with the HTTP baseline, + including topology refresh, stale owners, failures, digest costs, and the + existing 60-second active-read inactivity TTL. Keep placement, fencing, + durability, and publication rules in their existing components. +5. Expose native Dataset through a Python binding to the same Rust data plane. + Preserve the portable HTTP distribution without adding mandatory native + dependencies. Additional C/C++ business bindings are later work; reuse the + existing lease ABI rather than making a new business ABI a prerequisite. +6. Use the runtime's TCP/RDMA policy, with eligible pre-payload fallback only. + Native access does not automatically select HTTP on transport failure or + replay ambiguous mutations on another path. GPU delivery reuses R242's + host-copy lifetime contract; direct GPU memory remains later work. -- Implement direct access in the Rust SDK core and expose it through the same - Python wrapper used by the HTTP client. Routing, buffers, retries and optional - RDMA stay in Rust; Python does not implement another data plane. Additional - C/C++ bindings can be added later without making a C ABI a prerequisite. -- Document the SDK direct-client protocol over the same Dataset authority and Read - Plan contract used by HTTP. -- Compare direct Chunk-KV metadata reads and DiskIO/chunk range reads with the - HTTP baseline, including routing refresh, stale topology, failure retry, and - 60-second active-read TTL behavior. -- Define owner-backed buffer lifetime, bounded prefetch, cache admission, - cancellation, and checksum/MD5 verification without adding client-side global - locks or unbounded queues. -- Analyze which work belongs in the client versus Chunk-KV, DiskIO, and Dataset - library. The client cannot publish placement or bypass Dataset authority. -- Reserve GPU/direct-memory delivery for a later design after CPU direct access - is measured and correct. -- Use explicit server capabilities and authorization for direct Chunk-KV/chunk - access. Direct access never grants unrestricted internal metadata mutation. - Keep HTTP available for unsupported capabilities; choose transport before - starting an operation and never replay an ambiguous mutation on a new path. +## Dependencies -Dependencies: the existing public REST API and packaged HTTP SDK, CPU -E2E/container acceptance, and measured Chunk-KV/DiskIO read flows. R233 caching -is optional. Unlanded direct access does not block the first HTTP SDK release. +- Accepted R242 runtime/topology/lease APIs and first native S3 frontend. +- Existing public REST API, packaged HTTP SDK, Dataset CPU container + acceptance, and measured Chunk-KV/DiskIO flows. Keep HTTP available as an + explicitly selected client while native work is unlanded. +- R233 caching is optional. Dataset integration does not block the existing + HTTP SDK distribution or require a second transport/buffer implementation. -Acceptance: +## Acceptance -1. Given HTTP and direct-client prototypes over the same snapshot, produce a - comparison of p50/p99 latency, throughput, CPU, memory, and digest cost. - Measurement contract. Integration test. -2. Given stale routing or a disconnected DiskIO endpoint, assert the fat client - refreshes or retries without changing Dataset selection or snapshot identity. - Non-authoritative topology invariant. Integration test. -3. Given a slow consumer, assert direct prefetch and in-flight buffers remain - bounded and cancellation releases client buffers. Bounded delivery invariant. - Integration test. -4. Given unsupported direct capability or denied direct credentials, select - transport and attempt access; assert HTTP fallback where authorized, no - authority bypass and no cross-transport replay of ambiguous writes. - Transport authority invariant. Integration test. -5. Given the same direct read through Rust and the Python wrapper, cancel and - close the read while retaining returned data; assert equivalent logical - results, valid buffer lifetimes and isolated cleanup. Binding invariant. - Integration test. +- Given HTTP and native reads of the same Snapshot/Read Plan/Split, when each + executes, bytes, projection, worker partition, integrity, and cleanup agree. + Invariant: Dataset surface equivalence. Integration test. +- Given HTTP/native workloads over the same snapshot, when measured under + documented settings, report p50/p99 latency, throughput, CPU, memory, and + digest cost. Invariant: comparable measurement contract. Integration test. +- Given stale routing or a disconnected DiskIO endpoint, when a native read + executes, it refreshes, retries where eligible, or fails without changing + selection/snapshot or publishing placement. Invariant: non-authoritative + topology. Integration test. +- Given caller-held buffers and a slow consumer, when prefetch or cancellation + occurs, memory stays bounded and held data stays valid until final release. + Invariant: bounded owner-backed delivery. Integration test. +- Given a fetched but unacknowledged batch, when its buffers are released and + the read resumes, progress has not advanced; explicit acknowledgement alone + advances it. Invariant: processing acknowledgement. Integration test. +- Given a native Dataset facade sharing a runtime with S3, when either facade + executes or closes, resource sharing preserves Dataset authority and the + other's operations. Invariant: shared runtime, separate semantics. + Integration test. +- Given disabled RDMA, eligible setup failure, or accepted partial failure, + when direct access executes, TCP fallback obeys existing rules, with no + implicit HTTP retry or ambiguous mutation replay. Invariant: transport + failure boundary. Integration test. +- Given the same native read through Rust and Python, when cancellation and + close occur with retained data, logical results, buffer lifetime, isolated + cleanup, and explicit acknowledgement agree. Invariant: binding parity. + Integration test. -Verification commands: +Run after the R242 native crate is available: -- `pixi run rs-fmt-check` -- `pixi run rs-lint` -- `pixi run test-access-server` -- `pixi run test-unit` +```bash +pixi run -- cargo fmt --all -- --check +pixi run -- cargo test -p crowdb-sdk-native --all-targets +pixi run -- cargo test -p crowdb-access-dataset --all-targets +pixi run -- cargo test -p crowdb-access-server --all-targets +pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings +pixi run check-dataset-sdk-boundary +pixi run test-dataset-sdk +pixi run test-dataset-sdk-python +pixi run test-e2e-dataset +``` diff --git a/doc/backlog/R242-rdma-fat-client-sdk.md b/doc/backlog/R242-rdma-fat-client-sdk.md index 98d307510..8246ea637 100644 --- a/doc/backlog/R242-rdma-fat-client-sdk.md +++ b/doc/backlog/R242-rdma-fat-client-sdk.md @@ -1,62 +1,176 @@ -### R242: Access SDK — RDMA fat client and buffer leases +### R242: Access SDK — Trusted internal S3 fat client and shared RDMA runtime ## Problem -The current Access Server boundary adds an application-to-server socket and -async hop. A fat client should expose access semantics directly while reusing -the access core and allowing the same RDMA path used by the server. Returned -buffers need an explicit lifetime so the SDK can avoid copies without -recycling memory while the caller still reads it. +Data-processing and AI systems inside the trusted cluster need to embed +storage, inspect storage topology, schedule work near data, and control +concurrency and buffer lifetime. Routing every operation through Access Server +adds an application-to-server socket and async hop and hides this topology. + +A fat client is explicitly a trusted internal cluster component in the same +trust domain as Access Server, not an ordinary external-user SDK. It connects +directly to metadata and data services. It does not authenticate its own +operations by verifying SigV4 or receive the server's user-secret database. + +`crowdb-access-s3` already owns metadata, conditions, retrieval, publication, +and multipart components, but complete operations remain partly coupled to +HTTP handlers. The independent `sdk/` workspace is a portable Dataset HTTP +client. Existing RPC buffers carry lease metadata, but completion and access +adapters still include copying paths. Complete these boundaries rather than +replacing accepted storage clients or transport. + +The root contracts are [Access Server](../design/access-server/design-crowdb-access-server.md) +and [RDMA service interactions](../design/rpc/design-crowdb-rpc-rdma.md#7-service-interactions). ## Solution -1. Split the access core from the HTTP frontend and add a native SDK frontend. - Both frontends use the same planner, authorization, generation, aggregation, - Chunk-KV, DiskIO, and TCP/RDMA selection semantics. -2. Define an FFI-safe opaque buffer lease containing request identity, sequence, - logical offset, actual length, capacity class, and release operation. The - caller owns a completed host lease until release; release returns it to the - registered pool. -3. Deliver completion from the C++ RPC worker to Rust without exposing verbs - objects. Rust must not dereference a released lease, and C++ must not - recycle a caller-owned lease from a completion callback. -4. Support host memory first. A caller requesting GPU delivery receives host - leases and performs an explicit host-to-GPU copy; a future allocator can - supply GPU-capable memory behind the same lease contract. -5. Preserve bounded admission, cancellation, shutdown, and RDMA-to-TCP - fallback for SDK operations. +1. Provide a shared native cluster runtime in the main Rust workspace. Reuse + `crowdb-kv-client`, `crowdb-chunk-kv-client`, `crowdb-chunk-client`, + `crowdb-diskio-client`, and `crowdb-rpc-ffi` for discovery, routing, + TCP/RDMA connections, producer allocation, admission, and shutdown. Business + handles share these resources. Keep the existing `sdk/` distribution + independent of server/storage FFI dependencies. +2. Expose storage topology as a supported native API: service endpoints, + node/disk relationships, Chunk-KV owners, and resolved chunk placement for + caller scheduling and prefetch. Views retain existing source generations + and freshness information, without promising a globally atomic snapshot. + Stale views refresh or fail through existing routing/fencing rules. + Observing topology cannot publish placement or bypass generation checks, + durability, or publication rules. +3. Implement S3 first with typed bucket/object, HEAD/GET/range, PUT, listing, + delete, copy, and multipart operations backed by `crowdb-access-s3`. + Extract shared execution from `app/crowdb-access-server/src/s3/`. + HTTP retains parsing, SigV4 verification, and HTTP/XML encoding; native + callers supply explicit tenant and operation context. Both preserve the + same object visibility, conditions, integrity, generation, mutation + outcomes, and cleanup. Native execution needs neither a user-secret lookup + nor an Access Server authentication round trip. +4. Separate business APIs from common data delivery. Reads return typed + metadata and an ordered, pull-based host-buffer stream. Metadata or a + delivered prefix does not establish full-read success; successful + termination includes applicable checksum and length checks. A terminal + error remains observable; later polling cannot turn failure into success. + Writes accept bounded ordinary input or uniquely writable buffers frozen before + submit. Input acceptance is distinct from durable object publication; + only successful finalization establishes PUT or multipart completion. + Ambiguous publication retains existing uncertain-outcome semantics. +5. Complete the existing opaque RPC buffer lease contract through C++/Rust + completion and the native API. A lease exposes request identity, sequence, + logical offset, actual length, capacity class, and release without verbs + objects. Caller-owned storage stays valid until final release, including + after cancellation or runtime shutdown. Eligible payload paths transfer + ownership without a second SDK payload copy; decoding, alignment staging, + and EC reconstruction retain necessary copies. Wire-control fragments are + not object data. +6. Keep hot-path admission lock-free and bounded across requests, producers, + and caller-held buffer capacity. Shared views charge an allocation once + until final release; operation completion does not return held-buffer + credits. Exhaustion pauses production or returns a typed capacity/deadline + outcome without an unbounded waiting queue. Completion callbacks never + block RPC workers. Cancellation stops new work and resolves once; + uncertain in-flight storage stays retained or quarantined until safe. +7. Preserve TCP/RDMA path selection. Disabled RDMA uses TCP; eligible setup + or known pre-payload failures may fall back with the same request identity + under existing retry rules. Accepted/partial RDMA operations never append + a TCP suffix or replay ambiguous mutations. This fallback does not select + HTTP. GPU delivery initially means explicit host-to-GPU copy completion + before host-lease release, with no GPU pointer or RDMA READ/WRITE in the + base ABI. +8. Keep runtime, topology, and buffer APIs independent of S3 so Dataset can + attach after S3 acceptance under [R220](R220-dataset-fat-client-plan.md). + Dataset keeps Snapshot/Read Plan/Split and explicit acknowledgement in + `crowdb-access-dataset`; it reuses the runtime and data delivery without + encoding samples as S3 objects. Buffer release, operation completion, and + Dataset acknowledgement are separate events. R242 delivers the shared + foundation and S3; R220 owns the later Dataset adapter, measurements, and + Python integration. ## Dependencies - Accepted registered buffers/RDMA RPC and [Chunk-KV/access integration](../design/rpc/design-crowdb-rpc-rdma.md#7-service-interactions). -- Existing Access Server access-core and FFI conventions. -- R220 remains a separate Dataset SDK plan; this requirement supplies the - transport and lease contract it may reuse. +- Existing S3 library components, HTTP operation orchestration, ChunkIO + read/write policies, and RPC FFI ownership conventions. +- Existing Dataset HTTP SDK packaging and boundary checks are regression + controls, not a place to introduce native storage dependencies. +- R220 consumes the accepted runtime and S3-first API pattern. Its Dataset + measurements and bindings do not gate R242 or require a second transport, + topology cache, or buffer pool. +- Missing RDMA hardware uses the accepted bounded TCP path. Host-memory tests + and simulated asynchronous GPU copies need no GPU; real-HCA acceptance is + an explicit separate profile. ## Acceptance -- Given a native SDK access request, when the operation completes through RDMA, - the caller receives the exact ordered host buffers and releases each lease. - Invariant: caller-owned lifetime. Integration test. -- Given the caller holds all available leases, when another buffer is needed, - admission applies bounded backpressure and resumes after release. Invariant: - bounded SDK memory. Integration test. -- Given a request with host-to-GPU delivery selected, when the operation - completes, host data remains valid until the explicit copy completes and - release follows the copy. Invariant: copy dependency lifetime. Integration test. -- Given RDMA is unavailable or fails before payload, when the same SDK request - executes, TCP produces the same access result and request identity. - Invariant: frontend transport parity. Integration test. - -Run: +- Given cluster endpoints and a tenant but no S3 secret database or HTTP + listener, when native S3 executes an object operation, it reaches storage + directly. Invariant: trusted internal execution. Integration test. +- Given two handles sharing a runtime, when they execute and close + independently, they share connection/admission owners without invalidating + the other's data. Invariant: shared runtime lifetime. Integration test. +- Given service, disk, catalog, and object-placement observations, when the + caller inspects topology and schedules range reads, the API exposes their + identities/freshness and returns the requested logical bytes. Invariant: + supported topology surface. Integration test. +- Given a view whose owner or endpoint changes, when a request uses it, + routing refreshes or returns a typed stale/fenced outcome without changing + logical object identity or publishing placement. Invariant: + non-authoritative topology. Integration test. +- Given identical supported bucket/object, copy, and multipart operations, + when HTTP and native entry points encounter success, condition/range errors, + conflicts, or publication failures, logical bytes, metadata, and semantic + outcomes agree. Invariant: S3 frontend parity. Integration test. +- Given truncated data or a final checksum mismatch, when a caller drains a + stream after receiving a valid prefix and polls after the error, termination + remains failed rather than becoming successful EOF. + Invariant: prefix delivery is not full-read success. Integration test. +- Given writable producer storage, when frozen input is submitted, callers + cannot mutate it; final success waits for durability/publication and lost + publication replies retain uncertainty. Invariant: input ownership and + publication completion. Integration test. +- Given an eligible direct RDMA payload path, when a multi-buffer read + completes, ordered owner-backed views preserve request, sequence, offset, + and length without a second SDK payload copy. Invariant: caller-owned + payload lifetime. Integration test. +- Given opaque leases and shared views, when access and final release cross + FFI, ABI/access rules hold and each allocation is released once. Invariant: + lease ABI ownership. Unit test. +- Given all admitted capacity is caller-held, when another buffer is needed, + bounded backpressure resumes after final release or ends at its deadline; + stream completion alone does not return held credits. Invariant: bounded + SDK memory. Integration test. +- Given in-flight operations and held leases, when cancellation or shutdown + closes new admission, completion resolves once and held bytes stay valid + until release without reposting to destroyed transport. Invariant: + cancellation and shutdown ownership. Integration test. +- Given an asynchronous host-to-GPU copy simulation, when the stream closes + before copy completion, the host lease stays valid until copy completion and + release. Invariant: copy dependency lifetime. Integration test. +- Given disabled RDMA or an eligible pre-payload failure, when native access + runs, TCP preserves result/request identity; accepted or partial RDMA failure + instead fails without a TCP suffix or ambiguous mutation replay. Invariant: + transport parity and failure boundary. Integration test. +- Given the standalone Dataset SDK, when boundary and functional gates run + alongside the native client, HTTP behavior and independent packaging stay + intact. Invariant: independent HTTP SDK. Integration test. +- Given native and OCI S3 clients, when callers retain leases under pressure, + no-device TCP and shared-HCA profiles record actual topology/transport, + exact frontend results, bounded memory, and safe shutdown. Invariant: + embedded storage integration. E2E test. + +Run after registering the native crate: ```bash pixi run -- cargo fmt --all -- --check +pixi run -- cargo test -p crowdb-sdk-native --all-targets +pixi run -- cargo test -p crowdb-access-s3 --all-targets pixi run -- cargo test -p crowdb-rpc-ffi --all-targets pixi run -- cargo test -p crowdb-access-server --all-targets pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings +pixi run check-dataset-sdk-boundary +pixi run test-dataset-sdk +pixi run test-e2e-s3 ``` diff --git a/doc/backlog/backlog.md b/doc/backlog/backlog.md index d67c2fcc2..3249521fb 100644 --- a/doc/backlog/backlog.md +++ b/doc/backlog/backlog.md @@ -16,9 +16,10 @@ live in the linked document; implementation follows - **[R233](R233-sdk-local-cache-checkpoint.md)** — optional local disk read cache and recoverable checkpoint staging with explicit remote durability. **Area:** SDK. **Complexity:** High. **Status:** Deferred until dedicated HTTP workload measurements. -- **[R220](R220-dataset-fat-client-plan.md)** — Dataset SDK fat-client direct - access and optimization contract. **Area:** Dataset SDK. **Complexity:** High. - **Status:** Deferred until the core Dataset contract has workload measurements. +- **[R220](R220-dataset-fat-client-plan.md)** — trusted internal Dataset adapter + on the R242 runtime, measurements, and Python integration. + **Area:** Dataset SDK. **Complexity:** High. **Status:** Deferred until R242 + S3 acceptance and dedicated Dataset HTTP workload measurements. ## Access server and external data @@ -32,9 +33,11 @@ live in the linked document; implementation follows - **[R170](R170-s3-cuobject-rdma.md)** — optional cuObject RDMA data plane. **Area:** access server / S3 / DiskIO / RDMA. **Complexity:** High. **Status:** Deferred until the basic TCP S3 path is stable and measured. -- **[R242](R242-rdma-fat-client-sdk.md)** — native fat-client access SDK and - FFI buffer leases. **Area:** access server / SDK / RDMA. **Complexity:** High. - **Dependencies:** accepted RDMA RPC and Chunk-KV/access integration. +- **[R242](R242-rdma-fat-client-sdk.md)** — trusted internal cluster runtime, + exposed storage topology, native S3 first, and caller-owned buffer leases; + Dataset follows under R220. **Area:** access server / SDK / RDMA. + **Complexity:** High. **Dependencies:** accepted RDMA RPC and Chunk-KV/access + integration. - **[R185](R185-access-iceberg-cache-invalidation.md)** — bounded Iceberg cache and invalidation. **Area:** access server / Iceberg / Group 0 / Chunk-KV. **Complexity:** Medium. **Status:** Deferred pending focused cache measurements. diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index bc2e22c3e..f1177fe9c 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -1,111 +1,284 @@ -# RDMA fat client SDK and FFI lease Plan +# Trusted internal S3 fat client and shared RDMA runtime Plan Upstream: [R242](../backlog/R242-rdma-fat-client-sdk.md), -[root RDMA plan](plan-rdma.md), [registered buffers](../design/rpc/design-crowdb-rpc-rdma.md#4-registered-arenas-and-slot-ownership), -[RDMA RPC transport](../design/rpc/design-crowdb-rpc-rdma.md), and -[accepted Chunk-KV/access integration](../design/rpc/design-crowdb-rpc-rdma.md#7-service-interactions). - -Goal: expose Access Server semantics through a native fat-client SDK that can -consume the same RDMA stream directly, with caller-owned host buffer leases -and an explicit host-to-GPU copy simulation. - -## Phase 1: access-core split - -- [ ] **Map Access Server boundary**: identify HTTP-only parsing, authorization, - access planning, generation validation, aggregation, Chunk-KV calls, DiskIO - calls, and response publication. Files: `app/crowdb-access-server/` and - access design modules. -- [ ] **Extract access core**: move shared planning and operation semantics - behind a native interface while keeping HTTP frontend behavior unchanged. - The core must select the logical connection path through R239 and preserve - TCP fallback. -- [ ] **Define SDK operations**: specify open/read/write/range/metadata - operations, request ID, generation, cancellation, deadlines, and stream - sequence without exposing C++ transport or verbs types. - -## Phase 2: FFI lease interface - -- [ ] **Define opaque lease ABI**: expose handle, actual length, capacity, - request ID, sequence, logical offset, error/end status, and release. Use - fixed-width types and explicit null/error behavior. -- [ ] **Implement callback bridge**: C++ completion resolves ownership and - invokes the Rust FFI callback; Rust forwards to the future/stream consumer. - The callback must not block the RPC worker or retain a borrowed pointer past - release. -- [ ] **Implement release/drop**: make release idempotency and misuse visible, - return caller-owned buffers to R238 only after caller access ends, and keep - shutdown safe while leases remain held. -- [ ] **Implement cancellation**: stop new work, drain or fail submitted - requests, preserve in-flight buffer lifetime, and report cancellation once. - -## Phase 3: host and GPU-copy flow - -- [ ] **Host path**: deliver registered host leases directly to SDK callers; - do not copy payload into a second SDK buffer. -- [ ] **GPU-copy simulation**: provide an explicit copy hook or SDK operation - that holds the host lease until the caller confirms host-to-GPU copy - completion. Do not expose a GPU pointer in the base ABI. -- [ ] **Buffer pressure**: enforce SDK-held byte budget; pause producer/admission - when callers retain all leases and resume after release. - -## Phase 4: frontend parity - -- [ ] **HTTP parity**: run identical access requests through HTTP frontend and - native SDK and compare bytes, metadata, generation, errors, and cancellation. -- [ ] **Transport parity**: run SDK operations over RDMA and TCP with the same - request ID and verify fallback before payload submission. -- [ ] **Multi-message stream**: expose ordered 64 KiB metadata or 256 KiB - aggregate messages as one SDK stream without copying them into a monolithic - result. +[root RDMA plan](plan-rdma.md), +[registered buffers](../design/rpc/design-crowdb-rpc-rdma.md#4-registered-arenas-and-slot-ownership), +[service integration](../design/rpc/design-crowdb-rpc-rdma.md#7-service-interactions), +and subsequent [Dataset integration R220](../backlog/R220-dataset-fat-client-plan.md). + +Goal: embed trusted internal storage access through a shared topology-aware +native runtime, implement S3 first, and let Dataset attach to the accepted API. + +## Scope and current boundary + +- The embedding data-processing or AI system is in the Access Server trust + domain and can reach internal services. Native execution takes an explicit + tenant/context; it does not construct or verify SigV4, obtain a user-secret + database, or call HTTP for per-operation authorization. +- Reuse `crowdb-access-s3` metadata, conditions, `PreparedGet`, + `VerifiedGetStream`, publication, and multipart components. Complete S3 + operations still live partly in `app/crowdb-access-server/src/s3/`. +- `BufferLeaseMetadata`, `BufferOwnership`, `WritableBuffer`, and completion + callbacks already exist in `crowdb-rpc-ffi`. `standalone_buffer` and + `receive_data_buffer` have copy/detach branches; audit rather than assume + end-to-end ownership is already preserved. +- Add `lib/crowdb-sdk-native/` to the main workspace. Keep the independently + published `sdk/` HTTP crates and Python wheel free of native storage/FFI + dependencies. New Python/C++ business bindings are not R242 gates; the + existing C ABI buffer lease boundary remains in scope. +- R242 accepts shared runtime/topology/data APIs and native S3. Dataset + attaches afterward under R220; its measurements and Python work must not + block S3 or create a parallel native runtime. + +## Phase 1: shared runtime and native API + +- [ ] **Create native crate and baseline**: register `crowdb-sdk-native` with + `runtime`, `topology`, `buffer`, and `s3` modules and tests under `tests/`. + Run focused existing S3/RPC/access and standalone SDK baselines before + moving operation code. Files: `Cargo.toml`, `lib/crowdb-sdk-native/`, + `lib/crowdb-access-s3/tests/`, `app/crowdb-access-server/tests/`. +- [ ] **Define runtime ownership**: implement `ClientRuntime::connect(config)` + and shutdown, sharing discovery, compatible transport groups, producer + allocation, and hard request/byte limits across handles. Preserve distinct + S3/Dataset chunk types and writer policies; do not create a generic Repo + chunk type or mix incompatible writer pools. Let `S3StorageClients` accept + shared clients instead of reconnecting for each facade. Files: + `lib/crowdb-sdk-native/src/runtime.rs`, `lib/crowdb-access-s3/src/storage.rs`, + `lib/crowdb-chunk-client/src/client.rs`. +- [ ] **Define business context**: expose `S3Client::new(runtime, config)` with + tenant/write policy and typed per-operation request identity, deadline, + cancellation, conditions, and range. Preserve S3 errors and uncertain + outcomes without HTTP headers, XML, or `CredentialProvider`. Files: + `lib/crowdb-sdk-native/src/s3/`, `lib/crowdb-access-s3/src/error.rs`, + `lib/crowdb-access-s3/src/condition.rs`, `lib/crowdb-access-s3/src/range.rs`. +- [ ] **Define common delivery**: reads return metadata plus a pull-based + `next() -> Result, _>` body; `Ok(None)` means applicable + length/integrity checks passed. Preserve terminal failure across subsequent + polling; early drop is cancellation, not successful EOF. Define unique + writable storage, freeze/submit ownership, and write finalization. Keep business + results separate from common buffer types so later Dataset batches can own + sample/field streams. Files: `lib/crowdb-sdk-native/src/buffer.rs`, + `lib/crowdb-sdk-native/src/s3/`, `lib/crowdb-access-s3/src/retrieval.rs`. + +## Phase 2: supported topology and scheduling surface + +- [ ] **Expose retained topology views**: provide inspection and explicit + refresh of service endpoints, rack/node/disk membership, Chunk-KV range + owners, transport capabilities, and source generation/age. Reuse existing + catalog and route owners; independently observed generations must not + become an invented global topology epoch. Files: + `lib/crowdb-sdk-native/src/topology.rs`, + `lib/crowdb-chunk-kv-client/src/catalog.rs`, + `lib/crowdb-diskio-client/src/topology.rs`, + `lib/crowdb-diskio-client/src/status.rs`, + `lib/crowdb-diskio-client/src/semantic/native_routes.rs`. +- [ ] **Expose object locality**: resolve a logical object generation/range to + chunk placement and route observations for caller scheduling/prefetch. + Preserve the metadata-selected identity during execution instead of + re-resolving a mutable key to a different generation. Observation cannot + publish placement. Reuse `ChunkIoClient::native_storage_routes` where + suitable. Files: `lib/crowdb-sdk-native/src/s3/`, + `lib/crowdb-sdk-native/src/topology.rs`, + `lib/crowdb-access-s3/src/metadata/record.rs`, + `lib/crowdb-chunk-client/src/client.rs`. +- [ ] **Handle stale observations**: retain catalog invalidation, endpoint + refresh, ownership epochs, and generation fencing. Old views stay safe to + inspect but cannot force execution past validation. Return typed + stale/fenced/deadline outcomes when refresh fails; never replay ambiguous + writes. Files: `lib/crowdb-sdk-native/tests/`, + `lib/crowdb-chunk-kv-client/src/client.rs`, + `lib/crowdb-diskio-client/src/semantic/`. + +## Phase 3: shared S3 execution and first native frontend + +- [ ] **Extract bucket and metadata operations**: move typed bucket, HEAD, + list, and delete orchestration into the S3 library. Preserve bounded + listing, continuation scope, conditions, and shared-range cleanup. HTTP + parses/authenticates, calls shared execution, and encodes. Files: + `lib/crowdb-access-s3/src/`, + `app/crowdb-access-server/src/s3/operations.rs`, + `app/crowdb-access-server/src/s3/operations/delete.rs`, + `lib/crowdb-sdk-native/src/s3/`. +- [ ] **Extract streaming GET**: adapt `PreparedGet`/`VerifiedGetStream` to + typed metadata and a lazy native body with logical offsets. Preserve + selected generation, range checks, storage checksums, and final checksum + errors. Do not expose HTTP strings or RPC fragments as native object data. + Files: `lib/crowdb-access-s3/src/retrieval.rs`, + `app/crowdb-access-server/src/s3/operations.rs`, + `lib/crowdb-sdk-native/src/s3/`, + `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`. +- [ ] **Extract PUT and copy**: separate HTTP decoding from bounded input, + checksums, ChunkIO writes, publication, and definite-failure cleanup. Add + ordinary-input and frozen native-buffer entry points. Preserve + `PutOutcome::Timeout` uncertainty; final success requires durable data and + object publication. Files: `lib/crowdb-access-s3/src/streaming.rs`, + `lib/crowdb-access-s3/src/publication.rs`, + `app/crowdb-access-server/src/s3/operations/copy.rs`, + `app/crowdb-access-server/src/s3/operations/upload.rs`, + `lib/crowdb-sdk-native/src/s3/`. +- [ ] **Attach multipart**: reuse the repository and multipart engine for + begin, upload/list parts, list uploads, complete, and abort. Preserve upload + identity, validation, publication uncertainty, and deferred cleanup. Files: + `lib/crowdb-access-s3/src/metadata/multipart_repository/`, + `lib/crowdb-access-multipart/`, + `app/crowdb-access-server/src/s3/operations/multipart.rs`, + `lib/crowdb-sdk-native/src/s3/`. +- [ ] **Verify native TCP parity first**: compare supported operations through + HTTP and native entry points, then run native access without an HTTP + listener or S3 credential store. Compare bytes, metadata, conditions, + errors, publication, and cancellation before enabling RDMA. Files: + `lib/crowdb-sdk-native/tests/`, `lib/crowdb-access-s3/tests/`, + `app/crowdb-access-server/tests/`. + +## Phase 4: retained leases, RDMA, and bounded lifecycle + +- [ ] **Complete lease ABI**: reuse opaque `crowdb_rpc_buffer_t`, + `BufferLeaseMetadata`, and actual buffer length for native `BufferLease`. + Preserve request/stream identity, sequence, offset, capacity class, and + single final release. Rust borrowing prevents access after release; + document C ownership/misuse rather than promising repeated release of freed + handles is safe. Files: `lib/crowdb-rpc/ffi/src/buffer.rs`, + `lib/crowdb-rpc/ffi/src/sys.rs`, `lib/crowdb-rpc/include/crowdb-rpc/c_api.h`, + `lib/crowdb-sdk-native/src/buffer.rs`. +- [ ] **Preserve completion ownership**: audit C++ completion, + `on_complete_cb`, `receive_data_buffer`, DiskIO conversion, ChunkIO views, + and S3 adaptation. Retain eligible receive storage through caller release + rather than a copy/detach shortcut that loses credits. Keep control + decoding/assembly, alignment, and EC copies explicit; cover existing RPC + callers before changing ownership. Files: `lib/crowdb-rpc/ffi/src/client.rs`, + `lib/crowdb-diskio-client/src/client.rs`, + `lib/crowdb-diskio-client/src/semantic.rs`, + `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`, + `lib/crowdb-access-s3/src/retrieval.rs`. +- [ ] **Wire producer buffers**: reuse `ProducerBufferAllocator` and + `WritableBuffer`, expose only initialized frozen prefixes, and retain + submitted owners through transport/disk completion. Ordinary input + adapters stay bounded and account for copies. Files: + `lib/crowdb-rpc/ffi/src/producer_buffer.rs`, + `lib/crowdb-chunk-client/src/io.rs`, `lib/crowdb-sdk-native/src/buffer.rs`, + `lib/crowdb-access-s3/src/streaming.rs`. +- [ ] **Enforce held-capacity budgets**: charge actual allocation capacity + once across shared views until final release. Keep request/producer/held + limits explicit and lock-free; waits honor deadlines/cancellation without + blocking RPC workers. Release backing storage before waking waiters. Files: + `lib/crowdb-sdk-native/src/runtime.rs`, `lib/crowdb-sdk-native/src/buffer.rs`, + `lib/crowdb-rpc/ffi/tests/`, `lib/crowdb-sdk-native/tests/`. +- [ ] **Implement cancel and shutdown**: close admission, stop producers, + resolve operations once, and preserve storage through disk/CQ/caller + ownership. Release after shutdown retires slots without accessing destroyed + transport. Files: `lib/crowdb-sdk-native/src/runtime.rs`, + `lib/crowdb-sdk-native/src/buffer.rs`, `lib/crowdb-rpc/ffi/src/client.rs`, + `lib/crowdb-sdk-native/tests/`. +- [ ] **Verify fallback and GPU-copy lifetime**: retain existing TCP/RDMA + policy, request identity, and eligible pre-payload fallback; fail accepted + or partial RDMA operations without TCP suffixes or ambiguous replay. Add an + asynchronous host-to-GPU copy simulation holding a lease beyond stream + close until copy completion. Files: `lib/crowdb-sdk-native/tests/`, + `lib/crowdb-rpc/ffi/tests/`, `lib/crowdb-diskio-client/tests/rdma_failure_test.rs`. + +## Phase 5: S3 acceptance and Dataset handoff + +- [ ] **Run native and OCI profiles**: extend accepted launcher/runtime + conventions with the S3 SDK corpus, no-device TCP and shared-HCA profiles. + Record topology, actual path/classes, independent QPs, held-capacity high + water, errors, and exact HTTP/native parity. Native-only execution must not + require Access Server. Files: `tools/rdma/`, `e2e-test/tests/access_s3/`, + `lib/crowdb-sdk-native/tests/`. +- [ ] **Verify distribution and document API**: run standalone SDK boundary + and functional controls; document shared runtime, topology scheduling, + stream termination, lease release, publication, and shutdown. Record R242 + evidence and the shared API consumed by R220. Files: + `lib/crowdb-sdk-native/README.md`, `sdk/README.md`, + `tools/ci-checks/check-dataset-sdk-boundary.py`, + `doc/backlog/R220-dataset-fat-client-plan.md`. + +R242 exit: accepted runtime/topology/lease APIs and native S3 semantics, +TCP/RDMA parity, bounded pressure, and shutdown-safe caller ownership. Existing +transport acceptance alone cannot establish SDK completion. + +## Subsequent Dataset integration under R220 + +- [ ] **Attach Dataset after S3**: add a facade using the same runtime, + topology, and buffers. Reuse `DatasetAuthority`, `DatasetSessions`, upload, + composition, and Split logic; extract remaining common HTTP v1/v2 delivery + where needed. Files: `lib/crowdb-sdk-native/src/dataset/`, + `lib/crowdb-access-dataset/`, `app/crowdb-access-server/src/dataset/api/`. +- [ ] **Preserve Dataset completion**: retain Snapshot/Split binding, + selection/partition, active-read protection, explicit acknowledgement, + resume, and cleanup. Legacy `DatasetDirectClient::scan` progress behavior + is not the v2 contract. Buffer release and download completion never + acknowledge batches. Files: `lib/crowdb-access-dataset/src/read_session/`, + `lib/crowdb-sdk-native/tests/`, `app/crowdb-access-server/tests/`. +- [ ] **Complete Dataset follow-up**: perform R220's HTTP/direct workload + comparison and native Python integration using this runtime, preserving + the independent HTTP SDK distribution. Files: + `doc/backlog/R220-dataset-fat-client-plan.md`, `sdk/`, + `e2e-test/tests/access_dataset/`. + +These tasks belong to R220, follow S3 delivery, and are not prerequisites for +R242 acceptance or a second implementation of its runtime. ## Tests Unit tests: -- opaque ABI layout, fixed-width fields, null/error behavior; -- lease accessors, single release, release-after-shutdown, and misuse; -- callback does not block and does not publish unresolved ownership; -- cancellation resolves exactly once; -- host-to-GPU copy dependency keeps lease alive until completion. +- typed context/range/conditions and retained topology ownership; +- opaque ABI layout, fixed-width fields, initialized-length and null/error rules; +- unique write/freeze ownership, shared accounting, and single final release; +- nonblocking completion, single terminal result, and asynchronous copy ownership. Integration tests: -- native SDK uses the same access-core semantics as HTTP frontend; -- caller holds all leases and observes bounded admission/backpressure; -- multi-message metadata and aggregate responses preserve order and offsets; -- RDMA-disabled host uses TCP without changing SDK result; -- pre-payload RDMA failure retries eligible request with shared request ID; -- partial RDMA failure fails stream without accepting a TCP suffix. +- shared runtime handles with isolated operation cancellation; +- topology/locality inspection, stale owner/endpoint refresh, and fencing; +- S3 bucket/object/copy/multipart parity, conditions, and uncertain publication; +- native execution without HTTP listener, SigV4 verifier, or secret database; +- final integrity failure, lease order/offsets, and eligible no-copy payloads; +- held-capacity pressure, deadlines, cancel, shutdown, and release afterward; +- TCP-only results, pre-payload fallback, terminal accepted/partial RDMA failures; +- standalone Dataset HTTP SDK packaging and functional regression controls. E2E tests: -- single-node HCA local-loop OCI client/Access Server/DiskIO flow; -- TCP-only OCI profile with no RDMA devices; -- single-node local-loop multi-container SDK pressure and multiple logical streams; -- explicit host-to-GPU copy simulation with release after copy completion. +- native S3 against metadata/ChunkIO/DiskIO with explicit topology inspection; +- no-device TCP OCI and shared-HCA local-loop multi-container profiles; +- held-lease pressure, multiple streams, and simulated asynchronous GPU copy; +- HTTP/native parity separate from the native-only execution fixture. ## Consolidated files -- `app/crowdb-access-server/` -- Access core and native SDK modules -- `lib/crowdb-rpc/ffi/src/client.rs` -- `lib/crowdb-rpc/ffi/src/server.rs` -- `lib/crowdb-rpc/ffi/src/sys.rs` -- SDK integration tests and OCI E2E runner +- `Cargo.toml`, new `lib/crowdb-sdk-native/`, its `tests/` and README. +- `lib/crowdb-access-s3/`, `lib/crowdb-access-multipart/`, S3 HTTP operations. +- `lib/crowdb-chunk-kv-client/`, `lib/crowdb-chunk-client/`, `lib/crowdb-diskio-client/`. +- `lib/crowdb-rpc/ffi/`, `lib/crowdb-rpc/include/crowdb-rpc/c_api.h`, affected C++ ownership paths. +- Access Server tests, `e2e-test/tests/access_s3/`, `tools/rdma/`. +- `sdk/README.md` and standalone boundary controls; no new native dependency in `sdk/`. +- R220 follow-up: Dataset library/HTTP adapters, native Dataset facade and bindings. ## Verification gates +Run after registering the native crate; use existing-package gates for the +baseline. Root Cargo gates do not cover the independent SDK workspace. + ```bash pixi run -- cargo fmt --all -- --check +pixi run -- cargo test -p crowdb-sdk-native --all-targets +pixi run -- cargo test -p crowdb-access-s3 --all-targets pixi run -- cargo test -p crowdb-rpc-ffi --all-targets pixi run -- cargo test -p crowdb-access-server --all-targets pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings pixi run rs-lint pixi run tree-lint +pixi run check-dataset-sdk-boundary +pixi run test-dataset-sdk +pixi run sdk-fmt-check +pixi run sdk-lint +pixi run test-e2e-s3 ``` -Completion requires caller-controlled lease tests, HTTP/SDK parity, TCP/RDMA -parity, and safe shutdown with outstanding leases. +Enable `CROWDB_E2E_PRINT_SERVICE_ERRORS=1` in CI or while diagnosing failures. +Capture actual native/OCI profile and lease evidence; passing TCP gates do +not replace the real-HCA profile. diff --git a/doc/working/plan-rdma-mixed-size-send.md b/doc/working/plan-rdma-mixed-size-send.md new file mode 100644 index 000000000..6202743f5 --- /dev/null +++ b/doc/working/plan-rdma-mixed-size-send.md @@ -0,0 +1,336 @@ + + + +# RDMA mixed-size send efficiency Plan + +Upstream: user-requested transport optimization, +[RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md), +[accepted RDMA implementation](plan-rdma.md), +[RPC flow analysis](../design/rpc/rpc-flow-analysis.md), and +[physical-node validation](plan-rdma-multi-node.md). + +Goal: improve completed client requests per CPU cost under mixed message sizes, +using registered payloads, bounded small-message aggregation, batched WR posting, +and continuously replenished per-QP send windows without waiting to fill batches. + +Status: planned; code paths and external references inspected. No implementation +or new performance runs have been performed. Delete this temporary plan after +implementation, verification, and migration of accepted contracts/evidence. +Existing SDK/backlog work remains independent of this transport optimization. +After mixed-workload acceptance demonstrates a repeatable resource/throughput +benefit within the declared latency bounds, continue the existing +[R242 SDK plan](plan-rdma-fat-client-sdk.md), as authorized by the user. +Keep that conditional handoff distinct from transport completion; do not start +Dataset integration under R220 as part of R242. + +## Existing behavior and intended changes + +- Coroutine benchmark loaders are closed-loop: each has one outstanding request + and issues its next request after its response. Multiple loaders can run on + one completion thread. Loader count, producer thread count, CQ owner count, + QP count, and outstanding SEND count are different quantities. +- `ConnectionPools::send_queue` is bounded MPSC admission. Its fixed CQ owner + drains frames. `sends_in_flight` already caps posted but unreclaimed SEND WRs + at `min(sq_depth, effective_capacity.max_send_wr)` per QP. +- Both layouts currently post one WR per `ibv_post_send`; QP setup uses + `sq_sig_all = 1`. `poll_batch` is a loop budget, not a linked WR submission. +- `Contiguous` copies complete frames into one registered slot and can pack + several requests. `ControlData` uses a fixed 1024-byte control prefix and + actual-length payload; registered contiguous producer/receive ranges can be + borrowed. It currently rejects transport aggregation. +- Receive buffers are posted before the message arrives. Sender SGE boundaries + are not transferred to the peer. The fixed prefix and negotiated payload + capacity preserve scatter placement for variable lengths and data alignment. +- Current echo builds payloads through ordinary buffers, copies the server echo, + and validates response bytes. The existing fixed-size peak is a regression + control, not evidence of efficiency under mixed service traffic. + +## Execution rules + +- Retain RC SEND/RECV, control/data separation, setup-only TCP fallback, + request identities, stream sequencing, generation checks, and quarantine. +- Producers prepare immutable source buffers and publish through existing + lock-free admission. One CQ owner prepares each QP's pending aggregate, + ordered ready items, WR batches, and send retirement. Do not introduce a + shared mutable aggregate written by multiple producers or a new hot-path lock. +- Prepare direct items before publishing them as ready; the posting stage must + not discover that it needs to gather/copy an entire large payload. A proposed + ready-item record retains prefix, data ranges, original logical frames, + actual wire bytes, and completion owners. Symbol names are finalized in code. +- Keep one ordered stream per QP. A large/direct item seals the earlier pending + aggregate before entering ready. Control-stream fragments remain ordered and + cannot be interleaved with later application frames on that stream. +- Small aggregation may copy control and data. Eligibility includes complete + encoded size, message semantics, and alignment/lease requirements; mandatory + disk-aligned or directly retained payloads bypass aggregation. A large + registered payload bypasses packing and retains its source allocation. +- Aggregate byte/frame limits and WR batch size are upper bounds. Seal the + aggregate when full, when a direct/barrier item arrives, when the preparation + budget ends, or when available input is exhausted. Submit a single ready item + immediately when it is the only work. Do not wait for a future request. +- Bound preparation by items and copied bytes, and bound receive/callback and + post work per round. Continue CQ progress when SQ/memory admission is full; + a producer backlog must not starve completions or other QPs. Do not sleep + with a sealable aggregate or sendable ready item. A byte/WR-blocked ready + item may wait for the actual completion/release event without busy spinning. +- Treat software queue capacity, posted SEND WR count, retained bytes, and + client requests awaiting responses as separate budgets. Release each charge + only at its relevant ownership boundary; buffer views sharing an allocation + must not multiply-count physical retained bytes. + +## Phase 1: representative workload and baseline + +- [ ] **Add deterministic per-request sizes**: extend CLI workload arguments + and manifest cases with weighted sizes/seed and an explicit wire layout; + record expected length per loader/request instead of one global + `EchoWorkload::payload_bytes`. Validate ID, length, bytes, and response + control for every request. Preserve fixed-size mode and TCP parity. Files: + `app/crowdb-cli/src/commands/bench/rpc/{args,workload,runtime,report}.rs`, + `tools/benchmark/rpc_bench_cases.py`, `app/crowdb-cli/tests/rpc_bench_test.rs`, + `tools/benchmark/tests/rdma_regression_test.py`. +- [ ] **Capture service size distributions**: use existing HTTP/DiskIO and + Chunk-KV corpora to retain actual control/data lengths, class boundaries, + registration origin, alignment requirements, and lease-hold behavior. + Add reproducible synthetic mixes spanning control-only, 128 B, 1/4 KiB, + 64/256 KiB and near negotiated capacity; include non-power-of-two sizes. + Label synthetic mixes explicitly rather than claiming production traces. + Files: `tools/rdma/run-datapath.py`, `tools/rdma/run-chunk-kv.py`, + `app/crowdb-access-server/tests/common/rdma_datapath/`, + `app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv/`. +- [ ] **Retain comparable baselines**: save source/binary identity, machine + topology, CPU placement, actual paths, request mix/seed, worker/QP counts, + budgets and error counters. Measure single-loader latency, multi-loader + saturation, burst/idle transitions, and delayed consumer release. Keep + offered work and application behavior identical in before/after pairs. + Declare performance noise and acceptable per-size tail-latency bounds from + repeated baseline runs before selecting optimized defaults. Files: + `tools/benchmark/rpc_regression.py`, `tools/benchmark/rpc_bench_report.py`, + `tools/benchmark/bench-rdma-rpc-regression.sh`. + +## Phase 2: registered producers and copy accounting + +- [ ] **Generate payloads in registered buffers**: audit the existing + `allocate_producer_buffer` and Rust writable/frozen APIs, pass the transport + producer allocator through the benchmark runtime, and write variable-length + data into the final registered allocation. Retain an ordinary-buffer mode + for compatibility and a paired registered-buffer baseline. Avoid allocating + a Vec and then copying it merely to claim producer zero-copy. Files: + `lib/crowdb-rpc/src/transport/rdma/rdma_producer.cpp`, + `lib/crowdb-rpc/ffi/src/{buffer,co_bench}.rs`, + `app/crowdb-cli/src/commands/bench/rpc/{runtime,workload}.rs`. +- [ ] **Preserve direct source ownership**: prepare eligible payload ranges + using the existing producer/receive arena borrow checks; retain MR/PD, + allocation identity and immutable source through send retirement. Cover + nested views, shared mirror payloads, zero-length data and arbitrary + supported lengths. Unregistered or unsupported scattered input uses a + bounded, separately measured compatibility copy. Preserve usable pool + classes when a small allocation precedes a larger one. Files: + `lib/crowdb-rpc/src/transport/rdma/rdma_direct_send.cpp`, + `lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp`, + `lib/crowdb-rpc/tests/{producer_buffer,rdma_memory,rdma_lifetime}_test.cpp`. +- [ ] **Separate copy causes**: expose bytes copied for small aggregation, + unregistered/scattered compatibility staging, control framing, and echo + application work separately from borrowed direct payload bytes. Keep a + registered-source echo/retained-lease mode distinct from the existing + copy-echo workload; compare each mode against itself. Files: + `lib/crowdb-rpc/src/server/handler.cpp`, + `lib/crowdb-rpc/src/transport/rdma/{rdma_cq.h,rdma_status.cpp}`, + `app/crowdb-cli/src/commands/bench/rpc/{status,report}.rs`. + +## Phase 3: mixed direct and aggregate preparation + +- [ ] **Define the negotiated aggregate format**: extend the formal RDMA + contract before implementation. Retain the fixed control prefix; add an + unambiguous aggregate envelope and bounded packed data containing complete + logical control/data frames. Specify counts, lengths, offsets, maximum wire + bytes, supported message types, and whole-envelope validation. Direct data + retains its independent aligned allocation. Negotiate support explicitly; + older peers keep the existing direct format, without new-format payload + submission or midstream fallback. Files: + `doc/design/rpc/design-crowdb-rpc-rdma.md`, + `lib/crowdb-rpc/src/transport/rdma/{rdma_offer,rdma_direct_frame,rdma_aggregate}.{h,cpp}`, + `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h`, + `lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp`. +- [ ] **Add bounded preparation state**: extend `ConnectionPools` with an + owner-only pending aggregate and ordered ready items, reusing existing + input admission. Apply a small-message copy threshold and independent + aggregate byte/frame limits. Check local and peer capacities before copying; + preserve admission accounting through pending, ready and posted states. + Do not implement an additional unbounded queue or independent per-producer + aggregate that can retain a lone request indefinitely. Files: + `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h`, + `lib/crowdb-rpc/src/transport/rdma/{rdma_send,rdma_direct_send,rdma_control_send}.cpp`. +- [ ] **Dispatch mixed receive envelopes**: validate all packed frame bounds + before publishing any frame; create retained views for individual payloads. + A final caller release controls parent repost. Reject malformed envelopes + without partial RPC publication. Keep large logical control fragmentation, + handshake messages and alignment-sensitive service traffic on explicitly + compatible paths. Files: + `lib/crowdb-rpc/src/transport/rdma/{rdma_direct_receive,rdma_receive,rdma_aggregate}.cpp`, + `lib/crowdb-rpc/tests/{rdma_aggregate,rdma_direct_frame,rdma_offer,rdma_control_stream}_test.cpp`. + +## Phase 4: send windows and batched posting + +- [ ] **Make the send window explicit**: retain `sq_depth` as QP allocation + capacity and add a configurable software WR window no larger than effective + SQ capacity, plus explicit queued/prepared/posted retained-byte bounds. + Account control prefix, aggregation storage, and retained source allocations; + separate per-QP limits from shared-arena limits. Reject an item that can + never fit its configured admission rather than stalling forever. Keep + posted receives as implicit peer credit; local SQ completion does not imply + the peer has released/reposted its receive buffer. Files: + `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/{rdma_config,rdma_transport}.h`, + `lib/crowdb-rpc/src/transport/rdma/{rdma_config,rdma_send,rdma_completion}.cpp`. +- [ ] **Build linked WR batches**: post ready items from the same QP together, + bounded by ready count, post batch cap, free WR slots and byte admission. + Each WR retains its own variable-length SGEs and immutable owners. Keep WR + and SGE descriptor storage stable through the post call. For partial post + failure, use `bad_wr` to identify the accepted prefix; retire unposted items + separately, fail/drain the uncertain connection, and never replay accepted + operations as though they were unsubmitted. Files: + `lib/crowdb-rpc/src/transport/rdma/{rdma_send,rdma_direct_send,rdma_control_send}.cpp`, + `lib/crowdb-rpc/tests/{rdma_send,rdma_failure}_test.cpp`. +- [ ] **Retire sends by sequence**: change both active/passive QP setup to + `sq_sig_all = 0`. Give posted WRs ordered per-QP sequences while retaining + generation-tagged completion identity. Initially signal each actual batch + tail, including a singleton. A successful tail retires its covered prefix, + releasing WR/byte charges and owners exactly once. Keep multiple batches + outstanding and refill freed space without waiting for an entire window. + Handle error CQEs for unsignaled WRs, partial posts missing the planned + signaled tail, stale CQEs, cancellation and QP destruction. Preserve + conservative CQ/WR admission for error/flush completions. Files: + `lib/crowdb-rpc/src/transport/rdma/{rdma_active,rdma_connection,rdma_work_request,rdma_completion}.cpp`, + `lib/crowdb-rpc/tests/{rdma_completion,rdma_failure,rdma_lifetime}_test.cpp`. +- [ ] **Schedule bounded progress**: separate preparation, post and CQ budgets + instead of treating `poll_batch` as all three. Interleave CQ processing and + preparation/submission, round-robin QPs, and seal pending aggregates at a + round boundary. Full SQs or slow peers must not stop receive/response work. + Preserve idle arm/recheck/wake correctness, including caller-held leases + released on another thread. Files: + `lib/crowdb-rpc/src/transport/rdma/{rdma_poller,rdma_completion,rdma_send}.cpp`, + `lib/crowdb-rpc/src/transport/rdma/rdma_cq.h`, + `lib/crowdb-rpc/tests/{rdma_worker,rdma_send}_test.cpp`. +- [ ] **Batch receive replenishment**: collect only fully released receive + owners and post bounded linked RECV lists through the assigned owner. Flush + a partial replenish batch before idle or receive starvation. Preserve + posted-receive credits, data alignment, and caller lease lifetime; handle + partial post failure without losing or double-posting a slot. Files: + `lib/crowdb-rpc/src/transport/rdma/{rdma_transport,rdma_direct_receive,rdma_receive,rdma_poller}.cpp`, + `lib/crowdb-rpc/tests/{rdma_worker,rdma_lifetime,rdma_failure}_test.cpp`. +- [ ] **Expose consistent configuration and diagnostics**: thread new limits + through C ABI, Rust config, CLI and runtime manifests with compatible + defaults and validation. Report in-flight WR/bytes, ready/pending age and + high-water, actual WRs/post, aggregation copy bytes, SQ stalls, peer receive + pressure, and per-size latency. Prefer owner-sharded counters and sampled + age checks over per-frame shared atomics. Files: + `lib/crowdb-rpc/include/crowdb-rpc/c_api.h`, + `lib/crowdb-rpc/src/c_api/transport.cpp`, + `lib/crowdb-rpc/ffi/src/{sys,transport}.rs`, + `lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp`, + `app/crowdb-cli/src/commands/bench/rpc/`, `tools/benchmark/`. + +## Phase 5: practical acceptance and default selection + +- [ ] **Verify concurrency and low-load progress**: cover one loader/one owner, + many loaders/one owner, many producers on one QP, multiple QPs per owner and + multiple owners. Submit one small request with no successor; it must finish + without a batch-fill wait. Exercise continuous producers, small/large/small + ordering, asymmetric request/response sizes, and a busy QP alongside an idle + or slow QP. Files: `lib/crowdb-rpc/tests/rdma_worker_test.cpp`, + `lib/crowdb-rpc/tests/rdma_direct_hardware_test.cpp`, + `app/crowdb-cli/tests/rpc_bench_test.rs`, `tools/benchmark/`. +- [ ] **Verify bounded pressure and shutdown**: hold receive leases to exhaust + peer RQ, saturate WR and byte limits independently, then release and verify + recovery without hot spinning, leaks or premature reuse. Inject errors + before/within a posted batch, at signaled-tail boundaries, during control + fragmentation and at disconnect. Preserve typed uncertain outcomes and no + TCP suffix after accepted RDMA. Files: + `lib/crowdb-rpc/tests/{rdma_failure,rdma_lifetime,rdma_control_hardware}_test.cpp`, + `lib/crowdb-diskio-client/tests/rdma_failure_test.rs`. +- [ ] **Run service parity and topology controls**: run existing HTTP/DiskIO + and real Chunk-KV/tree/WAL corpora with mixed traffic and retained leases, + comparing exact results and errors against TCP. Reuse native, shared-HCA OCI + and no-device TCP profiles. Coordinate a physical two-host run through the + linked multi-node plan before claiming production-network performance; + local HCA traversal remains labeled as such. Record unavailable hardware + gates as pending rather than replacing them with a synthetic success. + Files: `tools/rdma/run-datapath.py`, `tools/rdma/run-chunk-kv.py`, + `tools/rdma/run-oci-profile.sh`, `tools/rdma/run-timed-load.py`, + `doc/working/plan-rdma-multi-node.md`. +- [ ] **Compare changes incrementally**: retain paired baseline, registered + direct, hybrid aggregation, WR batching and selective-completion results. + Use client completed requests/s as the primary throughput metric, plus CPU + seconds/request at matched throughput and throughput at matched CPU budget, + valid payload bytes/s, per-size P50/P99/P999, queue residence, registered and + retained memory, errors and fallback. Record mixed workloads at low load, + below saturation, saturation and overload/burst recovery. A bounded offered + load case records intended-arrival-to-completion delay, not only latency + of admitted requests. Do not infer open-load latency from closed-loop runs. + Files: `tools/benchmark/`, `app/crowdb-cli/src/commands/bench/rpc/report.rs`. +- [ ] **Select defaults from evidence**: require zero correctness errors, + bounded memory and eventual progress after pressure release. Demonstrate + reduced payload copying on eligible registered large messages and a + repeatable CPU-cost or request-throughput benefit on service-informed mixed + workloads within the baseline-declared tail limits. Small aggregation copy + is allowed and included in costs. Report regressions and leave unhelpful + options disabled; a larger 128-byte peak alone cannot close this plan. + Publish accepted contracts and results, then delete this plan. Files: + `doc/design/rpc/design-crowdb-rpc-rdma.md`, + `doc/design/rpc/rpc-flow-analysis.md`, changed config defaults. + +## Consolidated files and checks + +- Transport: `lib/crowdb-rpc/src/transport/rdma/` and + `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/`. +- Producer/C ABI/Rust: `lib/crowdb-rpc/src/c_api/`, + `lib/crowdb-rpc/include/crowdb-rpc/c_api.h`, `lib/crowdb-rpc/ffi/src/`. +- Workloads/reports: `app/crowdb-cli/src/commands/bench/rpc/`, + `tools/benchmark/`, `tools/rdma/` and the existing service fixtures. +- Unit: RDMA config/offer/aggregate/direct-frame/send/completion/worker/lifetime + and failure tests under `lib/crowdb-rpc/tests/`; Rust CLI/FFI contracts under + their crates' `tests/`; Python manifest/report tests in `tools/benchmark/tests/`. +- Integration: real-HCA direct/control hardware tests, TCP/no-verbs compatibility, + mixed-loader completion validation and slow-consumer lease pressure. +- E2E: HTTP/DiskIO and Chunk-KV native/OCI parity, no-device TCP, and separately + identified physical-host performance evidence. +- Run all implementation builds/tests/lints through `pixi run`, with focused + checks per phase before wider affected suites. Use Rust fmt/clippy, changed + C++ clang-format/tree-lint, C++/FFI/CLI and affected access/Chunk-KV/DiskIO + suites. Keep full errors; enable `CROWDB_E2E_PRINT_SERVICE_ERRORS=1` in CI or + when debugging. Start long hardware runs in the background and poll. +- For this plan-only edit, check Markdown links and `git diff --check`; do not + report implementation tests or performance acceptance as completed. + +## External implementation references + +These describe borrowed mechanisms, not CROWDB contracts or target QPS. +Inspected on 2026-10-23; pin the source revision when running comparisons. + +- [bRPC RDMA documentation](https://github.com/apache/brpc/blob/master/docs/cn/rdma.md): + registered IOBuf pools, RC, event/polling modes and peer receive-window ACKs. +- [bRPC RDMA endpoint](https://github.com/apache/brpc/blob/master/src/brpc/rdma/rdma_endpoint.cpp): + local SQ and remote RQ windows, reference-based SGE gathering, selective + signaling and solicited wakeups. Its inspected send loop posts WRs individually; + its event aggregation must not be described as linked-WR posting or CQE removal. +- [bRPC dispatch](https://github.com/apache/brpc/blob/master/src/brpc/rdma_transport.cpp): + polling defaults to dispatching application messages as bthreads; disabling + bthreads allows inline handling. Poller count is not the process's CPU budget. +- [bRPC performance client](https://github.com/apache/brpc/blob/master/example/rdma_performance/client.cpp) + and [server](https://github.com/apache/brpc/blob/master/example/rdma_performance/server.cpp): + callback-replenished request windows; `queue_depth` is application outstanding + requests, and `thread_num` starts bthreads. Attachments are prepared/reused; + echo is optional and the server appends IOBuf references. This is not a + mixed-size storage workload or a complete payload-validation benchmark. +- [bRPC general benchmark](https://brpc.apache.org/zh/docs/benchmark/): + do not relabel its general framework results as measured RDMA performance. + The inspected official RDMA guide does not publish a directly comparable + single-core QPS claim. Any future comparison must retain mode, all worker + CPU costs, attachment reuse/echo behavior, size and request window. +- [eRPC paper, sections 4 and 6](https://www.usenix.org/system/files/nsdi19-kalia.pdf): + registered application buffers, preallocated short responses and different + small-RPC/bandwidth experiment conditions. Its 10M headline combines outgoing + and incoming RPC processing in a symmetric experiment. +- [verbs posting](https://man7.org/linux/man-pages/man3/ibv_post_send.3.html) + and [InfiniBand section 10.8.6](https://www.afs.enea.it/asantoro/V1r1_2_1.Release_12062007.pdf): + partial posting and later completion on the same SQ confirming earlier + unsignaled sends. Keep error/drain ownership separate from normal retirement. diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index 1f2d2f3f7..d8295b9e2 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -106,7 +106,10 @@ Accepted registered host buffers and common RPC Accepted HTTP/ChunkIO/DiskIO and Chunk-KV/access | v - R242 fat client SDK and FFI lease + R242 shared runtime/topology, S3 SDK and leases + | + v + R220 Dataset adapter and measurements ``` The remaining child uses the existing [fat-client SDK plan](plan-rdma-fat-client-sdk.md) @@ -120,15 +123,20 @@ Transport echo acceptance alone does not prove SDK lease semantics. ## Phase 4: R242 fat client SDK -- [ ] **Execute R242 child plan**: split access core from HTTP frontend, expose - FFI-safe host leases, and reuse the accepted Chunk-KV/ChunkIO transports. +- [ ] **Execute R242 child plan**: build the shared trusted internal runtime, + expose storage topology and FFI-safe host leases, and deliver native S3 + using existing S3 components and Chunk-KV/ChunkIO transports. Native calls + take explicit tenant/context without SigV4 or a user-secret database; + Dataset attaches afterward under R220 using the same runtime. - [ ] **Host-to-GPU simulation**: hold each lease until the explicit host-to-GPU copy completes, then release it; no GPU pointer is exposed by the base API. - [ ] **SDK pressure tests**: hold all returned leases, verify bounded admission, cancellation, shutdown, and recovery after release. -Exit condition: native SDK and HTTP frontend return equivalent access results, -and the caller-controlled lease lifetime is visible and testable through FFI. +Exit condition: native S3 and HTTP return equivalent semantic results, topology +is exposed for caller scheduling with existing freshness/fencing rules, and +caller-controlled lease lifetime is testable through FFI. R220's later Dataset +adapter and measurements do not gate R242 or this RDMA umbrella acceptance. ## Phase 5: SDK OCI E2E matrix From b879f9e91db11361da2b0dafb0cd27e18b41ae2f Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 12:00:59 +0800 Subject: [PATCH 057/258] Optimize mixed-size RDMA sends with bounded batching and direct buffers --- .../tests/rdma_datapath_test.rs | 10 +- .../tests/common/rdma_chunk_kv.rs | 5 + .../tests/common/rdma_chunk_kv/corpus.rs | 1 + doc/design/rpc/design-crowdb-rpc-rdma.md | 123 ++++++- doc/design/rpc/rpc-flow-analysis.md | 50 +++ doc/working/plan-rdma-fat-client-sdk.md | 7 +- doc/working/plan-rdma-mixed-size-send.md | 336 ------------------ doc/working/plan-rdma.md | 98 ++++- lib/crowdb-rpc/ffi/src/producer_buffer.rs | 36 +- lib/crowdb-rpc/ffi/src/sys.rs | 18 + lib/crowdb-rpc/ffi/src/transport.rs | 39 +- .../ffi/tests/producer_buffer_test.rs | 24 ++ .../ffi/tests/transport_config_test.rs | 32 +- lib/crowdb-rpc/include/crowdb-rpc/c_api.h | 18 + .../crowdb-rpc/transport/rdma/rdma_config.h | 4 + .../transport/rdma/rdma_transport.h | 59 ++- lib/crowdb-rpc/src/buffer_arena.cpp | 24 -- lib/crowdb-rpc/src/buffer_arena.h | 1 + lib/crowdb-rpc/src/buffer_arena_borrow.cpp | 59 +++ lib/crowdb-rpc/src/c_api/transport.cpp | 32 +- .../src/transport/rdma/rdma_active.cpp | 44 ++- .../src/transport/rdma/rdma_buffer_pool.cpp | 9 + .../src/transport/rdma/rdma_capability.cpp | 3 +- .../src/transport/rdma/rdma_completion.cpp | 18 +- .../src/transport/rdma/rdma_config.cpp | 18 +- .../src/transport/rdma/rdma_connection.cpp | 5 +- .../src/transport/rdma/rdma_control_send.cpp | 15 +- lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp | 17 +- lib/crowdb-rpc/src/transport/rdma/rdma_cq.h | 48 ++- .../transport/rdma/rdma_direct_receive.cpp | 15 + .../src/transport/rdma/rdma_direct_send.cpp | 82 ++--- .../src/transport/rdma/rdma_offer.cpp | 3 +- .../src/transport/rdma/rdma_packed_frame.cpp | 76 ++++ .../src/transport/rdma/rdma_packed_frame.h | 17 + .../transport/rdma/rdma_packed_receive.cpp | 71 ++++ .../src/transport/rdma/rdma_packed_send.cpp | 77 ++++ .../src/transport/rdma/rdma_poller.cpp | 2 +- lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp | 2 + .../src/transport/rdma/rdma_receive.cpp | 36 +- .../src/transport/rdma/rdma_receive_batch.cpp | 106 ++++++ .../src/transport/rdma/rdma_send.cpp | 77 ++-- .../src/transport/rdma/rdma_send_batch.cpp | 179 ++++++++++ .../src/transport/rdma/rdma_status.cpp | 15 + .../src/transport/rdma/rdma_transport.cpp | 29 +- .../src/transport/rdma/rdma_work_request.cpp | 9 +- .../tests/common/test_rdma_transport.h | 37 +- lib/crowdb-rpc/tests/rdma_borrow_test.cpp | 90 +++++ lib/crowdb-rpc/tests/rdma_config_test.cpp | 33 +- .../tests/rdma_direct_hardware_test.cpp | 125 ++++++- lib/crowdb-rpc/tests/rdma_offer_test.cpp | 22 ++ .../tests/rdma_packed_frame_test.cpp | 59 +++ .../tests/rdma_receive_batch_test.cpp | 112 ++++++ .../tests/rdma_receive_budget_test.cpp | 120 +++++++ lib/crowdb-rpc/tests/rdma_send_test.cpp | 91 ++++- lib/crowdb-rpc/tests/rdma_startup_test.cpp | 56 ++- .../tests/transport_config_test.cpp | 31 ++ tools/benchmark/profiles/rdma-mixed-size.json | 47 +++ tools/benchmark/rpc_bench_cases.py | 18 +- tools/benchmark/rpc_bench_report.py | 38 ++ tools/benchmark/rpc_regression.py | 2 +- tools/benchmark/tests/rdma_posting_test.py | 34 ++ tools/benchmark/tests/rdma_regression_test.py | 34 ++ tools/rdma/posting.py | 35 ++ tools/rdma/run-chunk-kv.py | 6 + tools/rdma/run-datapath.py | 10 +- tools/rpc-test/src/args.rs | 45 ++- tools/rpc-test/src/workload.rs | 23 +- tools/rpc-test/src/workload/payload.rs | 64 ++++ tools/rpc-test/src/workload/report.rs | 10 +- tools/rpc-test/src/workload/runtime.rs | 17 +- tools/rpc-test/src/workload/workload.rs | 85 ++++- tools/rpc-test/tests/validated_echo_test.rs | 46 +++ 72 files changed, 2644 insertions(+), 595 deletions(-) delete mode 100644 doc/working/plan-rdma-mixed-size-send.md create mode 100644 lib/crowdb-rpc/src/buffer_arena_borrow.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_packed_frame.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_packed_frame.h create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_packed_receive.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_packed_send.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_receive_batch.cpp create mode 100644 lib/crowdb-rpc/src/transport/rdma/rdma_send_batch.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_borrow_test.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_packed_frame_test.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_receive_batch_test.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_receive_budget_test.cpp create mode 100644 tools/benchmark/profiles/rdma-mixed-size.json create mode 100644 tools/benchmark/tests/rdma_posting_test.py create mode 100644 tools/rdma/posting.py create mode 100644 tools/rpc-test/src/workload/payload.rs diff --git a/app/crowdb-access-server/tests/rdma_datapath_test.rs b/app/crowdb-access-server/tests/rdma_datapath_test.rs index 972ed6280..bef27d17a 100644 --- a/app/crowdb-access-server/tests/rdma_datapath_test.rs +++ b/app/crowdb-access-server/tests/rdma_datapath_test.rs @@ -24,6 +24,11 @@ fn test_rdma_config(enabled: bool) -> RdmaConfig { wire_layout: RdmaWireLayout::ControlData, wire_slot_capacity: (1 << 20) + 1024, sq_depth: 4, + send_batch_size: std::env::var("CROWDB_RDMA_SEND_BATCH_SIZE") + .map_or(1, |value| value.parse().unwrap()), + receive_batch_size: std::env::var("CROWDB_RDMA_RECEIVE_BATCH_SIZE") + .map_or(1, |value| value.parse().unwrap()), + aggregation_enabled: std::env::var("CROWDB_RDMA_AGGREGATION_ENABLED").as_deref() == Ok("1"), rq_depth: 4, posted_receive_target: 4, cq_depth: 16, @@ -146,7 +151,10 @@ async fn http_chunkio_round_trip_uses_selected_transport_and_exact_framed_bytes( let status: serde_json::Value = serde_json::from_slice(&client.transport_status_json().unwrap()).unwrap(); if std::env::var("CROWDB_RDMA_DATAPATH_TRANSPORT").unwrap() == "rdma" { let rdma = &status["rdma"]; - assert!(rdma["direct_payload_bytes"].as_u64().unwrap() >= 2 * cursor); + assert!( + rdma["direct_payload_bytes"].as_u64().unwrap() >= 2 * cursor, + "{rdma}" + ); for field in [ "copied_payload_bytes", "fallback_count", diff --git a/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv.rs b/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv.rs index 4ed16f1c5..efded3d34 100644 --- a/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv.rs +++ b/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv.rs @@ -38,6 +38,11 @@ impl TestChunkKvProfile { cq_count: 1, cq_depth: 16, sq_depth: 2, + send_batch_size: std::env::var("CROWDB_RDMA_SEND_BATCH_SIZE") + .map_or(1, |value| value.parse().unwrap()), + receive_batch_size: std::env::var("CROWDB_RDMA_RECEIVE_BATCH_SIZE") + .map_or(1, |value| value.parse().unwrap()), + aggregation_enabled: std::env::var("CROWDB_RDMA_AGGREGATION_ENABLED").as_deref() == Ok("1"), rq_depth: 2, posted_receive_target: 2, device: std::env::var("CROWDB_RDMA_DEVICE").unwrap_or_default(), diff --git a/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv/corpus.rs b/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv/corpus.rs index 9275e2e1e..21ea972e2 100644 --- a/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv/corpus.rs +++ b/app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv/corpus.rs @@ -124,6 +124,7 @@ pub(super) async fn run(transport: &ChunkKvRpcTransport, endpoint: &str) -> Valu assert_eq!(stale.unwrap_err().code, ChunkKvRpcErrorCode::RefreshRequired); json!({ "records": records.len(), "payload_bytes": records.iter().map(|(_, v)| v.len()).sum::(), + "value_sizes_bytes": records.iter().map(|(_, value)| value.len()).collect::>(), "values_sha256": format!("{:x}", Sha256::digest(serde_json::to_vec(&expected).unwrap())), "mutation_results_sha256": format!("{:x}", Sha256::digest(serde_json::to_vec(&results).unwrap())), "expected_typed_errors": 2, "typed_errors": [error.code, ChunkKvRpcErrorCode::RefreshRequired], "logical_mutation_sequences": (1..=8).collect::>(), diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index 0c1e0b029..41953fe8a 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -115,9 +115,37 @@ inside that region. Receive uses two SGEs with matching region capacities; data starts at its own registered allocation base. Data classes of 64 KiB and above have 4096-byte alignment. Control and data consume the configured receive arena budget; a 1 MiB data capacity requires `wire_slot_capacity = 1049600`. -Header and CQE lengths must match exactly before dispatch. This layout sends -one RPC per SEND and rejects transport aggregation or fewer than two provider -SGEs during setup. It leaves the HTTP/ChunkIO producer's aggregation intact. +Header and CQE lengths must match exactly before dispatch. A direct envelope +sends one RPC per SEND. Setup requires at least two provider SGEs. It leaves +the HTTP/ChunkIO producer's aggregation intact. + +Optional small-frame aggregation uses the same fixed prefix and separate data +region. A control/data capability and CM offer with `receive_frame_limit > 1` +explicitly advertise packed-envelope support; `1` permits direct envelopes +only. The advertised limit remains bound into the capability identity. A sender +uses packing only when enabled locally and the peer advertises at least two +frames. Direct envelopes remain valid on the same QP. Legacy peers reject the +extended advertisement before workload submission; use disabled aggregation +for their existing direct RDMA policy, or the explicit setup-only TCP fallback. + +The packed envelope uses reserved transport message type `0xFFFD`, one-way +flags, and the ordinary fixed-prefix header carrying its actual packed length. +Its padded control begins with `RDPK`, little-endian version 1 and a 16-bit frame +count. Remaining prefix bytes are zero. Its data region contains complete +ordinary `[header][control][data]` frames, bounded by local/peer capacity, +aggregate byte/frame limits, and the advertised receive frame count. The count, +all lengths, every inner frame and the complete suffix validate before any +logical frame is published. Nested envelopes, capability messages and control +fragments cannot appear inside it. + +Only complete frames at most 4096 bytes are eligible. DiskIO messages bypass +packing, including control-only READ requests whose receive allocation later +becomes the disk destination. Other services enabling aggregation accept +unaligned retained data views for small messages; services requiring independent +aligned allocations keep aggregation disabled. Large payloads remain direct. +The owner looks ahead only into available input, packs at least two eligible +adjacent frames and stops at a direct/control-stream barrier. A singleton goes +direct immediately. The final view release permits parent receive repost. Immutable contiguous data already owned by the device's producer arena or the connection's receive arena is borrowed by exact allocation range. SEND @@ -127,6 +155,21 @@ uses a bounded copy into the device arena. Per-CQ `direct_payload_bytes` and `copied_payload_bytes` distinguish these paths; `direct_sends` counts all SENDs using this layout. A failed SEND quarantines the canonical source allocation, including when the WR retains a nested view. +The immutable source descriptor identifies its canonical allocation and arena +directly, so the send path validates its range without scanning cached arena +descriptors. Nested view offsets and detached C/FFI leases retain that same +allocation and registration. +An external descriptor whose enclosing owner retains a native allocation uses +the bounded arena range lookup when direct descriptor identification is +unavailable. That lookup retains the canonical registered owner before posting; +such views do not require a compatibility staging copy. + +Producer sources and compatibility/packing staging use separate bounded +device arenas with independent descriptor admission. Both are registered at +device adoption under the existing two-arena memory reservation. A burst of +unretired staging SENDs cannot consume producer descriptors. Staging allocations +use the configured maximum data class so cached small slots cannot prevent +later larger transfers; actual copied and transmitted lengths remain unchanged. ### 3.1. Encoded logical control streams @@ -149,8 +192,10 @@ SEND carries only the actual fragment length. Rejected queue publication restores the caller's original control ownership. Control/data QP data capacity is `min(local_data_capacity, peer_data_capacity)`. -Active CM setup allocates and primes those receives after validating the peer -reply; passive setup already has the offer. Both actual RX allocation/posting +When TCP preflight supplies the bound peer capability, active CM setup allocates +and primes the negotiated receives before connect; the CM reply must confirm +that same capability before publication. Low-level diagnostics without preflight +prime after validating the reply. Passive setup already has the offer. Actual RX allocation/posting and SEND validation use this minimum, which immutable connection status exposes as `effective_data_capacity`. Contiguous slots retain their fixed local wire capacity and are posted before active CM connect, so an immediate peer response @@ -289,8 +334,8 @@ selection alone does not enable RDMA or change the explicit fallback policy. The device owner charges MR creation against `registered_memory_budget_bytes`. Zero derives a checked bound from all -CQ-admitted QPs, their send/receive arenas and the two compatibility copy -arenas. An explicit budget must admit those compatibility arenas and at least +CQ-admitted QPs, their send/receive arenas and the device producer/staging +arenas. An explicit budget must admit those device arenas and at least one QP. Invalid and overflowed limits fail configuration. Reservation and MR creation failure reject setup before connection publication and return the temporary reservation. Releasing CQ admission does not release registered @@ -343,6 +388,49 @@ leased until all frame/C++/FFI owners release it. Work cancellation and deferred receive retirement take the same exclusive CQ ownership gate. FFI receives a callback or buffer lease only after frame validation and ownership resolution. +The CQ owner prepares bounded lists of SEND WRs for one QP and submits each +list with one `ibv_post_send`. This batches posting without packing payloads: +each WR retains its own variable-length SGEs and source owners. Input exhaustion, +the software window, or the round budget flushes the actual list immediately, +including a singleton. No message waits for a future request to fill a batch. +One preparation round admits at most 64 SEND WRs, 256 logical frames/fragments +and 4 MiB of wire preparation. QPs rotate between rounds, which continue CQ +progress when preparation or SQ admission runs out. Packed envelopes also +stop at 32 frames or 64 KiB of packed bytes. These limits bound the work done +before returning to completions independently of the CQ poll budget. + +Receive dispatch independently admits at most 256 logical frames/fragments +and 4 MiB of wire data per round. A packed CQE counts every enclosed frame. +An envelope that cannot fit the remaining budget stays intact in the bounded +deferred receive list, retaining its posted slot until the next round. Later +receives preserve per-QP order while SEND CQEs can still retire. Control-stream +admission backpressure does not consume dispatch credits. Every valid envelope +fits a fresh round, so budget deferral always has a continuation with progress. + +Both QP directions use `sq_sig_all = 0`. The owner signals the last WR of every +actual posted batch and retains a FIFO of posted sends. A successful tail CQE +retires that WR and its preceding unretired sends on the same SQ. Multiple +batches may remain outstanding; completion replenishes WR and wire-byte +admission without waiting for the entire window. This reduces successful SEND +CQEs only; each completed RECV still produces its own CQE. Posted receive +capacity and final consumer release remain independent peer credits. + +A failed linked post uses `bad_wr` to retain the accepted prefix and release +only the unposted suffix. It closes the uncertain connection without replay. +An accepted prefix missing its planned signaled tail retains its owners until +error/flush completion or QP destruction and cancellation. Error CQEs use their +generation-tagged work records, including unsignaled WRs, rather than treating +an error as a successful FIFO watermark. Uncertain buffers are quarantined. + +Optional receive batching transfers fully released slots through a bounded +per-QP MPSC repost queue. The queue holds each slot's final reference; it +does not allocate replacement receive storage. The CQ owner rotates QPs, +prepares stable linked RECV lists and flushes the actual partial list at the +round boundary and before idle. A singleton or cross-thread final release +therefore cannot wait for another message. Cold receive priming also flushes +its tail before connection publication. Partial provider failure retains only +the accepted prefix, releases unposted owners and closes the connection. + ## 6. Failure and fallback WR flushes on a closed connection are counted as `shutdown_flushes` separately @@ -438,6 +526,10 @@ wire_slot_capacity = 1048576 send_arena_bytes = 67108864 receive_arena_bytes = 1073741824 sq_depth = 1024 +send_batch_size = 16 +receive_batch_size = 1 +send_window_size = 0 +send_window_bytes = 0 rq_depth = 1024 posted_receive_target = 1024 cq_count = 1 @@ -465,6 +557,23 @@ entry points. Rust `RdmaConfig::with_control_data_capacity` derives wire and SQ/RQ arena capacities from an explicit service data class. Logical message limits must be nonzero, fit FlatBuffer length bounds and fit each directional logical byte budget; these budgets are independent of registered MR bytes. +`crowdb_rpc_server_configure_rdma_v4` adds a separately versioned posting config; +earlier entry points select its defaults without changing their structures. +`send_batch_size` bounds each linked post (1 through 256). `send_window_size = 0` +uses SQ admission; a nonzero value cannot exceed allocated SQ depth and is +also capped by provider capacity. Prepared and posted WRs share this window. +`send_window_bytes = 0` disables the additional wire-byte cap; a nonzero cap +must fit the maximum configured wire message. It charges prepared and posted +wire bytes until retirement, independently of arena allocation budgets. +The matching environment names are `CROWDB_RDMA_SEND_BATCH_SIZE`, +`CROWDB_RDMA_SEND_WINDOW_SIZE`, and `CROWDB_RDMA_SEND_WINDOW_BYTES`. +`receive_batch_size` independently bounds linked RECV posts (1 through 256), +with `CROWDB_RDMA_RECEIVE_BATCH_SIZE` as its environment setting. A value of +one keeps immediate final-release posting. Status separates actual send/receive +post calls, successful send CQEs, logical frames, direct payload bytes, +aggregate payload copy bytes, compatibility staging bytes and control-framing +copy bytes. Copy counters include actual preparation work, including retries +after temporary admission pressure. `CROWDB_RDMA_WIRE_LAYOUT` uses 0 for contiguous and 1 for control/data. Owned status JSON contains requested settings, actual QP/CQ/provider capacities, diff --git a/doc/design/rpc/rpc-flow-analysis.md b/doc/design/rpc/rpc-flow-analysis.md index 31445a51f..5c14f1b6f 100644 --- a/doc/design/rpc/rpc-flow-analysis.md +++ b/doc/design/rpc/rpc-flow-analysis.md @@ -610,3 +610,53 @@ do not touch the RPC transport layer. This run confirms no transport-level regression: results are within noise (−1% to −5% vs 2026-08-31). Reference retained per regression policy. See the "Linux — 2026-09-02" subsection above for the full table. + +## Mixed-size posting and CPU comparison + +The mixed-size continuation uses registered producer allocation, optional bounded +small-frame packing, linked WR posting, selective tail completion and per-QP WR +and wire-byte windows. Its formal ownership and scheduling contract is in +[RDMA RPC](design-crowdb-rpc-rdma.md#5-shared-cq-and-completion-model), and the persistent +[RDMA execution record](../../working/plan-rdma.md#accepted-mixed-size-send-optimization) +retains binary identities, failed experiments, final per-size latency gates and +native/OCI service results. The checked profile is +`tools/benchmark/profiles/rdma-mixed-size.json`. + +A producer thread, closed-loop loader, outstanding RPC, posted SEND WR, packed +frame, CQE and CQ owner are distinct quantities. Completed validated client +requests/s remains the primary TPS metric, including useful small-message +aggregation. Both payload initialization and the unchanged application echo copy +are included in the paired CPU measurements. Large registered messages reuse +source owners; small eligible messages may copy into one aggregate. No future +arrival is required to flush a singleton or partial WR list. + +The 2026-10-23 repeated two-core mixed comparison improves median one-owner +throughput from 43,437.6 to 48,842.7 requests/s and total CPU cost from 26,743 to +19,224 ns/callback. Four QPs sharing an owner improve CPU cost from 30,944 to +23,153 ns/callback with similar throughput (50,180.5 to 51,657.7). Every size's +P99 passes the previously declared baseline plus 10%, with zero errors. This +synthetic mix and shared-HCA local-loop establish a bounded mixed-size benefit; +physical-network and intended-arrival latency need their own measurements. + +Primary comparison references inspected on 2026-10-23: + +- [bRPC RDMA guide](https://github.com/apache/brpc/blob/master/docs/cn/rdma.md) + and [endpoint implementation](https://github.com/apache/brpc/blob/master/src/brpc/rdma/rdma_endpoint.cpp) + describe registered IOBuf storage, RC, peer RQ windows, selective signaling + and event/polling modes. The inspected send loop posts individual WRs; + dispatch aggregation does not establish linked-WR submission or CQE removal. +- [bRPC RDMA dispatch](https://github.com/apache/brpc/blob/master/src/brpc/rdma_transport.cpp) + defaults to bthread application dispatch in polling mode. Its + [performance client](https://github.com/apache/brpc/blob/master/example/rdma_performance/client.cpp) + replenishes outstanding requests through callbacks; `thread_num` starts + bthreads and `queue_depth` is application concurrency. Attachments are reused + and echo is optional. Poller count alone does not establish process CPU cost. + The official guide supplies no directly comparable single-core RDMA QPS claim. +- [eRPC paper, sections 4 and 6](https://www.usenix.org/system/files/nsdi19-kalia.pdf) + uses registered application buffers and preallocated short responses. Its + 10M symmetric experiment combines outgoing and incoming RPC processing; + it is not a mixed-size storage benchmark or an identical client TPS measure. +- [ibv_post_send](https://man7.org/linux/man-pages/man3/ibv_post_send.3.html) + and [InfiniBand section 10.8.6](https://www.afs.enea.it/asantoro/V1r1_2_1.Release_12062007.pdf) + define partial posting and successful later completion of earlier unsignaled + sends on the same SQ. Error/flush teardown retains separate ownership rules. diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index f1177fe9c..5c0e3c24d 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -12,6 +12,11 @@ and subsequent [Dataset integration R220](../backlog/R220-dataset-fat-client-pla Goal: embed trusted internal storage access through a shared topology-aware native runtime, implement S3 first, and let Dataset attach to the accepted API. +Status: mixed-size transport acceptance passes with repeatable CPU/request +benefit and unchanged per-size latency bounds. Continue this requirement under +`/implement-requirement`; the accepted prerequisite is recorded in the root +RDMA plan. Native SDK implementation and acceptance remain outstanding. + ## Scope and current boundary - The embedding data-processing or AI system is in the Access Server trust @@ -35,7 +40,7 @@ native runtime, implement S3 first, and let Dataset attach to the accepted API. ## Phase 1: shared runtime and native API -- [ ] **Create native crate and baseline**: register `crowdb-sdk-native` with +- [~] **Create native crate and baseline**: register `crowdb-sdk-native` with `runtime`, `topology`, `buffer`, and `s3` modules and tests under `tests/`. Run focused existing S3/RPC/access and standalone SDK baselines before moving operation code. Files: `Cargo.toml`, `lib/crowdb-sdk-native/`, diff --git a/doc/working/plan-rdma-mixed-size-send.md b/doc/working/plan-rdma-mixed-size-send.md deleted file mode 100644 index 6202743f5..000000000 --- a/doc/working/plan-rdma-mixed-size-send.md +++ /dev/null @@ -1,336 +0,0 @@ - - - -# RDMA mixed-size send efficiency Plan - -Upstream: user-requested transport optimization, -[RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md), -[accepted RDMA implementation](plan-rdma.md), -[RPC flow analysis](../design/rpc/rpc-flow-analysis.md), and -[physical-node validation](plan-rdma-multi-node.md). - -Goal: improve completed client requests per CPU cost under mixed message sizes, -using registered payloads, bounded small-message aggregation, batched WR posting, -and continuously replenished per-QP send windows without waiting to fill batches. - -Status: planned; code paths and external references inspected. No implementation -or new performance runs have been performed. Delete this temporary plan after -implementation, verification, and migration of accepted contracts/evidence. -Existing SDK/backlog work remains independent of this transport optimization. -After mixed-workload acceptance demonstrates a repeatable resource/throughput -benefit within the declared latency bounds, continue the existing -[R242 SDK plan](plan-rdma-fat-client-sdk.md), as authorized by the user. -Keep that conditional handoff distinct from transport completion; do not start -Dataset integration under R220 as part of R242. - -## Existing behavior and intended changes - -- Coroutine benchmark loaders are closed-loop: each has one outstanding request - and issues its next request after its response. Multiple loaders can run on - one completion thread. Loader count, producer thread count, CQ owner count, - QP count, and outstanding SEND count are different quantities. -- `ConnectionPools::send_queue` is bounded MPSC admission. Its fixed CQ owner - drains frames. `sends_in_flight` already caps posted but unreclaimed SEND WRs - at `min(sq_depth, effective_capacity.max_send_wr)` per QP. -- Both layouts currently post one WR per `ibv_post_send`; QP setup uses - `sq_sig_all = 1`. `poll_batch` is a loop budget, not a linked WR submission. -- `Contiguous` copies complete frames into one registered slot and can pack - several requests. `ControlData` uses a fixed 1024-byte control prefix and - actual-length payload; registered contiguous producer/receive ranges can be - borrowed. It currently rejects transport aggregation. -- Receive buffers are posted before the message arrives. Sender SGE boundaries - are not transferred to the peer. The fixed prefix and negotiated payload - capacity preserve scatter placement for variable lengths and data alignment. -- Current echo builds payloads through ordinary buffers, copies the server echo, - and validates response bytes. The existing fixed-size peak is a regression - control, not evidence of efficiency under mixed service traffic. - -## Execution rules - -- Retain RC SEND/RECV, control/data separation, setup-only TCP fallback, - request identities, stream sequencing, generation checks, and quarantine. -- Producers prepare immutable source buffers and publish through existing - lock-free admission. One CQ owner prepares each QP's pending aggregate, - ordered ready items, WR batches, and send retirement. Do not introduce a - shared mutable aggregate written by multiple producers or a new hot-path lock. -- Prepare direct items before publishing them as ready; the posting stage must - not discover that it needs to gather/copy an entire large payload. A proposed - ready-item record retains prefix, data ranges, original logical frames, - actual wire bytes, and completion owners. Symbol names are finalized in code. -- Keep one ordered stream per QP. A large/direct item seals the earlier pending - aggregate before entering ready. Control-stream fragments remain ordered and - cannot be interleaved with later application frames on that stream. -- Small aggregation may copy control and data. Eligibility includes complete - encoded size, message semantics, and alignment/lease requirements; mandatory - disk-aligned or directly retained payloads bypass aggregation. A large - registered payload bypasses packing and retains its source allocation. -- Aggregate byte/frame limits and WR batch size are upper bounds. Seal the - aggregate when full, when a direct/barrier item arrives, when the preparation - budget ends, or when available input is exhausted. Submit a single ready item - immediately when it is the only work. Do not wait for a future request. -- Bound preparation by items and copied bytes, and bound receive/callback and - post work per round. Continue CQ progress when SQ/memory admission is full; - a producer backlog must not starve completions or other QPs. Do not sleep - with a sealable aggregate or sendable ready item. A byte/WR-blocked ready - item may wait for the actual completion/release event without busy spinning. -- Treat software queue capacity, posted SEND WR count, retained bytes, and - client requests awaiting responses as separate budgets. Release each charge - only at its relevant ownership boundary; buffer views sharing an allocation - must not multiply-count physical retained bytes. - -## Phase 1: representative workload and baseline - -- [ ] **Add deterministic per-request sizes**: extend CLI workload arguments - and manifest cases with weighted sizes/seed and an explicit wire layout; - record expected length per loader/request instead of one global - `EchoWorkload::payload_bytes`. Validate ID, length, bytes, and response - control for every request. Preserve fixed-size mode and TCP parity. Files: - `app/crowdb-cli/src/commands/bench/rpc/{args,workload,runtime,report}.rs`, - `tools/benchmark/rpc_bench_cases.py`, `app/crowdb-cli/tests/rpc_bench_test.rs`, - `tools/benchmark/tests/rdma_regression_test.py`. -- [ ] **Capture service size distributions**: use existing HTTP/DiskIO and - Chunk-KV corpora to retain actual control/data lengths, class boundaries, - registration origin, alignment requirements, and lease-hold behavior. - Add reproducible synthetic mixes spanning control-only, 128 B, 1/4 KiB, - 64/256 KiB and near negotiated capacity; include non-power-of-two sizes. - Label synthetic mixes explicitly rather than claiming production traces. - Files: `tools/rdma/run-datapath.py`, `tools/rdma/run-chunk-kv.py`, - `app/crowdb-access-server/tests/common/rdma_datapath/`, - `app/crowdb-chunk-kv-server/tests/common/rdma_chunk_kv/`. -- [ ] **Retain comparable baselines**: save source/binary identity, machine - topology, CPU placement, actual paths, request mix/seed, worker/QP counts, - budgets and error counters. Measure single-loader latency, multi-loader - saturation, burst/idle transitions, and delayed consumer release. Keep - offered work and application behavior identical in before/after pairs. - Declare performance noise and acceptable per-size tail-latency bounds from - repeated baseline runs before selecting optimized defaults. Files: - `tools/benchmark/rpc_regression.py`, `tools/benchmark/rpc_bench_report.py`, - `tools/benchmark/bench-rdma-rpc-regression.sh`. - -## Phase 2: registered producers and copy accounting - -- [ ] **Generate payloads in registered buffers**: audit the existing - `allocate_producer_buffer` and Rust writable/frozen APIs, pass the transport - producer allocator through the benchmark runtime, and write variable-length - data into the final registered allocation. Retain an ordinary-buffer mode - for compatibility and a paired registered-buffer baseline. Avoid allocating - a Vec and then copying it merely to claim producer zero-copy. Files: - `lib/crowdb-rpc/src/transport/rdma/rdma_producer.cpp`, - `lib/crowdb-rpc/ffi/src/{buffer,co_bench}.rs`, - `app/crowdb-cli/src/commands/bench/rpc/{runtime,workload}.rs`. -- [ ] **Preserve direct source ownership**: prepare eligible payload ranges - using the existing producer/receive arena borrow checks; retain MR/PD, - allocation identity and immutable source through send retirement. Cover - nested views, shared mirror payloads, zero-length data and arbitrary - supported lengths. Unregistered or unsupported scattered input uses a - bounded, separately measured compatibility copy. Preserve usable pool - classes when a small allocation precedes a larger one. Files: - `lib/crowdb-rpc/src/transport/rdma/rdma_direct_send.cpp`, - `lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp`, - `lib/crowdb-rpc/tests/{producer_buffer,rdma_memory,rdma_lifetime}_test.cpp`. -- [ ] **Separate copy causes**: expose bytes copied for small aggregation, - unregistered/scattered compatibility staging, control framing, and echo - application work separately from borrowed direct payload bytes. Keep a - registered-source echo/retained-lease mode distinct from the existing - copy-echo workload; compare each mode against itself. Files: - `lib/crowdb-rpc/src/server/handler.cpp`, - `lib/crowdb-rpc/src/transport/rdma/{rdma_cq.h,rdma_status.cpp}`, - `app/crowdb-cli/src/commands/bench/rpc/{status,report}.rs`. - -## Phase 3: mixed direct and aggregate preparation - -- [ ] **Define the negotiated aggregate format**: extend the formal RDMA - contract before implementation. Retain the fixed control prefix; add an - unambiguous aggregate envelope and bounded packed data containing complete - logical control/data frames. Specify counts, lengths, offsets, maximum wire - bytes, supported message types, and whole-envelope validation. Direct data - retains its independent aligned allocation. Negotiate support explicitly; - older peers keep the existing direct format, without new-format payload - submission or midstream fallback. Files: - `doc/design/rpc/design-crowdb-rpc-rdma.md`, - `lib/crowdb-rpc/src/transport/rdma/{rdma_offer,rdma_direct_frame,rdma_aggregate}.{h,cpp}`, - `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_capability.h`, - `lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp`. -- [ ] **Add bounded preparation state**: extend `ConnectionPools` with an - owner-only pending aggregate and ordered ready items, reusing existing - input admission. Apply a small-message copy threshold and independent - aggregate byte/frame limits. Check local and peer capacities before copying; - preserve admission accounting through pending, ready and posted states. - Do not implement an additional unbounded queue or independent per-producer - aggregate that can retain a lone request indefinitely. Files: - `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h`, - `lib/crowdb-rpc/src/transport/rdma/{rdma_send,rdma_direct_send,rdma_control_send}.cpp`. -- [ ] **Dispatch mixed receive envelopes**: validate all packed frame bounds - before publishing any frame; create retained views for individual payloads. - A final caller release controls parent repost. Reject malformed envelopes - without partial RPC publication. Keep large logical control fragmentation, - handshake messages and alignment-sensitive service traffic on explicitly - compatible paths. Files: - `lib/crowdb-rpc/src/transport/rdma/{rdma_direct_receive,rdma_receive,rdma_aggregate}.cpp`, - `lib/crowdb-rpc/tests/{rdma_aggregate,rdma_direct_frame,rdma_offer,rdma_control_stream}_test.cpp`. - -## Phase 4: send windows and batched posting - -- [ ] **Make the send window explicit**: retain `sq_depth` as QP allocation - capacity and add a configurable software WR window no larger than effective - SQ capacity, plus explicit queued/prepared/posted retained-byte bounds. - Account control prefix, aggregation storage, and retained source allocations; - separate per-QP limits from shared-arena limits. Reject an item that can - never fit its configured admission rather than stalling forever. Keep - posted receives as implicit peer credit; local SQ completion does not imply - the peer has released/reposted its receive buffer. Files: - `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/{rdma_config,rdma_transport}.h`, - `lib/crowdb-rpc/src/transport/rdma/{rdma_config,rdma_send,rdma_completion}.cpp`. -- [ ] **Build linked WR batches**: post ready items from the same QP together, - bounded by ready count, post batch cap, free WR slots and byte admission. - Each WR retains its own variable-length SGEs and immutable owners. Keep WR - and SGE descriptor storage stable through the post call. For partial post - failure, use `bad_wr` to identify the accepted prefix; retire unposted items - separately, fail/drain the uncertain connection, and never replay accepted - operations as though they were unsubmitted. Files: - `lib/crowdb-rpc/src/transport/rdma/{rdma_send,rdma_direct_send,rdma_control_send}.cpp`, - `lib/crowdb-rpc/tests/{rdma_send,rdma_failure}_test.cpp`. -- [ ] **Retire sends by sequence**: change both active/passive QP setup to - `sq_sig_all = 0`. Give posted WRs ordered per-QP sequences while retaining - generation-tagged completion identity. Initially signal each actual batch - tail, including a singleton. A successful tail retires its covered prefix, - releasing WR/byte charges and owners exactly once. Keep multiple batches - outstanding and refill freed space without waiting for an entire window. - Handle error CQEs for unsignaled WRs, partial posts missing the planned - signaled tail, stale CQEs, cancellation and QP destruction. Preserve - conservative CQ/WR admission for error/flush completions. Files: - `lib/crowdb-rpc/src/transport/rdma/{rdma_active,rdma_connection,rdma_work_request,rdma_completion}.cpp`, - `lib/crowdb-rpc/tests/{rdma_completion,rdma_failure,rdma_lifetime}_test.cpp`. -- [ ] **Schedule bounded progress**: separate preparation, post and CQ budgets - instead of treating `poll_batch` as all three. Interleave CQ processing and - preparation/submission, round-robin QPs, and seal pending aggregates at a - round boundary. Full SQs or slow peers must not stop receive/response work. - Preserve idle arm/recheck/wake correctness, including caller-held leases - released on another thread. Files: - `lib/crowdb-rpc/src/transport/rdma/{rdma_poller,rdma_completion,rdma_send}.cpp`, - `lib/crowdb-rpc/src/transport/rdma/rdma_cq.h`, - `lib/crowdb-rpc/tests/{rdma_worker,rdma_send}_test.cpp`. -- [ ] **Batch receive replenishment**: collect only fully released receive - owners and post bounded linked RECV lists through the assigned owner. Flush - a partial replenish batch before idle or receive starvation. Preserve - posted-receive credits, data alignment, and caller lease lifetime; handle - partial post failure without losing or double-posting a slot. Files: - `lib/crowdb-rpc/src/transport/rdma/{rdma_transport,rdma_direct_receive,rdma_receive,rdma_poller}.cpp`, - `lib/crowdb-rpc/tests/{rdma_worker,rdma_lifetime,rdma_failure}_test.cpp`. -- [ ] **Expose consistent configuration and diagnostics**: thread new limits - through C ABI, Rust config, CLI and runtime manifests with compatible - defaults and validation. Report in-flight WR/bytes, ready/pending age and - high-water, actual WRs/post, aggregation copy bytes, SQ stalls, peer receive - pressure, and per-size latency. Prefer owner-sharded counters and sampled - age checks over per-frame shared atomics. Files: - `lib/crowdb-rpc/include/crowdb-rpc/c_api.h`, - `lib/crowdb-rpc/src/c_api/transport.cpp`, - `lib/crowdb-rpc/ffi/src/{sys,transport}.rs`, - `lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp`, - `app/crowdb-cli/src/commands/bench/rpc/`, `tools/benchmark/`. - -## Phase 5: practical acceptance and default selection - -- [ ] **Verify concurrency and low-load progress**: cover one loader/one owner, - many loaders/one owner, many producers on one QP, multiple QPs per owner and - multiple owners. Submit one small request with no successor; it must finish - without a batch-fill wait. Exercise continuous producers, small/large/small - ordering, asymmetric request/response sizes, and a busy QP alongside an idle - or slow QP. Files: `lib/crowdb-rpc/tests/rdma_worker_test.cpp`, - `lib/crowdb-rpc/tests/rdma_direct_hardware_test.cpp`, - `app/crowdb-cli/tests/rpc_bench_test.rs`, `tools/benchmark/`. -- [ ] **Verify bounded pressure and shutdown**: hold receive leases to exhaust - peer RQ, saturate WR and byte limits independently, then release and verify - recovery without hot spinning, leaks or premature reuse. Inject errors - before/within a posted batch, at signaled-tail boundaries, during control - fragmentation and at disconnect. Preserve typed uncertain outcomes and no - TCP suffix after accepted RDMA. Files: - `lib/crowdb-rpc/tests/{rdma_failure,rdma_lifetime,rdma_control_hardware}_test.cpp`, - `lib/crowdb-diskio-client/tests/rdma_failure_test.rs`. -- [ ] **Run service parity and topology controls**: run existing HTTP/DiskIO - and real Chunk-KV/tree/WAL corpora with mixed traffic and retained leases, - comparing exact results and errors against TCP. Reuse native, shared-HCA OCI - and no-device TCP profiles. Coordinate a physical two-host run through the - linked multi-node plan before claiming production-network performance; - local HCA traversal remains labeled as such. Record unavailable hardware - gates as pending rather than replacing them with a synthetic success. - Files: `tools/rdma/run-datapath.py`, `tools/rdma/run-chunk-kv.py`, - `tools/rdma/run-oci-profile.sh`, `tools/rdma/run-timed-load.py`, - `doc/working/plan-rdma-multi-node.md`. -- [ ] **Compare changes incrementally**: retain paired baseline, registered - direct, hybrid aggregation, WR batching and selective-completion results. - Use client completed requests/s as the primary throughput metric, plus CPU - seconds/request at matched throughput and throughput at matched CPU budget, - valid payload bytes/s, per-size P50/P99/P999, queue residence, registered and - retained memory, errors and fallback. Record mixed workloads at low load, - below saturation, saturation and overload/burst recovery. A bounded offered - load case records intended-arrival-to-completion delay, not only latency - of admitted requests. Do not infer open-load latency from closed-loop runs. - Files: `tools/benchmark/`, `app/crowdb-cli/src/commands/bench/rpc/report.rs`. -- [ ] **Select defaults from evidence**: require zero correctness errors, - bounded memory and eventual progress after pressure release. Demonstrate - reduced payload copying on eligible registered large messages and a - repeatable CPU-cost or request-throughput benefit on service-informed mixed - workloads within the baseline-declared tail limits. Small aggregation copy - is allowed and included in costs. Report regressions and leave unhelpful - options disabled; a larger 128-byte peak alone cannot close this plan. - Publish accepted contracts and results, then delete this plan. Files: - `doc/design/rpc/design-crowdb-rpc-rdma.md`, - `doc/design/rpc/rpc-flow-analysis.md`, changed config defaults. - -## Consolidated files and checks - -- Transport: `lib/crowdb-rpc/src/transport/rdma/` and - `lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/`. -- Producer/C ABI/Rust: `lib/crowdb-rpc/src/c_api/`, - `lib/crowdb-rpc/include/crowdb-rpc/c_api.h`, `lib/crowdb-rpc/ffi/src/`. -- Workloads/reports: `app/crowdb-cli/src/commands/bench/rpc/`, - `tools/benchmark/`, `tools/rdma/` and the existing service fixtures. -- Unit: RDMA config/offer/aggregate/direct-frame/send/completion/worker/lifetime - and failure tests under `lib/crowdb-rpc/tests/`; Rust CLI/FFI contracts under - their crates' `tests/`; Python manifest/report tests in `tools/benchmark/tests/`. -- Integration: real-HCA direct/control hardware tests, TCP/no-verbs compatibility, - mixed-loader completion validation and slow-consumer lease pressure. -- E2E: HTTP/DiskIO and Chunk-KV native/OCI parity, no-device TCP, and separately - identified physical-host performance evidence. -- Run all implementation builds/tests/lints through `pixi run`, with focused - checks per phase before wider affected suites. Use Rust fmt/clippy, changed - C++ clang-format/tree-lint, C++/FFI/CLI and affected access/Chunk-KV/DiskIO - suites. Keep full errors; enable `CROWDB_E2E_PRINT_SERVICE_ERRORS=1` in CI or - when debugging. Start long hardware runs in the background and poll. -- For this plan-only edit, check Markdown links and `git diff --check`; do not - report implementation tests or performance acceptance as completed. - -## External implementation references - -These describe borrowed mechanisms, not CROWDB contracts or target QPS. -Inspected on 2026-10-23; pin the source revision when running comparisons. - -- [bRPC RDMA documentation](https://github.com/apache/brpc/blob/master/docs/cn/rdma.md): - registered IOBuf pools, RC, event/polling modes and peer receive-window ACKs. -- [bRPC RDMA endpoint](https://github.com/apache/brpc/blob/master/src/brpc/rdma/rdma_endpoint.cpp): - local SQ and remote RQ windows, reference-based SGE gathering, selective - signaling and solicited wakeups. Its inspected send loop posts WRs individually; - its event aggregation must not be described as linked-WR posting or CQE removal. -- [bRPC dispatch](https://github.com/apache/brpc/blob/master/src/brpc/rdma_transport.cpp): - polling defaults to dispatching application messages as bthreads; disabling - bthreads allows inline handling. Poller count is not the process's CPU budget. -- [bRPC performance client](https://github.com/apache/brpc/blob/master/example/rdma_performance/client.cpp) - and [server](https://github.com/apache/brpc/blob/master/example/rdma_performance/server.cpp): - callback-replenished request windows; `queue_depth` is application outstanding - requests, and `thread_num` starts bthreads. Attachments are prepared/reused; - echo is optional and the server appends IOBuf references. This is not a - mixed-size storage workload or a complete payload-validation benchmark. -- [bRPC general benchmark](https://brpc.apache.org/zh/docs/benchmark/): - do not relabel its general framework results as measured RDMA performance. - The inspected official RDMA guide does not publish a directly comparable - single-core QPS claim. Any future comparison must retain mode, all worker - CPU costs, attachment reuse/echo behavior, size and request window. -- [eRPC paper, sections 4 and 6](https://www.usenix.org/system/files/nsdi19-kalia.pdf): - registered application buffers, preallocated short responses and different - small-RPC/bandwidth experiment conditions. Its 10M headline combines outgoing - and incoming RPC processing in a symmetric experiment. -- [verbs posting](https://man7.org/linux/man-pages/man3/ibv_post_send.3.html) - and [InfiniBand section 10.8.6](https://www.afs.enea.it/asantoro/V1r1_2_1.Release_12062007.pdf): - partial posting and later completion on the same SQ confirming earlier - unsignaled sends. Keep error/drain ownership separate from normal retirement. diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index d8295b9e2..8822864c6 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -57,10 +57,100 @@ Access construction is covered by the affected Rust suites. Formal service contracts and complete acceptance evidence are linked from the RDMA design and RPC flow analysis. -The requested foundation, transport, datapath and Chunk-KV continuation is -complete. Only the fat-client SDK child remains. R242 and physical multi-node -validation are outside the completed continuation; no SDK implementation is -started by this plan update. +The foundation, transport, datapath and Chunk-KV continuation is complete. +The subsequent mixed-size send optimization also passes its local-HCA +acceptance, as recorded below. The user authorized continuation with R242 +after a repeatable benefit; its implementation now follows the existing +[SDK plan](plan-rdma-fat-client-sdk.md). Physical multi-node validation remains +in its separate plan. + +## Accepted mixed-size send optimization + +- Registered producer storage is filled once in place. A retained native + descriptor supplies a constant-time ownership check; external HTTP/FFI + wrappers retain the bounded canonical range lookup. Compatibility staging + preserves usable allocation classes across small/large transitions. +- The existing second device MR now owns producer descriptors separately from + staging/packing, without raising the registered-memory budget. Both native + and external-view direct reuse are covered by HCA regressions. +- Linked SEND lists retain stable WR/SGE records and signal their actual tail, + including a singleton. FIFO success retirement replenishes independent WR + and wire-byte windows while several batches remain in flight. Partial post + and error completions retain uncertain owners until drain, without replay. +- Control/data separation remains. Optional negotiated packing copies only + adjacent eligible small messages, at most 32 frames / 64 KiB, without a + batch-fill timer. Large data and alignment-sensitive DiskIO stay direct. + SEND preparation and receive dispatch each have separate logical-frame and + byte quanta; SEND, receive repost and CQ polling remain independently bounded. +- TCP preflight permits active ControlData RQs to be primed before CM connect. + Repeated idle singleton tests caught and then verified the cold-start race + without extending their one-second completion deadline. +- Default SEND batch is 16. Receive batching remains optional with default 1; + its additional CPU benefit was absent. Packing remains opt-in on both peers. + The reproducible mixed profile is + `tools/benchmark/profiles/rdma-mixed-size.json`. +- Accepted final binary hashes: CLI + `0df668817118cfab7dfc12966690afad9cd05ab4504741284ec9ceef3c1104c9`; + native server + `92794c08a31250f57ecbffb3755b1eeaf7e2e83b45011d27fd4f7b03f8ee1197`. + Binaries, tracked diff and new sources are retained under + `.crowdb-runtime/rdma-check/binaries/mixed-send-final/`. +- Final correctness gates: 209 native RPC cases pass with three explicit + environment/process-pair skips; 128 no-verbs cases pass with one hardware + skip; all FFI tests, five CLI cases, seventeen runner cases, Rust fmt/clippy, + C++ formatting and scoped tree-lint pass. Scoped tree-lint still reports + existing style warnings and exits zero. +- Actual multi-producer progress passes with four producers sharing one QP + and with four QPs / 128 outstanding requests. Mixed coroutine, Tokio and + four-CQ cases all validate every payload with no errors. +- Final repeated fixed-CPU mixed evidence is in + `.crowdb-runtime/rdma-check/artifacts/bench/mixed-steady-final/`, with the + declared original thresholds in `mixed-pinned-baseline/acceptance-limits.json`. + Each peer is pinned to one distinct physical core (client 2, server 4), on a + Ryzen 5950X and one shared 100G HCA. The deterministic synthetic mix includes + 0, 128, 513, 4096, 65535, 262144 and 1048576 bytes; seed 93; warmup 5 s, + measurement 10 s, three repeats. No production-frequency claim is made. +- Against `mixed-steady-baseline`, median completed requests/s are singleton + 16,655.9 -> 16,919.2, 64 loaders / one QP 43,437.6 -> 48,842.7, and 128 + loaders / four QPs 50,180.5 -> 51,657.7. Total client+server CPU ns/callback + over warmup/measurement/drain are 62,097 -> 60,587, 26,743 -> 19,224, and + 30,944 -> 23,153. Every size's P99 passes the declared baseline plus 10%; + all rows have zero allocation, validation, timeout or transport errors. + Copy echo is unchanged in both versions and included in CPU cost. P999 is + recorded, without inferring open-arrival latency from closed-loop loaders. +- The fixed 128-byte regression control uses the same two-core pinning, + 512 loaders / four QPs, unchanged copy echo and packing in both versions. + Its two repeats are 702,145 / 696,953 requests/s before and 877,811 / 879,331 + after, retained in `mixed-fixed-baseline/` and `mixed-fixed-final/`. + These CPU-constrained controls are distinct from the earlier unrestricted + 3M headline; both final rows validate all requests with zero errors. +- Failed experiments remain retained: shared-owner baseline staging exhaustion, + scalar and double-fill registered initializers, shared producer/staging + descriptor exhaustion, and a fast-only borrow check that missed external + HTTP/FFI owners. No failed row is counted as accepted performance. +- Service revalidation with TX/RX 16 and packing enabled preserves HTTP bytes, + aligned DiskIO and zero copied payload bytes in native/OCI HCA profiles + `rdma-datapath/c562dc95cc9841d6921b2dce5334a57f` and + `rdma-datapath/f6a02c215e8d4f39ac695d4b98f1c4a6`. Native partial-download + disconnect `670211f0f3d345269342eebd3d045717` terminates without a TCP suffix; + no-device OCI TCP `926068fccb004ed9981e0e82cdf3af22` preserves exact bytes. + Native uses io_uring; the container's existing EPERM fallback uses blocking + disk I/O and is functional evidence only. +- Real Chunk-KV/tree/WAL passes native RDMA + `rdma-chunk-kv/36d69f7a97764016950e28753b54340a`, final OCI RDMA + `778a5ee288b543f38433f696afd148fa`, OCI TCP + `b6333c5e1ddd4a859ed53720052de6fc`, native setup-only fallback + `372dd806ab434dc28b917f841cfb1613`, and OCI partial-response disconnect + `7f4dc5c2c0e54678b9d0a523a8b0bbed`. Corpus values cross the 64/256 KiB + classes and 1 MiB control-fragment boundary. These are service-path tests + with injected catalog/storage seams, not physical cross-host measurements. + +Further tuning remains separate from this accepted mechanism: intended-arrival +open-load latency, sampled queue-age/peer-pressure telemetry and a retained-echo +benchmark mode. Transport counters already distinguish direct payload, +aggregation, compatibility staging and framing/control copies; application echo +copy remains explicitly identified. Physical multi-node performance continues +under `plan-rdma-multi-node.md`. ## Fixed design decisions diff --git a/lib/crowdb-rpc/ffi/src/producer_buffer.rs b/lib/crowdb-rpc/ffi/src/producer_buffer.rs index 72be6c29f..6b213b6b8 100644 --- a/lib/crowdb-rpc/ffi/src/producer_buffer.rs +++ b/lib/crowdb-rpc/ffi/src/producer_buffer.rs @@ -36,17 +36,19 @@ impl ProducerBufferAllocator { #[derive(Debug)] pub struct WritableBuffer { buffer: Buffer, + capacity: usize, + registration_id: u32, } impl WritableBuffer { #[must_use] pub fn capacity(&self) -> usize { - self.buffer.lease_metadata().capacity as usize + self.capacity } #[must_use] pub fn registration_id(&self) -> u32 { - self.buffer.lease_metadata().registration_id + self.registration_id } /// Spare storage for an I/O producer. No initialized bytes are published. @@ -79,6 +81,31 @@ impl WritableBuffer { Ok(self.buffer) } + /// Fill a prefix once, then initialize fields in the final allocation. + /// + /// # Errors + /// Rejects an empty prefix or a length exceeding capacity. + pub fn initialize_with( + mut self, + length: usize, + initial: u8, + initialize: impl FnOnce(&mut [u8]), + ) -> Result { + if length == 0 || length > self.capacity() { + return Err(RpcError::InvalidArg); + } + let storage = &mut self.uninit_bytes_mut()[..length]; + storage.fill(MaybeUninit::new(initial)); + // SAFETY: the prefix was initialized above and this unique mutable + // borrow cannot escape the initializer callback. + let initialized = + unsafe { std::slice::from_raw_parts_mut(storage.as_mut_ptr().cast::(), length) }; + initialize(initialized); + // SAFETY: every published byte was initialized above; the unique + // mutable borrow ended before ownership is frozen by assume_init. + unsafe { self.assume_init(length) } + } + /// Publish an initialized prefix without copying it. /// /// # Safety @@ -119,6 +146,11 @@ impl RpcServer { }; (!handle.is_null()).then(|| WritableBuffer { buffer: Buffer::from_raw(handle), + // SAFETY: this newly allocated unique handle retains immutable + // capacity and registration metadata until publication. + capacity: unsafe { sys::crowdb_rpc_buffer_capacity(handle) } as usize, + // SAFETY: the same unique allocation owns this registration. + registration_id: unsafe { sys::crowdb_rpc_buffer_registration_id(handle) }, }) } } diff --git a/lib/crowdb-rpc/ffi/src/sys.rs b/lib/crowdb-rpc/ffi/src/sys.rs index c72dd640c..fc929cbb3 100644 --- a/lib/crowdb-rpc/ffi/src/sys.rs +++ b/lib/crowdb-rpc/ffi/src/sys.rs @@ -90,6 +90,16 @@ pub struct CrowdbRpcRdmaStreamConfig { pub control_receive_budget_bytes: u64, } +#[repr(C)] +pub struct CrowdbRpcRdmaPostConfig { + pub struct_size: u32, + pub version: u32, + pub send_batch_size: u32, + pub receive_batch_size: u32, + pub send_window_size: u32, + pub send_window_bytes: u64, +} + #[repr(C)] #[derive(Default, Debug)] pub struct CrowdbRpcClientCounters { @@ -162,6 +172,14 @@ pub type crowdb_rpc_co_on_response = Option< extern "C" { pub fn crowdb_rpc_rdma_config_init(config: *mut CrowdbRpcRdmaConfig); pub fn crowdb_rpc_rdma_stream_config_init(config: *mut CrowdbRpcRdmaStreamConfig); + pub fn crowdb_rpc_rdma_post_config_init(config: *mut CrowdbRpcRdmaPostConfig); + pub fn crowdb_rpc_server_configure_rdma_v4( + server: crowdb_rpc_server_t, + config: *const CrowdbRpcRdmaConfig, + wire_layout: u32, + streams: *const CrowdbRpcRdmaStreamConfig, + sends: *const CrowdbRpcRdmaPostConfig, + ) -> crowdb_rpc_status; pub fn crowdb_rpc_server_configure_rdma_v3( server: crowdb_rpc_server_t, config: *const CrowdbRpcRdmaConfig, diff --git a/lib/crowdb-rpc/ffi/src/transport.rs b/lib/crowdb-rpc/ffi/src/transport.rs index 85934279b..f3a91a14b 100644 --- a/lib/crowdb-rpc/ffi/src/transport.rs +++ b/lib/crowdb-rpc/ffi/src/transport.rs @@ -32,6 +32,10 @@ pub struct RdmaConfig { pub cq_count: u32, pub cq_depth: u32, pub sq_depth: u32, + pub send_batch_size: u32, + pub receive_batch_size: u32, + pub send_window_size: u32, + pub send_window_bytes: u64, pub rq_depth: u32, pub posted_receive_target: u32, pub wire_slot_capacity: u32, @@ -73,10 +77,20 @@ fn native_stream_defaults() -> sys::CrowdbRpcRdmaStreamConfig { } } +fn native_post_defaults() -> sys::CrowdbRpcRdmaPostConfig { + let mut config = MaybeUninit::uninit(); + // SAFETY: initializer fills every field of the send configuration ABI. + unsafe { + sys::crowdb_rpc_rdma_post_config_init(config.as_mut_ptr()); + config.assume_init() + } +} + impl Default for RdmaConfig { fn default() -> Self { let raw = native_defaults(); let streams = native_stream_defaults(); + let sends = native_post_defaults(); Self { enabled: raw.enabled != 0, allow_tcp_fallback: raw.allow_tcp_fallback != 0, @@ -85,6 +99,10 @@ impl Default for RdmaConfig { cq_count: raw.cq_count, cq_depth: raw.cq_depth, sq_depth: raw.sq_depth, + send_batch_size: sends.send_batch_size, + receive_batch_size: sends.receive_batch_size, + send_window_size: sends.send_window_size, + send_window_bytes: sends.send_window_bytes, rq_depth: raw.rq_depth, posted_receive_target: raw.posted_receive_target, wire_slot_capacity: raw.wire_slot_capacity, @@ -115,9 +133,9 @@ impl RdmaConfig { /// Existing `Default` and native service defaults remain unchanged. /// /// # Errors - /// Rejects classes outside 64 KiB/256 KiB/1 MiB and transport aggregation. + /// Rejects classes outside 64 KiB/256 KiB/1 MiB. pub fn with_control_data_capacity(mut self, capacity: u32) -> Result { - if !matches!(capacity, 65_536 | 262_144 | 1_048_576) || self.aggregation_enabled { + if !matches!(capacity, 65_536 | 262_144 | 1_048_576) { return Err(RpcError::InvalidArg); } self.wire_layout = RdmaWireLayout::ControlData; @@ -173,13 +191,28 @@ impl RpcServer { raw.source_address = source.as_ptr(); raw.gid = gid.as_ptr(); let mut streams = native_stream_defaults(); + let sends = sys::CrowdbRpcRdmaPostConfig { + struct_size: u32::try_from(std::mem::size_of::()) + .expect("fixed ABI size"), + version: 1, + send_batch_size: config.send_batch_size, + receive_batch_size: config.receive_batch_size, + send_window_size: config.send_window_size, + send_window_bytes: config.send_window_bytes, + }; streams.max_control_message_bytes = config.max_control_message_bytes; streams.control_assembly_timeout_ms = config.control_assembly_timeout_ms; streams.control_send_budget_bytes = config.control_send_budget_bytes; streams.control_receive_budget_bytes = config.control_receive_budget_bytes; // SAFETY: handle is borrowed, struct and CStrings live through the call. let status = unsafe { - sys::crowdb_rpc_server_configure_rdma_v3(self.handle(), &raw, config.wire_layout as u32, &streams) + sys::crowdb_rpc_server_configure_rdma_v4( + self.handle(), + &raw, + config.wire_layout as u32, + &streams, + &sends, + ) }; if status == sys::CROWDB_RPC_OK { Ok(()) diff --git a/lib/crowdb-rpc/ffi/tests/producer_buffer_test.rs b/lib/crowdb-rpc/ffi/tests/producer_buffer_test.rs index 2f9b4f7ff..244b98f8e 100644 --- a/lib/crowdb-rpc/ffi/tests/producer_buffer_test.rs +++ b/lib/crowdb-rpc/ffi/tests/producer_buffer_test.rs @@ -36,3 +36,27 @@ fn oversize_initialization_releases_producer_admission() { assert!(buffer.write(b"too large").is_err()); assert!(server.allocate_producer_buffer(1, false).is_some()); } + +#[test] +fn generated_prefix_keeps_storage_and_freezes_only_initialized_bytes() { + let pool = BufferPool::new(1); + let server = RpcServer::new(Some(&pool)); + server.configure_rdma(&RdmaConfig::default()).unwrap(); + let mut writable = server.allocate_producer_buffer(512, false).unwrap(); + let pointer = writable.uninit_bytes_mut().as_ptr().cast::(); + let buffer = writable + .initialize_with(127, 0xAB, |bytes| { + assert!(bytes.iter().all(|byte| *byte == 0xAB)); + for (index, byte) in bytes.iter_mut().enumerate() { + *byte = u8::try_from(index).unwrap(); + } + }) + .unwrap(); + assert_eq!(buffer.bytes().as_ptr(), pointer); + assert_eq!(buffer.bytes(), (0_u8..127).collect::>()); + assert!(server.allocate_producer_buffer(1, false).is_none()); + drop(server); + assert_eq!(buffer.bytes().len(), 127); + drop(buffer); + assert!(pool.alloc_buffer(1).is_some()); +} diff --git a/lib/crowdb-rpc/ffi/tests/transport_config_test.rs b/lib/crowdb-rpc/ffi/tests/transport_config_test.rs index 6b13dfdfe..4b54cc8c7 100644 --- a/lib/crowdb-rpc/ffi/tests/transport_config_test.rs +++ b/lib/crowdb-rpc/ffi/tests/transport_config_test.rs @@ -39,6 +39,32 @@ fn invalid_queue_bounds_fail_before_start() { assert!(server.configure_rdma(&config).is_err()); } +#[test] +fn send_post_batch_and_software_window_are_independent_bounded_settings() { + let server = RpcServer::new(None); + let mut config = RdmaConfig { + send_batch_size: 16, + receive_batch_size: 8, + send_window_size: 4, + ..RdmaConfig::default() + }; + config.send_window_bytes = u64::from(config.wire_slot_capacity); + server.configure_rdma(&config).unwrap(); + let encoded = serde_json::to_value(&config).unwrap(); + assert_eq!(serde_json::from_value::(encoded).unwrap(), config); + config.send_window_size = config.sq_depth + 1; + assert!(server.configure_rdma(&config).is_err()); + config.send_window_size = 4; + config.send_window_bytes -= 1; + assert!(server.configure_rdma(&config).is_err()); + config.send_window_bytes = 0; + config.receive_batch_size = 0; + assert!(server.configure_rdma(&config).is_err()); + config.receive_batch_size = 8; + config.send_batch_size = 0; + assert!(server.configure_rdma(&config).is_err()); +} + #[test] fn logical_control_limits_are_typed_and_bounded_in_both_directions() { let server = RpcServer::new(None); @@ -86,7 +112,7 @@ fn serialized_control_data_settings_derive_the_selected_sq_and_rq_windows() { ..RdmaConfig::default() } .with_control_data_capacity(65_536) - .is_err()); + .is_ok()); assert!(serde_json::from_str::(r#"{"receive_queue_length": 1024}"#).is_err()); } @@ -100,7 +126,7 @@ fn service_layout_defaults_validate_capacity_and_preserve_explicit_tcp() { } #[test] -fn typed_control_data_layout_accounts_for_control_and_forbids_aggregation() { +fn typed_control_data_layout_accounts_for_both_regions_with_optional_packing() { let server = RpcServer::new(None); let mut config = RdmaConfig { wire_layout: RdmaWireLayout::ControlData, @@ -110,7 +136,7 @@ fn typed_control_data_layout_accounts_for_control_and_forbids_aggregation() { }; server.configure_rdma(&config).unwrap(); config.aggregation_enabled = true; - assert!(server.configure_rdma(&config).is_err()); + server.configure_rdma(&config).unwrap(); config.aggregation_enabled = false; config.receive_arena_bytes -= 1; assert!(server.configure_rdma(&config).is_err()); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h index 94c91936d..f3e1d046a 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h @@ -104,6 +104,19 @@ typedef struct crowdb_rpc_rdma_config } crowdb_rpc_rdma_config_t; void crowdb_rpc_rdma_config_init(crowdb_rpc_rdma_config_t *config); + +// Posting settings are separate to preserve earlier config layouts. +typedef struct crowdb_rpc_rdma_post_config +{ + uint32_t struct_size; + uint32_t version; + uint32_t send_batch_size; + uint32_t receive_batch_size; + uint32_t send_window_size; + uint64_t send_window_bytes; +} crowdb_rpc_rdma_post_config_t; + +void crowdb_rpc_rdma_post_config_init(crowdb_rpc_rdma_post_config_t *config); // Explicit cold configuration overrides environment; rejected after listen. crowdb_rpc_status crowdb_rpc_server_configure_rdma(crowdb_rpc_server_t server, const crowdb_rpc_rdma_config_t *config); // Version-two configuration adds the negotiated wire layout without changing @@ -123,6 +136,11 @@ typedef struct crowdb_rpc_rdma_stream_config uint64_t control_receive_budget_bytes; } crowdb_rpc_rdma_stream_config_t; +crowdb_rpc_status crowdb_rpc_server_configure_rdma_v4(crowdb_rpc_server_t server, + const crowdb_rpc_rdma_config_t *config, uint32_t wire_layout, + const crowdb_rpc_rdma_stream_config_t *streams, + const crowdb_rpc_rdma_post_config_t *sends); + void crowdb_rpc_rdma_stream_config_init(crowdb_rpc_rdma_stream_config_t *config); crowdb_rpc_status crowdb_rpc_server_configure_rdma_v3(crowdb_rpc_server_t server, const crowdb_rpc_rdma_config_t *config, uint32_t wire_layout, diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h index 53007b183..eab9f9ca6 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_config.h @@ -27,6 +27,10 @@ struct RdmaConfig uint64_t receive_arena_bytes = 1ULL << 30; uint64_t registered_memory_budget_bytes = 0; uint32_t sq_depth = 1024; + uint32_t send_batch_size = 16; + uint32_t receive_batch_size = 1; + uint32_t send_window_size = 0; + uint64_t send_window_bytes = 0; uint32_t rq_depth = 1024; uint32_t qp_max_send_sge = 2; uint32_t qp_max_recv_sge = 2; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h index 7edcb471b..fbabf847f 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/rdma/rdma_transport.h @@ -14,6 +14,7 @@ # include # include +# include # include # include # include @@ -51,6 +52,7 @@ class RdmaBufferPool : public BufferPool Buffer *alloc(uint32_t capacity) override; Buffer *borrow_range(const uint8_t *data, uint32_t size); + Buffer *borrow_owned_range(Buffer *source, const uint8_t *data, uint32_t size); void recycle(Buffer *buf) override; [[nodiscard]] uint64_t quarantined_bytes() const; @@ -73,6 +75,8 @@ class RdmaTransport : public Transport struct Stats { std::atomic sends{0}; + std::atomic send_post_calls{0}; + std::atomic send_completions{0}; std::atomic receives{0}; std::atomic cq_errors{0}; std::atomic fallback_count{0}; @@ -107,9 +111,13 @@ class RdmaTransport : public Transport std::atomic send_queue_high_water{0}; std::atomic send_errors{0}; std::atomic receive_post_errors{0}; + std::atomic receive_post_calls{0}; std::atomic direct_sends{0}; std::atomic direct_payload_bytes{0}; std::atomic copied_payload_bytes{0}; + std::atomic copied_aggregate_payload_bytes{0}; + std::atomic copied_staging_payload_bytes{0}; + std::atomic copied_framing_control_bytes{0}; std::atomic control_fragments_sent{0}; std::atomic control_fragments_received{0}; std::atomic copied_control_send_bytes{0}; @@ -228,7 +236,7 @@ class RdmaTransport : public Transport BufferPool *application_pool_; std::unique_ptr send_pool_; std::atomic producer_pool_{nullptr}; - std::unique_ptr recv_pool_; + std::unique_ptr source_pool_; ConnectionHandler connection_handler_; std::atomic running_{false}; @@ -245,7 +253,8 @@ class RdmaTransport : public Transport bool create_connection_pools(ConnectionPools &pools); void configure_peer(ConnectionPools &pools, const RdmaCmOffer &peer); std::shared_ptr create_active_connection(rdma_cm_id *id, const std::string &addr, - std::shared_ptr &pools); + std::shared_ptr &pools, + const RdmaEndpointCapability *expected_peer); bool establish_active_connection(rdma_cm_id *id, rdma_event_channel *channel, const std::string &addr, const RdmaEndpointCapability *expected_peer, RdmaCmOffer &peer_offer); @@ -260,10 +269,17 @@ class RdmaTransport : public Transport Buffer *prefix = nullptr; OutFrame *frame = nullptr; std::shared_ptr control_send; - uint64_t request_id = 0; - uint64_t stream_id = 0; - uint64_t sequence = 0; - uint64_t logical_offset = 0; + uint64_t request_id = 0; + uint64_t stream_id = 0; + uint64_t sequence = 0; + uint64_t logical_offset = 0; + WorkRequest *send_next = nullptr; + bool send_posted = false; + uint32_t send_frame_count = 0; + uint32_t send_wire_bytes = 0; + ibv_send_wr send_wr{}; + ibv_recv_wr receive_wr{}; + std::array regions{}; }; static constexpr uint64_t WORK_RESERVED = UINT64_MAX; @@ -298,14 +314,24 @@ class RdmaTransport : public Transport void notify_queue(uint32_t index); void install_connection_notifiers(const std::shared_ptr &connection, uint32_t index); uint32_t flush_sends(CompletionQueue &queue, uint32_t budget); - uint32_t flush_connection(Connection *connection, ConnectionPools &pools, uint32_t budget); - uint32_t flush_direct_connection(Connection *connection, ConnectionPools &pools, uint32_t budget); + bool prepare_send(Connection *connection, ConnectionPools &pools, Buffer *slot, Buffer *prefix, OutFrame *frames, + uint32_t frame_count, uint32_t wire_bytes, std::shared_ptr control_send = {}); + bool post_prepared(Connection *connection, ConnectionPools &pools); + bool send_window_available(const ConnectionPools &pools, uint32_t bytes = 0) const; + void complete_send(WorkRequest *work, const ibv_wc &completion); + static void unlink_send(WorkRequest *work); + uint32_t flush_connection(Connection *connection, ConnectionPools &pools, uint32_t budget); + uint32_t flush_direct_connection(Connection *connection, ConnectionPools &pools, uint32_t budget); + bool prepare_packed_send(Connection *connection, ConnectionPools &pools, bool &handled); bool post_direct(Connection *connection, ConnectionPools &pools, Buffer *slot, Buffer *prefix, OutFrame *frame, std::shared_ptr control_send = {}); bool flush_control_fragment(Connection *connection, ConnectionPools &pools); Buffer *admit_control_frame(const OutFrame &frame, const ConnectionPools &pools); void check_control_deadlines(CompletionQueue &queue); bool post_direct_receive(Connection *connection, Buffer *slot); + bool stage_receive(WorkRequest *work, std::span regions); + bool post_prepared_receives(Connection *connection, ConnectionPools &pools); + uint32_t flush_receive_reposts(CompletionQueue &queue, uint32_t budget); void retire_sends(CompletionQueue &queue, Connection *connection); bool post_aggregate(Connection *connection, ConnectionPools &pools, Buffer *slot, OutFrame *frames, uint32_t frame_count); @@ -370,7 +396,9 @@ class RdmaTransport : public Transport struct ConnectionPools : std::enable_shared_from_this { - explicit ConnectionPools(uint32_t queue_capacity = 4096) : send_queue(queue_capacity) + explicit ConnectionPools(uint32_t queue_capacity = 4096, uint32_t receive_capacity = 0) + : send_queue(queue_capacity), + receive_reposts(std::max(2U, receive_capacity)) { } @@ -392,9 +420,17 @@ class RdmaTransport : public Transport rdma_cm_id *cm_id = nullptr; // immutable after publication std::atomic disconnect_requested{false}; SendQueue send_queue; + crowdb::common::MpscQueue receive_reposts; + std::vector prepared_receives; std::atomic queued_frames{0}; - OutFrame *pending_frame = nullptr; - uint32_t sends_in_flight = 0; // CQ owner only + OutFrame *pending_frame = nullptr; + OutFrame *lookahead_frame = nullptr; + uint32_t sends_in_flight = 0; // CQ owner only + uint64_t send_bytes_in_flight = 0; + uint64_t prepared_send_bytes = 0; + std::vector prepared_sends; + WorkRequest *send_head = nullptr; + WorkRequest *send_tail = nullptr; uint64_t next_control_stream_id = 1; std::shared_ptr control_send; std::shared_ptr control_receive; @@ -414,6 +450,7 @@ class RdmaTransport : public Transport void release_completed_frame(WorkRequest *work, const struct ibv_wc &wc); bool release_completed_receive(WorkRequest *work, const struct ibv_wc &wc); bool release_completed_direct_receive(WorkRequest *work, const struct ibv_wc &wc); + bool release_completed_packed_receive(WorkRequest *work, const struct ibv_wc &wc); void record_completion_error(const struct ibv_wc &wc, bool expected_flush = false); static void repost_frame_release(void *context, Buffer *slot); bool adopt_cm_context(struct ibv_context *context); diff --git a/lib/crowdb-rpc/src/buffer_arena.cpp b/lib/crowdb-rpc/src/buffer_arena.cpp index a8a595789..def30aca7 100644 --- a/lib/crowdb-rpc/src/buffer_arena.cpp +++ b/lib/crowdb-rpc/src/buffer_arena.cpp @@ -147,30 +147,6 @@ Buffer *BufferArena::alloc(uint32_t requested) return buffer; } -Buffer *BufferArena::borrow_range(const uint8_t *data, uint32_t size) -{ - if (region == nullptr || region->registration_handle == 0 || entries_ == nullptr || size == 0) - return nullptr; - const auto address = reinterpret_cast(data); - const auto begin = reinterpret_cast(region->data); - if (address < begin || size > bytes || address - begin > bytes - size) - return nullptr; - const auto count = next_entry_.load(std::memory_order_acquire); - for (uint32_t index = 0; index < count; ++index) { - auto *buffer = entries_[index].buffer.load(std::memory_order_acquire); - if (buffer == nullptr) - continue; - const auto start = reinterpret_cast(buffer->allocation_data); - if (address >= start && address - start <= buffer->allocation_capacity && - size <= buffer->allocation_capacity - (address - start)) { - // The immutable source owner still pins this allocation. Retain - // that same allocation, not merely any MR containing the address. - return buffer->slice(static_cast(address - start), size); - } - } - return nullptr; -} - void BufferArena::recycle(Buffer *buffer) { // Retain this receiver until after publication; another allocator may diff --git a/lib/crowdb-rpc/src/buffer_arena.h b/lib/crowdb-rpc/src/buffer_arena.h index c18cd4afb..d8f7a7550 100644 --- a/lib/crowdb-rpc/src/buffer_arena.h +++ b/lib/crowdb-rpc/src/buffer_arena.h @@ -40,6 +40,7 @@ class BufferArena : public BufferPool, public std::enable_shared_from_this +// Licensed under the Apache License, Version 2.0. + +#include "buffer_arena.h" + +namespace crowdb::rpc +{ +Buffer *BufferArena::borrow_owned_range(Buffer *source, const uint8_t *data, uint32_t size) +{ + if (source == nullptr || size == 0 || region == nullptr || region->registration_handle == 0) { + return nullptr; + } + auto *root = source->allocation_owner == nullptr ? source : source->allocation_owner; + if (root->pool_owner.get() != this || root->registration_handle != region->registration_handle || + root->ownership.load(std::memory_order_acquire) == BufferOwnership::Quarantined) { + return nullptr; + } + const auto address = reinterpret_cast(data); + const auto start = reinterpret_cast(root->allocation_data); + if (address < start || address - start > root->allocation_capacity || + size > root->allocation_capacity - (address - start)) { + return nullptr; + } + // The immutable source pins this exact allocation, including detached + // leases. Retain its canonical owner without scanning other descriptors. + auto *view = root->slice(0, size); + if (view != nullptr) { + // Address is relative to allocation_data, independent of a canonical + // receive owner's temporary payload offset. + view->data = root->allocation_data + static_cast(address - start); + } + return view; +} + +Buffer *BufferArena::borrow_range(const uint8_t *data, uint32_t size) +{ + if (region == nullptr || region->registration_handle == 0 || entries_ == nullptr || size == 0) { + return nullptr; + } + const auto address = reinterpret_cast(data); + const auto begin = reinterpret_cast(region->data); + if (address < begin || size > bytes || address - begin > bytes - size) { + return nullptr; + } + const auto count = next_entry_.load(std::memory_order_acquire); + for (uint32_t index = 0; index < count; ++index) { + auto *buffer = entries_[index].buffer.load(std::memory_order_acquire); + if (buffer == nullptr) { + continue; + } + const auto start = reinterpret_cast(buffer->allocation_data); + if (address >= start && address - start <= buffer->allocation_capacity && + size <= buffer->allocation_capacity - (address - start)) { + return borrow_owned_range(buffer, data, size); + } + } + return nullptr; +} +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/c_api/transport.cpp b/lib/crowdb-rpc/src/c_api/transport.cpp index b984e7df1..53ad033b1 100644 --- a/lib/crowdb-rpc/src/c_api/transport.cpp +++ b/lib/crowdb-rpc/src/c_api/transport.cpp @@ -74,11 +74,37 @@ extern "C" crowdb_rpc_status crowdb_rpc_server_configure_rdma_v3(crowdb_rpc_serv const crowdb_rpc_rdma_config_t *config, uint32_t wire_layout, const crowdb_rpc_rdma_stream_config_t *streams) +{ + crowdb_rpc_rdma_post_config_t sends; + crowdb_rpc_rdma_post_config_init(&sends); + return crowdb_rpc_server_configure_rdma_v4(server, config, wire_layout, streams, &sends); +} + +extern "C" void crowdb_rpc_rdma_post_config_init(crowdb_rpc_rdma_post_config_t *config) +{ + if (config == nullptr) { + return; + } + const RdmaConfig defaults; + *config = {.struct_size = sizeof(*config), + .version = 1, + .send_batch_size = defaults.send_batch_size, + .receive_batch_size = defaults.receive_batch_size, + .send_window_size = defaults.send_window_size, + .send_window_bytes = defaults.send_window_bytes}; +} + +extern "C" crowdb_rpc_status crowdb_rpc_server_configure_rdma_v4(crowdb_rpc_server_t server, + const crowdb_rpc_rdma_config_t *config, + uint32_t wire_layout, + const crowdb_rpc_rdma_stream_config_t *streams, + const crowdb_rpc_rdma_post_config_t *sends) { if (server == nullptr || config == nullptr || config->struct_size != sizeof(*config) || config->version != 1 || config->enabled > 1 || config->allow_tcp_fallback > 1 || config->listen_enabled > 1 || config->aggregation_enabled > 1 || config->port > UINT8_MAX || streams == nullptr || - streams->struct_size != sizeof(*streams) || streams->version != 1) + streams->struct_size != sizeof(*streams) || streams->version != 1 || sends == nullptr || + sends->struct_size != sizeof(*sends) || sends->version != 1) return CROWDB_RPC_ERR_INVALID_ARG; try { RdmaConfig native; @@ -93,6 +119,10 @@ extern "C" crowdb_rpc_status crowdb_rpc_server_configure_rdma_v3(crowdb_rpc_serv native.control_assembly_timeout_ms = streams->control_assembly_timeout_ms; native.control_send_budget_bytes = streams->control_send_budget_bytes; native.control_receive_budget_bytes = streams->control_receive_budget_bytes; + native.send_batch_size = sends->send_batch_size; + native.receive_batch_size = sends->receive_batch_size; + native.send_window_size = sends->send_window_size; + native.send_window_bytes = sends->send_window_bytes; #define COPY_VALUE(name) native.name = config->name; CONFIG_NUMBERS(COPY_VALUE) #undef COPY_VALUE diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_active.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_active.cpp index 2f0b3a5de..051ca6f02 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_active.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_active.cpp @@ -84,7 +84,8 @@ bool resolve_active_route(rdma_cm_id *id, rdma_event_channel *channel, const std } // namespace std::shared_ptr RdmaTransport::create_active_connection(rdma_cm_id *id, const std::string &addr, - std::shared_ptr &pools) + std::shared_ptr &pools, + const RdmaEndpointCapability *expected_peer) { if (!adopt_cm_context(id->verbs)) { return nullptr; @@ -110,7 +111,7 @@ std::shared_ptr RdmaTransport::create_active_connection(rdma_cm_id * qp_attr.cap.max_send_sge = std::min(config_.qp_max_send_sge, static_cast(device_attributes.max_sge)); qp_attr.cap.max_recv_sge = std::min(config_.qp_max_recv_sge, static_cast(device_attributes.max_sge)); qp_attr.cap.max_inline_data = 0; - qp_attr.sq_sig_all = 1; + qp_attr.sq_sig_all = 0; qp_attr.qp_context = id; if (rdma_create_qp(id, pd_, &qp_attr) != 0) { @@ -122,7 +123,8 @@ std::shared_ptr RdmaTransport::create_active_connection(rdma_cm_id * rdma_destroy_qp(id); return nullptr; } - pools = std::make_shared(config_.send_queue_capacity); + pools = std::make_shared(config_.send_queue_capacity, + config_.receive_batch_size > 1 ? config_.posted_receive_target : 0); pools->effective_capacity = qp_attr.cap; pools->cq_index = reservation.queue->index; pools->reserved_capacity = reservation.capacity; @@ -132,10 +134,20 @@ std::shared_ptr RdmaTransport::create_active_connection(rdma_cm_id * connection->set_path(ConnectionPath::Rdma); connection->transport_handle = reinterpret_cast(id->qp); install_connection_notifiers(connection, pools->cq_index); - // Contiguous RQ slots have a fixed local wire capacity. Prepost before CM - // connect so an immediate response cannot enter an RNR retry cycle. - if (config_.wire_layout == RdmaWireLayout::Contiguous) { - pools->effective_data_capacity = config_.data_slot_capacity(); + // Preflight already binds the direct receive class. Prepost its negotiated + // slots before CM connect so an immediate peer response finds RQ credits. + if (config_.wire_layout == RdmaWireLayout::Contiguous || expected_peer != nullptr) { + if (config_.wire_layout == RdmaWireLayout::ControlData) { + RdmaCmOffer peer{.receive_frame_limit = static_cast(expected_peer->receive_frame_limit), + .receive_slot_capacity = expected_peer->receive_slot_capacity, + .wire_layout = expected_peer->wire_layout, + .generation = expected_peer->generation, + .max_control_message_bytes = expected_peer->max_control_message_bytes}; + configure_peer(*pools, peer); + } + else { + pools->effective_data_capacity = config_.data_slot_capacity(); + } if (!create_connection_pools(*pools) || !prime_receive(connection)) { destroy_connection_qp(connection.get(), id); return nullptr; @@ -187,7 +199,10 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int const RdmaEndpointCapability *expected_peer) { if (!ready() || addr.empty() || port <= 0 || port > 65535 || - (config_.require_tcp_preflight && expected_peer == nullptr)) { + (config_.require_tcp_preflight && expected_peer == nullptr) || + (expected_peer != nullptr && + (expected_peer->wire_layout != config_.wire_layout || + !validate_rdma_capability(*expected_peer, RDMA_PROTOCOL_VERSION, expected_peer->generation, "")))) { CRB_LOG_WARN("rdma connect: transport unavailable addr={} port={}", addr, port); return nullptr; @@ -221,15 +236,20 @@ std::shared_ptr RdmaTransport::connect(const std::string &addr, int if (!resolve_active_route(id, setup_channel.get(), addr, port, config_.source_address)) return nullptr; std::shared_ptr pools; - connection = create_active_connection(id, addr, pools); + connection = create_active_connection(id, addr, pools, expected_peer); if (connection == nullptr) return nullptr; RdmaCmOffer peer_offer; if (!establish_active_connection(id, setup_channel.get(), addr, expected_peer, peer_offer)) return nullptr; - configure_peer(*pools, peer_offer); - if (pools->recv == nullptr && (!create_connection_pools(*pools) || !prime_receive(connection))) { - return nullptr; + if (pools->recv == nullptr) { + configure_peer(*pools, peer_offer); + if (!create_connection_pools(*pools) || !prime_receive(connection)) { + return nullptr; + } + } + else if (config_.wire_layout == RdmaWireLayout::Contiguous) { + configure_peer(*pools, peer_offer); } pools->qp_number = id->qp->qp_num; ibv_qp_attr receive_attributes{}; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp index c914a24ca..9e1056493 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_buffer_pool.cpp @@ -69,6 +69,15 @@ Buffer *RdmaBufferPool::borrow_range(const uint8_t *data, uint32_t size) return arena_->borrow_range(data, size); } +Buffer *RdmaBufferPool::borrow_owned_range(Buffer *source, const uint8_t *data, uint32_t size) +{ + auto *borrowed = arena_->borrow_owned_range(source, data, size); + // External FFI views may retain a registered native allocation through + // their enclosing owner rather than its native descriptor. Preserve the + // bounded range lookup for those views before considering a staging copy. + return borrowed == nullptr ? arena_->borrow_range(data, size) : borrowed; +} + void RdmaBufferPool::recycle(Buffer *buffer) { arena_->recycle(buffer); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp index f9610d640..e973c1fa2 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_capability.cpp @@ -147,8 +147,7 @@ bool validate_rdma_capability(const RdmaEndpointCapability &capability, uint16_t ::inet_pton(AF_INET6, capability.gid.c_str(), &gid) != 1 || capability.generation == 0 || !valid_rdma_wire_capacity(capability.receive_slot_capacity, capability.wire_layout) || capability.receive_frame_limit == 0 || capability.receive_frame_limit > 256 || - (capability.wire_layout == RdmaWireLayout::ControlData && - (expected_protocol < 3 || capability.receive_frame_limit != 1))) + (capability.wire_layout == RdmaWireLayout::ControlData && expected_protocol < 3)) return false; } return (expected_protocol < 4 || capability.max_control_message_bytes != 0) && diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp index 41220c757..a95816fa0 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_completion.cpp @@ -46,7 +46,11 @@ int RdmaTransport::drain_queue(uint32_t index, uint32_t max_completions, bool wa } if (max_completions == 0) max_completions = std::max(1U, config_.poll_batch); + queue.receive_frames_left = CompletionQueue::RECEIVE_FRAMES_PER_ROUND; + queue.receive_bytes_left = queue.receive_byte_limit; + queue.receive_budget_exhausted = false; check_control_deadlines(queue); + uint32_t reposted = flush_receive_reposts(queue, CompletionQueue::SENDS_PER_ROUND); const uint32_t sent = flush_sends(queue, max_completions); uint32_t drained = process_deferred(queue, max_completions); uint32_t empty_polls = 0; @@ -82,21 +86,23 @@ int RdmaTransport::drain_queue(uint32_t index, uint32_t max_completions, bool wa } } poll_one(queue.cq); - if (managed && (drained > 0 || sent > 0)) { + reposted += flush_receive_reposts(queue, CompletionQueue::SENDS_PER_ROUND); + if (managed && (drained > 0 || sent > 0 || reposted > 0)) { queue.polling.clear(std::memory_order_release); - return static_cast(drained + sent); + return static_cast(drained + sent + reposted); } (void)ibv_req_notify_cq(queue.cq, 0); // Completions arriving between the empty poll and notification arming // need one final poll. A full batch gets an explicit epoll continuation. empty_polls = 0; poll_one(queue.cq); + reposted += flush_receive_reposts(queue, CompletionQueue::SENDS_PER_ROUND); if (!managed && drained == max_completions && queue.continuation_fd >= 0) { const uint64_t value = 1; (void)::write(queue.continuation_fd, &value, sizeof(value)); } queue.polling.clear(std::memory_order_release); - return static_cast(drained + sent); + return static_cast(drained + sent + reposted); } void RdmaTransport::register_events(SocketEngine *engine) @@ -176,7 +182,7 @@ bool RdmaTransport::prime_receive(const std::shared_ptr &connection) // owns the slot after the caller reference is released. slot->release(); } - return true; + return post_prepared_receives(connection.get(), *pools); } void RdmaTransport::record_completion_error(const ibv_wc &wc, bool expected_flush) @@ -211,8 +217,10 @@ void RdmaTransport::release_completed_frame(WorkRequest *work, const ibv_wc &wc) const auto pools = std::move(work->pools); Buffer *slot = work->slot; Buffer *prefix = work->prefix; - if (pools != nullptr && pools->sends_in_flight != 0) + if (pools != nullptr && work->send_posted && pools->sends_in_flight != 0) { --pools->sends_in_flight; + pools->send_bytes_in_flight -= work->send_wire_bytes; + } if (wc.status != IBV_WC_SUCCESS) { record_completion_error(wc, connection != nullptr && !connection->is_open()); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp index 7a6d45cfc..93461c7e0 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_config.cpp @@ -47,7 +47,11 @@ bool RdmaConfig::apply_environment() return false; wire_layout = static_cast(layout); return (port != 1 || read("CROWDB_RDMA_PORT", port)) && read("CROWDB_RDMA_SQ_DEPTH", sq_depth) && - read("CROWDB_RDMA_RQ_DEPTH", rq_depth) && read("CROWDB_RDMA_POSTED_RECEIVE_TARGET", posted_receive_target) && + read("CROWDB_RDMA_SEND_BATCH_SIZE", send_batch_size) && + read("CROWDB_RDMA_RECEIVE_BATCH_SIZE", receive_batch_size) && + read("CROWDB_RDMA_SEND_WINDOW_SIZE", send_window_size) && + read("CROWDB_RDMA_SEND_WINDOW_BYTES", send_window_bytes) && read("CROWDB_RDMA_RQ_DEPTH", rq_depth) && + read("CROWDB_RDMA_POSTED_RECEIVE_TARGET", posted_receive_target) && read("CROWDB_RDMA_WIRE_SLOT_CAPACITY", wire_slot_capacity) && read("CROWDB_RDMA_SEND_ARENA_BYTES", send_arena_bytes) && read("CROWDB_RDMA_RECEIVE_ARENA_BYTES", receive_arena_bytes) && read("CROWDB_RDMA_CQ_DEPTH", cq_depth) && @@ -68,11 +72,13 @@ bool RdmaConfig::apply_environment() bool RdmaConfig::valid() const { const bool direct = wire_layout == RdmaWireLayout::ControlData; - return port > 0 && valid_rdma_wire_capacity(wire_slot_capacity, wire_layout) && - (!direct || (!aggregation_enabled && qp_max_send_sge >= 2 && qp_max_recv_sge >= 2)) && sq_depth > 0 && - rq_depth > 0 && posted_receive_target > 0 && posted_receive_target <= rq_depth && - cq_depth > posted_receive_target && cq_depth <= static_cast(INT32_MAX) && cq_count > 0 && - cq_count <= 256 && static_cast(cq_count) * cq_depth < UINT32_MAX && poll_batch > 0 && + return send_batch_size > 0 && send_batch_size <= 256 && receive_batch_size > 0 && receive_batch_size <= 256 && + send_window_size <= sq_depth && (send_window_bytes == 0 || send_window_bytes >= wire_slot_capacity) && + port > 0 && valid_rdma_wire_capacity(wire_slot_capacity, wire_layout) && + (!direct || (qp_max_send_sge >= 2 && qp_max_recv_sge >= 2)) && sq_depth > 0 && rq_depth > 0 && + posted_receive_target > 0 && posted_receive_target <= rq_depth && cq_depth > posted_receive_target && + cq_depth <= static_cast(INT32_MAX) && cq_count > 0 && cq_count <= 256 && + static_cast(cq_count) * cq_depth < UINT32_MAX && poll_batch > 0 && poll_batch <= static_cast(INT32_MAX) && verbs_poll_batch > 0 && verbs_poll_batch <= 256 && poll_empty_budget > 0 && send_arena_bytes >= data_slot_capacity() && receive_arena_bytes >= static_cast(posted_receive_target) * wire_slot_capacity && diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp index bd0b23a51..682a0388d 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp @@ -143,14 +143,15 @@ std::shared_ptr RdmaTransport::accept() qp_attr.cap.max_send_sge = config_.qp_max_send_sge; qp_attr.cap.max_recv_sge = config_.qp_max_recv_sge; qp_attr.cap.max_inline_data = 0; - qp_attr.sq_sig_all = 1; + qp_attr.sq_sig_all = 0; qp_attr.qp_context = id; if (rdma_create_qp(id, pd_, &qp_attr) != 0) { rdma_destroy_id(id); return nullptr; } - auto pools = std::make_shared(config_.send_queue_capacity); + auto pools = std::make_shared(config_.send_queue_capacity, + config_.receive_batch_size > 1 ? config_.posted_receive_target : 0); configure_peer(*pools, peer_offer); pools->qp_number = id->qp->qp_num; pools->effective_capacity = qp_attr.cap; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_control_send.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_control_send.cpp index 4cb408a1e..11f0a8b07 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_control_send.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_control_send.cpp @@ -7,6 +7,7 @@ #ifdef CROWDB_RPC_HAVE_RDMA # include "rdma_aggregate.h" +# include "rdma_cq.h" # include # include @@ -73,10 +74,15 @@ bool RdmaTransport::flush_control_fragment(Connection *connection, ConnectionPoo f.message_flags = state->frame->header.flags; pools.control_send = std::move(state); } - auto state = pools.control_send; - auto &f = state->fragment; - const auto length = static_cast(std::min(f.capacity, f.total - f.offset)); - auto release = [](Buffer *buffer) { buffer->release(); }; + auto state = pools.control_send; + auto &f = state->fragment; + const auto length = static_cast(std::min(f.capacity, f.total - f.offset)); + auto &queue = *completion_queues_[pools.cq_index]; + if (!send_window_available(pools, RDMA_CONTROL_CAPACITY + length) || + !queue.can_prepare(RDMA_CONTROL_CAPACITY + length)) { + return false; + } + auto release = [](Buffer *buffer) { buffer->release(); }; std::unique_ptr prefix(pools.send->alloc(RDMA_CONTROL_CAPACITY), release); // Keep every staging descriptor in the configured data class. Smaller // cached allocations cannot grow and would strand a later larger stream @@ -93,6 +99,7 @@ bool RdmaTransport::flush_control_fragment(Connection *connection, ConnectionPoo } prefix->len = RDMA_CONTROL_CAPACITY; std::memcpy(data->data, state->frame->control->data + f.offset, length); + queue.charge_preparation(RDMA_CONTROL_CAPACITY + length); data->len = length; data->set_metadata(f.request_id, f.stream_id, f.sequence, f.offset, data->registration_id); auto *data_slot = data.release(); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp index 4b2dd30ef..20c954489 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.cpp @@ -66,23 +66,30 @@ bool RdmaTransport::create_completion_queues(ibv_context *context) void RdmaTransport::refresh_poll_stats() const { - uint64_t polls = 0; - uint64_t empty = 0; - uint64_t direct_sends = 0; - uint64_t direct_bytes = 0; - uint64_t copied_bytes = 0; + uint64_t polls = 0; + uint64_t empty = 0; + uint64_t direct_sends = 0; + uint64_t direct_bytes = 0; + uint64_t copied_bytes = 0; + uint64_t aggregate_bytes = 0, staging_bytes = 0, control_bytes = 0; for (const auto &queue : completion_queues_) { polls += queue->poll_calls.load(std::memory_order_relaxed); empty += queue->empty_polls.load(std::memory_order_relaxed); direct_sends += queue->direct_sends.load(std::memory_order_relaxed); direct_bytes += queue->direct_payload_bytes.load(std::memory_order_relaxed); copied_bytes += queue->copied_payload_bytes.load(std::memory_order_relaxed); + aggregate_bytes += queue->copied_aggregate_payload_bytes.load(std::memory_order_relaxed); + staging_bytes += queue->copied_staging_payload_bytes.load(std::memory_order_relaxed); + control_bytes += queue->copied_framing_control_bytes.load(std::memory_order_relaxed); } stats_.cq_polls.store(polls, std::memory_order_relaxed); stats_.cq_empty_polls.store(empty, std::memory_order_relaxed); stats_.direct_sends.store(direct_sends, std::memory_order_relaxed); stats_.direct_payload_bytes.store(direct_bytes, std::memory_order_relaxed); stats_.copied_payload_bytes.store(copied_bytes, std::memory_order_relaxed); + stats_.copied_aggregate_payload_bytes.store(aggregate_bytes, std::memory_order_relaxed); + stats_.copied_staging_payload_bytes.store(staging_bytes, std::memory_order_relaxed); + stats_.copied_framing_control_bytes.store(control_bytes, std::memory_order_relaxed); stats_.registered_memory_budget_bytes.store(memory_stats_->budget, std::memory_order_relaxed); stats_.registered_bytes.store(memory_stats_->current.load(std::memory_order_acquire), std::memory_order_relaxed); stats_.registered_bytes_high_water.store(memory_stats_->high_water.load(std::memory_order_relaxed), diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h index 1996f0e07..1fa284e98 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_cq.h @@ -12,6 +12,48 @@ namespace crowdb::rpc { struct RdmaTransport::CompletionQueue { + static constexpr uint32_t SENDS_PER_ROUND = 64; + static constexpr uint32_t PREPARE_FRAMES_PER_ROUND = 256; + static constexpr uint32_t PREPARE_BYTES_PER_ROUND = 4U << 20; + static constexpr uint32_t RECEIVE_FRAMES_PER_ROUND = 256; + static constexpr uint32_t RECEIVE_BYTES_PER_ROUND = 4U << 20; + uint32_t prepare_byte_limit = PREPARE_BYTES_PER_ROUND; + uint32_t prepare_frames_left = PREPARE_FRAMES_PER_ROUND; + uint32_t prepare_bytes_left = PREPARE_BYTES_PER_ROUND; + + bool preparation_exhausted = false; + + uint32_t receive_byte_limit = RECEIVE_BYTES_PER_ROUND; + uint32_t receive_frames_left = RECEIVE_FRAMES_PER_ROUND; + uint32_t receive_bytes_left = RECEIVE_BYTES_PER_ROUND; + bool receive_budget_exhausted = false; + + bool can_receive(uint32_t bytes, uint32_t frames = 1) + { + const bool fits = frames <= receive_frames_left && bytes <= receive_bytes_left; + receive_budget_exhausted |= !fits; + return fits; + } + + void charge_receive(uint32_t bytes, uint32_t frames = 1) + { + receive_bytes_left -= bytes; + receive_frames_left -= frames; + } + + bool can_prepare(uint32_t bytes, uint32_t frames = 1) + { + const bool fits = frames <= prepare_frames_left && bytes <= prepare_bytes_left; + preparation_exhausted |= !fits; + return fits; + } + + void charge_preparation(uint32_t bytes, uint32_t frames = 1) + { + prepare_bytes_left -= bytes; + prepare_frames_left -= frames; + } + uint32_t index = 0; std::atomic effective_depth{0}; ibv_cq *cq = nullptr; @@ -36,7 +78,8 @@ struct RdmaTransport::CompletionQueue }; std::vector connections; // protected by CQ polling ownership - size_t next_connection = 0; + size_t next_connection = 0; + size_t next_repost_connection = 0; // Producers publish PENDING and clear IDLE atomically. Only the first // producer observing IDLE writes the wake fd; active traffic needs none. static constexpr uint32_t IDLE = 1; @@ -55,6 +98,9 @@ struct RdmaTransport::CompletionQueue std::atomic direct_sends{0}; std::atomic direct_payload_bytes{0}; std::atomic copied_payload_bytes{0}; + std::atomic copied_aggregate_payload_bytes{0}; + std::atomic copied_staging_payload_bytes{0}; + std::atomic copied_framing_control_bytes{0}; }; } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_direct_receive.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_receive.cpp index 26639f2ab..dce3bf96a 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_direct_receive.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_receive.cpp @@ -3,7 +3,9 @@ #include "crowdb-rpc/transport/rdma/rdma_transport.h" #include "rdma_control_codec.h" +#include "rdma_cq.h" #include "rdma_direct_frame.h" +#include "rdma_packed_frame.h" #ifdef CROWDB_RPC_HAVE_RDMA namespace crowdb::rpc @@ -45,11 +47,16 @@ bool RdmaTransport::post_direct_receive(Connection *connection, Buffer *slot) std::array regions{}; regions[0] = {reinterpret_cast(prefix->data), RDMA_CONTROL_CAPACITY, prefix->registration_key}; regions[1] = {reinterpret_cast(slot->data), pools->effective_data_capacity, slot->registration_key}; + if (config_.receive_batch_size > 1) { + (void)prefix.release(); + return stage_receive(work, regions); + } ibv_recv_wr request{}; request.wr_id = work->token.load(std::memory_order_acquire); request.sg_list = regions.data(); request.num_sge = 2; ibv_recv_wr *bad = nullptr; + stats_.receive_post_calls.fetch_add(1, std::memory_order_relaxed); if (ibv_post_recv(reinterpret_cast(connection->transport_handle), &request, &bad) != 0) { release_work(work); rollback(); @@ -69,6 +76,13 @@ bool RdmaTransport::release_completed_direct_receive(WorkRequest *work, const ib bool valid = completion.status == IBV_WC_SUCCESS && work->conn->is_open() && parse_rdma_direct_control(std::span(prefix->data, RDMA_CONTROL_CAPACITY), work->pools->effective_data_capacity, completion.byte_len, header); + if (valid && header.msg_type == RDMA_PACKED_MSG_TYPE) { + return release_completed_packed_receive(work, completion); + } + auto &queue = *completion_queues_[work->pools->cq_index]; + if (valid && !queue.can_receive(completion.byte_len)) { + return false; + } std::unique_ptr logical; const bool fragment = valid && header.msg_type == RPC_CONTROL_FRAGMENT_MSG_TYPE; if (fragment) { @@ -113,6 +127,7 @@ bool RdmaTransport::release_completed_direct_receive(WorkRequest *work, const ib prefix->release(); return true; } + queue.charge_receive(completion.byte_len); auto connection = std::move(work->conn); auto pools = std::move(work->pools); slot->last_release_cb = &RdmaTransport::repost_frame_release; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_direct_send.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_send.cpp index 1f89b0f06..cab676c37 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_direct_send.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_send.cpp @@ -14,7 +14,7 @@ namespace crowdb::rpc { namespace { -const uint8_t *contiguous_data(const OutFrame &frame) +const uint8_t *contiguous_data(const OutFrame &frame, Buffer *&owner) { const uint8_t *first = nullptr; uintptr_t next = 0; @@ -25,6 +25,7 @@ const uint8_t *contiguous_data(const OutFrame &frame) const auto address = reinterpret_cast(view->payload_data()); if (first == nullptr) { first = view->payload_data(); + owner = frame.data_view(index); next = address; } if (address != next || view->len > UINTPTR_MAX - next) @@ -40,8 +41,7 @@ uint32_t RdmaTransport::flush_direct_connection(Connection *connection, Connecti const auto limit = std::min(config_.wire_slot_capacity, pools.peer_receive_slot_capacity); uint32_t sent = 0; auto &queue = *completion_queues_[pools.cq_index]; - while (sent < budget && connection->is_open() && - pools.sends_in_flight < std::min(config_.sq_depth, pools.effective_capacity.max_send_wr)) { + while (sent < budget && connection->is_open() && send_window_available(pools)) { if (pools.control_send == nullptr && pools.pending_frame == nullptr && pools.send_queue.drain(&pools.pending_frame, 1) == 0) break; @@ -52,30 +52,47 @@ uint32_t RdmaTransport::flush_direct_connection(Connection *connection, Connecti ++sent; continue; } + bool packed = false; + if (!prepare_packed_send(connection, pools, packed)) { + break; + } + if (packed) { + ++sent; + continue; + } auto *frame = pools.pending_frame; uint32_t bytes = 0; if (!rdma_direct_frame_size(*frame, limit, bytes)) { connection->close(); break; } + if (!send_window_available(pools, bytes) || !queue.can_prepare(bytes)) { + break; + } auto *prefix = pools.send->alloc(RDMA_CONTROL_CAPACITY); if (prefix == nullptr) break; write_rdma_direct_control(prefix->data, *frame); + queue.copied_framing_control_bytes.fetch_add(frame->control->len, std::memory_order_relaxed); prefix->len = RDMA_CONTROL_CAPACITY; const auto length = bytes - RDMA_CONTROL_CAPACITY; Buffer *data = nullptr; bool borrowed = false; if (length != 0) { - const auto *source = contiguous_data(*frame); + Buffer *source_owner = nullptr; + const auto *source = contiguous_data(*frame, source_owner); if (source != nullptr) { - data = send_pool_->borrow_range(source, length); + data = send_pool_->borrow_owned_range(source_owner, source, length); + if (data == nullptr) + data = source_pool_->borrow_owned_range(source_owner, source, length); if (data == nullptr) - data = pools.recv->borrow_range(source, length); + data = pools.recv->borrow_owned_range(source_owner, source, length); } borrowed = data != nullptr; if (data == nullptr) { - data = send_pool_->alloc(length); + // Staging descriptors must remain usable for every length in + // this data class after a burst fills the bounded pool. + data = send_pool_->alloc(config_.data_slot_capacity()); if (data != nullptr) { uint32_t offset = 0; for (uint8_t index = 0; index < frame->data_views(); ++index) { @@ -86,6 +103,7 @@ uint32_t RdmaTransport::flush_direct_connection(Connection *connection, Connecti } data->len = length; queue.copied_payload_bytes.fetch_add(length, std::memory_order_relaxed); + queue.copied_staging_payload_bytes.fetch_add(length, std::memory_order_relaxed); } } if (data == nullptr) { @@ -96,7 +114,8 @@ uint32_t RdmaTransport::flush_direct_connection(Connection *connection, Connecti (void)data->transition(BufferOwnership::CallerOwned, BufferOwnership::Available); data->request_id = frame->request_id; } - pools.pending_frame = nullptr; + pools.pending_frame = std::exchange(pools.lookahead_frame, nullptr); + queue.charge_preparation(bytes); pools.queued_frames.fetch_sub(1, std::memory_order_release); frame->next_aggregate = nullptr; if (!post_direct(connection, pools, data == nullptr ? prefix : data, data == nullptr ? nullptr : prefix, frame)) @@ -112,49 +131,10 @@ uint32_t RdmaTransport::flush_direct_connection(Connection *connection, Connecti bool RdmaTransport::post_direct(Connection *connection, ConnectionPools &pools, Buffer *slot, Buffer *prefix, OutFrame *frame, std::shared_ptr control_send) { - Operation posting(pools.posting_state); - WorkRequest *work = nullptr; - if (posting && connection->is_open() && connection->transport_handle != 0 && - slot->transition(BufferOwnership::Available, BufferOwnership::SendInFlight) && - (prefix == nullptr || prefix->transition(BufferOwnership::Available, BufferOwnership::SendInFlight))) - work = acquire_work(WorkKind::Send, connection, slot, frame, prefix); - if (work != nullptr) { - work->control_send = std::move(control_send); - auto *control = prefix == nullptr ? slot : prefix; - std::array regions{}; - regions[0] = {reinterpret_cast(control->data), RDMA_CONTROL_CAPACITY, control->registration_key}; - if (prefix != nullptr) - regions[1] = {reinterpret_cast(slot->payload_data()), slot->len, slot->registration_key}; - ibv_send_wr request{}; - request.wr_id = work->token.load(std::memory_order_acquire); - request.sg_list = regions.data(); - request.num_sge = prefix == nullptr ? 1 : 2; - request.opcode = IBV_WR_SEND; - request.send_flags = IBV_SEND_SIGNALED; - ibv_send_wr *bad = nullptr; - if (ibv_post_send(reinterpret_cast(connection->transport_handle), &request, &bad) == 0) { - ++pools.sends_in_flight; - stats_.sends.fetch_add(1, std::memory_order_relaxed); - if (work->control_send == nullptr || - work->control_send->fragment.offset + slot->len == work->control_send->fragment.total) { - stats_.rpc_frames_sent.fetch_add(1, std::memory_order_relaxed); - } - stats_.wire_bytes_sent.fetch_add(RDMA_CONTROL_CAPACITY + (prefix == nullptr ? 0 : slot->len), - std::memory_order_relaxed); - return true; - } - release_work(work); - } - stats_.send_errors.fetch_add(1, std::memory_order_relaxed); - connection->close(); - (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); - slot->release(); - if (prefix != nullptr) { - (void)prefix->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); - prefix->release(); - } - release_rdma_frames(frame); - return false; + const bool last = + control_send == nullptr || control_send->fragment.offset + slot->len == control_send->fragment.total; + return prepare_send(connection, pools, slot, prefix, frame, last ? 1U : 0U, + RDMA_CONTROL_CAPACITY + (prefix == nullptr ? 0 : slot->len), std::move(control_send)); } } // namespace crowdb::rpc #endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_offer.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_offer.cpp index 693572f68..d79f754cc 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_offer.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_offer.cpp @@ -57,8 +57,7 @@ bool parse_rdma_offer(std::span bytes, RdmaCmOffer *out) offer.expected_identity = get(bytes, 40, 8); offer.max_control_message_bytes = static_cast(get(bytes, 48, 4)); if (!valid_rdma_wire_capacity(offer.receive_slot_capacity, offer.wire_layout) || offer.receive_frame_limit == 0 || - offer.receive_frame_limit > 256 || offer.generation == 0 || - (offer.wire_layout == RdmaWireLayout::ControlData && offer.receive_frame_limit != 1) || offer.identity == 0 || + offer.receive_frame_limit > 256 || offer.generation == 0 || offer.identity == 0 || (offer.expected_generation == 0) != (offer.expected_identity == 0) || offer.max_control_message_bytes == 0) return false; *out = offer; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_packed_frame.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_packed_frame.cpp new file mode 100644 index 000000000..facb0a77e --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_packed_frame.cpp @@ -0,0 +1,76 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_packed_frame.h" + +#include "crowdb-rpc/transport/rdma/rdma_capability.h" +#include "msg_type_generated.h" +#include "rdma_direct_frame.h" + +#include + +namespace crowdb::rpc +{ +namespace +{ +constexpr std::array ENVELOPE_PREFIX{'R', 'D', 'P', 'K', 1, 0}; + +bool packable_type(uint16_t type) +{ + // Disk READ also needs an independent receive allocation as its later + // I/O destination, even though its request has no incoming data payload. + return type != RDMA_HANDSHAKE_MSG_TYPE && type != RDMA_PACKED_MSG_TYPE && + type != proto::FBMsgType_ERpcControlFragment && + !(type >= proto::FBMsgType_EDiskWriteRequest && type <= proto::FBMsgType_EDiskFsyncResponse); +} +} // namespace + +bool rdma_packable_frame(const OutFrame &frame, uint32_t limit, uint32_t &size) +{ + return packable_type(frame.header.msg_type) && + rdma_frame_size(frame, std::min(limit, RDMA_SMALL_FRAME_BYTES), size); +} + +void write_rdma_packed_control(uint8_t *destination, uint32_t data_length, uint32_t count) +{ + std::fill_n(destination, RDMA_CONTROL_CAPACITY, uint8_t{0}); + Header header; + header.msg_type = RDMA_PACKED_MSG_TYPE; + header.msg_size = RDMA_CONTROL_CAPACITY - HEADER_SIZE; + header.data_size = data_length; + header.flags = FLAG_ONE_WAY; + serialize_header(destination, header); + std::copy(ENVELOPE_PREFIX.begin(), ENVELOPE_PREFIX.end(), destination + HEADER_SIZE); + destination[HEADER_SIZE + 6] = static_cast(count); + destination[HEADER_SIZE + 7] = static_cast(count >> 8); +} + +bool parse_rdma_packed_frame(std::span control, std::span data, uint32_t frame_limit, + RdmaAggregate &aggregate) +{ + if (data.size() > RDMA_DATAPATH_CAPACITY) { + return false; + } + Header header; + if (!parse_rdma_direct_control(control, static_cast(data.size()), + RDMA_CONTROL_CAPACITY + static_cast(data.size()), header) || + header.msg_type != RDMA_PACKED_MSG_TYPE || header.flags != FLAG_ONE_WAY || + !std::equal(ENVELOPE_PREFIX.begin(), ENVELOPE_PREFIX.end(), control.begin() + HEADER_SIZE)) { + return false; + } + const uint32_t count = control[HEADER_SIZE + 6] | (uint32_t{control[HEADER_SIZE + 7]} << 8); + if (count < 2 || count > frame_limit || + !std::all_of(control.begin() + HEADER_SIZE + 8, control.end(), [](uint8_t byte) { return byte == 0; }) || + !parse_rdma_aggregate(data, aggregate) || aggregate.count != count) { + return false; + } + for (uint32_t index = 0; index < count; ++index) { + const auto &view = aggregate.frames[index]; + if (!packable_type(view.header.msg_type) || + uint64_t{HEADER_SIZE} + view.header.msg_size + view.header.data_size > RDMA_SMALL_FRAME_BYTES) { + return false; + } + } + return true; +} +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_packed_frame.h b/lib/crowdb-rpc/src/transport/rdma/rdma_packed_frame.h new file mode 100644 index 000000000..1c5152a78 --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_packed_frame.h @@ -0,0 +1,17 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "rdma_aggregate.h" + +namespace crowdb::rpc +{ +constexpr uint16_t RDMA_PACKED_MSG_TYPE = 0xFFFD; +constexpr uint32_t RDMA_SMALL_FRAME_BYTES = 4096; + +bool rdma_packable_frame(const OutFrame &frame, uint32_t limit, uint32_t &size); +void write_rdma_packed_control(uint8_t *destination, uint32_t data_length, uint32_t count); +bool parse_rdma_packed_frame(std::span control, std::span data, uint32_t frame_limit, + RdmaAggregate &aggregate); +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_packed_receive.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_packed_receive.cpp new file mode 100644 index 000000000..e6eb4c14d --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_packed_receive.cpp @@ -0,0 +1,71 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/transport/rdma/rdma_transport.h" +#include "rdma_control_stream.h" +#include "rdma_cq.h" +#include "rdma_packed_frame.h" + +#ifdef CROWDB_RPC_HAVE_RDMA +namespace crowdb::rpc +{ +bool RdmaTransport::release_completed_packed_receive(WorkRequest *work, const ibv_wc &wc) +{ + auto *slot = work->slot; + auto *prefix = work->prefix; + RdmaAggregate aggregate; + const uint32_t limit = config_.aggregation_enabled ? config_.aggregation_max_frames : 1; + bool valid = wc.byte_len >= RDMA_CONTROL_CAPACITY && + wc.byte_len - RDMA_CONTROL_CAPACITY <= work->pools->effective_data_capacity && + (work->pools->control_receive == nullptr || !work->pools->control_receive->active()) && + parse_rdma_packed_frame(std::span(prefix->data, RDMA_CONTROL_CAPACITY), + std::span(slot->data, wc.byte_len - RDMA_CONTROL_CAPACITY), limit, aggregate); + auto &queue = *completion_queues_[work->pools->cq_index]; + if (valid && !queue.can_receive(wc.byte_len, aggregate.count)) { + return false; + } + valid = valid && slot->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight) && + prefix->transition(BufferOwnership::PostedReceive, BufferOwnership::Available) && + slot->transition(BufferOwnership::ReceiveInFlight, BufferOwnership::CallerOwned); + if (!valid) { + auto connection = work->conn; + stats_.protocol_errors.fetch_add(1, std::memory_order_relaxed); + (void)slot->quarantine(); + (void)prefix->quarantine(); + release_work(work); + connection->close(); + slot->release(); + prefix->release(); + return true; + } + queue.charge_receive(wc.byte_len, aggregate.count); + auto connection = std::move(work->conn); + auto pools = std::move(work->pools); + slot->last_release_cb = &RdmaTransport::repost_frame_release; + slot->last_release_ctx = new RepostContext{repost_state_, connection, std::move(pools)}; + release_work(work); + prefix->release(); + stats_.receives.fetch_add(1, std::memory_order_relaxed); + stats_.rpc_frames_received.fetch_add(aggregate.count, std::memory_order_relaxed); + stats_.wire_bytes_received.fetch_add(wc.byte_len, std::memory_order_relaxed); + for (uint32_t index = 0; index < aggregate.count; ++index) { + const auto &view = aggregate.frames[index]; + auto *frame = new Frame; + frame->header = view.header; + const auto *control = slot->data + view.offset + HEADER_SIZE; + frame->control.assign(control, control + view.header.msg_size); + extract_control_fields(control, view.header.msg_size, frame->request_id, frame->rpc_create_nano, + frame->response_create_nano); + frame->data_buf = slot->slice(view.data_offset, view.header.data_size); + if (connection->is_open()) { + connection->on_frame(frame); + } + else { + delete frame; + } + } + slot->release(); + return true; +} +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_packed_send.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_packed_send.cpp new file mode 100644 index 000000000..5b90e5dfb --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_packed_send.cpp @@ -0,0 +1,77 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_cq.h" +#include "rdma_packed_frame.h" + +#ifdef CROWDB_RPC_HAVE_RDMA +# include + +namespace crowdb::rpc +{ +bool RdmaTransport::prepare_packed_send(Connection *connection, ConnectionPools &pools, bool &handled) +{ + handled = false; + auto &queue = *completion_queues_[pools.cq_index]; + if (!config_.aggregation_enabled || config_.aggregation_max_frames < 2 || pools.peer_receive_frame_limit < 2) { + return true; + } + const auto bytes_limit = std::min({config_.aggregation_max_bytes, pools.effective_data_capacity, 65536U}); + const auto small_limit = std::min(RDMA_SMALL_FRAME_BYTES, bytes_limit / 2); + uint32_t first_bytes = 0, second_bytes = 0; + if (!rdma_packable_frame(*pools.pending_frame, small_limit, first_bytes)) { + return true; + } + if (pools.lookahead_frame == nullptr) { + (void)pools.send_queue.drain(&pools.lookahead_frame, 1); + } + if (pools.lookahead_frame == nullptr || !rdma_packable_frame(*pools.lookahead_frame, small_limit, second_bytes) || + !send_window_available(pools, RDMA_CONTROL_CAPACITY + first_bytes + second_bytes) || + !queue.can_prepare(RDMA_CONTROL_CAPACITY + first_bytes + second_bytes, 2)) { + return true; + } + auto *prefix = pools.send->alloc(RDMA_CONTROL_CAPACITY); + auto *data = prefix == nullptr ? nullptr : send_pool_->alloc(config_.data_slot_capacity()); + if (data == nullptr) { + if (prefix != nullptr) { + prefix->release(); + } + return false; + } + handled = true; + data->len = 0; + OutFrame *first = nullptr; + OutFrame **tail = &first; + uint32_t count = 0; + queue.charge_preparation(RDMA_CONTROL_CAPACITY, 0); + const auto frame_limit = std::min({config_.aggregation_max_frames, pools.peer_receive_frame_limit, 32U}); + while (pools.pending_frame != nullptr && count < frame_limit) { + auto *frame = pools.pending_frame; + uint32_t bytes = 0; + if (!rdma_packable_frame(*frame, small_limit, bytes) || bytes > bytes_limit - data->len || + !send_window_available(pools, RDMA_CONTROL_CAPACITY + data->len + bytes) || !queue.can_prepare(bytes)) { + break; + } + copy_rdma_frame(data->data + data->len, *frame); + queue.charge_preparation(bytes); + const auto payload_bytes = bytes - HEADER_SIZE - frame->control->len; + queue.copied_payload_bytes.fetch_add(payload_bytes, std::memory_order_relaxed); + queue.copied_aggregate_payload_bytes.fetch_add(payload_bytes, std::memory_order_relaxed); + queue.copied_framing_control_bytes.fetch_add(frame->control->len, std::memory_order_relaxed); + data->len += bytes; + frame->next_aggregate = nullptr; + *tail = frame; + tail = &frame->next_aggregate; + pools.queued_frames.fetch_sub(1, std::memory_order_release); + pools.pending_frame = std::exchange(pools.lookahead_frame, nullptr); + if (pools.pending_frame == nullptr) { + (void)pools.send_queue.drain(&pools.pending_frame, 1); + } + ++count; + } + write_rdma_packed_control(prefix->data, data->len, count); + prefix->len = RDMA_CONTROL_CAPACITY; + return prepare_send(connection, pools, data, prefix, first, count, RDMA_CONTROL_CAPACITY + data->len); +} +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp index 036913b3d..408947121 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_poller.cpp @@ -107,7 +107,7 @@ uint32_t RdmaTransport::process_deferred(CompletionQueue &queue, uint32_t max_co auto pools = pending.work->pools; if (completed < max_completions && (!pending.work->conn->is_open() || - (pending.work->conn->has_frame_handler() && + (!queue.receive_budget_exhausted && pending.work->conn->has_frame_handler() && (pools == nullptr || pools->deferred_blocked_epoch != queue.deferred_epoch)))) { if (!pending.work->conn->is_open()) pending.completion.status = IBV_WC_WR_FLUSH_ERR; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp index 51e289817..80981423d 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_qp.cpp @@ -16,6 +16,8 @@ namespace crowdb::rpc bool RdmaTransport::create_connection_pools(ConnectionPools &pools) { + pools.prepared_sends.reserve(config_.send_batch_size); + pools.prepared_receives.reserve(config_.receive_batch_size); const auto control_bytes = config_.receive_control_bytes(); const uint64_t data_bytes = config_.wire_layout == RdmaWireLayout::ControlData ? uint64_t{config_.posted_receive_target} * pools.effective_data_capacity diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp index 8ee8d4324..be4833ce7 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp @@ -3,6 +3,7 @@ #include "crowdb-rpc/transport/rdma/rdma_transport.h" #include "rdma_aggregate.h" +#include "rdma_cq.h" #ifdef CROWDB_RPC_HAVE_RDMA @@ -25,18 +26,26 @@ bool RdmaTransport::release_completed_receive(WorkRequest *work, const ibv_wc &w connection->close(); slot->release(); }; - if (wc.status != IBV_WC_SUCCESS || !work->conn->is_open() || - !work->slot->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight)) { + if (wc.status != IBV_WC_SUCCESS || !work->conn->is_open()) { fail(); return true; } RdmaAggregate aggregate; if (wc.byte_len > work->slot->capacity || - !parse_rdma_aggregate(std::span(work->slot->data, wc.byte_len), aggregate) || + !parse_rdma_aggregate(std::span(work->slot->data, wc.byte_len), aggregate)) { + fail(); + return true; + } + auto &queue = *completion_queues_[work->pools == nullptr ? 0 : work->pools->cq_index]; + if (!queue.can_receive(wc.byte_len, aggregate.count)) { + return false; + } + if (!work->slot->transition(BufferOwnership::PostedReceive, BufferOwnership::ReceiveInFlight) || !work->slot->transition(BufferOwnership::ReceiveInFlight, BufferOwnership::CallerOwned)) { fail(); return true; } + queue.charge_receive(wc.byte_len, aggregate.count); auto connection = std::move(work->conn); auto pools = std::move(work->pools); @@ -78,10 +87,23 @@ void RdmaTransport::repost_frame_release(void *context, Buffer *slot) { Operation operation(repost->state); if (operation && repost->connection->is_open() && - slot->ownership.load(std::memory_order_acquire) == BufferOwnership::Available && - !repost->state->transport->post_receive(repost->connection.get(), slot)) { - repost->state->transport->stats_.receive_post_errors.fetch_add(1, std::memory_order_relaxed); - repost->connection->close(); + slot->ownership.load(std::memory_order_acquire) == BufferOwnership::Available) { + auto *transport = repost->state->transport; + if (transport->config_.receive_batch_size > 1 && repost->pools != nullptr) { + Operation posting(repost->pools->posting_state); + if (posting && repost->pools->receive_reposts.try_push(slot)) { + transport->notify_queue(repost->pools->cq_index); + return; + } + if (posting) { + transport->stats_.receive_post_errors.fetch_add(1, std::memory_order_relaxed); + repost->connection->close(); + } + } + else if (!transport->post_receive(repost->connection.get(), slot)) { + transport->stats_.receive_post_errors.fetch_add(1, std::memory_order_relaxed); + repost->connection->close(); + } } } slot->release(); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_receive_batch.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_receive_batch.cpp new file mode 100644 index 000000000..6102fa3ec --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_receive_batch.cpp @@ -0,0 +1,106 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_cq.h" + +#ifdef CROWDB_RPC_HAVE_RDMA +# include +# include + +namespace crowdb::rpc +{ +bool RdmaTransport::stage_receive(WorkRequest *work, std::span regions) +{ + auto &pools = *work->pools; + std::copy(regions.begin(), regions.end(), work->regions.begin()); + work->receive_wr = {}; + work->receive_wr.wr_id = work->token.load(std::memory_order_acquire); + work->receive_wr.sg_list = work->regions.data(); + work->receive_wr.num_sge = static_cast(regions.size()); + if (!pools.prepared_receives.empty()) { + pools.prepared_receives.back()->receive_wr.next = &work->receive_wr; + } + pools.prepared_receives.push_back(work); + return pools.prepared_receives.size() < config_.receive_batch_size || + post_prepared_receives(work->conn.get(), pools); +} + +bool RdmaTransport::post_prepared_receives(Connection *connection, ConnectionPools &pools) +{ + auto &batch = pools.prepared_receives; + if (batch.empty()) { + return true; + } + Operation operation(repost_state_); + Operation posting(pools.posting_state); + const bool can_post = operation && posting && connection->is_open() && connection->transport_handle != 0; + ibv_recv_wr *bad = &batch.front()->receive_wr; + int error = EINVAL; + if (can_post) { + stats_.receive_post_calls.fetch_add(1, std::memory_order_relaxed); + error = + ibv_post_recv(reinterpret_cast(connection->transport_handle), &batch.front()->receive_wr, &bad); + } + bool accepted = error == 0 || (can_post && bad != &batch.front()->receive_wr); + for (auto *work : batch) { + if (error != 0 && &work->receive_wr == bad) { + accepted = false; + } + if (accepted) { + stats_.posted_receives.fetch_add(1, std::memory_order_relaxed); + continue; + } + auto *slot = work->slot; + auto *prefix = work->prefix; + (void)slot->transition(BufferOwnership::PostedReceive, BufferOwnership::Available); + if (prefix != nullptr) { + (void)prefix->transition(BufferOwnership::PostedReceive, BufferOwnership::Available); + } + release_work(work); + slot->release(); + if (prefix != nullptr) { + prefix->release(); + } + } + batch.clear(); + if (error != 0) { + connection->close(); + } + return error == 0; +} + +uint32_t RdmaTransport::flush_receive_reposts(CompletionQueue &queue, uint32_t budget) +{ + if (config_.receive_batch_size == 1) { + return 0; + } + uint32_t count = 0; + std::array slots{}; + for (size_t visited = 0; visited < queue.connections.size() && count < budget; ++visited) { + const auto &entry = queue.connections[queue.next_repost_connection % queue.connections.size()]; + queue.next_repost_connection = (queue.next_repost_connection + 1) % queue.connections.size(); + auto &pools = *entry.pools; + Operation posting(pools.posting_state); + if (!posting || !entry.connection->is_open()) { + continue; + } + const auto limit = std::min(config_.receive_batch_size, budget - count); + const auto taken = static_cast(pools.receive_reposts.drain(slots.data(), static_cast(limit))); + bool valid = true; + for (uint32_t index = 0; index < taken; ++index) { + if (valid) { + valid = post_receive(entry.connection.get(), slots[index]); + } + slots[index]->release(); + } + valid = post_prepared_receives(entry.connection.get(), pools) && valid; + if (!valid) { + stats_.receive_post_errors.fetch_add(1, std::memory_order_relaxed); + entry.connection->close(); + } + count += taken; + } + return count; +} +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp index f89e3a7cb..639cbb70f 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_send.cpp @@ -5,6 +5,7 @@ #include "rdma_control_send.h" #include "rdma_cq.h" #include "rdma_direct_frame.h" +#include "rdma_packed_frame.h" #ifdef CROWDB_RPC_HAVE_RDMA # include @@ -14,9 +15,14 @@ namespace crowdb::rpc RdmaTransport::ConnectionPools::~ConnectionPools() { release_rdma_frames(pending_frame); + release_rdma_frames(lookahead_frame); OutFrame *frame = nullptr; while (send_queue.drain(&frame, 1) != 0) release_rdma_frames(frame); + Buffer *slot = nullptr; + while (receive_reposts.drain(&slot, 1) != 0) { + slot->release(); + } } bool RdmaTransport::submit(Connection *connection, OutFrame *frame) @@ -27,7 +33,7 @@ bool RdmaTransport::submit(Connection *connection, OutFrame *frame) }; Operation operation(repost_state_); if (!operation || connection == nullptr || frame == nullptr || !connection->is_open() || - frame->header.msg_type == RPC_CONTROL_FRAGMENT_MSG_TYPE) + frame->header.msg_type == RPC_CONTROL_FRAGMENT_MSG_TYPE || frame->header.msg_type == RDMA_PACKED_MSG_TYPE) return reject(); auto *pools = pools_for(connection); if (pools == nullptr || pools->send == nullptr || pools->cq_index >= completion_queues_.size()) @@ -57,6 +63,9 @@ bool RdmaTransport::submit(Connection *connection, OutFrame *frame) else if (!(direct ? rdma_direct_frame_size(*frame, limit, bytes) : rdma_frame_size(*frame, limit, bytes))) { return reject(); } + if (!stream && config_.send_window_bytes != 0 && bytes > config_.send_window_bytes) { + return reject(); + } auto restore = [&] { if (stream) { auto *charged = frame->control; @@ -90,11 +99,16 @@ uint32_t RdmaTransport::flush_connection(Connection *connection, ConnectionPools std::min({config_.wire_slot_capacity, pools.peer_receive_slot_capacity, config_.aggregation_max_bytes}); const auto frame_limit = config_.aggregation_enabled ? std::min(config_.aggregation_max_frames, pools.peer_receive_frame_limit) : 1U; - uint32_t sent = 0; - while (sent < budget && connection->is_open() && - pools.sends_in_flight < std::min(config_.sq_depth, pools.effective_capacity.max_send_wr)) { + auto &queue = *completion_queues_[pools.cq_index]; + uint32_t sent = 0; + while (sent < budget && connection->is_open() && send_window_available(pools)) { if (pools.pending_frame == nullptr && pools.send_queue.drain(&pools.pending_frame, 1) == 0) break; + uint32_t next_bytes = 0; + if (!rdma_frame_size(*pools.pending_frame, byte_limit, next_bytes) || + !send_window_available(pools, next_bytes) || !queue.can_prepare(next_bytes)) { + break; + } auto *slot = pools.send->alloc(config_.wire_slot_capacity); if (slot == nullptr) break; @@ -110,11 +124,17 @@ uint32_t RdmaTransport::flush_connection(Connection *connection, ConnectionPools connection->close(); break; } - if (bytes > byte_limit - slot->len) + if (bytes > byte_limit - slot->len || !send_window_available(pools, slot->len + bytes) || + !queue.can_prepare(bytes)) break; copy_rdma_frame(slot->data + slot->len, *frame); + queue.charge_preparation(bytes); completion_queues_[pools.cq_index]->copied_payload_bytes.fetch_add( bytes - HEADER_SIZE - frame->control->len, std::memory_order_relaxed); + completion_queues_[pools.cq_index]->copied_aggregate_payload_bytes.fetch_add( + bytes - HEADER_SIZE - frame->control->len, std::memory_order_relaxed); + completion_queues_[pools.cq_index]->copied_framing_control_bytes.fetch_add(frame->control->len, + std::memory_order_relaxed); slot->len += bytes; frame->next_aggregate = nullptr; *tail = frame; @@ -140,48 +160,24 @@ uint32_t RdmaTransport::flush_connection(Connection *connection, ConnectionPools bool RdmaTransport::post_aggregate(Connection *connection, ConnectionPools &pools, Buffer *slot, OutFrame *frames, uint32_t frame_count) { - Operation posting(pools.posting_state); - WorkRequest *work = nullptr; - if (posting && connection->is_open() && connection->transport_handle != 0 && - slot->transition(BufferOwnership::Available, BufferOwnership::SendInFlight)) - work = acquire_work(WorkKind::Send, connection, slot, frames); - if (work != nullptr) { - ibv_sge sge{}; - sge.addr = reinterpret_cast(slot->data); - sge.length = slot->len; - sge.lkey = slot->registration_key; - ibv_send_wr wr{}; - wr.wr_id = work->token.load(std::memory_order_acquire); - wr.sg_list = &sge; - wr.num_sge = 1; - wr.opcode = IBV_WR_SEND; - wr.send_flags = IBV_SEND_SIGNALED; - ibv_send_wr *bad = nullptr; - if (ibv_post_send(reinterpret_cast(connection->transport_handle), &wr, &bad) == 0) { - ++pools.sends_in_flight; - stats_.sends.fetch_add(1, std::memory_order_relaxed); - stats_.rpc_frames_sent.fetch_add(frame_count, std::memory_order_relaxed); - stats_.wire_bytes_sent.fetch_add(slot->len, std::memory_order_relaxed); - return true; - } - release_work(work); - } - stats_.send_errors.fetch_add(1, std::memory_order_relaxed); - connection->close(); - (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); - slot->release(); - release_rdma_frames(frames); - return false; + return prepare_send(connection, pools, slot, nullptr, frames, frame_count, slot->len); } uint32_t RdmaTransport::flush_sends(CompletionQueue &queue, uint32_t budget) { - uint32_t sent = 0; - for (size_t visited = 0; visited < queue.connections.size() && sent < budget; ++visited) { + queue.prepare_bytes_left = queue.prepare_byte_limit; + queue.prepare_frames_left = CompletionQueue::PREPARE_FRAMES_PER_ROUND; + queue.preparation_exhausted = false; + budget = std::min(budget, CompletionQueue::SENDS_PER_ROUND); + uint32_t sent = 0; + for (size_t visited = 0; visited < queue.connections.size() && sent < budget && !queue.preparation_exhausted; + ++visited) { const auto &entry = queue.connections[queue.next_connection % queue.connections.size()]; queue.next_connection = (queue.next_connection + 1) % queue.connections.size(); - if (entry.connection->is_open()) + if (entry.connection->is_open()) { sent += flush_connection(entry.connection.get(), *entry.pools, budget - sent); + (void)post_prepared(entry.connection.get(), *entry.pools); + } } // Closed producers finish publishing before their bounded queue retires. for (size_t index = 0; index < queue.connections.size();) { @@ -211,6 +207,7 @@ void RdmaTransport::retire_sends(CompletionQueue &queue, Connection *connection) pools.control_receive->reset(); } release_rdma_frames(std::exchange(pools.pending_frame, nullptr)); + release_rdma_frames(std::exchange(pools.lookahead_frame, nullptr)); OutFrame *frame = nullptr; while (pools.send_queue.drain(&frame, 1) != 0) release_rdma_frames(frame); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_send_batch.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_send_batch.cpp new file mode 100644 index 000000000..b3e684b7d --- /dev/null +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_send_batch.cpp @@ -0,0 +1,179 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "rdma_aggregate.h" +#include "rdma_cq.h" + +#ifdef CROWDB_RPC_HAVE_RDMA +# include +# include + +namespace crowdb::rpc +{ +bool RdmaTransport::send_window_available(const ConnectionPools &pools, uint32_t bytes) const +{ + const auto requested = config_.send_window_size == 0 ? config_.sq_depth : config_.send_window_size; + const auto limit = std::min(requested, pools.effective_capacity.max_send_wr); + if (pools.sends_in_flight + pools.prepared_sends.size() >= limit) { + return false; + } + const uint64_t used = pools.send_bytes_in_flight + pools.prepared_send_bytes; + return config_.send_window_bytes == 0 || + (used <= config_.send_window_bytes && bytes <= config_.send_window_bytes - used); +} + +bool RdmaTransport::prepare_send(Connection *connection, ConnectionPools &pools, Buffer *slot, Buffer *prefix, + OutFrame *frames, uint32_t frame_count, uint32_t wire_bytes, + std::shared_ptr control_send) +{ + Operation posting(pools.posting_state); + WorkRequest *work = nullptr; + if (posting && connection->is_open() && connection->transport_handle != 0 && + slot->transition(BufferOwnership::Available, BufferOwnership::SendInFlight) && + (prefix == nullptr || prefix->transition(BufferOwnership::Available, BufferOwnership::SendInFlight))) { + work = acquire_work(WorkKind::Send, connection, slot, frames, prefix); + } + if (work == nullptr) { + stats_.send_errors.fetch_add(1, std::memory_order_relaxed); + connection->close(); + (void)slot->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); + slot->release(); + if (prefix != nullptr) { + (void)prefix->transition(BufferOwnership::SendInFlight, BufferOwnership::Available); + prefix->release(); + } + release_rdma_frames(frames); + return false; + } + work->control_send = std::move(control_send); + work->send_frame_count = frame_count; + work->send_wire_bytes = wire_bytes; + auto *control = prefix == nullptr ? slot : prefix; + work->regions[0] = {.addr = reinterpret_cast(control->data), + .length = prefix == nullptr ? wire_bytes : RDMA_CONTROL_CAPACITY, + .lkey = control->registration_key}; + if (prefix != nullptr) { + work->regions[1] = {.addr = reinterpret_cast(slot->payload_data()), + .length = slot->len, + .lkey = slot->registration_key}; + } + work->send_wr = {}; + work->send_wr.wr_id = work->token.load(std::memory_order_acquire); + work->send_wr.sg_list = work->regions.data(); + work->send_wr.num_sge = prefix == nullptr ? 1 : 2; + work->send_wr.opcode = IBV_WR_SEND; + if (!pools.prepared_sends.empty()) { + pools.prepared_sends.back()->send_wr.next = &work->send_wr; + } + pools.prepared_sends.push_back(work); + pools.prepared_send_bytes += wire_bytes; + return pools.prepared_sends.size() < config_.send_batch_size || post_prepared(connection, pools); +} + +bool RdmaTransport::post_prepared(Connection *connection, ConnectionPools &pools) +{ + if (pools.prepared_sends.empty()) { + return true; + } + Operation posting(pools.posting_state); + auto &batch = pools.prepared_sends; + batch.back()->send_wr.send_flags = IBV_SEND_SIGNALED; + ibv_send_wr *bad = &batch.front()->send_wr; + const bool can_post = posting && connection->is_open() && connection->transport_handle != 0; + int error = EINVAL; + if (can_post) { + stats_.send_post_calls.fetch_add(1, std::memory_order_relaxed); + error = ibv_post_send(reinterpret_cast(connection->transport_handle), &batch.front()->send_wr, &bad); + } + bool accepted = error == 0 || (can_post && bad != &batch.front()->send_wr); + for (auto *work : batch) { + if (error != 0 && &work->send_wr == bad) { + accepted = false; + } + if (accepted) { + work->send_posted = true; + if (pools.send_tail != nullptr) { + pools.send_tail->send_next = work; + } + else { + pools.send_head = work; + } + pools.send_tail = work; + ++pools.sends_in_flight; + pools.send_bytes_in_flight += work->send_wire_bytes; + stats_.sends.fetch_add(1, std::memory_order_relaxed); + stats_.rpc_frames_sent.fetch_add(work->send_frame_count, std::memory_order_relaxed); + stats_.wire_bytes_sent.fetch_add(work->send_wire_bytes, std::memory_order_relaxed); + } + else { + ibv_wc completion{}; + completion.status = IBV_WC_SUCCESS; + release_completed_frame(work, completion); + } + } + batch.clear(); + pools.prepared_send_bytes = 0; + if (error != 0) { + stats_.send_errors.fetch_add(1, std::memory_order_relaxed); + connection->close(); + } + return error == 0; +} + +void RdmaTransport::unlink_send(WorkRequest *work) +{ + auto pools = work->pools; + if (pools == nullptr || !work->send_posted) { + return; + } + WorkRequest *previous = nullptr; + for (auto *entry = pools->send_head; entry != nullptr; entry = entry->send_next) { + if (entry == work) { + if (previous == nullptr) { + pools->send_head = entry->send_next; + } + else { + previous->send_next = entry->send_next; + } + if (pools->send_tail == work) { + pools->send_tail = previous; + } + return; + } + previous = entry; + } +} + +void RdmaTransport::complete_send(WorkRequest *work, const ibv_wc &completion) +{ + auto pools = work->pools; + if (completion.status != IBV_WC_SUCCESS || !work->send_posted || pools == nullptr) { + unlink_send(work); + release_completed_frame(work, completion); + return; + } + stats_.send_completions.fetch_add(1, std::memory_order_relaxed); + while (pools->send_head != nullptr) { + auto *entry = pools->send_head; + const bool tail = entry == work; + if (!tail && claim_work(entry->token.load(std::memory_order_acquire)) != entry) { + work->conn->close(); + unlink_send(work); + release_completed_frame(work, completion); + return; + } + pools->send_head = entry->send_next; + if (pools->send_head == nullptr) { + pools->send_tail = nullptr; + } + release_completed_frame(entry, completion); + if (tail) { + return; + } + } + stats_.send_errors.fetch_add(1, std::memory_order_relaxed); + work->conn->close(); + release_completed_frame(work, completion); +} +} // namespace crowdb::rpc +#endif diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp index d4086d9dc..f9a2ab1d6 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_status.cpp @@ -31,6 +31,15 @@ std::string RdmaTransport::status_json() const CONFIG_FIELD(cq_count); CONFIG_FIELD(cq_depth); CONFIG_FIELD(sq_depth); + CONFIG_FIELD(send_batch_size); + CONFIG_FIELD(receive_batch_size); + CONFIG_FIELD(send_window_size); + CONFIG_FIELD(send_window_bytes); + status.number("sends_per_round", CompletionQueue::SENDS_PER_ROUND); + status.number("prepare_frames_per_round", CompletionQueue::PREPARE_FRAMES_PER_ROUND); + status.number("prepare_bytes_per_round", CompletionQueue::PREPARE_BYTES_PER_ROUND); + status.number("receive_frames_per_round", CompletionQueue::RECEIVE_FRAMES_PER_ROUND); + status.number("receive_bytes_per_round", CompletionQueue::RECEIVE_BYTES_PER_ROUND); CONFIG_FIELD(rq_depth); CONFIG_FIELD(posted_receive_target); CONFIG_FIELD(wire_slot_capacity); @@ -51,6 +60,8 @@ std::string RdmaTransport::status_json() const COUNTER_FIELD(sends); COUNTER_FIELD(receives); COUNTER_FIELD(rpc_frames_sent); + COUNTER_FIELD(send_post_calls); + COUNTER_FIELD(send_completions); COUNTER_FIELD(rpc_frames_received); COUNTER_FIELD(wire_bytes_sent); COUNTER_FIELD(wire_bytes_received); @@ -58,9 +69,13 @@ std::string RdmaTransport::status_json() const COUNTER_FIELD(protocol_errors); COUNTER_FIELD(send_errors); COUNTER_FIELD(receive_post_errors); + COUNTER_FIELD(receive_post_calls); COUNTER_FIELD(direct_sends); COUNTER_FIELD(direct_payload_bytes); COUNTER_FIELD(copied_payload_bytes); + COUNTER_FIELD(copied_aggregate_payload_bytes); + COUNTER_FIELD(copied_staging_payload_bytes); + COUNTER_FIELD(copied_framing_control_bytes); COUNTER_FIELD(submit_rejects); COUNTER_FIELD(send_queue_high_water); COUNTER_FIELD(fallback_count); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp index 5a53d88b1..1de90efa6 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp @@ -92,9 +92,10 @@ RdmaTransport::RdmaTransport(RdmaConfig config, BufferPool *application_pool) local_capability_.receive_slot_capacity = config_.wire_slot_capacity; local_capability_.wire_layout = config_.wire_layout; local_capability_.max_control_message_bytes = config_.max_control_message_bytes; - local_capability_.receive_frame_limit = - config_.wire_layout == RdmaWireLayout::ControlData ? 1 : RDMA_MAX_AGGREGATE_FRAMES; - local_capability_.cm_address = config_.source_address; + local_capability_.receive_frame_limit = config_.wire_layout == RdmaWireLayout::ControlData + ? (config_.aggregation_enabled ? config_.aggregation_max_frames : 1) + : RDMA_MAX_AGGREGATE_FRAMES; + local_capability_.cm_address = config_.source_address; int device_count = 0; ibv_device **devices = ibv_get_device_list(&device_count); @@ -141,7 +142,7 @@ RdmaTransport::~RdmaTransport() { RdmaTransport::shutdown(); send_pool_.reset(); - recv_pool_.reset(); + source_pool_.reset(); destroy_completion_queues(); for (const auto &queue : completion_queues_) { if (queue->continuation_fd >= 0) @@ -172,18 +173,18 @@ bool RdmaTransport::adopt_cm_context(struct ibv_context *context) if (!create_completion_queues(context)) { return false; } - context_ = context; - device_ = std::move(device); - pd_ = device_->pd; - send_pool_ = make_pool(config_.send_arena_bytes, config_.sq_depth); - recv_pool_ = make_pool(config_.send_arena_bytes, config_.sq_depth); - if (send_pool_ == nullptr || recv_pool_ == nullptr) { + context_ = context; + device_ = std::move(device); + pd_ = device_->pd; + send_pool_ = make_pool(config_.send_arena_bytes, config_.sq_depth); + source_pool_ = make_pool(config_.send_arena_bytes, config_.sq_depth); + if (send_pool_ == nullptr || source_pool_ == nullptr) { send_pool_.reset(); - recv_pool_.reset(); + source_pool_.reset(); capability_available_.store(false, std::memory_order_release); return false; } - producer_pool_.store(send_pool_.get(), std::memory_order_release); + producer_pool_.store(source_pool_.get(), std::memory_order_release); register_events(event_engine_); return ready(); } @@ -238,12 +239,16 @@ bool RdmaTransport::post_receive(Connection *conn, Buffer *slot) slot->release(); return false; } + if (config_.receive_batch_size > 1) { + return stage_receive(work, std::span(&sge, 1)); + } ibv_recv_wr wr{}; wr.wr_id = work->token.load(std::memory_order_acquire); wr.sg_list = &sge; wr.num_sge = 1; ibv_recv_wr *bad = nullptr; int post_rc = -1; + stats_.receive_post_calls.fetch_add(1, std::memory_order_relaxed); if (slot_posted) post_rc = ibv_post_recv(reinterpret_cast(conn->transport_handle), &wr, &bad); if (!slot_posted || post_rc != 0) { diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp index a404421ca..32fe89254 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_work_request.cpp @@ -35,6 +35,8 @@ RdmaTransport::WorkRequest *RdmaTransport::acquire_work(WorkKind kind, Connectio work.slot = slot; work.prefix = prefix; work.frame = frame; + work.send_next = nullptr; + work.send_posted = false; work.request_id = frame == nullptr ? slot->request_id : frame->request_id; work.stream_id = slot->stream_id; work.sequence = slot->sequence; @@ -77,8 +79,9 @@ void RdmaTransport::dispatch_completion(const ibv_wc &wc) return; // Error completions do not provide a trustworthy opcode. The retained // work record owns dispatch, even after disconnect or QP destruction. - bool deferred = work->kind == WorkKind::Receive && wc.status == IBV_WC_SUCCESS && work->pools != nullptr && - work->pools->deferred_receives != 0; + auto &queue = *completion_queues_[static_cast(wc.wr_id) / config_.cq_depth]; + bool deferred = work->kind == WorkKind::Receive && wc.status == IBV_WC_SUCCESS && + (queue.receive_budget_exhausted || (work->pools != nullptr && work->pools->deferred_receives != 0)); if (!deferred && work->kind == WorkKind::Receive && wc.status == IBV_WC_SUCCESS && !work->conn->has_frame_handler()) { RdmaAggregate aggregate; @@ -99,7 +102,7 @@ void RdmaTransport::dispatch_completion(const ibv_wc &wc) deferred = application; } if (work->kind != WorkKind::Receive) { - release_completed_frame(work, wc); + complete_send(work, wc); return; } if (deferred || !release_completed_receive(work, wc)) { diff --git a/lib/crowdb-rpc/tests/common/test_rdma_transport.h b/lib/crowdb-rpc/tests/common/test_rdma_transport.h index 9701583ae..f5e401fab 100644 --- a/lib/crowdb-rpc/tests/common/test_rdma_transport.h +++ b/lib/crowdb-rpc/tests/common/test_rdma_transport.h @@ -32,6 +32,34 @@ struct TestRdmaTransport transport.config_.aggregation_enabled = enabled; } + static void set_send_policy_for_tests(RdmaTransport &transport, uint32_t batch, uint32_t window = 0, + uint64_t bytes = 0) + { + transport.config_.send_batch_size = batch; + transport.config_.send_window_size = window; + transport.config_.send_window_bytes = bytes; + } + + static void set_receive_policy_for_tests(RdmaTransport &transport, uint32_t batch) + { + transport.config_.receive_batch_size = batch; + } + + static void set_preparation_budget_for_tests(RdmaTransport &transport, uint32_t bytes) + { + transport.completion_queues_[0]->prepare_byte_limit = bytes; + } + + static void set_receive_budget_for_tests(RdmaTransport &transport, uint32_t bytes) + { + transport.completion_queues_[0]->receive_byte_limit = bytes; + } + + static bool flush_receive_batch_for_tests(RdmaTransport &transport, Connection *connection) + { + return transport.post_prepared_receives(connection, *RdmaTransport::pools_for(connection)); + } + static void bind_sender_for_tests(RdmaTransport &transport, const std::shared_ptr &connection, std::unique_ptr pool, ibv_qp *qp, uint32_t peer_slot, uint32_t peer_frames) @@ -143,7 +171,14 @@ struct TestRdmaTransport static uint32_t flush_for_tests(RdmaTransport &transport, Connection *connection, uint32_t budget) { - return transport.flush_connection(connection, *RdmaTransport::pools_for(connection), budget); + auto *pools = RdmaTransport::pools_for(connection); + auto &queue = *transport.completion_queues_[pools->cq_index]; + queue.prepare_bytes_left = RdmaTransport::CompletionQueue::PREPARE_BYTES_PER_ROUND; + queue.prepare_frames_left = RdmaTransport::CompletionQueue::PREPARE_FRAMES_PER_ROUND; + queue.preparation_exhausted = false; + const auto count = transport.flush_connection(connection, *pools, budget); + (void)transport.post_prepared(connection, *pools); + return count; } static bool post_fragment_for_tests(RdmaTransport &transport, Connection *connection, diff --git a/lib/crowdb-rpc/tests/rdma_borrow_test.cpp b/lib/crowdb-rpc/tests/rdma_borrow_test.cpp new file mode 100644 index 000000000..d2e98636a --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_borrow_test.cpp @@ -0,0 +1,90 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "common/test_rdma_device.h" +#include "crowdb-rpc/transport/rdma/rdma_transport.h" + +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) +# include + +using namespace crowdb::rpc; + +TEST(RdmaBufferPoolTest, OwnedBorrowKeepsExactNestedOffsetAndRejectsOtherArena) +{ + auto device = TestRdmaDevice::open(); + if (device == nullptr) { + GTEST_SKIP() << "no accessible HCA"; + } + RdmaBufferPool pool(device->pd, 4096, 4, device), other(device->pd, 4096, 4, device); + auto *root = pool.alloc(128); + ASSERT_NE(root, nullptr); + auto *base = root->data; + for (uint32_t index = 0; index < 128; ++index) { + base[index] = static_cast(index); + } + root->len = 128; + root->view_offset = 17; + auto *first = root->slice(4, 40); + ASSERT_NE(first, nullptr); + auto *nested = first->slice(3, 13); + ASSERT_NE(nested, nullptr); + auto *borrowed = pool.borrow_owned_range(nested, nested->payload_data(), 13); + ASSERT_NE(borrowed, nullptr); + EXPECT_EQ(borrowed->payload_data(), base + 24); + EXPECT_EQ(borrowed->payload_data()[12], 36); + EXPECT_EQ(other.borrow_owned_range(nested, nested->payload_data(), 13), nullptr); + EXPECT_EQ(pool.borrow_owned_range(nested, base + 1020, 8), nullptr); + root->release(); + first->release(); + nested->release(); + EXPECT_FALSE(pool.quiescent()); + EXPECT_EQ(borrowed->payload_data()[0], 24); + borrowed->release(); + EXPECT_TRUE(pool.quiescent()); +} + +TEST(RdmaBufferPoolTest, OwnedBorrowRetainsDetachedRegistrationAfterFacadeDestruction) +{ + auto device = TestRdmaDevice::open(); + if (device == nullptr) { + GTEST_SKIP() << "no accessible HCA"; + } + auto pool = std::make_unique(device->pd, 4096, 4, device); + auto *source = pool->alloc(128); + ASSERT_NE(source, nullptr); + source->write("detached payload", 16); + ASSERT_TRUE(source->pool->detach(source)); + EXPECT_EQ(source->pool, nullptr); + auto *borrowed = pool->borrow_owned_range(source, source->payload_data(), 16); + ASSERT_NE(borrowed, nullptr); + pool.reset(); + source->release(); + EXPECT_EQ(std::memcmp(borrowed->payload_data(), "detached payload", 16), 0); + EXPECT_NE(borrowed->registration_handle, 0U); + borrowed->release(); +} + +TEST(RdmaBufferPoolTest, ExternalViewCanRetainTheRegisteredAllocationWithoutACopy) +{ + auto device = TestRdmaDevice::open(); + if (device == nullptr) { + GTEST_SKIP() << "no accessible HCA"; + } + RdmaBufferPool pool(device->pd, 4096, 4, device); + auto *source = pool.alloc(128); + ASSERT_NE(source, nullptr); + source->write("external payload", 16); + // The external descriptor's enclosing owner pins the native source, + // as the HTTP framed-body adapter does across the Rust/C ABI boundary. + Buffer external; + external.data = source->payload_data(); + external.len = external.capacity = 16; + auto *borrowed = pool.borrow_owned_range(&external, external.data, external.len); + ASSERT_NE(borrowed, nullptr); + EXPECT_EQ(borrowed->payload_data(), source->payload_data()); + source->release(); + EXPECT_EQ(std::memcmp(borrowed->payload_data(), "external payload", 16), 0); + borrowed->release(); + EXPECT_TRUE(pool.quiescent()); +} +#endif diff --git a/lib/crowdb-rpc/tests/rdma_config_test.cpp b/lib/crowdb-rpc/tests/rdma_config_test.cpp index cc1ea3361..1b0af07e1 100644 --- a/lib/crowdb-rpc/tests/rdma_config_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_config_test.cpp @@ -10,6 +10,37 @@ using crowdb::rpc::RdmaConfig; +TEST(RdmaConfigTest, SendWindowAndBatchRemainBoundedIndependentlyOfAllocatedSq) +{ + RdmaConfig config; + config.send_batch_size = 0; + EXPECT_FALSE(config.valid()); + config.send_batch_size = 257; + EXPECT_FALSE(config.valid()); + config.send_batch_size = 16; + config.send_window_size = config.sq_depth + 1; + EXPECT_FALSE(config.valid()); + config.send_window_size = 1; + config.send_window_bytes = config.wire_slot_capacity - 1; + EXPECT_FALSE(config.valid()); + config.send_window_bytes = config.wire_slot_capacity; + EXPECT_TRUE(config.valid()); + const TestEnvironment batch("CROWDB_RDMA_SEND_BATCH_SIZE", "8"); + const TestEnvironment receive_batch("CROWDB_RDMA_RECEIVE_BATCH_SIZE", "16"); + const TestEnvironment window("CROWDB_RDMA_SEND_WINDOW_SIZE", "32"); + const TestEnvironment bytes("CROWDB_RDMA_SEND_WINDOW_BYTES", "8589934592"); + ASSERT_TRUE(config.apply_environment()); + EXPECT_EQ(config.send_batch_size, 8U); + EXPECT_EQ(config.receive_batch_size, 16U); + EXPECT_EQ(config.send_window_size, 32U); + EXPECT_EQ(config.send_window_bytes, 8ULL << 30); + EXPECT_TRUE(config.valid()); + config.receive_batch_size = 0; + EXPECT_FALSE(config.valid()); + config.receive_batch_size = 257; + EXPECT_FALSE(config.valid()); +} + TEST(RdmaConfigTest, ControlDataChargesBothRegionsAndRequiresTwoSges) { RdmaConfig config; @@ -23,7 +54,7 @@ TEST(RdmaConfigTest, ControlDataChargesBothRegionsAndRequiresTwoSges) EXPECT_FALSE(config.valid()); ++config.receive_arena_bytes; config.aggregation_enabled = true; - EXPECT_FALSE(config.valid()); + EXPECT_TRUE(config.valid()); config.aggregation_enabled = false; config.qp_max_send_sge = 1; EXPECT_FALSE(config.valid()); diff --git a/lib/crowdb-rpc/tests/rdma_direct_hardware_test.cpp b/lib/crowdb-rpc/tests/rdma_direct_hardware_test.cpp index 4bcd85afd..012055629 100644 --- a/lib/crowdb-rpc/tests/rdma_direct_hardware_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_direct_hardware_test.cpp @@ -48,7 +48,8 @@ TEST(RdmaHardwareTest, DirectDataPreservesBytesAlignmentAndLeaseAdmission) const auto *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; - const auto config = direct_config(); + auto config = direct_config(); + config.send_batch_size = 4; SystemBufferPool application; auto server = std::make_unique(config, &application); auto client = std::make_unique(config, &application); @@ -176,4 +177,126 @@ TEST(RdmaHardwareTest, LayoutMismatchFailsBeforeCmOrUsesExplicitTcpFallback) EXPECT_EQ(stats->fallback_count.load(), static_cast(fallback)); } } + +static void test_mixed_staging(bool packed, uint32_t receive_batch = 1) +{ + const auto *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) + GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; + auto config = direct_config(); + config.send_batch_size = 4; + config.aggregation_enabled = packed; + config.receive_batch_size = receive_batch; + SystemBufferPool application; + RdmaTransport server(config, &application), client(config, &application); + ASSERT_TRUE(server.listen(address, 19392)); + std::shared_ptr incoming; + std::thread acceptor([&] { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (incoming == nullptr && std::chrono::steady_clock::now() < deadline) { + incoming = server.accept(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + }); + auto outgoing = client.connect(address, 19392); + acceptor.join(); + ASSERT_NE(outgoing, nullptr); + ASSERT_NE(incoming, nullptr); + uint32_t replies = 0; + incoming->set_on_frame([&](Frame *frame, Connection *connection) { + auto *reply = + build_out_frame(frame->request_id, 100, build_ping_response(&application, frame->request_id, 0, 0), + frame->data_buf->ref_clone()); + delete frame; + const bool accepted = server.submit(connection, reply); + EXPECT_TRUE(accepted); + if (!accepted) + release_rdma_frames(reply); + }); + std::vector held; + outgoing->set_on_frame([&](Frame *frame, Connection *) { + EXPECT_EQ(frame->request_id, replies + 1); + const uint32_t expected = frame->request_id <= 4 ? 128 : 65536; + EXPECT_EQ(frame->data_buf->len, expected); + EXPECT_TRUE(std::all_of(frame->data_buf->payload_data(), frame->data_buf->payload_data() + expected, + [id = frame->request_id](uint8_t byte) { return byte == id; })); + if (packed && frame->request_id <= 4) { + held.push_back(frame->data_buf->ref_clone()); + } + if (expected == 65536) { + EXPECT_EQ(reinterpret_cast(frame->data_buf->payload_data()) % 4096, 0U); + } + ++replies; + delete frame; + }); + auto submit = [&](uint64_t id, uint32_t size) { + auto *data = application.alloc(size); + EXPECT_NE(data, nullptr); + if (data == nullptr) + return false; + std::memset(data->data, static_cast(id), size); + data->len = size; + auto *frame = build_out_frame(id, 100, build_ping_request(&application, id, 0), data); + if (client.submit(outgoing.get(), frame)) + return true; + release_rdma_frames(frame); + return false; + }; + for (uint64_t id = 1; id <= 4; ++id) + ASSERT_TRUE(submit(id, 128)); + if (!packed) { + ASSERT_EQ(TestRdmaTransport::flush_for_tests(client, outgoing.get(), 4), 4U); + // Compatibility staging cannot consume the registered producer's + // independent admission while completed SENDs await CQ retirement. + auto *producer = client.allocate_producer_buffer(128); + ASSERT_NE(producer, nullptr); + producer->release(); + } + ASSERT_TRUE(drain_until(client, server, [&] { return replies == 4; })); + EXPECT_EQ(client.stats().sends.load(), packed ? 1U : 4U); + EXPECT_EQ(client.stats().send_post_calls.load(), 1U); + EXPECT_EQ(client.stats().send_completions.load(), 1U); + ASSERT_TRUE(submit(5, 65536)); + ASSERT_TRUE(drain_until(client, server, [&] { return replies == 5; })); + EXPECT_EQ(client.stats().send_post_calls.load(), 2U); + EXPECT_EQ(client.stats().copied_payload_bytes.load(), 4U * 128 + 65536); + EXPECT_EQ(client.stats().copied_aggregate_payload_bytes.load(), packed ? 4U * 128 : 0U); + EXPECT_EQ(client.stats().copied_staging_payload_bytes.load(), packed ? 65536U : 4U * 128 + 65536); + if (packed) { + const auto posted = client.stats().posted_receives.load(); + ASSERT_EQ(held.size(), 4U); + for (size_t index = 0; index < held.size(); ++index) { + EXPECT_EQ(held[index]->payload_data()[127], index + 1); + if (index == 3 && receive_batch > 1) { + std::thread release([buffer = held[index]] { buffer->release(); }); + release.join(); + } + else { + held[index]->release(); + } + EXPECT_EQ(client.stats().posted_receives.load(), + posted + static_cast(index == 3 && receive_batch == 1)); + } + client.drain_cq(); + EXPECT_EQ(client.stats().posted_receives.load(), posted + 1); + } + EXPECT_EQ(client.stats().send_errors.load(), 0U); + EXPECT_EQ(client.stats().cq_errors.load(), 0U); +} + +TEST(RdmaHardwareTest, DirectStagingReusesSmallBurstSlotsForLaterLargePayload) +{ + test_mixed_staging(false); +} + +TEST(RdmaHardwareTest, PackedSmallBurstKeepsOrderAlignmentAndSharedReceiveLease) +{ + test_mixed_staging(true); +} + +TEST(RdmaHardwareTest, BatchedReceiveRepostsFlushSingletonAndCrossThreadFinalLease) +{ + test_mixed_staging(true, 4); + test_mixed_staging(false, 4); +} #endif diff --git a/lib/crowdb-rpc/tests/rdma_offer_test.cpp b/lib/crowdb-rpc/tests/rdma_offer_test.cpp index c42b66566..7bc1a6b46 100644 --- a/lib/crowdb-rpc/tests/rdma_offer_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_offer_test.cpp @@ -130,6 +130,28 @@ TEST(RdmaOfferTest, ControlDataLayoutIsBoundByBothCapabilityAndPrivateOffer) EXPECT_TRUE(validate_rdma_capability(decoded, 2, 11, "test-device")); } +TEST(RdmaOfferTest, PackedControlDataAdmissionIsExplicitAndBoundIntoTheOffer) +{ + auto client = test_capability(11); + auto server = test_capability(17); + client.wire_layout = server.wire_layout = RdmaWireLayout::ControlData; + client.receive_slot_capacity = server.receive_slot_capacity = RDMA_METADATA_CAPACITY + RDMA_CONTROL_CAPACITY; + client.receive_frame_limit = 32; + server.receive_frame_limit = 1; + RdmaEndpointCapability decoded; + ASSERT_TRUE(parse_rdma_handshake(serialize_rdma_handshake(client), &decoded)); + EXPECT_EQ(decoded.receive_frame_limit, 32U); + EXPECT_TRUE(validate_rdma_capability(decoded, RDMA_PROTOCOL_VERSION, 11, "test-device")); + RdmaCmOffer offer; + ASSERT_TRUE(parse_rdma_offer(serialize_rdma_offer(client, &server), &offer)); + EXPECT_TRUE(validate_rdma_offer(offer, server, &client, true)); + EXPECT_EQ(offer.receive_frame_limit, 32U); + offer.receive_frame_limit = 1; + EXPECT_FALSE(validate_rdma_offer(offer, server, &client, true)); + client.receive_frame_limit = 257; + EXPECT_FALSE(parse_rdma_offer(serialize_rdma_offer(client, &server), &offer)); +} + TEST(RdmaOfferTest, LogicalControlLimitIsExplicitAndBoundIntoBothIdentities) { auto client = test_capability(31); diff --git a/lib/crowdb-rpc/tests/rdma_packed_frame_test.cpp b/lib/crowdb-rpc/tests/rdma_packed_frame_test.cpp new file mode 100644 index 000000000..abc1d67dc --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_packed_frame_test.cpp @@ -0,0 +1,59 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "../src/transport/rdma/rdma_packed_frame.h" +#include "crowdb-rpc/server/message.h" +#include "msg_type_generated.h" + +#include + +using namespace crowdb::rpc; + +TEST(RdmaPackedFrameTest, EnvelopeValidatesEveryFrameAndRejectsMalformedSuffixBeforePublication) +{ + SystemBufferPool pool; + std::vector data; + for (uint64_t id = 1; id <= 3; ++id) { + auto *payload = pool.alloc(static_cast(id * 127)); + ASSERT_NE(payload, nullptr); + std::memset(payload->data, static_cast(id), id * 127); + payload->len = static_cast(id * 127); + auto *frame = build_out_frame(id, 100, build_ping_request(&pool, id, 0), payload); + uint32_t bytes = 0; + ASSERT_TRUE(rdma_packable_frame(*frame, 4096, bytes)); + const auto offset = data.size(); + data.resize(offset + bytes); + copy_rdma_frame(data.data() + offset, *frame); + release_rdma_frames(frame); + } + std::array control; + write_rdma_packed_control(control.data(), static_cast(data.size()), 3); + RdmaAggregate aggregate; + ASSERT_TRUE(parse_rdma_packed_frame(control, data, 3, aggregate)); + ASSERT_EQ(aggregate.count, 3U); + for (uint32_t index = 0; index < 3; ++index) { + EXPECT_EQ(data[aggregate.frames[index].data_offset], index + 1); + } + EXPECT_FALSE(parse_rdma_packed_frame(control, data, 2, aggregate)); + EXPECT_FALSE(parse_rdma_packed_frame(control, std::span(data).first(data.size() - 1), 3, aggregate)); + control.back() = 1; + EXPECT_FALSE(parse_rdma_packed_frame(control, data, 3, aggregate)); + control.back() = 0; + data[aggregate.frames[2].offset] ^= 1; + EXPECT_FALSE(parse_rdma_packed_frame(control, data, 3, aggregate)); +} + +TEST(RdmaPackedFrameTest, AlignmentSensitiveAndTransportMessagesRemainDirect) +{ + SystemBufferPool pool; + auto *frame = build_out_frame(1, 100, build_ping_request(&pool, 1, 0), nullptr); + uint32_t size = 0; + EXPECT_TRUE(rdma_packable_frame(*frame, 4096, size)); + for (uint16_t type : {static_cast(proto::FBMsgType_EDiskReadRequest), + static_cast(proto::FBMsgType_EDiskWriteRequest), + static_cast(proto::FBMsgType_ERpcControlFragment), RDMA_PACKED_MSG_TYPE}) { + frame->header.msg_type = type; + EXPECT_FALSE(rdma_packable_frame(*frame, 4096, size)); + } + release_rdma_frames(frame); +} diff --git a/lib/crowdb-rpc/tests/rdma_receive_batch_test.cpp b/lib/crowdb-rpc/tests/rdma_receive_batch_test.cpp new file mode 100644 index 000000000..446968b31 --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_receive_batch_test.cpp @@ -0,0 +1,112 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "common/test_rdma_transport.h" + +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) +# include + +using namespace crowdb::rpc; + +namespace +{ +RdmaConfig test_receive_config() +{ + RdmaConfig config; + config.device = "test-no-device"; + config.wire_slot_capacity = 1024; + config.sq_depth = config.rq_depth = config.posted_receive_target = 4; + config.cq_depth = 8; + config.send_arena_bytes = config.receive_arena_bytes = 4096; + config.receive_batch_size = 4; + return config; +} + +class RdmaReceiveBatchTest : public ::testing::Test +{ + protected: + RegisteredBufferPool slots{4096, 4}; + RdmaTransport transport{test_receive_config()}; + std::shared_ptr connection = std::make_shared(1, "receiver", &slots); + std::shared_ptr pools; + ibv_context context{}; + ibv_qp qp{}; + std::vector accepted; + uint32_t fail_at = UINT32_MAX; + + void SetUp() override + { + pools = TestRdmaTransport::bind_cq(connection, 0); + qp.context = &context; + qp.qp_context = this; + context.ops.post_recv = [](ibv_qp *qp, ibv_recv_wr *wr, ibv_recv_wr **bad) { + auto &self = *static_cast(qp->qp_context); + uint32_t index = 0; + for (auto *entry = wr; entry != nullptr; entry = entry->next, ++index) { + if (index == self.fail_at) { + *bad = entry; + return EIO; + } + EXPECT_EQ(entry->num_sge, 1); + EXPECT_EQ(entry->sg_list[0].length, 1024U); + self.accepted.push_back(entry->wr_id); + } + return 0; + }; + connection->transport_handle = reinterpret_cast(&qp); + } + + void TearDown() override + { + connection->close(); + TestRdmaTransport::cancel(transport); + } + + bool post() + { + auto *slot = slots.alloc(1024); + if (slot == nullptr) { + ADD_FAILURE() << "receive slot admission exhausted"; + return false; + } + const bool result = transport.post_receive(connection.get(), slot); + slot->release(); + return result; + } +}; + +TEST_F(RdmaReceiveBatchTest, ActualPartialListPostsWithoutWaitingForAFullBatch) +{ + ASSERT_TRUE(post()); + ASSERT_TRUE(post()); + EXPECT_TRUE(accepted.empty()); + ASSERT_TRUE(TestRdmaTransport::flush_receive_batch_for_tests(transport, connection.get())); + EXPECT_EQ(accepted.size(), 2U); + EXPECT_EQ(transport.stats().receive_post_calls.load(), 1U); + EXPECT_EQ(transport.stats().posted_receives.load(), 2U); +} + +TEST_F(RdmaReceiveBatchTest, PartialProviderFailureKeepsOnlyAcceptedReceiveOwners) +{ + fail_at = 2; + ASSERT_TRUE(post()); + ASSERT_TRUE(post()); + ASSERT_TRUE(post()); + EXPECT_FALSE(post()); + EXPECT_FALSE(connection->is_open()); + ASSERT_EQ(accepted.size(), 2U); + for (auto token : accepted) { + EXPECT_TRUE(TestRdmaTransport::pending(transport, token)); + } + auto *first = slots.alloc(1024); + auto *second = slots.alloc(1024); + ASSERT_NE(first, nullptr); + ASSERT_NE(second, nullptr); + EXPECT_EQ(slots.alloc(1024), nullptr); + first->release(); + second->release(); + TestRdmaTransport::cancel(transport); + EXPECT_EQ(transport.stats().shutdown_flushes.load(), 2U); +} +} // namespace +#endif diff --git a/lib/crowdb-rpc/tests/rdma_receive_budget_test.cpp b/lib/crowdb-rpc/tests/rdma_receive_budget_test.cpp new file mode 100644 index 000000000..2721a5b5d --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_receive_budget_test.cpp @@ -0,0 +1,120 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "../src/transport/rdma/rdma_aggregate.h" +#include "common/test_rdma_transport.h" +#include "crowdb-rpc/server/message.h" + +#if defined(CROWDB_RPC_HAVE_RDMA) && defined(CROWDB_RPC_TEST_UTIL) +# include + +using namespace crowdb::rpc; + +namespace +{ +RdmaConfig test_receive_budget_config() +{ + RdmaConfig config; + config.device = "test-no-device"; + config.sq_depth = config.rq_depth = config.posted_receive_target = 4; + config.cq_depth = 8; + config.poll_empty_budget = 1; + return config; +} + +class RdmaReceiveBudgetTest : public ::testing::Test +{ + protected: + RegisteredBufferPool slots{4U << 20, 8}; + SystemBufferPool buffers; + TestCompletionQueue queue; + RdmaTransport transport{test_receive_budget_config()}; + std::shared_ptr connection = std::make_shared(1, "receiver", &slots); + std::vector> held; + uint64_t received = 0; + uint32_t data_bytes = 1; + + void SetUp() override + { + (void)TestRdmaTransport::bind_cq(connection, 0); + TestRdmaTransport::set_cq(transport, &queue.cq); + connection->set_on_frame([&](Frame *frame, Connection *) { + EXPECT_EQ(frame->request_id, ++received); + EXPECT_EQ(frame->data_buf->len, data_bytes); + const auto value = static_cast(frame->request_id); + EXPECT_TRUE(std::all_of(frame->data_buf->payload_data(), frame->data_buf->payload_data() + data_bytes, + [value](uint8_t byte) { return byte == value; })); + delete frame; + }); + } + + void TearDown() override + { + connection->close(); + transport.stop(); + TestRdmaTransport::set_cq(transport, nullptr); + TestRdmaTransport::cancel(transport); + held.clear(); + } + + void enqueue(uint32_t count, uint64_t first_id) + { + auto *slot = slots.alloc(RDMA_METADATA_CAPACITY); + ASSERT_NE(slot, nullptr); + held.emplace_back(slot, [](Buffer *buffer) { buffer->release(); }); + slot->len = 0; + for (uint64_t id = first_id; id < first_id + count; ++id) { + auto *data = buffers.alloc(data_bytes); + ASSERT_NE(data, nullptr); + std::memset(data->data, static_cast(id), data_bytes); + data->len = data_bytes; + auto release = [](OutFrame *frame) { release_rdma_frames(frame); }; + std::unique_ptr frame( + build_out_frame(id, 100, build_ping_request(&buffers, id, 0), data), release); + uint32_t bytes = 0; + ASSERT_TRUE(rdma_frame_size(*frame, slot->capacity - slot->len, bytes)); + copy_rdma_frame(slot->data + slot->len, *frame); + slot->len += bytes; + } + ASSERT_TRUE(slot->transition(BufferOwnership::Available, BufferOwnership::PostedReceive)); + auto *work_slot = slot->ref_clone(); + const auto token = TestRdmaTransport::receive(transport, connection, work_slot); + if (token == 0) { + work_slot->release(); + FAIL() << "receive work table exhausted"; + } + auto completion = TestCompletionQueue::completion(token); + completion.byte_len = slot->len; + queue.ready.push_back(completion); + } +}; + +TEST_F(RdmaReceiveBudgetTest, AggregateFrameCountBoundsCallbacksAndContinuesNextRoundInOrder) +{ + enqueue(256, 1); + enqueue(1, 257); + EXPECT_GT(transport.drain_cq(8), 0); + EXPECT_EQ(received, 256U); + EXPECT_EQ(transport.stats().pending_deferred_receives.load(), 1U); + EXPECT_GT(transport.drain_cq(8), 0); + EXPECT_EQ(received, 257U); + EXPECT_EQ(transport.stats().pending_deferred_receives.load(), 0U); + EXPECT_EQ(transport.stats().protocol_errors.load(), 0U); +} + +TEST_F(RdmaReceiveBudgetTest, WireBytesBoundCallbacksAndContinueNextRoundWithoutConsumingSlots) +{ + data_bytes = 400; + TestRdmaTransport::set_receive_budget_for_tests(transport, 512); + enqueue(1, 1); + enqueue(1, 2); + EXPECT_GT(transport.drain_cq(8), 0); + EXPECT_EQ(received, 1U); + EXPECT_EQ(transport.stats().pending_deferred_receives.load(), 1U); + EXPECT_GT(transport.drain_cq(8), 0); + EXPECT_EQ(received, 2U); + EXPECT_EQ(transport.stats().pending_deferred_receives.load(), 0U); + EXPECT_EQ(transport.stats().protocol_errors.load(), 0U); +} +} // namespace +#endif diff --git a/lib/crowdb-rpc/tests/rdma_send_test.cpp b/lib/crowdb-rpc/tests/rdma_send_test.cpp index bf90e4bd8..03dc0b742 100644 --- a/lib/crowdb-rpc/tests/rdma_send_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_send_test.cpp @@ -34,14 +34,16 @@ class RdmaSendTest : public ::testing::Test ibv_context context{}; ibv_qp qp{}; SystemBufferPool buffers; + std::shared_ptr device; RdmaTransport transport{test_send_config(), &buffers}; std::shared_ptr connection = std::make_shared(1, "sender", &buffers); std::vector> sends; bool fail_send = false; + uint32_t fail_at = UINT32_MAX; void SetUp() override { - auto device = TestRdmaDevice::open(); + device = TestRdmaDevice::open(); if (device == nullptr) { GTEST_SKIP() << "no accessible HCA for registered send slots"; } @@ -55,9 +57,18 @@ class RdmaSendTest : public ::testing::Test *bad = wr; return EIO; } - const auto *data = reinterpret_cast(wr->sg_list[0].addr); - self.sends.emplace_back(data, data + wr->sg_list[0].length); - self.queue.ready.push_back(TestCompletionQueue::completion(wr->wr_id)); + uint32_t index = 0; + for (auto *entry = wr; entry != nullptr; entry = entry->next, ++index) { + if (index == self.fail_at) { + *bad = entry; + return EIO; + } + const auto *data = reinterpret_cast(entry->sg_list[0].addr); + self.sends.emplace_back(data, data + entry->sg_list[0].length); + if ((entry->send_flags & IBV_SEND_SIGNALED) != 0) { + self.queue.ready.push_back(TestCompletionQueue::completion(entry->wr_id)); + } + } return 0; }; TestRdmaTransport::bind_sender_for_tests(transport, connection, std::move(pool), &qp, 1024, 3); @@ -116,6 +127,78 @@ TEST_F(RdmaSendTest, PeerSlotOversizeRejectsBeforeQueueOrSend) EXPECT_EQ(transport.stats().send_queue_high_water.load(), 0U); } +TEST_F(RdmaSendTest, LinkedBatchSignalsEachActualTailAndRetiresTheCoveredOwners) +{ + TestRdmaTransport::set_aggregation_for_tests(transport, false); + TestRdmaTransport::set_send_policy_for_tests(transport, 3); + for (uint64_t id = 1; id <= 4; ++id) { + ASSERT_TRUE(transport.submit(connection.get(), frame(id))); + } + EXPECT_GT(transport.drain_cq(8), 0); + EXPECT_EQ(sends.size(), 4U); + EXPECT_EQ(transport.stats().send_post_calls.load(), 2U); + EXPECT_EQ(transport.stats().send_completions.load(), 2U); + EXPECT_EQ(transport.stats().sends.load(), 4U); + EXPECT_TRUE(connection->is_open()); + ASSERT_TRUE(transport.submit(connection.get(), frame(5))); + EXPECT_GT(transport.drain_cq(8), 0); + EXPECT_EQ(sends.size(), 5U) << "a singleton must post without waiting for another request"; +} + +TEST_F(RdmaSendTest, SoftwareWindowPostsPartialBatchAndRefillsAfterCompletion) +{ + TestRdmaTransport::set_aggregation_for_tests(transport, false); + TestRdmaTransport::set_send_policy_for_tests(transport, 4, 2, 1024); + for (uint64_t id = 1; id <= 4; ++id) { + ASSERT_TRUE(transport.submit(connection.get(), frame(id, 400))); + } + EXPECT_GT(transport.drain_cq(8), 0); + EXPECT_EQ(sends.size(), 2U); + EXPECT_EQ(transport.stats().send_completions.load(), 1U); + EXPECT_GT(transport.drain_cq(8), 0); + EXPECT_EQ(sends.size(), 4U); + EXPECT_EQ(transport.stats().send_completions.load(), 2U); +} + +TEST_F(RdmaSendTest, PreparationByteBudgetRotatesBeforeABusyQpConsumesTheNextRound) +{ + TestRdmaTransport::set_aggregation_for_tests(transport, false); + TestRdmaTransport::set_preparation_budget_for_tests(transport, 512); + auto second = std::make_shared(2, "second", &buffers); + auto pool = std::make_unique(device->pd, 4096, 4, device); + ASSERT_TRUE(pool->ready()); + TestRdmaTransport::bind_sender_for_tests(transport, second, std::move(pool), &qp, 1024, 1); + for (uint64_t id = 1; id <= 4; ++id) { + ASSERT_TRUE(transport.submit(connection.get(), frame(id, 400))); + } + ASSERT_TRUE(transport.submit(second.get(), frame(91, 400))); + EXPECT_GT(transport.drain_cq(8), 0); + ASSERT_EQ(sends.size(), 1U); + EXPECT_GT(transport.drain_cq(8), 0); + ASSERT_EQ(sends.size(), 2U); + RdmaAggregate decoded; + ASSERT_TRUE(parse_rdma_aggregate(sends[1], decoded)); + EXPECT_EQ(sends[1][decoded.frames[0].data_offset], 91); +} + +TEST_F(RdmaSendTest, PartiallyAcceptedBatchDoesNotReplayOrPrematurelyReleaseThePrefix) +{ + TestRdmaTransport::set_aggregation_for_tests(transport, false); + TestRdmaTransport::set_send_policy_for_tests(transport, 4); + fail_at = 2; + for (uint64_t id = 1; id <= 4; ++id) { + ASSERT_TRUE(transport.submit(connection.get(), frame(id))); + } + EXPECT_GT(transport.drain_cq(8), 0); + EXPECT_FALSE(connection->is_open()); + EXPECT_EQ(sends.size(), 2U); + EXPECT_EQ(transport.stats().sends.load(), 2U); + EXPECT_EQ(transport.stats().send_errors.load(), 1U); + EXPECT_EQ(transport.stats().send_completions.load(), 0U); + TestRdmaTransport::cancel(transport); + EXPECT_EQ(transport.stats().shutdown_flushes.load(), 2U); +} + TEST_F(RdmaSendTest, SlotByteLimitSplitsWithoutReorderingPendingFrame) { for (uint64_t id = 1; id <= 4; ++id) diff --git a/lib/crowdb-rpc/tests/rdma_startup_test.cpp b/lib/crowdb-rpc/tests/rdma_startup_test.cpp index 2e77b5d2a..9172dbca3 100644 --- a/lib/crowdb-rpc/tests/rdma_startup_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_startup_test.cpp @@ -40,14 +40,14 @@ void complete_first_response(uint64_t id, crowdb_rpc_buffer_t control, crowdb_rp } } // namespace -TEST(RdmaHardwareTest, ContiguousFirstResponseCompletesWithinTheWarmupWindow) +static void test_first_response(RdmaWireLayout layout) { const auto *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; RdmaConfig config; config.use_environment = false; - config.wire_layout = RdmaWireLayout::Contiguous; + config.wire_layout = layout; config.source_address = address; if (const auto *device = std::getenv("CROWDB_RDMA_DEVICE")) config.device = device; @@ -55,6 +55,15 @@ TEST(RdmaHardwareTest, ContiguousFirstResponseCompletesWithinTheWarmupWindow) config.sq_depth = config.rq_depth = config.posted_receive_target = 1024; config.cq_depth = 2048; config.send_arena_bytes = config.receive_arena_bytes = 1024 * 1024; + if (layout == RdmaWireLayout::ControlData) { + config.wire_slot_capacity = RDMA_DATAPATH_CAPACITY + RDMA_CONTROL_CAPACITY; + config.sq_depth = config.rq_depth = config.posted_receive_target = 4; + config.cq_depth = 8; + config.send_arena_bytes = 4 * RDMA_DATAPATH_CAPACITY; + config.receive_arena_bytes = uint64_t{4} * config.wire_slot_capacity; + config.send_batch_size = config.receive_batch_size = 16; + config.aggregation_enabled = true; + } TestFirstResponse result; RpcClient caller; RpcServer server; @@ -69,21 +78,42 @@ TEST(RdmaHardwareTest, ContiguousFirstResponseCompletesWithinTheWarmupWindow) ASSERT_NE(connection, nullptr); ASSERT_EQ(connection->path(), ConnectionPath::Rdma); caller.attach(connection.get()); - auto *data = client.pool()->alloc(128); - ASSERT_NE(data, nullptr); - std::memset(data->data, 0xab, 128); - data->len = 128; - ASSERT_TRUE(caller.send(&client, connection.get(), 42, build_ping_request(client.pool(), 42, 0), data, 100, - complete_first_response, &result)); - const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(1); - while (!result.completed.load(std::memory_order_acquire) && std::chrono::steady_clock::now() < deadline) - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - const bool completed_in_time = result.completed.load(std::memory_order_acquire); + bool completed_in_time = true; + for (uint32_t round = 0; round < 3; ++round) { + result.completed.store(false, std::memory_order_release); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + auto *data = client.pool()->alloc(128); + ASSERT_NE(data, nullptr); + std::memset(data->data, 0xab, 128); + data->len = 128; + ASSERT_TRUE(caller.send(&client, connection.get(), 42, build_ping_request(client.pool(), 42, 0), data, 100, + complete_first_response, &result)); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(1); + while (!result.completed.load(std::memory_order_acquire) && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + completed_in_time = result.completed.load(std::memory_order_acquire); + if (!completed_in_time) { + break; + } + EXPECT_TRUE(result.matches); + } + EXPECT_TRUE(completed_in_time) << "singleton exceeded the one-second progress bound; client=" + << client.status_json() << " server=" << server.transport()->status_json(); client.stop(); server.stop(); caller.fail_all(nullptr, RpcError::ConnectionClosed); - EXPECT_TRUE(completed_in_time) << "first RPC exceeded the unchanged one-second benchmark warmup"; EXPECT_TRUE(result.matches); EXPECT_EQ(client.rdma_stats_for_tests()->fallback_count.load(), 0U); } + +TEST(RdmaHardwareTest, ContiguousFirstResponseCompletesWithinTheWarmupWindow) +{ + test_first_response(RdmaWireLayout::Contiguous); +} + +TEST(RdmaHardwareTest, ControlDataSingletonAfterIdleDoesNotWaitForTheNextMessage) +{ + test_first_response(RdmaWireLayout::ControlData); +} #endif diff --git a/lib/crowdb-rpc/tests/transport_config_test.cpp b/lib/crowdb-rpc/tests/transport_config_test.cpp index fdafe09d2..27c5cb466 100644 --- a/lib/crowdb-rpc/tests/transport_config_test.cpp +++ b/lib/crowdb-rpc/tests/transport_config_test.cpp @@ -81,6 +81,37 @@ TEST(TransportConfigTest, VersionedLogicalControlBoundsRejectBeforeConfiguration crowdb_rpc_server_destroy(server); } +TEST(TransportConfigTest, SendExtensionPreservesEarlierAbiAndRejectsMalformedBounds) +{ + const auto server = crowdb_rpc_server_create(nullptr); + ASSERT_NE(server, nullptr); + crowdb_rpc_rdma_config_t config; + crowdb_rpc_rdma_config_init(&config); + crowdb_rpc_rdma_stream_config_t streams; + crowdb_rpc_rdma_stream_config_init(&streams); + crowdb_rpc_rdma_post_config_t sends; + crowdb_rpc_rdma_post_config_init(&sends); + EXPECT_EQ(sends.struct_size, sizeof(sends)); + EXPECT_EQ(sends.version, 1U); + EXPECT_EQ(crowdb_rpc_server_configure_rdma_v4(server, &config, 0, &streams, nullptr), CROWDB_RPC_ERR_INVALID_ARG); + sends.version = 2; + EXPECT_EQ(crowdb_rpc_server_configure_rdma_v4(server, &config, 0, &streams, &sends), CROWDB_RPC_ERR_INVALID_ARG); + sends.version = 1; + sends.send_batch_size = 0; + EXPECT_EQ(crowdb_rpc_server_configure_rdma_v4(server, &config, 0, &streams, &sends), CROWDB_RPC_ERR_INVALID_ARG); + sends.send_batch_size = 16; + sends.receive_batch_size = 0; + EXPECT_EQ(crowdb_rpc_server_configure_rdma_v4(server, &config, 0, &streams, &sends), CROWDB_RPC_ERR_INVALID_ARG); + sends.receive_batch_size = 8; + sends.send_window_size = config.sq_depth + 1; + EXPECT_EQ(crowdb_rpc_server_configure_rdma_v4(server, &config, 0, &streams, &sends), CROWDB_RPC_ERR_INVALID_ARG); + sends.send_window_size = 4; + sends.send_window_bytes = config.wire_slot_capacity; + EXPECT_EQ(crowdb_rpc_server_configure_rdma_v4(server, &config, 0, &streams, &sends), CROWDB_RPC_OK); + EXPECT_EQ(crowdb_rpc_server_configure_rdma_v3(server, &config, 0, &streams), CROWDB_RPC_OK); + crowdb_rpc_server_destroy(server); +} + TEST(TransportConfigTest, ClientErrorCountersSurviveMetricWindowFlush) { crowdb_rpc_client_counters_t before{}, pending{}, flushed{}; diff --git a/tools/benchmark/profiles/rdma-mixed-size.json b/tools/benchmark/profiles/rdma-mixed-size.json new file mode 100644 index 000000000..5787bf0ea --- /dev/null +++ b/tools/benchmark/profiles/rdma-mixed-size.json @@ -0,0 +1,47 @@ +{ + "schema_version": 1, + "description": "Synthetic mixed-size controls; no claim of a production trace. Same-QP sizes vary per request.", + "defaults": { + "wire_layout": "control-data", + "wire_slot_capacity": 1049600, + "sq_depth": 256, + "rq_depth": 64, + "posted_receive_target": 64, + "warmup_secs": 5, + "duration_secs": 10, + "payload_seed": 93, + "payload_mix": "0:5,128:45,513:15,4096:20,65535:10,262144:4,1048576:1", + "registered_payload": true, + "send_batch_size": 16, + "receive_batch_size": 1, + "aggregation_enabled": true + }, + "cases": [ + { + "case_id": "mixed-single", + "loader_count": 1 + }, + { + "case_id": "mixed-one-owner", + "loader_count": 64 + }, + { + "case_id": "mixed-shared-owner", + "loader_count": 128, + "connection_count": 4 + }, + { + "case_id": "mixed-four-owners", + "loader_count": 128, + "connection_count": 4, + "cq_count": 4 + }, + { + "case_id": "mixed-tokio-producers", + "mode": "tokio", + "loader_count": 128, + "connection_count": 1, + "io_workers": 4 + } + ] +} diff --git a/tools/benchmark/rpc_bench_cases.py b/tools/benchmark/rpc_bench_cases.py index 6ce47e6a0..588cf10e7 100644 --- a/tools/benchmark/rpc_bench_cases.py +++ b/tools/benchmark/rpc_bench_cases.py @@ -17,6 +17,9 @@ request_timeout_secs=5, registered_memory_budget_bytes=0) ENV_FIELDS = dict(cq_count="CQ_COUNT", cq_depth="CQ_DEPTH", sq_depth="SQ_DEPTH", + send_batch_size="SEND_BATCH_SIZE", send_window_size="SEND_WINDOW_SIZE", + receive_batch_size="RECEIVE_BATCH_SIZE", + send_window_bytes="SEND_WINDOW_BYTES", rq_depth="RQ_DEPTH", posted_receive_target="POSTED_RECEIVE_TARGET", wire_slot_capacity="WIRE_SLOT_CAPACITY", send_arena_bytes="SEND_ARENA_BYTES", receive_arena_bytes="RECEIVE_ARENA_BYTES", poll_batch="POLL_BATCH", @@ -28,6 +31,13 @@ def resolve(case, defaults=None): resolved = DEFAULTS | (defaults or {}) | case + if resolved.get("mode", "coroutine") not in ("coroutine", "tokio"): + raise ValueError("invalid client mode") + layout = resolved.get("wire_layout", "contiguous") + if layout not in ("contiguous", "control-data"): + raise ValueError("invalid wire layout") + if layout == "control-data" and resolved["wire_slot_capacity"] <= 1024: + raise ValueError("control-data requires payload capacity") qps, cqs = resolved["connection_count"], resolved["cq_count"] if not 0 < cqs <= 256 or not 0 < qps <= resolved["loader_count"]: raise ValueError("invalid CQ, QP/connection or loader count") @@ -39,7 +49,7 @@ def resolve(case, defaults=None): if not resolved.get("receive_arena_bytes"): resolved["receive_arena_bytes"] = slot * resolved["posted_receive_target"] if not resolved.get("aggregation_max_bytes"): - resolved["aggregation_max_bytes"] = slot + resolved["aggregation_max_bytes"] = min(slot, 1 << 20) return resolved @@ -84,14 +94,18 @@ def server_environment(case, transport, address, device, hca_port, port, runtime CROWDB_RDMA_ALLOW_TCP_FALLBACK="0", CROWDB_RDMA_E2E_PORT=str(port), CROWDB_RDMA_DEVICE=device, CROWDB_RDMA_SOURCE_ADDR=address, CROWDB_RDMA_PORT=str(hca_port), CROWDB_PORT_OFFSET="0") + env["CROWDB_RDMA_WIRE_LAYOUT"] = str(int(case.get("wire_layout") == "control-data")) for field, suffix in ENV_FIELDS.items(): + if field not in case: + continue value = case[field] env[f"CROWDB_RDMA_{suffix}"] = str(int(value)) if isinstance(value, bool) else str(value) return env def client_arguments(case): - keys = set(DEFAULTS) | set(ENV_FIELDS) | {"case_id", "repeat_index"} + keys = set(DEFAULTS) | set(ENV_FIELDS) | {"case_id", "repeat_index", "payload_mix", "payload_seed", + "wire_layout", "registered_payload"} args = [] for key in sorted(keys): if key not in case: diff --git a/tools/benchmark/rpc_bench_report.py b/tools/benchmark/rpc_bench_report.py index 202c0fa99..faea93000 100644 --- a/tools/benchmark/rpc_bench_report.py +++ b/tools/benchmark/rpc_bench_report.py @@ -44,6 +44,36 @@ def provider_counters(device, port): return result or None +def validate_sizes(row, case): + sizes = row.get("size_results") + expected = {int(part.split(":")[0]) for part in case["payload_mix"].split(",")} + if not isinstance(sizes, list) or len(sizes) != len(expected): + return ["missing/incomplete per-size results"] + seen, completed, payload = set(), 0, 0 + for item in sizes: + if not isinstance(item, dict): + return ["invalid per-size results"] + size, count = item.get("payload_bytes"), item.get("completed") + if (type(size) is not int or size not in expected or size in seen + or type(count) is not int or count < 0 + or item.get("payload_bytes_completed") != size * count): + return ["invalid per-size class/count/bytes"] + latency = [item.get(key) for key in ("p50_ns", "p99_ns", "p999_ns", "max_ns")] + if any(type(value) is not int or value < 0 for value in latency) or latency != sorted(latency): + return ["invalid per-size latency bounds"] + seen.add(size) + completed += count + payload += size * count + if completed != row.get("completed_in_window") or payload != row.get("payload_bytes_completed"): + return ["mixed completion/byte totals mismatch"] + duration = row.get("measure_s") + bandwidth = row.get("payload_Bps") + if (type(duration) is not int or duration <= 0 or type(bandwidth) not in (int, float) + or not math.isfinite(bandwidth) or not math.isclose(bandwidth, payload / duration, rel_tol=1e-9)): + return ["mixed payload bandwidth mismatch"] + return [] + + def validate(row, case, transport): errors = [] if row.get("schema_version") != 1: @@ -59,6 +89,8 @@ def validate(row, case, transport): errors.append(f"missing/invalid {name}") if row.get("actual_path") != transport or row.get("allow_tcp_fallback") is not False: errors.append("actual path/fallback policy mismatch") + if "mode" in case and row.get("client_mode") != case["mode"]: + errors.append("client mode mismatch") if row.get("fallback_count") != 0: errors.append("missing/nonzero fallback count") if row.get("case_id") != case["case_id"] or row.get("measure_s") != case["duration_secs"]: @@ -68,6 +100,12 @@ def validate(row, case, transport): errors.append(f"workload mismatch: {name}") if row.get("latency_count") != row.get("completed_in_window"): errors.append("latency/completion count mismatch") + if case.get("payload_mix"): + parameters = row.get("parameters") or {} + if (parameters.get("payload_mix") != case["payload_mix"] + or parameters.get("payload_seed") != case.get("payload_seed", 1)): + errors.append("payload mix/seed mismatch") + errors.extend(validate_sizes(row, case)) measure, completed = row.get("measure_s", 0), row.get("completed_in_window", 0) tps = row.get("rpc_tps") if (type(tps) not in (int, float) or not math.isfinite(tps) diff --git a/tools/benchmark/rpc_regression.py b/tools/benchmark/rpc_regression.py index 37e9d3426..51c4be9cc 100644 --- a/tools/benchmark/rpc_regression.py +++ b/tools/benchmark/rpc_regression.py @@ -69,7 +69,7 @@ def run_case(args, case, index, metadata): if case["enable_nagle"]: server_argv.append("--enable_nagle") server_command = runtime.command(server_argv, env, "server", args.server_cpus) - client_argv = [args.cli_bin, "--log-dir", directory / "client-log", "--mode", "coroutine", + client_argv = [args.cli_bin, "--log-dir", directory / "client-log", "--mode", case.get("mode", "coroutine"), "--server-host", args.address, "--server-port", args.port, "--output", report_file, "--device", args.device, "--source-address", args.address, "--hca-port", args.hca_port, *client_arguments(case)] diff --git a/tools/benchmark/tests/rdma_posting_test.py b/tools/benchmark/tests/rdma_posting_test.py new file mode 100644 index 000000000..f41464240 --- /dev/null +++ b/tools/benchmark/tests/rdma_posting_test.py @@ -0,0 +1,34 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +import argparse +from pathlib import Path +import sys +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "rdma")) +from posting import add_posting_arguments, batch_size, check_posting, posting_environment, posting_settings + + +class RdmaPostingTest(unittest.TestCase): + def test_requested_policy_reaches_both_service_snapshots(self): + parser = argparse.ArgumentParser() + add_posting_arguments(parser) + args = parser.parse_args(["--send-batch-size", "16", "--receive-batch-size", "8", "--aggregation-enabled"]) + settings = posting_settings(args) + self.assertEqual(posting_environment(settings), dict(CROWDB_RDMA_SEND_BATCH_SIZE="16", + CROWDB_RDMA_RECEIVE_BATCH_SIZE="8", CROWDB_RDMA_AGGREGATION_ENABLED="1")) + check_posting(settings, dict(rdma=dict(settings)), dict(rdma=None)) + with self.assertRaises(RuntimeError): + check_posting(settings, dict(rdma=settings | dict(receive_batch_size=1))) + with self.assertRaises(RuntimeError): + check_posting(settings, dict(rdma=settings | dict(aggregation_enabled=1))) + + def test_batch_limits_reject_invalid_arguments_before_launching_services(self): + for value in ("0", "257", "-1"): + with self.assertRaises(argparse.ArgumentTypeError): + batch_size(value) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/benchmark/tests/rdma_regression_test.py b/tools/benchmark/tests/rdma_regression_test.py index a14fcff23..aec08ee63 100644 --- a/tools/benchmark/tests/rdma_regression_test.py +++ b/tools/benchmark/tests/rdma_regression_test.py @@ -9,11 +9,27 @@ from pathlib import Path import shutil import subprocess +import sys import tempfile import unittest class RdmaRegressionTest(unittest.TestCase): + def test_mixed_control_data_profile_keeps_wire_and_aggregate_bounds_distinct(self): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from rpc_bench_cases import client_arguments, resolve, server_environment + case = resolve(dict(wire_layout="control-data", wire_slot_capacity=1049600, + payload_mix="0:5,127:70,1048576:25", payload_seed=93)) + self.assertEqual(case["aggregation_max_bytes"], 1048576) + args = client_arguments(case) + self.assertEqual(args[args.index("--payload-mix") + 1], case["payload_mix"]) + env = server_environment(case, "rdma", "10.200.1.1", "rocep35s0f0", 1, 19101, Path("/tmp/test")) + self.assertEqual(env["CROWDB_RDMA_WIRE_LAYOUT"], "1") + packed = resolve(case | dict(aggregation_enabled=True)) + self.assertIn("--aggregation-enabled", client_arguments(packed)) + with self.assertRaises(ValueError): + resolve(dict(wire_layout="control-data", aggregation_enabled=True)) + def setUp(self): temporary = tempfile.TemporaryDirectory(prefix="rdma-runner-") self.addCleanup(temporary.cleanup) @@ -58,6 +74,24 @@ def setUp(self): self.manifest.write_text(json.dumps(dict(schema_version=1, cases=[ dict(case_id="fixture", loader_count=4, connection_count=2, cq_count=2)]))) + def test_mixed_report_rejects_missing_classes_and_fabricated_byte_totals(self): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from rpc_bench_report import validate_sizes + case = dict(payload_mix="0:1,127:1") + row = dict(completed_in_window=10, payload_bytes_completed=635, measure_s=10, + payload_Bps=63.5, size_results=[ + dict(payload_bytes=size, completed=5, payload_bytes_completed=size * 5, + p50_ns=1, p99_ns=2, p999_ns=3, max_ns=4) for size in (0, 127)]) + self.assertEqual(validate_sizes(row, case), []) + row["size_results"][1]["payload_bytes"] = 0 + self.assertTrue(validate_sizes(row, case)) + row["size_results"][1]["payload_bytes"] = 127 + row["payload_bytes_completed"] += 1 + self.assertTrue(validate_sizes(row, case)) + row["payload_bytes_completed"] -= 1 + row["size_results"][1]["completed"] = -1 + self.assertTrue(validate_sizes(row, case)) + def run_case(self, report, client_exit=0, server_mode="ok", transport="rdma"): prefix = "RDMA" if transport == "rdma" else "RPC" environment = os.environ | { diff --git a/tools/rdma/posting.py b/tools/rdma/posting.py new file mode 100644 index 000000000..96388663d --- /dev/null +++ b/tools/rdma/posting.py @@ -0,0 +1,35 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +"""Explicit posting policy shared by the real-service validation launchers.""" + +import argparse + + +def batch_size(value): + parsed = int(value) + if not 1 <= parsed <= 256: + raise argparse.ArgumentTypeError("batch size must be 1..256") + return parsed + + +def add_posting_arguments(parser): + parser.add_argument("--send-batch-size", type=batch_size, default=1) + parser.add_argument("--receive-batch-size", type=batch_size, default=1) + parser.add_argument("--aggregation-enabled", action="store_true") + + +def posting_settings(args): + return {key: getattr(args, key) for key in ("send_batch_size", "receive_batch_size", "aggregation_enabled")} + + +def posting_environment(settings): + return {"CROWDB_RDMA_" + key.upper(): str(int(value)) for key, value in settings.items()} + + +def check_posting(settings, *snapshots): + for snapshot in snapshots: + if snapshot["rdma"] is not None: + for key, expected in settings.items(): + if type(snapshot["rdma"].get(key)) is not type(expected) or snapshot["rdma"].get(key) != expected: + raise RuntimeError(f"requested {key}={expected}, actual {snapshot['rdma'].get(key)}") diff --git a/tools/rdma/run-chunk-kv.py b/tools/rdma/run-chunk-kv.py index 97b53c193..7901ebdee 100644 --- a/tools/rdma/run-chunk-kv.py +++ b/tools/rdma/run-chunk-kv.py @@ -18,6 +18,7 @@ ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "tools/benchmark")) from rpc_bench_process import Runtime, start # noqa: E402 +from posting import add_posting_arguments, posting_settings, posting_environment, check_posting CASE = "real_chunk_kv_service_preserves_large_controls_and_typed_results" @@ -92,7 +93,9 @@ def main(): parser.add_argument("--address", default="10.200.1.1") parser.add_argument("--device", default="rocep35s0f0") parser.add_argument("--image", default="crowdb-node:dev") + add_posting_arguments(parser) args = parser.parse_args() + posting = posting_settings(args) pixi = shutil.which("pixi") if not pixi: raise RuntimeError("pixi is required") @@ -111,9 +114,11 @@ def main(): env.update(CROWDB_RUNTIME_ROOT=str(output / "runtime"), CROWDB_RDMA_DEVICE=args.device, CROWDB_RDMA_CHUNK_KV_ENDPOINT=f"{address}:{port}", CROWDB_RDMA_CHUNK_KV_OUTPUT=str(output), CROWDB_RDMA_CHUNK_KV_PROFILE=args.profile) + env.update(posting_environment(posting)) topology = "containers-tcp" if args.topology == "containers" and args.profile == "tcp" else args.topology runtime = Runtime(ROOT, output, pixi, topology, args.image) manifest = {"profile": args.profile, "topology": args.topology, "physical_hosts": 1, + "posting": posting, "image": args.image if args.topology == "containers" else None, "source": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip(), "source_dirty": bool(subprocess.check_output(["git", "status", "--porcelain"], cwd=ROOT, text=True)), @@ -147,6 +152,7 @@ def main(): raise RuntimeError("Chunk-KV server failed") result = json.loads((output / "client.json").read_text()) server = json.loads((output / "server-final.json").read_text()) + check_posting(posting, server, *(result["transport"][group]["transport"] for group in ("metadata", "aggregate"))) check(result, server, args.profile) if args.topology == "containers" and result["effective_uid"] == "0": raise RuntimeError("client container must be non-root") diff --git a/tools/rdma/run-datapath.py b/tools/rdma/run-datapath.py index b725e15ef..f40ffa496 100644 --- a/tools/rdma/run-datapath.py +++ b/tools/rdma/run-datapath.py @@ -18,6 +18,7 @@ ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "tools/benchmark")) from rpc_bench_process import Runtime, start # noqa: E402 +from posting import add_posting_arguments, posting_settings, posting_environment, check_posting def build(pixi, output): @@ -66,8 +67,9 @@ def ready(server, log, rdma): raise RuntimeError("DiskIO readiness deadline expired") -def checked_result(output, transport, final, fault): +def checked_result(output, transport, final, fault, posting): result = json.loads((output / "client.json").read_text()) + check_posting(posting, final, result["transport"]) if result["errors"] != int(fault) or result["verification_errors"] != 0 or result["payload_bytes"] <= 3 << 20: raise RuntimeError("datapath corpus or error count is invalid") if transport == "rdma": @@ -95,7 +97,9 @@ def main(): parser.add_argument("--device", default=os.environ.get("CROWDB_RDMA_DEVICE", "rocep35s0f0")) parser.add_argument("--image", default="crowdb-node:dev") parser.add_argument("--fault-disconnect", action="store_true") + add_posting_arguments(parser) args = parser.parse_args() + posting = posting_settings(args) pixi = shutil.which("pixi") if pixi is None: raise RuntimeError("pixi is required") @@ -134,9 +138,11 @@ def main(): CROWDB_RDMA_DATAPATH_PAUSED=str(output / "download-paused"), CROWDB_RDMA_DATAPATH_RESUME=str(output / "download-resume"), CROWDB_RDMA_DATAPATH_TOPOLOGY=args.topology + ("_hca_local_loop" if rdma else "_tcp_loopback")) + env.update(posting_environment(posting)) topology = args.topology if rdma or args.topology == "native" else "containers-tcp" runtime = Runtime(ROOT, output, pixi, topology, args.image) manifest = {"transport": args.transport, "topology": env["CROWDB_RDMA_DATAPATH_TOPOLOGY"], + "posting": posting, "image": args.image if args.topology == "containers" else None, "source": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip(), "source_dirty": bool(subprocess.check_output(["git", "status", "--porcelain"], cwd=ROOT, text=True)), @@ -182,7 +188,7 @@ def main(): final = status(server_log, "crowdb-diskio transport final: ") if final is None: raise RuntimeError("DiskIO did not retain its final transport counters") - checked_result(output, args.transport, final, args.fault_disconnect) + checked_result(output, args.transport, final, args.fault_disconnect, posting) manifest["errors"] = int(args.fault_disconnect) manifest["verification_errors"] = 0 except Exception as error: diff --git a/tools/rpc-test/src/args.rs b/tools/rpc-test/src/args.rs index e79e78bc7..29a16f353 100644 --- a/tools/rpc-test/src/args.rs +++ b/tools/rpc-test/src/args.rs @@ -9,6 +9,7 @@ use crowdb_rpc_ffi::RdmaConfig; use serde::Serialize; use clap::ValueEnum; +use crate::workload::payload::PayloadProfile; #[derive(clap::Args, Debug, Serialize)] pub struct RpcArgs { @@ -31,6 +32,14 @@ pub struct RpcArgs { pub connection_count: u32, #[arg(long, visible_alias = "value-size", default_value_t = 128)] pub payload_bytes: usize, + /// Per-request sizes and relative weights, for example 0:5,128:70,4096:25. + #[arg(long)] + pub payload_mix: Option, + #[arg(long, default_value_t = 1)] + pub payload_seed: u64, + /// Generate payloads directly in the connected transport's registered arena. + #[arg(long)] + pub registered_payload: bool, #[arg(long, default_value_t = 1)] pub io_engines: u32, #[arg(long, default_value_t = 1)] @@ -61,6 +70,8 @@ pub struct RpcArgs { #[derive(clap::Args, Debug, Serialize)] pub struct QueueArgs { + #[arg(long, default_value = "contiguous", value_parser = ["contiguous", "control-data"])] + pub wire_layout: String, #[arg(long, default_value = "")] pub device: String, #[arg(long, default_value = "")] @@ -76,6 +87,16 @@ pub struct QueueArgs { pub cq_depth: u32, #[arg(long, default_value_t = 1024)] pub sq_depth: u32, + #[arg(long, default_value_t = 16)] + pub send_batch_size: u32, + #[arg(long, default_value_t = 1)] + pub receive_batch_size: u32, + /// Zero uses the configured SQ depth. + #[arg(long, default_value_t = 0)] + pub send_window_size: u32, + /// Wire bytes awaiting send completion; zero uses the WR limit only. + #[arg(long, default_value_t = 0)] + pub send_window_bytes: u64, #[arg(long, default_value_t = 1024)] pub rq_depth: u32, #[arg(long, default_value_t = 1024)] @@ -108,6 +129,10 @@ pub struct QueueArgs { } impl RpcArgs { + pub(super) fn payload_profile(&self) -> Result { + PayloadProfile::new(self.payload_bytes, self.payload_mix.as_deref(), self.payload_seed) + } + pub(super) fn resolve(&mut self, rdma: bool) -> Result { if self.duration_secs == 0 || self.duration_secs > 3600 @@ -127,7 +152,19 @@ impl RpcArgs { { return Err("invalid duration, loader/connection count, workers, timeout or payload size".into()); } + self.payload_profile()?; + if self.registered_payload && !rdma { + return Err("registered payload generation requires strict RDMA".into()); + } let q = &mut self.queues; + let layout = if q.wire_layout == "control-data" { + if q.wire_slot_capacity <= 1024 { + return Err("control-data requires payload capacity".into()); + } + crowdb_rpc_ffi::RdmaWireLayout::ControlData + } else { + crowdb_rpc_ffi::RdmaWireLayout::Contiguous + }; if q.cq_count == 0 || q.cq_count > 256 { return Err("cq_count must be in 1..=256".into()); } @@ -149,7 +186,7 @@ impl RpcArgs { q.receive_arena_bytes = u64::from(q.posted_receive_target) * u64::from(q.wire_slot_capacity); } if q.aggregation_max_bytes == 0 { - q.aggregation_max_bytes = q.wire_slot_capacity; + q.aggregation_max_bytes = q.wire_slot_capacity.min(1 << 20); } Ok(RdmaConfig { enabled: rdma, @@ -162,10 +199,14 @@ impl RpcArgs { cq_count: q.cq_count, cq_depth: q.cq_depth, sq_depth: q.sq_depth, + send_batch_size: q.send_batch_size, + receive_batch_size: q.receive_batch_size, + send_window_size: q.send_window_size, + send_window_bytes: q.send_window_bytes, rq_depth: q.rq_depth, posted_receive_target: q.posted_receive_target, wire_slot_capacity: q.wire_slot_capacity, - wire_layout: crowdb_rpc_ffi::RdmaWireLayout::Contiguous, + wire_layout: layout, send_arena_bytes: q.send_arena_bytes, receive_arena_bytes: q.receive_arena_bytes, registered_memory_budget_bytes: q.registered_memory_budget_bytes, diff --git a/tools/rpc-test/src/workload.rs b/tools/rpc-test/src/workload.rs index e455df431..6a9df96c3 100644 --- a/tools/rpc-test/src/workload.rs +++ b/tools/rpc-test/src/workload.rs @@ -4,6 +4,7 @@ //! Shared timed echo benchmark for explicit TCP and strict RDMA transports. mod control; +pub(super) mod payload; mod report; mod runtime; mod status; @@ -17,7 +18,7 @@ use std::time::Duration; use crowdb_common::logging::open_named_log; use crowdb_common::metrics::{MetricsRegistry, MetricsRunner}; -use crowdb_rpc_ffi::RdmaConfig; +use crowdb_rpc_ffi::{ProducerBufferAllocator, RdmaConfig, RpcServer}; use serde_json::{json, Value}; use std::path::Path; @@ -92,7 +93,7 @@ async fn run_case( status::validate(&report["server_before"], rdma)?; let before = session.client.counters(); let mut runner = metrics_runner(log_dir, args)?; - let handler = create_workload(args, &owners, runner.as_ref()); + let handler = create_workload(args, &owners, runner.as_ref(), session.server.clone())?; if let Some(runner) = &mut runner { runner.start(); } @@ -121,21 +122,31 @@ async fn run_case( Ok(()) } -fn create_workload(args: &RpcArgs, owners: &[usize], runner: Option<&MetricsRunner>) -> Arc { +fn create_workload( + args: &RpcArgs, + owners: &[usize], + runner: Option<&MetricsRunner>, + server: Arc, +) -> Result, String> { + let profile = args.payload_profile()?; + let allocator = args + .registered_payload + .then(|| ProducerBufferAllocator::new(server, true)); let create = |registry: &mut MetricsRegistry| { Arc::new(EchoWorkload::new( args.loader_count, owners, - args.payload_bytes, + profile.clone(), + allocator.clone(), Duration::from_secs(args.warmup_secs), Duration::from_secs(args.duration_secs), registry, )) }; if let Some(runner) = runner { - create(&mut runner.registry().lock().expect("metrics registry")) + Ok(create(&mut runner.registry().lock().expect("metrics registry"))) } else { - create(&mut MetricsRegistry::new()) + Ok(create(&mut MetricsRegistry::new())) } } diff --git a/tools/rpc-test/src/workload/payload.rs b/tools/rpc-test/src/workload/payload.rs new file mode 100644 index 000000000..d5d243c63 --- /dev/null +++ b/tools/rpc-test/src/workload/payload.rs @@ -0,0 +1,64 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Reproducible request sizes independent of callback and thread ordering. + +#[derive(Clone)] +pub(crate) struct PayloadProfile { + sizes: Vec, + cumulative: Vec, + seed: u64, +} + +impl PayloadProfile { + pub fn new(fixed: usize, mix: Option<&str>, seed: u64) -> Result { + let mut profile = Self { + sizes: Vec::new(), + cumulative: Vec::new(), + seed, + }; + let mut total = 0_u64; + if let Some(mix) = mix { + for entry in mix.split(',') { + let (size, weight) = entry + .split_once(':') + .ok_or("payload mix must contain comma-separated bytes:weight entries")?; + let size = size.parse::().map_err(|_| "invalid payload size")?; + let weight = weight.parse::().map_err(|_| "invalid payload weight")?; + if size > (1 << 20) || weight == 0 || profile.sizes.contains(&size) { + return Err("payload sizes must be unique, at most 1 MiB, with positive weights".into()); + } + total += u64::from(weight); + profile.sizes.push(size); + profile.cumulative.push(total); + if profile.sizes.len() > 16 { + return Err("payload mix supports at most 16 sizes".into()); + } + } + } else { + profile.sizes.push(fixed); + profile.cumulative.push(1); + } + Ok(profile) + } + + pub fn sizes(&self) -> &[usize] { + &self.sizes + } + + pub fn maximum(&self) -> usize { + self.sizes.iter().copied().max().unwrap_or(0) + } + + pub fn select(&self, id: u64) -> usize { + if self.sizes.len() == 1 { + return 0; + } + let mut value = id.wrapping_add(self.seed).wrapping_add(0x9e37_79b9_7f4a_7c15); + value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + value ^= value >> 31; + let target = value % self.cumulative.last().copied().unwrap_or(1); + self.cumulative.partition_point(|&weight| weight <= target) + } +} diff --git a/tools/rpc-test/src/workload/report.rs b/tools/rpc-test/src/workload/report.rs index d23faaaf4..18337bd5e 100644 --- a/tools/rpc-test/src/workload/report.rs +++ b/tools/rpc-test/src/workload/report.rs @@ -149,6 +149,7 @@ pub(super) fn finish( .map(|field| sum_rdma_delta(report, field)) .sum(); let errors = native.total_errors + + get(Count::BuildErrors) + get(Count::PayloadMismatches) + get(Count::Duplicates) + counters.resp_missed @@ -157,6 +158,11 @@ pub(super) fn finish( + transport_errors + u64::from(reconcile); let tps = ratio(histogram.count, args.duration_secs); + let sizes = workload.size_results(); + let completed_bytes: u64 = sizes + .iter() + .filter_map(|row| row["payload_bytes_completed"].as_u64()) + .sum(); let fields = json!({ "submitted": accepted, "build_attempts": get(Count::Attempts), "submitted_in_window": get(Count::SubmittedInWindow), @@ -165,7 +171,8 @@ pub(super) fn finish( "callbacks_total": get(Count::Responses), "native_total_ops": native.total_ops, "native_total_errors": native.total_errors, "started_loaders": workload.started_loaders(), "late_loaders": get(Count::LateLoaders), "rpc_tps": tps, "logical_ops_s": tps, - "payload_Bps": tps * f64::from(u32::try_from(args.payload_bytes).unwrap_or(u32::MAX)), "latency_count": histogram.count, + "payload_Bps": ratio(completed_bytes, args.duration_secs), + "payload_bytes_completed": completed_bytes, "size_results": sizes, "latency_count": histogram.count, "avg_ns": histogram.avg, "p50_ns": histogram.p50, "p90_ns": histogram.p90, "p95_ns": histogram.p95, "p99_ns": histogram.p99, "p999_ns": histogram.p999, "min_ns": histogram.min, "max_ns": histogram.max, "histogram_zero_count": histogram.zero_count, @@ -174,6 +181,7 @@ pub(super) fn finish( extend(report, &fields); let fields = json!({ "errors_total": errors, "submit_rejects": rejects, "response_errors": get(Count::ResponseErrors), + "build_errors": get(Count::BuildErrors), "timeouts": get(Count::Timeouts), "reaped": counters.reaped, "connection_errors": get(Count::ConnectionErrors), "cq_errors": cq_errors, "payload_mismatches": get(Count::PayloadMismatches), "duplicate_completions": get(Count::Duplicates), diff --git a/tools/rpc-test/src/workload/runtime.rs b/tools/rpc-test/src/workload/runtime.rs index e21b84a81..8b037a09c 100644 --- a/tools/rpc-test/src/workload/runtime.rs +++ b/tools/rpc-test/src/workload/runtime.rs @@ -79,14 +79,23 @@ pub(super) fn validate_wire_bound(args: &RpcArgs, connections: &Value, rdma: boo ); builder.finish(response, None); let control_bytes = builder.finished_data().len().max(build_control(u64::MAX).len()); - let frame_bytes = 14 + control_bytes + args.payload_bytes; + let maximum = args.payload_profile()?.maximum(); + let direct = args.queues.wire_layout == "control-data"; + let frame_bytes = if direct { + 1024 + maximum + } else { + 14 + control_bytes + maximum + }; for connection in connections.as_array().ok_or("missing connection statuses")? { let peer = connection["info"]["peer_receive_slot_capacity"] .as_u64() .ok_or("missing peer receive admission")?; - let bound = peer - .min(u64::from(args.queues.wire_slot_capacity)) - .min(u64::from(args.queues.aggregation_max_bytes)); + let bound = peer.min(u64::from(args.queues.wire_slot_capacity)); + let bound = if direct { + bound + } else { + bound.min(u64::from(args.queues.aggregation_max_bytes)) + }; if frame_bytes as u64 > bound { return Err(format!( "frame_bytes={frame_bytes} exceeds negotiated wire bound={bound} (header/control included)" diff --git a/tools/rpc-test/src/workload/workload.rs b/tools/rpc-test/src/workload/workload.rs index a10e56008..e377acca3 100644 --- a/tools/rpc-test/src/workload/workload.rs +++ b/tools/rpc-test/src/workload/workload.rs @@ -3,7 +3,7 @@ //! Bounded per-loader correlation and per-completion-owner measurement. -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -11,8 +11,11 @@ use crowdb_common::metrics::histogram::HistogramData; use crowdb_common::metrics::{HistogramSnapshot, LatencyHistogram, MetricsRegistry}; use crowdb_protocol::fb::{ConnectionPingRequest, ConnectionPingRequestArgs, ConnectionPingResponse}; use crowdb_rpc_ffi::co_bench::CoBenchHandler; -use crowdb_rpc_ffi::{Buffer, RpcError}; +use crowdb_rpc_ffi::{Buffer, ProducerBufferAllocator, RpcError}; use flatbuffers::FlatBufferBuilder; +use serde_json::{json, Value}; + +use super::payload::PayloadProfile; #[derive(Clone, Copy)] pub(super) enum Count { @@ -28,14 +31,16 @@ pub(super) enum Count { Duplicates, SubmittedInWindow, LateLoaders, + BuildErrors, } -const COUNT: usize = 12; +const COUNT: usize = 13; const EMPTY: u64 = u64::MAX; struct Shard { counts: [AtomicU64; COUNT], latency: Arc, + sizes: Vec>, } struct Slot { @@ -43,13 +48,15 @@ struct Slot { started: AtomicBool, submitted_in_window: AtomicBool, shard: usize, + expected_class: AtomicUsize, } pub(super) struct EchoWorkload { slots: Vec, shards: Vec, mask: u64, - payload_bytes: usize, + profile: PayloadProfile, + allocator: Option, pub measure_start: Instant, pub measure_end: Instant, } @@ -58,7 +65,8 @@ impl EchoWorkload { pub fn new( loaders: u32, owners: &[usize], - payload_bytes: usize, + profile: PayloadProfile, + allocator: Option, warmup: Duration, measure: Duration, registry: &mut MetricsRegistry, @@ -72,16 +80,25 @@ impl EchoWorkload { started: AtomicBool::new(false), submitted_in_window: AtomicBool::new(false), shard: owners[i as usize % owners.len()], + expected_class: AtomicUsize::new(0), }) .collect(), shards: (0..shard_count) .map(|i| Shard { counts: std::array::from_fn(|_| AtomicU64::new(0)), latency: registry.register_histogram(format!("bench.rpc.owner{i}.latency.lh")), + sizes: profile + .sizes() + .iter() + .map(|size| { + registry.register_histogram(format!("bench.rpc.owner{i}.size{size}.latency.lh")) + }) + .collect(), }) .collect(), mask: u64::from(loaders.next_power_of_two() - 1), - payload_bytes, + profile, + allocator, measure_start, measure_end: measure_start + measure, } @@ -119,6 +136,25 @@ impl EchoWorkload { data.snapshot(0) } + pub fn size_results(&self) -> Vec { + self.profile + .sizes() + .iter() + .enumerate() + .map(|(index, size)| { + let mut data = HistogramData::default(); + for shard in &self.shards { + data.merge(&shard.sizes[index].capture_total()); + } + let snapshot = data.snapshot(0); + json!({"payload_bytes": size, "completed": snapshot.count, + "payload_bytes_completed": snapshot.count * *size as u64, + "p50_ns": snapshot.p50, "p99_ns": snapshot.p99, + "p999_ns": snapshot.p999, "max_ns": snapshot.max}) + }) + .collect() + } + fn complete( &self, id: u64, @@ -157,12 +193,14 @@ impl EchoWorkload { _ => {} } } else { - let valid = correlated && valid_response(id, self.payload_bytes, control, data); + let class = slot.expected_class.load(Ordering::Relaxed); + let valid = correlated && valid_response(id, self.profile.sizes()[class], control, data); if !valid { self.inc(slot, Count::PayloadMismatches); } if valid && now >= self.measure_start && now < self.measure_end { self.shards[slot.shard].latency.observe(latency_ns); + self.shards[slot.shard].sizes[class].observe(latency_ns); } } now < self.measure_end @@ -176,17 +214,42 @@ impl CoBenchHandler for EchoWorkload { return None; } let slot = self.slot(id); + let class = self.profile.select(id); + slot.expected_class.store(class, Ordering::Relaxed); if !slot.started.swap(true, Ordering::Relaxed) && now >= self.measure_start { self.inc(slot, Count::LateLoaders); } slot.expected.store(id, Ordering::Release); slot.submitted_in_window .store(now >= self.measure_start, Ordering::Relaxed); + let length = self.profile.sizes()[class]; + let data = if let Some(allocator) = &self.allocator { + if length == 0 { + Some(Buffer::from_vec(Vec::new())) + } else { + allocator + .allocate(u32::try_from(length).ok()?) + .and_then(|buffer| { + buffer + .initialize_with(length, 0xAB, |bytes| { + let prefix = bytes.len().min(8); + bytes[..prefix].copy_from_slice(&id.to_le_bytes()[..prefix]); + }) + .ok() + }) + } + } else { + let mut bytes = vec![0xAB; length]; + let prefix = bytes.len().min(8); + bytes[..prefix].copy_from_slice(&id.to_le_bytes()[..prefix]); + Some(Buffer::from_vec(bytes)) + }; + let Some(data) = data else { + self.inc(slot, Count::BuildErrors); + return None; + }; self.inc(slot, Count::Attempts); - let mut bytes = vec![0xAB; self.payload_bytes]; - let prefix = bytes.len().min(8); - bytes[..prefix].copy_from_slice(&id.to_le_bytes()[..prefix]); - Some((Buffer::from_vec(build_control(id)), Buffer::from_vec(bytes))) + Some((Buffer::from_vec(build_control(id)), data)) } fn on_response(&self, id: u64, status: Result<(), RpcError>, latency_ns: u64) -> bool { diff --git a/tools/rpc-test/tests/validated_echo_test.rs b/tools/rpc-test/tests/validated_echo_test.rs index e237e870f..5ac9722fb 100644 --- a/tools/rpc-test/tests/validated_echo_test.rs +++ b/tools/rpc-test/tests/validated_echo_test.rs @@ -144,6 +144,9 @@ fn strict_rdma_and_invalid_input_produce_setup_failure_reports() { for (verb, args) in [ ("rdma", vec!["--device", "crowdb-missing-device"]), ("rpc", vec!["--loader-count", "0"]), + ("rpc", vec!["--payload-mix", "128:0"]), + ("rpc", vec!["--payload-mix", "128:1,128:2"]), + ("rpc", vec!["--payload-mix", "1048577:1"]), ] { let (output, report) = run(&server, verb, &args); assert!(!output.status.success(), "{output:?}"); @@ -154,3 +157,46 @@ fn strict_rdma_and_invalid_input_produce_setup_failure_reports() { assert!(report["submit_rejects"].is_null()); } } + +#[test] +fn mixed_sizes_validate_each_request_and_account_actual_completed_bytes() { + let server = TestServer::new(); + server.0.register_echo_handler(100); + server.0.start(); + for mode in ["coroutine", "tokio"] { + let (output, report) = run( + &server, + "rpc", + &[ + "--loader-count", + "8", + "--connections", + "2", + "--mode", + mode, + "--payload-mix", + "0:1,127:2,4097:1", + "--payload-seed", + "93", + ], + ); + assert!(output.status.success(), "{output:?}"); + assert_eq!(report["valid"], true, "{report}"); + let sizes = report["size_results"].as_array().unwrap(); + assert_eq!(sizes.len(), 3); + let mut completed = 0; + let mut bytes = 0; + for (row, size) in sizes.iter().zip([0, 127, 4097]) { + assert_eq!(row["payload_bytes"], size); + let count = row["completed"].as_u64().unwrap(); + assert!(count > 0, "{report}"); + completed += count; + bytes += count * row["payload_bytes"].as_u64().unwrap(); + assert!(row["p99_ns"].as_u64().unwrap() <= row["max_ns"].as_u64().unwrap()); + } + assert_eq!(report["completed_in_window"], completed); + assert_eq!(report["payload_bytes_completed"], bytes); + let expected = f64::from(u32::try_from(bytes).unwrap()); + assert!((report["payload_Bps"].as_f64().unwrap() - expected).abs() < f64::EPSILON); + } +} From fd0eab3d31bfdaa311b933e829c8d2ff9e5fef81 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 12:31:51 +0800 Subject: [PATCH 058/258] Add shared native runtime resources and S3 namespace execution --- Cargo.lock | 24 +++ Cargo.toml | 1 + app/crowdb-access-server/src/s3/operations.rs | 86 ++------ doc/working/plan-rdma-fat-client-sdk.md | 34 ++- lib/crowdb-access-s3/src/execution.rs | 145 +++++++++++++ lib/crowdb-access-s3/src/lib.rs | 1 + lib/crowdb-chunk-client/src/client.rs | 2 + .../src/client/observation.rs | 25 +++ .../src/client/writer_policy.rs | 40 ++++ lib/crowdb-chunk-client/src/config.rs | 2 +- .../src/disk_io/routing.rs | 9 + lib/crowdb-diskio-client/src/lib.rs | 4 +- lib/crowdb-diskio-client/src/semantic.rs | 2 + .../src/semantic/observation.rs | 57 +++++ lib/crowdb-sdk-native/Cargo.toml | 39 ++++ lib/crowdb-sdk-native/src/admission.rs | 60 ++++++ lib/crowdb-sdk-native/src/buffer.rs | 187 ++++++++++++++++ lib/crowdb-sdk-native/src/context.rs | 91 ++++++++ lib/crowdb-sdk-native/src/error.rs | 45 ++++ lib/crowdb-sdk-native/src/lib.rs | 19 ++ lib/crowdb-sdk-native/src/runtime.rs | 111 ++++++++++ .../src/runtime/attachment.rs | 52 +++++ .../src/runtime/resources.rs | 151 +++++++++++++ lib/crowdb-sdk-native/src/runtime/storage.rs | 84 ++++++++ .../src/runtime/writer_policy.rs | 63 ++++++ lib/crowdb-sdk-native/src/s3.rs | 103 +++++++++ lib/crowdb-sdk-native/src/topology.rs | 140 ++++++++++++ .../tests/common/metadata.rs | 200 ++++++++++++++++++ lib/crowdb-sdk-native/tests/common/storage.rs | 68 ++++++ lib/crowdb-sdk-native/tests/resources_test.rs | 187 ++++++++++++++++ lib/crowdb-sdk-native/tests/runtime_test.rs | 76 +++++++ 31 files changed, 2036 insertions(+), 72 deletions(-) create mode 100644 lib/crowdb-access-s3/src/execution.rs create mode 100644 lib/crowdb-chunk-client/src/client/observation.rs create mode 100644 lib/crowdb-chunk-client/src/client/writer_policy.rs create mode 100644 lib/crowdb-diskio-client/src/semantic/observation.rs create mode 100644 lib/crowdb-sdk-native/Cargo.toml create mode 100644 lib/crowdb-sdk-native/src/admission.rs create mode 100644 lib/crowdb-sdk-native/src/buffer.rs create mode 100644 lib/crowdb-sdk-native/src/context.rs create mode 100644 lib/crowdb-sdk-native/src/error.rs create mode 100644 lib/crowdb-sdk-native/src/lib.rs create mode 100644 lib/crowdb-sdk-native/src/runtime.rs create mode 100644 lib/crowdb-sdk-native/src/runtime/attachment.rs create mode 100644 lib/crowdb-sdk-native/src/runtime/resources.rs create mode 100644 lib/crowdb-sdk-native/src/runtime/storage.rs create mode 100644 lib/crowdb-sdk-native/src/runtime/writer_policy.rs create mode 100644 lib/crowdb-sdk-native/src/s3.rs create mode 100644 lib/crowdb-sdk-native/src/topology.rs create mode 100644 lib/crowdb-sdk-native/tests/common/metadata.rs create mode 100644 lib/crowdb-sdk-native/tests/common/storage.rs create mode 100644 lib/crowdb-sdk-native/tests/resources_test.rs create mode 100644 lib/crowdb-sdk-native/tests/runtime_test.rs diff --git a/Cargo.lock b/Cargo.lock index ef1b7a699..b18465fea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1274,6 +1274,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "crowdb-sdk-native" +version = "0.3.0" +dependencies = [ + "arc-swap", + "async-trait", + "bincode", + "bytes", + "crowdb-access-multipart", + "crowdb-access-s3", + "crowdb-chunk-client", + "crowdb-chunk-kv-client", + "crowdb-diskio-client", + "crowdb-kv-client", + "crowdb-protocol", + "crowdb-rpc-ffi", + "crowdb-sdk-native", + "futures", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tracing", +] + [[package]] name = "crowdb-test-harness" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index 7d8855b18..e79f46297 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ members = [ "lib/crowdb-access-iceberg", "lib/crowdb-access-dataset", "lib/crowdb-access-multipart", + "lib/crowdb-sdk-native", ] exclude = ["third-party/hyper", "sdk"] # : crowdb-tree/ffi moved from `exclude` into `members` now that diff --git a/app/crowdb-access-server/src/s3/operations.rs b/app/crowdb-access-server/src/s3/operations.rs index 47d7a6f90..3da8a01d8 100644 --- a/app/crowdb-access-server/src/s3/operations.rs +++ b/app/crowdb-access-server/src/s3/operations.rs @@ -7,14 +7,14 @@ use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use base64::Engine as _; -use crowdb_access_s3::bucket::{self, BucketError, DeleteBucketResult, RandomBucketIdGenerator}; use crowdb_access_s3::condition::ObjectConditions; use crowdb_access_s3::continuation::ContinuationTokenSigner; use crowdb_access_s3::error::{S3Error, S3ErrorCode}; +use crowdb_access_s3::execution::S3Executor; use crowdb_access_s3::metadata::{ObjectRecord, TenantId}; use crowdb_access_s3::object::{self, ListObjectsV2Request, ObjectMetadataError}; use crowdb_access_s3::publication::PublicationRequest; -use crowdb_access_s3::retrieval::{self, ObjectHeaders, RetrievalError}; +use crowdb_access_s3::retrieval::{ObjectHeaders, RetrievalError}; use crowdb_access_s3::route::{S3Operation, S3Route}; use crowdb_access_s3::streaming::{publish_completed_locations, PutErrorCode, PutOutcome}; use crowdb_chunk_client::{ @@ -99,7 +99,7 @@ pub struct ProductionS3Operations { pub(super) storage: S3StorageClients, pub(super) config: S3ServiceConfig, signer: ContinuationTokenSigner, - bucket_ids: RandomBucketIdGenerator, + execution: S3Executor, metrics: Option>, } @@ -113,10 +113,10 @@ impl ProductionS3Operations { let signer = ContinuationTokenSigner::new(config.continuation_key.clone()) .map_err(|_| S3ErrorCode::InvalidRequest)?; Ok(Self { + execution: S3Executor::new(storage.clone(), config.tenant.clone()), storage, config, signer, - bucket_ids: RandomBucketIdGenerator, metrics: None, }) } @@ -165,50 +165,25 @@ impl ProductionS3Operations { async fn create_bucket(&self, route: S3Route) -> Result, S3ErrorCode> { let name = required_bucket(&route)?; - bucket::create_bucket( - &self.storage.metadata, - &self.bucket_ids, - &self.config.tenant, - name, - ) - .await - .map_err(|error| map_bucket_error(&error))?; + self.execution.create_bucket(name).await?; response(StatusCode::OK, Vec::new()) } async fn head_bucket(&self, route: S3Route) -> Result, S3ErrorCode> { let name = required_bucket(&route)?; - let exists = bucket::head_bucket(&self.storage.metadata, &self.config.tenant, name) - .await - .map_err(|error| map_bucket_error(&error))? - .is_some(); - if !exists { - return Err(S3ErrorCode::NoSuchBucket); - } + self.execution.resolve_bucket(name).await?; response(StatusCode::OK, Vec::new()) } async fn list_buckets(&self) -> Result, S3ErrorCode> { - let buckets = bucket::list_buckets(&self.storage.metadata, &self.config.tenant, DEFAULT_LIST_LIMIT) - .await - .map_err(|error| { - tracing::error!(%error, "S3 bucket listing failed; returning a service error"); - map_bucket_error(&error) - })?; + let buckets = self.execution.list_buckets(DEFAULT_LIST_LIMIT).await?; xml_response(wire::list_buckets(self.config.tenant.as_bytes(), &buckets)) } async fn delete_bucket(&self, route: S3Route) -> Result, S3ErrorCode> { let name = required_bucket(&route)?; - match bucket::delete_bucket(&self.storage.metadata, &self.config.tenant, name) - .await - .map_err(|error| map_bucket_error(&error))? - { - DeleteBucketResult::Deleted => response(StatusCode::NO_CONTENT, Vec::new()), - DeleteBucketResult::Missing => Err(S3ErrorCode::NoSuchBucket), - DeleteBucketResult::NotEmpty => Err(S3ErrorCode::BucketNotEmpty), - DeleteBucketResult::Conflict => Err(S3ErrorCode::ServiceUnavailable), - } + self.execution.delete_bucket(name).await?; + response(StatusCode::NO_CONTENT, Vec::new()) } async fn put_object( @@ -309,7 +284,7 @@ impl ProductionS3Operations { request: &Request, ) -> Result, S3ErrorCode> { let record = self.object(&route).await?; - let headers = match retrieval::prepare_head(&record, &conditions(request)) { + let headers = match self.execution.prepare_head(&record, &conditions(request)) { Ok(headers) => headers, Err(RetrievalError::NotModified) => return not_modified(&record.etag), Err(error) => return Err(map_retrieval(&error)), @@ -324,12 +299,10 @@ impl ProductionS3Operations { request_id: &str, ) -> Result, S3ErrorCode> { let record = self.object(&route).await?; - let prepared = match retrieval::prepare_get( - &self.storage.chunks, - &record, - header(request, RANGE), - &conditions(request), - ) { + let prepared = match self + .execution + .prepare_get(&record, header(request, RANGE), &conditions(request)) + { Ok(prepared) => prepared, Err(RetrievalError::NotModified) => return not_modified(&record.etag), Err(error) => return Err(map_retrieval(&error)), @@ -416,31 +389,18 @@ impl ProductionS3Operations { async fn delete_object(&self, route: S3Route) -> Result, S3ErrorCode> { let bucket_name = required_bucket(&route)?; let key = required_key(&route)?; - let bucket_id = self.resolve_bucket(bucket_name).await?; - object::delete(&self.storage.metadata, &self.config.tenant, bucket_id, key) - .await - .map_err(|error| map_object_error(&error))?; + self.execution.delete_object(bucket_name, key).await?; response(StatusCode::NO_CONTENT, Vec::new()) } async fn resolve_bucket(&self, name: &[u8]) -> Result { - bucket::head_bucket(&self.storage.metadata, &self.config.tenant, name) - .await - .map_err(|error| map_bucket_error(&error))? - .ok_or(S3ErrorCode::NoSuchBucket) + self.execution.resolve_bucket(name).await } async fn object(&self, route: &S3Route) -> Result { - let bucket = self.resolve_bucket(required_bucket(route)?).await?; - object::head( - &self.storage.metadata, - &self.config.tenant, - bucket, - required_key(route)?, - ) - .await - .map_err(|error| map_object_error(&error))? - .ok_or(S3ErrorCode::NoSuchKey) + self.execution + .object(required_bucket(route)?, required_key(route)?) + .await } async fn prepare_writer( @@ -668,14 +628,6 @@ fn map_retrieval(error: &RetrievalError) -> S3ErrorCode { } } -fn map_bucket_error(error: &BucketError) -> S3ErrorCode { - match error { - BucketError::Key(_) => S3ErrorCode::InvalidRequest, - BucketError::Record(_) => S3ErrorCode::InternalError, - BucketError::Store(_) => S3ErrorCode::ServiceUnavailable, - } -} - fn map_object_error(error: &ObjectMetadataError) -> S3ErrorCode { match error { ObjectMetadataError::Key(_) diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index 5c0e3c24d..0b48f38fd 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -40,12 +40,12 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. ## Phase 1: shared runtime and native API -- [~] **Create native crate and baseline**: register `crowdb-sdk-native` with +- [x] **Create native crate and baseline**: register `crowdb-sdk-native` with `runtime`, `topology`, `buffer`, and `s3` modules and tests under `tests/`. Run focused existing S3/RPC/access and standalone SDK baselines before moving operation code. Files: `Cargo.toml`, `lib/crowdb-sdk-native/`, `lib/crowdb-access-s3/tests/`, `app/crowdb-access-server/tests/`. -- [ ] **Define runtime ownership**: implement `ClientRuntime::connect(config)` +- [~] **Define runtime ownership**: implement `ClientRuntime::connect(config)` and shutdown, sharing discovery, compatible transport groups, producer allocation, and hard request/byte limits across handles. Preserve distinct S3/Dataset chunk types and writer policies; do not create a generic Repo @@ -204,6 +204,36 @@ R242 exit: accepted runtime/topology/lease APIs and native S3 semantics, TCP/RDMA parity, bounded pressure, and shutdown-safe caller ownership. Existing transport acceptance alone cannot establish SDK completion. +## Execution evidence + +- Mixed-size optimization prerequisite is committed as `f3de1109`; repeated + final HCA measurements and service parity pass under the root RDMA plan. +- Existing S3 library all-target tests, Access Server all-target tests, and + standalone SDK boundary/functional controls pass before extraction. +- `crowdb-sdk-native` is registered in the main workspace. Initial tests cover + actual Vec capacity, shared-slice accounting, frozen native producer owners, + request cancellation/deadline, shutdown-safe held data, two tenant handles + without an HTTP listener/credential store, and separate typed writer groups. + Nine native cases pass, including asynchronous-copy and interrupted-mutation + outcomes. The S3, ChunkIO and DiskIO all-target suites, affected HTTP tests + and focused four-crate all-target clippy also pass. +- Runtime connection and attachment reuse control/catalog/chunk clients. + Compatible writer policies share a bounded group; incompatible policies get + separate small-writer pools sharing transports, reader and SDK budgets. + Shutdown currently closes SDK admission and drains small writers. Completing + native transport/completion teardown remains in Phase 4. +- Disk routes now have a retained inspection view with local generation and + publication age. Native catalog/service/transport observations do not invent + a global epoch. Attached catalog age is explicitly unknown until refreshed. + Full stale-view/locality acceptance remains outstanding. +- Basic bucket and point object lookup/delete orchestration is extracted into + `S3Executor`, and HTTP delegates to it. Streaming GET/PUT, copy, multipart, + typed native results and real HTTP/native parity are not complete. +- The existing FFI completion still has detach/copy branches. The initial + `BufferLease` tests establish its own storage/credit behavior only; they do + not establish end-to-end native RDMA receive ownership or actual-capacity + propagation through DiskIO/ChunkIO. Phase 4 must complete that audit and path. + ## Subsequent Dataset integration under R220 - [ ] **Attach Dataset after S3**: add a facade using the same runtime, diff --git a/lib/crowdb-access-s3/src/execution.rs b/lib/crowdb-access-s3/src/execution.rs new file mode 100644 index 000000000..565f553b1 --- /dev/null +++ b/lib/crowdb-access-s3/src/execution.rs @@ -0,0 +1,145 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Trusted S3 execution shared by HTTP parsing and native business handles. + +use std::sync::Arc; + +use crate::bucket::{self, BucketError, DeleteBucketResult, RandomBucketIdGenerator}; +use crate::condition::ObjectConditions; +use crate::error::S3ErrorCode; +use crate::metadata::{BucketId, BucketNameRecord, ObjectRecord, TenantId}; +use crate::object::{self, ObjectMetadataError}; +use crate::retrieval::{self, ObjectHeaders, PreparedGet, RetrievalError}; +use crate::storage::S3StorageClients; + +/// Namespace and storage owners only; credential verification belongs to the +/// frontend. Cloned executors retain exactly the same connection groups. +#[derive(Clone)] +pub struct S3Executor { + pub(crate) storage: S3StorageClients, + tenant: TenantId, +} + +impl S3Executor { + #[must_use] + pub fn new(storage: S3StorageClients, tenant: TenantId) -> Self { + Self { storage, tenant } + } + + #[must_use] + pub fn chunks(&self) -> &Arc { + &self.storage.chunks + } + + #[must_use] + pub fn tenant(&self) -> &TenantId { + &self.tenant + } + + /// # Errors + /// Returns namespace validation or routed metadata failures. + pub async fn create_bucket(&self, name: &[u8]) -> Result { + bucket::create_bucket( + &self.storage.metadata, + &RandomBucketIdGenerator, + &self.tenant, + name, + ) + .await + .map_err(|error| bucket_error(&error)) + } + + /// # Errors + /// Returns a missing bucket or routed metadata failure. + pub async fn resolve_bucket(&self, name: &[u8]) -> Result { + bucket::head_bucket(&self.storage.metadata, &self.tenant, name) + .await + .map_err(|error| bucket_error(&error))? + .ok_or(S3ErrorCode::NoSuchBucket) + } + + /// # Errors + /// Returns namespace validation or bounded metadata scan failure. + pub async fn list_buckets(&self, limit: usize) -> Result, S3ErrorCode> { + if !(1..=1000).contains(&limit) { + return Err(S3ErrorCode::InvalidRequest); + } + bucket::list_buckets(&self.storage.metadata, &self.tenant, limit) + .await + .map_err(|error| bucket_error(&error)) + } + + /// # Errors + /// Returns missing/nonempty bucket, conflict or metadata failure. + pub async fn delete_bucket(&self, name: &[u8]) -> Result<(), S3ErrorCode> { + match bucket::delete_bucket(&self.storage.metadata, &self.tenant, name) + .await + .map_err(|error| bucket_error(&error))? + { + DeleteBucketResult::Deleted => Ok(()), + DeleteBucketResult::Missing => Err(S3ErrorCode::NoSuchBucket), + DeleteBucketResult::NotEmpty => Err(S3ErrorCode::BucketNotEmpty), + DeleteBucketResult::Conflict => Err(S3ErrorCode::ServiceUnavailable), + } + } + + /// Resolves one immutable object generation. Subsequent reads retain it. + /// # Errors + /// Returns missing bucket/key, invalid metadata or routed storage failure. + pub async fn object(&self, bucket: &[u8], key: &[u8]) -> Result { + let bucket_id = self.resolve_bucket(bucket).await?; + object::head(&self.storage.metadata, &self.tenant, bucket_id, key) + .await + .map_err(|error| object_error(&error))? + .ok_or(S3ErrorCode::NoSuchKey) + } + + /// # Errors + /// Returns namespace or metadata mutation failure. Physical reclamation is deferred. + pub async fn delete_object(&self, bucket: &[u8], key: &[u8]) -> Result<(), S3ErrorCode> { + let bucket_id = self.resolve_bucket(bucket).await?; + object::delete(&self.storage.metadata, &self.tenant, bucket_id, key) + .await + .map_err(|error| object_error(&error)) + } + + /// # Errors + /// Returns the same conditional outcome for HTTP and native callers. + pub fn prepare_head( + &self, + record: &ObjectRecord, + conditions: &ObjectConditions<'_>, + ) -> Result { + retrieval::prepare_head(record, conditions) + } + + /// # Errors + /// Rejects invalid ranges/conditions without replacing the selected generation. + pub fn prepare_get( + &self, + record: &ObjectRecord, + range: Option<&str>, + conditions: &ObjectConditions<'_>, + ) -> Result { + retrieval::prepare_get(&self.storage.chunks, record, range, conditions) + } +} + +pub(crate) fn bucket_error(error: &BucketError) -> S3ErrorCode { + match error { + BucketError::Key(_) => S3ErrorCode::InvalidRequest, + BucketError::Record(_) => S3ErrorCode::InternalError, + BucketError::Store(_) => S3ErrorCode::ServiceUnavailable, + } +} + +pub(crate) fn object_error(error: &ObjectMetadataError) -> S3ErrorCode { + match error { + ObjectMetadataError::Key(_) + | ObjectMetadataError::Continuation(_) + | ObjectMetadataError::InvalidListLimit => S3ErrorCode::InvalidRequest, + ObjectMetadataError::Record(_) => S3ErrorCode::InternalError, + ObjectMetadataError::Store(_) => S3ErrorCode::ServiceUnavailable, + } +} diff --git a/lib/crowdb-access-s3/src/lib.rs b/lib/crowdb-access-s3/src/lib.rs index ae09590dd..bbce6e01f 100644 --- a/lib/crowdb-access-s3/src/lib.rs +++ b/lib/crowdb-access-s3/src/lib.rs @@ -10,6 +10,7 @@ pub mod continuation; pub mod copy; pub mod delete; pub mod error; +pub mod execution; pub mod inspection; pub mod integrity; pub mod metadata; diff --git a/lib/crowdb-chunk-client/src/client.rs b/lib/crowdb-chunk-client/src/client.rs index 37d670144..5bd8dfd1a 100644 --- a/lib/crowdb-chunk-client/src/client.rs +++ b/lib/crowdb-chunk-client/src/client.rs @@ -9,6 +9,8 @@ use disk_metrics::MetricsDiskWriter; mod readiness; #[cfg(feature = "test-util")] pub use readiness::observe_readiness_phase_for_tests; +mod observation; +mod writer_policy; use std::collections::VecDeque; use std::sync::Arc; diff --git a/lib/crowdb-chunk-client/src/client/observation.rs b/lib/crowdb-chunk-client/src/client/observation.rs new file mode 100644 index 000000000..89aa94079 --- /dev/null +++ b/lib/crowdb-chunk-client/src/client/observation.rs @@ -0,0 +1,25 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::ChunkIoClient; + +impl ChunkIoClient { + /// Retains existing disk routes without opening any new connection group. + #[must_use] + pub fn disk_topology(&self) -> Option { + self.topology + .as_ref() + .map(|topology| topology.disk_writer.topology()) + } + + /// Actual native transport configuration and counters for these routes. + /// # Errors + /// Returns unavailable topology for clients constructed from low-level seams. + pub fn transport_status_json(&self) -> crate::Result> { + self.topology + .as_ref() + .ok_or_else(|| crate::IoError::Topology("client has no discovered production topology".into()))? + .disk_writer + .transport_status_json() + } +} diff --git a/lib/crowdb-chunk-client/src/client/writer_policy.rs b/lib/crowdb-chunk-client/src/client/writer_policy.rs new file mode 100644 index 000000000..92166ba28 --- /dev/null +++ b/lib/crowdb-chunk-client/src/client/writer_policy.rs @@ -0,0 +1,40 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use super::{ChunkIoClient, FailedDiskList, SmallWriteMetrics, SmallWritePolicy, SmallWritePool}; + +impl ChunkIoClient { + /// The immutable policy owned by this concrete writer pool. + #[must_use] + pub fn small_write_policy(&self) -> &Arc { + &self.small_pool.policy + } + + /// Shares discovered transports, routes and read credits while constructing + /// a separate small-writer pool for an explicitly selected chunk purpose. + /// # Errors + /// Rejects invalid writer policy before any background manager starts. + pub fn with_writer_policy(&self, policy: SmallWritePolicy) -> crate::Result { + let failed_disks = Arc::new(FailedDiskList::new(policy.failed_disk_ttl)); + let small_pool = SmallWritePool::new( + Arc::clone(&self.allocator), + Arc::clone(&self.disk_writer), + policy, + Arc::new(SmallWriteMetrics::default()), + Arc::clone(&failed_disks), + )?; + Ok(Self { + allocator: Arc::clone(&self.allocator), + disk_writer: Arc::clone(&self.disk_writer), + topology: self.topology.clone(), + metrics: self.metrics.clone(), + small_pool, + reader: self.reader.clone(), + failed_disks, + large_write_repair: Arc::clone(&self.large_write_repair), + large_write_buffer: Arc::clone(&self.large_write_buffer), + }) + } +} diff --git a/lib/crowdb-chunk-client/src/config.rs b/lib/crowdb-chunk-client/src/config.rs index b9783b6d0..da36c3ac3 100644 --- a/lib/crowdb-chunk-client/src/config.rs +++ b/lib/crowdb-chunk-client/src/config.rs @@ -16,7 +16,7 @@ use crowdb_protocol::chunkdb::rpc::ChunkType; use crate::IoError; /// Bounded aggregation and elasticity policy for shared small writes. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct SmallWritePolicy { /// Type assigned to every chunk owned by this pool. pub chunk_type: ChunkType, diff --git a/lib/crowdb-chunk-client/src/disk_io/routing.rs b/lib/crowdb-chunk-client/src/disk_io/routing.rs index ff8baedb2..0913cb380 100644 --- a/lib/crowdb-chunk-client/src/disk_io/routing.rs +++ b/lib/crowdb-chunk-client/src/disk_io/routing.rs @@ -21,6 +21,15 @@ pub struct RoutedDiskWriter { } impl RoutedDiskWriter { + #[must_use] + pub fn topology(&self) -> crowdb_diskio_client::DiskTopologyView { + self.client.topology() + } + + pub fn transport_status_json(&self) -> Result> { + self.client.transport_status_json().map_err(map_topology_error) + } + /// Reuse an already connected semantic client in an embedded storage stack. #[must_use] pub fn from_client(client: Arc) -> Self { diff --git a/lib/crowdb-diskio-client/src/lib.rs b/lib/crowdb-diskio-client/src/lib.rs index 69a10cdc7..5cf446040 100644 --- a/lib/crowdb-diskio-client/src/lib.rs +++ b/lib/crowdb-diskio-client/src/lib.rs @@ -13,8 +13,8 @@ mod topology; pub use address::{DiskId, SegmentTarget}; pub use error::{DiskioError, DiskioResult}; pub use semantic::{ - DiskioClient, DiskioClientConfig, Durability, NativeDiskIoRoutes, NativeDiskRouteResolver, - OperationOptions, TrafficLane, + DiskPlacement, DiskTopologyView, DiskioClient, DiskioClientConfig, Durability, NativeDiskIoRoutes, + NativeDiskRouteResolver, OperationOptions, TrafficLane, }; pub use status::DiskioStatus; diff --git a/lib/crowdb-diskio-client/src/semantic.rs b/lib/crowdb-diskio-client/src/semantic.rs index a769e9db0..8a0c28b82 100644 --- a/lib/crowdb-diskio-client/src/semantic.rs +++ b/lib/crowdb-diskio-client/src/semantic.rs @@ -23,8 +23,10 @@ use crate::{DiskId, DiskioError, DiskioResult, DiskioStatus, SegmentTarget}; mod connection; mod native_routes; +mod observation; pub use native_routes::NativeDiskRouteResolver; mod drain; +pub use observation::{DiskPlacement, DiskTopologyView}; mod write; mod write_views; diff --git a/lib/crowdb-diskio-client/src/semantic/observation.rs b/lib/crowdb-diskio-client/src/semantic/observation.rs new file mode 100644 index 000000000..8e2fe667a --- /dev/null +++ b/lib/crowdb-diskio-client/src/semantic/observation.rs @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; +use std::time::Duration; + +use super::{DiskioClient, RouteSnapshot}; +use crate::DiskId; + +/// An inspectable retained route observation, never a placement authority. +#[derive(Clone, Debug)] +pub struct DiskTopologyView { + snapshot: Arc, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DiskPlacement { + pub disk_id: DiskId, + pub rack_id: u64, + pub node_id: u64, + pub disk_group_id: u64, + pub instance_id: u64, + pub endpoint: Arc, +} + +impl DiskTopologyView { + /// Process-local route-cache generation, independent of catalog epochs. + #[must_use] + pub fn generation(&self) -> u64 { + self.snapshot.generation + } + + #[must_use] + pub fn age(&self) -> Duration { + self.snapshot.published_at.elapsed() + } + + pub fn disks(&self) -> impl Iterator + '_ { + self.snapshot.routes.iter().map(|(disk_id, route)| DiskPlacement { + disk_id: *disk_id, + rack_id: route.rack_id, + node_id: route.node_id, + disk_group_id: route.disk_group_id, + instance_id: route.instance_id, + endpoint: Arc::clone(&route.endpoint), + }) + } +} + +impl DiskioClient { + #[must_use] + pub fn topology(&self) -> DiskTopologyView { + DiskTopologyView { + snapshot: self.routes.load_full(), + } + } +} diff --git a/lib/crowdb-sdk-native/Cargo.toml b/lib/crowdb-sdk-native/Cargo.toml new file mode 100644 index 000000000..f32abbed0 --- /dev/null +++ b/lib/crowdb-sdk-native/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "crowdb-sdk-native" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Trusted embedded CROWDB runtime and native S3 client" + +[lints] +workspace = true + +[features] +test-util = [] + +[dependencies] +bytes = "1" +arc-swap = "1.9" +bincode = "1" +crowdb-access-s3 = { path = "../crowdb-access-s3" } +crowdb-access-multipart = { path = "../crowdb-access-multipart" } +crowdb-chunk-client = { path = "../crowdb-chunk-client" } +crowdb-chunk-kv-client = { path = "../crowdb-chunk-kv-client" } +crowdb-diskio-client = { path = "../crowdb-diskio-client" } +crowdb-kv-client = { path = "../crowdb-kv-client" } +crowdb-protocol = { path = "../crowdb-protocol" } +crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi" } +futures = "0.3" +thiserror = { workspace = true } +tokio = { workspace = true, features = ["time", "sync", "macros"] } +tracing = { workspace = true } + +[dev-dependencies] +async-trait = "0.1" +crowdb-sdk-native = { path = ".", features = ["test-util"] } +crowdb-chunk-client = { path = "../crowdb-chunk-client", features = ["test-util"] } +crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi", features = ["test-util"] } +tokio = { workspace = true, features = ["rt-multi-thread", "test-util"] } +serde_json = "1" diff --git a/lib/crowdb-sdk-native/src/admission.rs b/lib/crowdb-sdk-native/src/admission.rs new file mode 100644 index 000000000..ea4478ea0 --- /dev/null +++ b/lib/crowdb-sdk-native/src/admission.rs @@ -0,0 +1,60 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Lock-free credits; shared views retain one allocation charge. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use crate::{CapacityKind, Error, Result}; + +pub(crate) struct Budget { + limit: usize, + used: AtomicUsize, + high_water: AtomicUsize, + kind: CapacityKind, +} + +impl Budget { + pub fn new(limit: usize, kind: CapacityKind) -> Arc { + Arc::new(Self { + limit, + used: AtomicUsize::new(0), + high_water: AtomicUsize::new(0), + kind, + }) + } + + pub fn charge(self: &Arc, amount: usize) -> Result { + let before = self + .used + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |used| { + used.checked_add(amount).filter(|after| *after <= self.limit) + }) + .map_err(|_| Error::Capacity(self.kind))?; + self.high_water.fetch_max(before + amount, Ordering::Relaxed); + Ok(Credit { + budget: Arc::clone(self), + amount, + }) + } + + pub fn used(&self) -> usize { + self.used.load(Ordering::Acquire) + } + + pub fn high_water(&self) -> usize { + self.high_water.load(Ordering::Relaxed) + } +} + +pub(crate) struct Credit { + budget: Arc, + amount: usize, +} + +impl Drop for Credit { + fn drop(&mut self) { + self.budget.used.fetch_sub(self.amount, Ordering::AcqRel); + } +} diff --git a/lib/crowdb-sdk-native/src/buffer.rs b/lib/crowdb-sdk-native/src/buffer.rs new file mode 100644 index 000000000..962b0944a --- /dev/null +++ b/lib/crowdb-sdk-native/src/buffer.rs @@ -0,0 +1,187 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Immutable host leases whose shared views charge one backing allocation. + +use std::ops::Range; +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_rpc_ffi::{Buffer, ProducerBufferAllocator, WritableBuffer}; + +use crate::admission::Credit; +use crate::{CapacityKind, Error, Result, RuntimeResources}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct LeaseMetadata { + pub request_id: u64, + pub stream_id: u64, + pub sequence: u64, + pub logical_offset: u64, + pub length: usize, + pub allocation_capacity: usize, + pub registration_id: u32, +} + +struct Backing { + // Storage is dropped before admission is returned to another producer. + bytes: Bytes, + _credit: Credit, +} + +#[derive(Clone)] +pub struct BufferLease { + backing: Arc, + range: Range, + metadata: LeaseMetadata, +} + +impl BufferLease { + #[must_use] + pub fn as_slice(&self) -> &[u8] { + &self.backing.bytes[self.range.clone()] + } + + #[must_use] + pub fn metadata(&self) -> LeaseMetadata { + self.metadata + } + + /// # Errors + /// Rejects ranges outside the logical payload or overflowing offsets. + pub fn slice(&self, range: Range) -> Result { + if range.start > range.end || range.end > self.range.len() { + return Err(Error::BufferRange); + } + let mut metadata = self.metadata; + metadata.logical_offset = metadata + .logical_offset + .checked_add(range.start as u64) + .ok_or(Error::BufferRange)?; + metadata.length = range.len(); + Ok(Self { + backing: Arc::clone(&self.backing), + range: self.range.start + range.start..self.range.start + range.end, + metadata, + }) + } + + /// Transfers the same backing owner to a storage writer or asynchronous copy. + #[must_use] + pub fn into_bytes(self) -> Bytes { + Bytes::from_owner(self) + } + + fn new(bytes: Bytes, metadata: LeaseMetadata, credit: Credit) -> Self { + Self { + range: 0..bytes.len(), + backing: Arc::new(Backing { + bytes, + _credit: credit, + }), + metadata, + } + } +} + +impl AsRef<[u8]> for BufferLease { + fn as_ref(&self) -> &[u8] { + self.as_slice() + } +} + +pub struct WritableLease { + buffer: WritableBuffer, + // Unique writable storage is released before its capacity credit. + credit: Credit, +} + +impl WritableLease { + #[must_use] + pub fn capacity(&self) -> usize { + self.buffer.capacity() + } + + /// Initializes a final allocation once and freezes it for immutable submission. + /// # Errors + /// Rejects empty or oversized initialized prefixes. + pub fn initialize_with( + self, + length: usize, + initial: u8, + initialize: impl FnOnce(&mut [u8]), + ) -> Result { + let buffer = self + .buffer + .initialize_with(length, initial, initialize) + .map_err(|_| Error::BufferRange)?; + Ok(freeze(buffer, self.credit)) + } + + /// # Errors + /// Rejects empty input or input larger than the unique allocation. + pub fn write(self, bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Err(Error::BufferRange); + } + let buffer = self.buffer.write(bytes).map_err(|_| Error::BufferRange)?; + Ok(freeze(buffer, self.credit)) + } +} + +fn freeze(buffer: Buffer, credit: Credit) -> BufferLease { + let native = buffer.lease_metadata(); + let metadata = LeaseMetadata { + request_id: native.request_id, + stream_id: native.stream_id, + sequence: native.sequence, + logical_offset: native.logical_offset, + length: buffer.as_slice().len(), + allocation_capacity: native.capacity as usize, + registration_id: native.registration_id, + }; + BufferLease::new(Bytes::from_owner(buffer), metadata, credit) +} + +impl RuntimeResources { + /// Retains ordinary caller storage with its actual Vec allocation capacity. + /// # Errors + /// Rejects closed admission, invalid identity/offset or exhausted held capacity. + pub fn retain_vec( + &self, + data: Vec, + request_id: u64, + stream_id: u64, + sequence: u64, + logical_offset: u64, + ) -> Result { + self.check_open()?; + if request_id == 0 || logical_offset.checked_add(data.len() as u64).is_none() { + return Err(Error::BufferRange); + } + let metadata = LeaseMetadata { + request_id, + stream_id, + sequence, + logical_offset, + length: data.len(), + allocation_capacity: data.capacity(), + registration_id: 0, + }; + let credit = self.held.charge(data.capacity())?; + Ok(BufferLease::new(Bytes::from(data), metadata, credit)) + } + + /// Reuses a service's bounded producer allocator; no staging Vec is created. + /// # Errors + /// Returns producer capacity exhaustion, invalid size or closed admission. + pub fn allocate(&self, allocator: &ProducerBufferAllocator, capacity: u32) -> Result { + self.check_open()?; + let buffer = allocator + .allocate(capacity) + .ok_or(Error::Capacity(CapacityKind::Producers))?; + let credit = self.producers.charge(buffer.capacity())?; + self.check_open()?; + Ok(WritableLease { buffer, credit }) + } +} diff --git a/lib/crowdb-sdk-native/src/context.rs b/lib/crowdb-sdk-native/src/context.rs new file mode 100644 index 000000000..b5df01dee --- /dev/null +++ b/lib/crowdb-sdk-native/src/context.rs @@ -0,0 +1,91 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Caller request identity, deadline and cancellation ownership. + +use std::future::Future; +use std::num::NonZeroU64; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Instant; + +use tokio::sync::Notify; + +use crate::{Error, Result}; + +#[derive(Default)] +struct CancellationState { + cancelled: AtomicBool, + wake: Notify, +} + +#[derive(Clone, Default)] +pub struct Cancellation(Arc); + +impl Cancellation { + pub fn cancel(&self) { + self.0.cancelled.store(true, Ordering::Release); + self.0.wake.notify_waiters(); + } + + #[must_use] + pub fn is_cancelled(&self) -> bool { + self.0.cancelled.load(Ordering::Acquire) + } + + pub async fn cancelled(&self) { + loop { + let wake = self.0.wake.notified(); + tokio::pin!(wake); + wake.as_mut().enable(); + if self.is_cancelled() { + return; + } + wake.await; + } + } +} + +#[derive(Clone)] +pub struct OperationContext { + pub request_id: NonZeroU64, + pub deadline: Instant, + pub cancellation: Cancellation, +} + +impl OperationContext { + #[must_use] + pub fn new(request_id: NonZeroU64, deadline: Instant) -> Self { + Self { + request_id, + deadline, + cancellation: Cancellation::default(), + } + } + + /// # Errors + /// Returns cancellation or an expired total deadline before starting work. + pub fn check(&self) -> Result<()> { + if self.cancellation.is_cancelled() { + Err(Error::Cancelled) + } else if Instant::now() >= self.deadline { + Err(Error::Deadline) + } else { + Ok(()) + } + } + + /// Bounds a cancellable read or wait. Mutation publication must retain its + /// own uncertain outcome if this future is interrupted after acceptance. + /// # Errors + /// Returns cancellation, deadline, or the supplied operation's failure. + pub async fn run(&self, future: impl Future>) -> Result { + self.check()?; + tokio::select! { + biased; + () = self.cancellation.cancelled() => Err(Error::Cancelled), + () = tokio::time::sleep_until(self.deadline.into()) => Err(Error::Deadline), + result = future => result, + } + } +} diff --git a/lib/crowdb-sdk-native/src/error.rs b/lib/crowdb-sdk-native/src/error.rs new file mode 100644 index 000000000..82a7f15fd --- /dev/null +++ b/lib/crowdb-sdk-native/src/error.rs @@ -0,0 +1,45 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Native scheduling and semantic failures, independent of HTTP encoding. + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CapacityKind { + Requests, + Producers, + HeldBuffers, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MutationInterruption { + Cancelled, + Deadline, + Shutdown, +} + +#[derive(Clone, Debug, thiserror::Error, Eq, PartialEq)] +pub enum Error { + #[error("invalid native client configuration: {0}")] + Configuration(String), + #[error("native runtime is shut down")] + Shutdown, + #[error("operation was cancelled")] + Cancelled, + #[error("operation deadline expired")] + Deadline, + #[error("native capacity exhausted: {0:?}")] + Capacity(CapacityKind), + #[error("native storage unavailable: {0}")] + Storage(String), + #[error("invalid buffer range or initialized length")] + BufferRange, + #[error("S3 operation failed: {0:?}")] + S3(crowdb_access_s3::S3ErrorCode), + #[error("mutation outcome is uncertain: request={request_id} reason={reason:?}")] + UncertainMutation { + request_id: u64, + reason: MutationInterruption, + }, +} + +pub type Result = std::result::Result; diff --git a/lib/crowdb-sdk-native/src/lib.rs b/lib/crowdb-sdk-native/src/lib.rs new file mode 100644 index 000000000..e6ccb926e --- /dev/null +++ b/lib/crowdb-sdk-native/src/lib.rs @@ -0,0 +1,19 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Trusted internal storage access with shared routing and retained host memory. +//! +//! This crate belongs to the main workspace. The independently distributed +//! HTTP SDK has no dependency on this native storage and FFI stack. + +mod admission; +pub mod buffer; +pub mod context; +pub mod error; +pub mod runtime; +pub mod s3; +pub mod topology; + +pub use context::{Cancellation, OperationContext}; +pub use error::{CapacityKind, Error, Result}; +pub use runtime::{ClientRuntime, RuntimeConfig, RuntimeLimits, RuntimeResources}; diff --git a/lib/crowdb-sdk-native/src/runtime.rs b/lib/crowdb-sdk-native/src/runtime.rs new file mode 100644 index 000000000..39b8b7e7f --- /dev/null +++ b/lib/crowdb-sdk-native/src/runtime.rs @@ -0,0 +1,111 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Shared cluster clients and admission lifetime, independent of object handles. + +mod attachment; +mod resources; +mod storage; +mod writer_policy; + +use std::sync::Arc; +use std::time::Duration; + +use arc_swap::ArcSwap; + +use crowdb_chunk_client::{ChunkReadPolicy, SmallWritePolicy}; +use crowdb_chunk_kv_client::ChunkKvRpcTransportConfig; +use crowdb_protocol::chunkdb::rpc::ChunkType; + +use crate::{Error, Result}; + +pub use attachment::RuntimeClients; +pub use resources::{RequestLease, ResourceStatus, RuntimeLimits, RuntimeResources}; +pub(crate) use storage::Storage; + +pub struct RuntimeConfig { + pub management_seeds: Vec, + pub diskio_connections_per_endpoint: usize, + pub diskio_rpc_workers: u32, + pub small_write: SmallWritePolicy, + pub read_policy: ChunkReadPolicy, + pub chunk_kv_transport: ChunkKvRpcTransportConfig, + pub limits: RuntimeLimits, + pub connect_timeout: Duration, +} + +impl RuntimeConfig { + #[must_use] + pub fn new(management_seeds: Vec) -> Self { + Self { + management_seeds, + diskio_connections_per_endpoint: 1, + diskio_rpc_workers: 1, + small_write: SmallWritePolicy::new(ChunkType::S3), + read_policy: ChunkReadPolicy::default(), + chunk_kv_transport: ChunkKvRpcTransportConfig::default(), + limits: RuntimeLimits::default(), + connect_timeout: Duration::from_secs(10), + } + } +} + +#[derive(Clone)] +pub struct ClientRuntime { + pub(crate) storage: Arc, + pub(crate) resources: Arc, + pub(crate) groups: Arc>>>, +} + +impl ClientRuntime { + /// Connects discovery and storage without an Access Server listener or + /// credential lookup. Business handles clone these same client owners. + /// # Errors + /// Returns invalid configuration or initial discovery/connection failures. + pub async fn connect(config: RuntimeConfig) -> Result { + let resources = RuntimeResources::new(config.limits)?; + if config.management_seeds.is_empty() { + return Err(Error::Configuration("management endpoints are empty".into())); + } + let timeout = config.connect_timeout; + if timeout.is_zero() { + return Err(Error::Configuration("connect timeout must be positive".into())); + } + let storage = Arc::new( + tokio::time::timeout(timeout, Storage::connect(config)) + .await + .map_err(|_| Error::Deadline)??, + ); + Ok(Self { + groups: Arc::new(ArcSwap::from_pointee(vec![Arc::clone(&storage)])), + storage, + resources, + }) + } + + #[must_use] + pub fn resources(&self) -> &Arc { + &self.resources + } + + #[must_use] + pub fn writer_group_count(&self) -> usize { + self.groups.load().len() + } + + /// Closes shared admission and wakes outstanding cancellable operations. + /// Caller-owned leases retain their backing storage after this call. + /// # Errors + /// Returns a writer group's durability/drain failure after closing admission. + pub async fn shutdown(&self) -> Result<()> { + self.resources.shutdown(); + let groups = self.groups.load_full(); + let mut failure = None; + for group in groups.iter() { + if let Err(error) = group.chunks.shutdown_small_writes().await { + failure.get_or_insert_with(|| Error::Storage(error.to_string())); + } + } + failure.map_or(Ok(()), Err) + } +} diff --git a/lib/crowdb-sdk-native/src/runtime/attachment.rs b/lib/crowdb-sdk-native/src/runtime/attachment.rs new file mode 100644 index 000000000..8993ab01d --- /dev/null +++ b/lib/crowdb-sdk-native/src/runtime/attachment.rs @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use arc_swap::ArcSwap; +use crowdb_chunk_client::ChunkIoClient; +use crowdb_chunk_kv_client::{ChunkKvClient, ChunkKvRpcTransport}; +use crowdb_kv_client::CrowdbKvClient; + +use super::{ClientRuntime, RuntimeLimits, RuntimeResources, Storage}; +use crate::topology::CatalogObservation; +use crate::{Error, Result}; + +/// Existing client owners attached by an embedding system. The catalog must +/// already be validated; this path never reconnects or creates HTTP listeners. +pub struct RuntimeClients { + pub control: Arc, + pub metadata: Arc, + pub metadata_transport: Arc, + pub chunks: Arc, +} + +impl ClientRuntime { + /// # Errors + /// Rejects invalid resource limits or a missing initial catalog observation. + pub fn from_clients(clients: RuntimeClients, limits: RuntimeLimits) -> Result { + let catalog = clients + .metadata + .cached_catalog() + .ok_or_else(|| Error::Configuration("attached metadata catalog is not initialized".into()))?; + let policy = clients.chunks.small_write_policy().as_ref().clone(); + let storage = Arc::new(Storage { + control: clients.control, + metadata: clients.metadata, + chunks: clients.chunks, + metadata_transport: clients.metadata_transport, + chunk_type: policy.chunk_type, + policy, + catalog: Arc::new(ArcSwap::from_pointee(CatalogObservation { + catalog, + observed_at: None, + })), + services: Arc::new(ArcSwap::from_pointee(Vec::new())), + }); + Ok(Self { + groups: Arc::new(ArcSwap::from_pointee(vec![Arc::clone(&storage)])), + storage, + resources: RuntimeResources::new(limits)?, + }) + } +} diff --git a/lib/crowdb-sdk-native/src/runtime/resources.rs b/lib/crowdb-sdk-native/src/runtime/resources.rs new file mode 100644 index 000000000..aaa1175d0 --- /dev/null +++ b/lib/crowdb-sdk-native/src/runtime/resources.rs @@ -0,0 +1,151 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::future::Future; +use std::sync::Arc; + +use crate::admission::{Budget, Credit}; +use crate::{Cancellation, CapacityKind, Error, OperationContext, Result}; + +#[derive(Clone, Copy, Debug)] +pub struct RuntimeLimits { + pub requests: usize, + pub producer_bytes: usize, + pub held_bytes: usize, +} + +impl Default for RuntimeLimits { + fn default() -> Self { + Self { + requests: 1024, + producer_bytes: 64 << 20, + held_bytes: 256 << 20, + } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct ResourceStatus { + pub closed: bool, + pub requests: usize, + pub producer_bytes: usize, + pub producer_bytes_high_water: usize, + pub held_bytes: usize, + pub held_bytes_high_water: usize, +} + +pub struct RuntimeResources { + pub(crate) producers: Arc, + pub(crate) held: Arc, + requests: Arc, + closing: Cancellation, +} + +impl RuntimeResources { + /// Builds shared hard limits. No background worker or waiting queue is created. + /// # Errors + /// Rejects zero, overflowing or excessive request admission. + pub fn new(limits: RuntimeLimits) -> Result> { + if limits.requests == 0 + || limits.requests > 1 << 20 + || limits.producer_bytes == 0 + || limits.held_bytes == 0 + || limits.producer_bytes > isize::MAX as usize + || limits.held_bytes > isize::MAX as usize + { + return Err(Error::Configuration("invalid request or memory limits".into())); + } + Ok(Arc::new(Self { + requests: Budget::new(limits.requests, CapacityKind::Requests), + producers: Budget::new(limits.producer_bytes, CapacityKind::Producers), + held: Budget::new(limits.held_bytes, CapacityKind::HeldBuffers), + closing: Cancellation::default(), + })) + } + + /// # Errors + /// Rejects closed runtime, expired/cancelled work or exhausted request slots. + pub fn admit(self: &Arc, context: OperationContext) -> Result { + context.check()?; + self.check_open()?; + let credit = self.requests.charge(1)?; + self.check_open()?; + Ok(RequestLease { + resources: Arc::clone(self), + context, + _credit: credit, + }) + } + + pub fn shutdown(&self) { + self.closing.cancel(); + } + + pub(crate) fn check_open(&self) -> Result<()> { + if self.closing.is_cancelled() { + Err(Error::Shutdown) + } else { + Ok(()) + } + } + + #[must_use] + pub fn status(&self) -> ResourceStatus { + ResourceStatus { + closed: self.closing.is_cancelled(), + requests: self.requests.used(), + producer_bytes: self.producers.used(), + producer_bytes_high_water: self.producers.high_water(), + held_bytes: self.held.used(), + held_bytes_high_water: self.held.high_water(), + } + } +} + +/// Request admission lasts through a stream's terminal result or cancellation. +/// Its release is independent of any buffer leases delivered to the caller. +pub struct RequestLease { + resources: Arc, + context: OperationContext, + _credit: Credit, +} + +impl RequestLease { + #[must_use] + pub fn context(&self) -> &OperationContext { + &self.context + } + + /// # Errors + /// Returns shutdown, cancellation, deadline or the underlying operation error. + pub async fn run(&self, future: impl Future>) -> Result { + self.resources.check_open()?; + tokio::select! { + biased; + () = self.resources.closing.cancelled() => Err(Error::Shutdown), + result = self.context.run(future) => result, + } + } + + /// Preserves uncertainty if cancellation interrupts an accepted mutation. + /// There is no implicit replay of the supplied future. + /// # Errors + /// Returns definite pre-admission errors, semantic failures or uncertainty. + pub async fn run_mutation(&self, future: impl Future>) -> Result { + use crate::error::MutationInterruption; + self.context.check()?; + self.resources.check_open()?; + self.run(future).await.map_err(|error| { + let reason = match error { + Error::Cancelled => MutationInterruption::Cancelled, + Error::Deadline => MutationInterruption::Deadline, + Error::Shutdown => MutationInterruption::Shutdown, + other => return other, + }; + Error::UncertainMutation { + request_id: self.context.request_id.get(), + reason, + } + }) + } +} diff --git a/lib/crowdb-sdk-native/src/runtime/storage.rs b/lib/crowdb-sdk-native/src/runtime/storage.rs new file mode 100644 index 000000000..2febb23fc --- /dev/null +++ b/lib/crowdb-sdk-native/src/runtime/storage.rs @@ -0,0 +1,84 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; +use std::time::Instant; + +use arc_swap::ArcSwap; + +use crowdb_chunk_client::{ChunkIoClient, ChunkIoClientConfig, SmallWritePolicy}; +use crowdb_chunk_kv_client::{ + ChunkKvClient, ChunkKvRpcTransport, ClientConfig, Group0ChunkKvRangeCatalogSource, +}; +use crowdb_kv_client::{ClientConfig as KvConfig, CrowdbKvClient}; +use crowdb_protocol::chunkdb::rpc::ChunkType; + +use super::RuntimeConfig; +use crate::topology::{CatalogObservation, ServiceObservation}; +use crate::{Error, Result}; + +pub(crate) struct Storage { + pub control: Arc, + pub metadata: Arc, + pub chunks: Arc, + pub chunk_type: ChunkType, + pub policy: SmallWritePolicy, + pub metadata_transport: Arc, + pub catalog: Arc>, + pub services: Arc>>, +} + +impl Storage { + pub async fn connect(config: RuntimeConfig) -> Result { + let policy = config.small_write.clone(); + let chunk_type = config.small_write.chunk_type; + let control = Arc::new(CrowdbKvClient::new(KvConfig::new( + config.management_seeds.clone(), + ))); + let catalog = Arc::new(Group0ChunkKvRangeCatalogSource::from_shared(Arc::clone(&control))); + let metadata_config = ClientConfig::default(); + let transport = Arc::new( + ChunkKvRpcTransport::with_config( + metadata_config.max_owner_connections, + 1, + 2, + &config.chunk_kv_transport, + ) + .map_err(|error| Error::Storage(error.to_string()))?, + ); + let metadata = Arc::new( + ChunkKvClient::new(metadata_config, catalog, transport.clone()) + .map_err(|error| Error::Storage(error.to_string()))?, + ); + let catalog = metadata + .refresh_catalog() + .await + .map_err(|error| Error::Storage(error.to_string()))?; + let catalog_observed_at = Instant::now(); + let chunks = ChunkIoClient::connect_with_kv_read_policy( + ChunkIoClientConfig { + management_seeds: config.management_seeds, + diskio_connections_per_endpoint: config.diskio_connections_per_endpoint, + diskio_rpc_workers: config.diskio_rpc_workers, + small_write: config.small_write, + }, + Arc::clone(&control), + config.read_policy, + ) + .await + .map_err(|error| Error::Storage(error.to_string()))?; + Ok(Self { + control, + metadata, + chunks: Arc::new(chunks), + chunk_type, + policy, + metadata_transport: transport, + catalog: Arc::new(ArcSwap::from_pointee(CatalogObservation { + catalog, + observed_at: Some(catalog_observed_at), + })), + services: Arc::new(ArcSwap::from_pointee(Vec::new())), + }) + } +} diff --git a/lib/crowdb-sdk-native/src/runtime/writer_policy.rs b/lib/crowdb-sdk-native/src/runtime/writer_policy.rs new file mode 100644 index 000000000..e339727f4 --- /dev/null +++ b/lib/crowdb-sdk-native/src/runtime/writer_policy.rs @@ -0,0 +1,63 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_chunk_client::SmallWritePolicy; + +use super::{ClientRuntime, Storage}; +use crate::{Error, Result}; + +impl ClientRuntime { + /// Reuses a compatible writer group, or creates a separate bounded pool + /// while retaining the same discovery, transports, read and SDK credits. + /// # Errors + /// Rejects closed admission, invalid policy or more than 32 writer groups. + pub fn with_writer_policy(&self, policy: &SmallWritePolicy) -> Result { + policy + .validate() + .map_err(|error| Error::Configuration(error.to_string()))?; + loop { + self.resources.check_open()?; + let observed = self.groups.load_full(); + if let Some(storage) = observed.iter().find(|group| &group.policy == policy) { + return Ok(Self { + storage: Arc::clone(storage), + resources: Arc::clone(&self.resources), + groups: Arc::clone(&self.groups), + }); + } + if observed.len() >= 32 { + return Err(Error::Configuration("writer group limit reached".into())); + } + let storage = Arc::new(Storage { + control: Arc::clone(&self.storage.control), + metadata: Arc::clone(&self.storage.metadata), + chunks: Arc::new( + self.storage + .chunks + .with_writer_policy(policy.clone()) + .map_err(|error| Error::Configuration(error.to_string()))?, + ), + chunk_type: policy.chunk_type, + policy: policy.clone(), + metadata_transport: Arc::clone(&self.storage.metadata_transport), + catalog: Arc::clone(&self.storage.catalog), + services: Arc::clone(&self.storage.services), + }); + let mut next = observed.as_ref().clone(); + next.push(Arc::clone(&storage)); + let previous = self.groups.compare_and_swap(&observed, Arc::new(next)); + if Arc::ptr_eq(&previous, &observed) { + // A new pool has no running manager until its first write. + // A racing shutdown cannot leave an untracked active producer. + self.resources.check_open()?; + return Ok(Self { + storage, + resources: Arc::clone(&self.resources), + groups: Arc::clone(&self.groups), + }); + } + } + } +} diff --git a/lib/crowdb-sdk-native/src/s3.rs b/lib/crowdb-sdk-native/src/s3.rs new file mode 100644 index 000000000..eb878edb5 --- /dev/null +++ b/lib/crowdb-sdk-native/src/s3.rs @@ -0,0 +1,103 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! S3 business handles over the shared trusted runtime, without HTTP authentication. + +use crowdb_access_s3::execution::S3Executor; +use crowdb_access_s3::metadata::{BucketId, BucketNameRecord, ChunkKvMetadataStore, ObjectRecord, TenantId}; +use crowdb_access_s3::storage::S3StorageClients; +use crowdb_protocol::chunkdb::rpc::ChunkType; + +use crate::{ClientRuntime, Error, OperationContext, Result}; +use std::sync::Arc; + +#[derive(Clone)] +pub struct S3Client { + runtime: ClientRuntime, + execution: S3Executor, +} + +impl S3Client { + /// Creates a tenant handle using the runtime's existing storage owners. + /// # Errors + /// Rejects closed runtime or a writer group for another concrete chunk type. + pub fn new(runtime: ClientRuntime, tenant: TenantId) -> Result { + runtime.resources.check_open()?; + if runtime.storage.chunk_type != ChunkType::S3 { + return Err(Error::Configuration("S3 requires an S3 writer group".into())); + } + let storage = S3StorageClients { + control: Arc::clone(&runtime.storage.control), + metadata: Arc::new(ChunkKvMetadataStore::new(Arc::clone(&runtime.storage.metadata))), + chunks: Arc::clone(&runtime.storage.chunks), + }; + Ok(Self { + execution: S3Executor::new(storage, tenant), + runtime, + }) + } + + /// # Errors + /// Returns admission, deadline, cancellation or S3 namespace/storage failures. + pub async fn create_bucket(&self, context: OperationContext, name: &[u8]) -> Result { + let request = self.runtime.resources.admit(context)?; + request + .run_mutation(async { self.execution.create_bucket(name).await.map_err(Error::S3) }) + .await + } + + /// # Errors + /// Returns admission, deadline, cancellation or a missing bucket. + pub async fn head_bucket(&self, context: OperationContext, name: &[u8]) -> Result { + let request = self.runtime.resources.admit(context)?; + request + .run(async { self.execution.resolve_bucket(name).await.map_err(Error::S3) }) + .await + } + + /// # Errors + /// Returns admission or bounded metadata scan failure. + pub async fn list_buckets( + &self, + context: OperationContext, + limit: usize, + ) -> Result> { + let request = self.runtime.resources.admit(context)?; + request + .run(async { self.execution.list_buckets(limit).await.map_err(Error::S3) }) + .await + } + + /// # Errors + /// Returns nonempty/missing bucket or a storage/admission failure. + pub async fn delete_bucket(&self, context: OperationContext, name: &[u8]) -> Result<()> { + let request = self.runtime.resources.admit(context)?; + request + .run_mutation(async { self.execution.delete_bucket(name).await.map_err(Error::S3) }) + .await + } + + /// Captures one published object identity for inspection or a later read. + /// # Errors + /// Returns admission or missing/corrupt object metadata. + pub async fn resolve_object( + &self, + context: OperationContext, + bucket: &[u8], + key: &[u8], + ) -> Result { + let request = self.runtime.resources.admit(context)?; + request + .run(async { self.execution.object(bucket, key).await.map_err(Error::S3) }) + .await + } + + /// # Errors + /// Returns namespace, admission or routed metadata mutation failure. + pub async fn delete_object(&self, context: OperationContext, bucket: &[u8], key: &[u8]) -> Result<()> { + let request = self.runtime.resources.admit(context)?; + request + .run_mutation(async { self.execution.delete_object(bucket, key).await.map_err(Error::S3) }) + .await + } +} diff --git a/lib/crowdb-sdk-native/src/topology.rs b/lib/crowdb-sdk-native/src/topology.rs new file mode 100644 index 000000000..9468c9994 --- /dev/null +++ b/lib/crowdb-sdk-native/src/topology.rs @@ -0,0 +1,140 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Retained independent routing observations for caller scheduling. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crowdb_chunk_kv_client::ChunkKvRangeCatalogMap; +use crowdb_diskio_client::DiskTopologyView; +use crowdb_kv_client::ServiceRegistryClient; +use crowdb_protocol::common::InstanceValue; + +use crate::{ClientRuntime, Error, OperationContext, Result}; + +pub(crate) struct CatalogObservation { + pub catalog: Arc, + pub observed_at: Option, +} + +#[derive(Clone, Debug)] +pub struct ServiceObservation { + pub service: &'static str, + pub observed_at: Instant, + pub instances: Vec, +} + +/// The sources do not share a global epoch or an atomic cluster snapshot. +/// Execution always goes through current routing and generation checks. +#[derive(Clone)] +pub struct TopologyView { + catalog: Arc, + pub disks: Option, + pub services: Arc>, + pub metadata_transport_json: Vec, + pub data_transport_json: Option>, +} + +impl TopologyView { + #[must_use] + pub fn catalog(&self) -> &Arc { + &self.catalog.catalog + } + + #[must_use] + pub fn catalog_age(&self) -> Option { + self.catalog.observed_at.map(|observed| observed.elapsed()) + } +} + +impl ClientRuntime { + /// Inspects retained observations without dialing or publishing placement. + /// Service observations remain empty until an explicit refresh. + /// # Errors + /// Returns native status allocation or topology-observation failures. + pub fn topology(&self) -> Result { + let disks = self.storage.chunks.disk_topology(); + let data_transport_json = if disks.is_some() { + Some( + self.storage + .chunks + .transport_status_json() + .map_err(|error| Error::Storage(error.to_string()))?, + ) + } else { + None + }; + Ok(TopologyView { + catalog: self.storage.catalog.load_full(), + disks, + services: self.storage.services.load_full(), + metadata_transport_json: self + .storage + .metadata_transport + .transport_status_json() + .map_err(|error| Error::Storage(error.to_string()))?, + data_transport_json, + }) + } + + /// Refreshes each authority independently; old views remain safe to inspect. + /// This never uses a caller-supplied view to override current placement. + /// # Errors + /// Returns admission, deadline, cancellation, catalog or endpoint failure. + pub async fn refresh_topology(&self, context: OperationContext) -> Result { + let request = self.resources.admit(context)?; + request + .run(async { + let catalog = self + .storage + .metadata + .refresh_catalog() + .await + .map_err(|error| Error::Storage(error.to_string()))?; + self.record_catalog(&catalog); + self.storage + .chunks + .refresh_chunkdb_routes() + .await + .map_err(|error| Error::Storage(error.to_string()))?; + self.storage + .chunks + .refresh_diskio_routes() + .await + .map_err(|error| Error::Storage(error.to_string()))?; + let registry = ServiceRegistryClient::from_shared(Arc::clone(&self.storage.control)); + let mut services = Vec::with_capacity(3); + for service in ["chunkdb", "chunk-kv", "diskio"] { + let instances = registry + .read_all_instances(service) + .await + .map_err(|error| Error::Storage(error.to_string()))?; + services.push(ServiceObservation { + service, + observed_at: Instant::now(), + instances: instances.into_iter().map(|(_, instance)| instance).collect(), + }); + } + self.storage.services.store(Arc::new(services)); + self.topology() + }) + .await + } + + fn record_catalog(&self, catalog: &Arc) { + loop { + let observed = self.storage.catalog.load_full(); + if catalog.generation() < observed.catalog.generation() { + return; + } + let next = Arc::new(CatalogObservation { + catalog: Arc::clone(catalog), + observed_at: Some(Instant::now()), + }); + if Arc::ptr_eq(&self.storage.catalog.compare_and_swap(&observed, next), &observed) { + return; + } + } + } +} diff --git a/lib/crowdb-sdk-native/tests/common/metadata.rs b/lib/crowdb-sdk-native/tests/common/metadata.rs new file mode 100644 index 000000000..b929f1171 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/common/metadata.rs @@ -0,0 +1,200 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use async_trait::async_trait; +use crowdb_chunk_kv_client::{ + ChunkKvClient, ChunkKvRangeCatalogSource, ChunkKvTransport, ClientConfig, Result, +}; +use crowdb_protocol::chunk_kv::*; +use crowdb_protocol::chunk_stream::StreamName; +use tokio::sync::{mpsc, oneshot}; + +struct TestCatalog(ChunkKvRangeCatalogHead, Vec); + +#[async_trait] +impl ChunkKvRangeCatalogSource for TestCatalog { + async fn load(&self) -> Result<(ChunkKvRangeCatalogHead, Vec)> { + Ok((self.0.clone(), self.1.clone())) + } +} + +enum TestCommand { + Point(PointOperation, oneshot::Sender), + Scan(ScanRequest, oneshot::Sender), +} + +pub struct TestMetadataTransport(mpsc::Sender); + +#[async_trait] +impl ChunkKvTransport for TestMetadataTransport { + async fn point(&self, _: &str, request: &PointRequest) -> Result { + let (sender, receiver) = oneshot::channel(); + self.0 + .send(TestCommand::Point(request.operation.clone(), sender)) + .await + .unwrap(); + Ok(receiver.await.unwrap()) + } + + async fn seek(&self, _: &str, _: &SeekRequest) -> Result { + panic!("unexpected seek") + } + + async fn scan(&self, _: &str, request: &ScanRequest) -> Result { + let (sender, receiver) = oneshot::channel(); + self.0 + .send(TestCommand::Scan(request.clone(), sender)) + .await + .unwrap(); + Ok(receiver.await.unwrap()) + } +} + +fn matches(observed: Option<&RpcValue>, condition: &RpcCompareCondition) -> bool { + observed.is_some_and(|observed| match condition { + RpcCompareCondition::Value(value) => observed.value == *value, + RpcCompareCondition::Revision(revision) => observed.revision == *revision, + }) +} + +fn point( + operation: PointOperation, + values: &mut BTreeMap, RpcValue>, + revision: &mut u64, +) -> OperationResult { + let key = operation.key().to_vec(); + let observed = values.get(&key).cloned(); + let (applied, value) = match operation { + PointOperation::Get { .. } => return OperationResult::Value(observed), + PointOperation::Put { value, .. } => (true, Some(value)), + PointOperation::PutIfAbsent { value, .. } => (observed.is_none(), Some(value)), + PointOperation::Delete { .. } => (observed.is_some(), None), + PointOperation::CompareExchange { value, condition, .. } => { + (matches(observed.as_ref(), &condition), Some(value)) + } + PointOperation::ConditionalDelete { condition, .. } => (matches(observed.as_ref(), &condition), None), + }; + if applied { + *revision += 1; + if let Some(value) = value { + values.insert( + key.clone(), + RpcValue { + key, + value, + revision: *revision, + }, + ); + } else { + values.remove(&key); + } + } + OperationResult::Mutation { + applied, + revision: applied.then_some(*revision), + observed, + } +} + +fn scan(request: &ScanRequest, values: &BTreeMap, RpcValue>) -> OperationResult { + let mut items: Vec<_> = values + .values() + .filter(|value| { + request.start.as_ref().map_or(true, |start| value.key >= *start) + && request.end.as_ref().map_or(true, |end| value.key < *end) + && request + .continuation + .as_ref() + .map_or(true, |token| value.key > token.last_key) + }) + .cloned() + .collect(); + if request.direction == ScanDirection::Reverse { + items.reverse(); + } + let truncated = items.len() > request.limit as usize; + items.truncate(request.limit as usize); + let continuation = if truncated { + items.last().map(|last| ScanContinuation { + direction: request.direction, + last_key: last.key.clone(), + partition_id: request.routing.partition_id, + owner_epoch: request.routing.owner_epoch, + map_revision: request.routing.map_revision, + }) + } else { + None + }; + OperationResult::Scan { items, continuation } +} + +pub async fn client() -> Arc { + let (sender, mut receiver) = mpsc::channel(64); + tokio::spawn(async move { + let mut values = BTreeMap::new(); + let mut revision = 0; + while let Some(command) = receiver.recv().await { + let (result, response) = match command { + TestCommand::Point(operation, response) => { + (point(operation, &mut values, &mut revision), response) + } + TestCommand::Scan(request, response) => (scan(&request, &values), response), + }; + let _ = response.send(ChunkKvResponse { + map_revision: 1, + journal_position: None, + result: Ok(result), + }); + } + }); + let mut page = ChunkKvRangeCatalogPage { + generation: 1, + page_index: 0, + entries: vec![ChunkKvRangeCatalogEntry { + partition_id: Id128 { high: 1, low: 1 }, + range: KeyRange { + start: Vec::new(), + end: None, + }, + owner: OwnerDescriptor { + instance_id: 1, + rpc_endpoint: "test-owner".into(), + }, + owner_epoch: 1, + state: ChunkKvRangeCatalogPartitionState::Serving, + artifact: PartitionArtifact { + tree_id: 1, + stream_name: StreamName { high: 1, low: 1 }, + tail_overlay: None, + }, + transition_id: None, + }], + checksum: [0; 32], + }; + page.seal().unwrap(); + let mut head = ChunkKvRangeCatalogHead { + generation: 1, + previous_generation: None, + pages: vec![ChunkKvRangeCatalogPageRef { + page_generation: 1, + page_index: 0, + first_key: Vec::new(), + page_checksum: page.checksum, + }], + checksum: [0; 32], + }; + head.seal().unwrap(); + let client = Arc::new( + ChunkKvClient::new( + ClientConfig::default(), + Arc::new(TestCatalog(head, vec![page])), + Arc::new(TestMetadataTransport(sender)), + ) + .unwrap(), + ); + client.refresh_catalog().await.unwrap(); + client +} diff --git a/lib/crowdb-sdk-native/tests/common/storage.rs b/lib/crowdb-sdk-native/tests/common/storage.rs new file mode 100644 index 000000000..3a01d7063 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/common/storage.rs @@ -0,0 +1,68 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_chunk_client::{ChunkAllocator, ChunkIoClient, DiskWriter, Result}; +use crowdb_chunk_kv_client::{ChunkKvRpcTransport, ChunkKvRpcTransportConfig}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_protocol::chunkdb::rpc::*; +use crowdb_protocol::diskdb::rpc::Segment; +use crowdb_sdk_native::runtime::RuntimeClients; +use crowdb_sdk_native::{ClientRuntime, RuntimeLimits}; + +struct TestNoPayload; + +#[async_trait] +impl ChunkAllocator for TestNoPayload { + async fn allocate_chunk(&self, _: AllocateChunkRequest) -> Result { + panic!("metadata operation allocated data") + } + async fn append_chunk(&self, _: AppendChunkRequest) -> Result { + panic!("metadata operation appended data") + } + async fn seal_chunk(&self, _: SealChunkRequest) -> Result { + panic!("metadata operation sealed data") + } + async fn delete_chunk(&self, _: DeleteChunkRequest) -> Result { + panic!("metadata operation deleted data") + } + async fn update_chunk_strip(&self, _: UpdateChunkStripRequest) -> Result { + panic!("metadata operation updated data") + } + async fn query_chunk(&self, _: QueryChunkRequest) -> Result { + panic!("metadata operation read payload") + } +} + +#[async_trait] +impl DiskWriter for TestNoPayload { + async fn write(&self, _: &Segment, _: u64, _: Bytes) -> Result<()> { + panic!("metadata operation wrote data") + } + async fn write_at_byte_offset(&self, _: &Segment, _: u64, _: u64, _: Bytes) -> Result<()> { + panic!("metadata operation wrote data") + } +} + +pub async fn runtime() -> ClientRuntime { + let metadata = super::metadata::client().await; + let metadata_transport = + Arc::new(ChunkKvRpcTransport::with_config(8, 1, 1, &ChunkKvRpcTransportConfig::default()).unwrap()); + ClientRuntime::from_clients( + RuntimeClients { + control: Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))), + metadata, + metadata_transport, + chunks: Arc::new(ChunkIoClient::from_parts( + Arc::new(TestNoPayload), + Arc::new(TestNoPayload), + ChunkType::S3, + )), + }, + RuntimeLimits::default(), + ) + .unwrap() +} diff --git a/lib/crowdb-sdk-native/tests/resources_test.rs b/lib/crowdb-sdk-native/tests/resources_test.rs new file mode 100644 index 000000000..8116a7f51 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/resources_test.rs @@ -0,0 +1,187 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::num::NonZeroU64; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crowdb_rpc_ffi::{BufferPool, ProducerBufferAllocator, RdmaConfig, RpcServer}; +use crowdb_sdk_native::{CapacityKind, Error, OperationContext, RuntimeLimits, RuntimeResources}; + +fn resources(requests: usize, bytes: usize) -> Arc { + RuntimeResources::new(RuntimeLimits { + requests, + producer_bytes: bytes, + held_bytes: bytes, + }) + .unwrap() +} + +fn context(id: u64) -> OperationContext { + OperationContext::new( + NonZeroU64::new(id).unwrap(), + Instant::now() + Duration::from_secs(1), + ) +} + +#[test] +fn cloned_views_charge_actual_allocation_once_until_last_external_owner() { + let resources = resources(2, 1024); + let mut data = Vec::with_capacity(1024); + data.extend_from_slice(b"0123456789"); + let pointer = data.as_ptr(); + let lease = resources.retain_vec(data, 42, 81, 3, 100).unwrap(); + assert_eq!(lease.as_slice().as_ptr(), pointer); + let tail = lease.slice(4..10).unwrap(); + assert_eq!(tail.metadata().logical_offset, 104); + assert_eq!(tail.metadata().length, 6); + assert_eq!(tail.metadata().allocation_capacity, 1024); + assert!(matches!( + resources.retain_vec(vec![1], 43, 82, 0, 0), + Err(Error::Capacity(CapacityKind::HeldBuffers)) + )); + let external = tail.into_bytes(); + drop(lease); + assert_eq!(resources.status().held_bytes, 1024); + resources.shutdown(); + assert_eq!(external, b"456789"[..]); + drop(external); + assert_eq!(resources.status().held_bytes, 0); + assert_eq!(resources.status().held_bytes_high_water, 1024); +} + +#[test] +fn frozen_native_input_retains_producer_credit_and_backing_after_shutdown() { + let pool = BufferPool::new(2); + let server = Arc::new(RpcServer::new(Some(&pool))); + server.configure_rdma(&RdmaConfig::default()).unwrap(); + let allocator = ProducerBufferAllocator::new(Arc::clone(&server), false); + let resources = resources(1, 512); + let writable = resources.allocate(&allocator, 512).unwrap(); + let lease = writable + .initialize_with(127, 0xAB, |bytes| { + bytes[..8].copy_from_slice(&42_u64.to_le_bytes()); + }) + .unwrap(); + assert_eq!(lease.metadata().allocation_capacity, 512); + assert!(matches!( + resources.allocate(&allocator, 1), + Err(Error::Capacity(CapacityKind::Producers)) + )); + let final_view = lease.slice(8..127).unwrap().into_bytes(); + drop(lease); + resources.shutdown(); + drop(allocator); + drop(server); + drop(pool); + assert_eq!(resources.status().producer_bytes, 512); + assert!(final_view.iter().all(|byte| *byte == 0xAB)); + drop(final_view); + assert_eq!(resources.status().producer_bytes, 0); +} + +#[tokio::test] +async fn request_admission_cancellation_and_shared_shutdown_are_independent_of_held_bytes() { + let resources = resources(1, 64); + let first_context = context(1); + let first = resources.admit(first_context.clone()).unwrap(); + assert!(matches!( + resources.admit(context(2)), + Err(Error::Capacity(CapacityKind::Requests)) + )); + let lease = resources.retain_vec(vec![1; 64], 1, 1, 0, 0).unwrap(); + first_context.cancellation.cancel(); + assert_eq!( + first.run(std::future::pending::>()).await, + Err(Error::Cancelled) + ); + drop(first); + assert_eq!(resources.status().requests, 0); + assert_eq!(resources.status().held_bytes, 64); + let second = resources.admit(context(2)).unwrap(); + let running = tokio::spawn(async move { second.run(std::future::pending::>()).await }); + tokio::task::yield_now().await; + resources.shutdown(); + assert_eq!(running.await.unwrap(), Err(Error::Shutdown)); + assert!(matches!(resources.admit(context(3)), Err(Error::Shutdown))); + assert_eq!(lease.as_slice(), &[1; 64]); + drop(lease); + assert_eq!(resources.status().held_bytes, 0); +} + +#[tokio::test] +async fn total_deadline_stops_a_wait_without_returning_caller_held_credit() { + let resources = resources(1, 8); + let context = OperationContext::new( + NonZeroU64::new(7).unwrap(), + Instant::now() + Duration::from_millis(10), + ); + let request = resources.admit(context).unwrap(); + let held = resources.retain_vec(vec![9; 8], 7, 7, 0, 0).unwrap(); + assert_eq!( + request.run(std::future::pending::>()).await, + Err(Error::Deadline) + ); + drop(request); + assert_eq!(resources.status().held_bytes, 8); + drop(held); + assert_eq!(resources.status().held_bytes, 0); +} + +#[test] +fn rejected_initialized_prefix_returns_storage_and_producer_admission() { + let pool = BufferPool::new(1); + let server = Arc::new(RpcServer::new(Some(&pool))); + let allocator = ProducerBufferAllocator::new(server, false); + let resources = resources(1, 64); + assert!(resources + .allocate(&allocator, 64) + .unwrap() + .initialize_with(65, 0, |_| {}) + .is_err()); + assert_eq!(resources.status().producer_bytes, 0); + assert!(resources.allocate(&allocator, 64).is_ok()); +} + +#[tokio::test] +async fn asynchronous_copy_retains_host_storage_beyond_stream_and_runtime_close() { + let resources = resources(1, 1024); + let host = resources.retain_vec(vec![0xAB; 1024], 90, 91, 0, 0).unwrap(); + let (complete_copy, copied) = tokio::sync::oneshot::channel(); + let copy = tokio::spawn(async move { + copied.await.unwrap(); + assert!(host.as_slice().iter().all(|byte| *byte == 0xAB)); + }); + resources.shutdown(); + assert_eq!(resources.status().held_bytes, 1024); + complete_copy.send(()).unwrap(); + copy.await.unwrap(); + assert_eq!(resources.status().held_bytes, 0); +} + +#[tokio::test] +async fn an_interrupted_mutation_exposes_uncertainty_and_never_replays() { + let resources = resources(1, 8); + let context = context(81); + let cancellation = context.cancellation.clone(); + let request = resources.admit(context).unwrap(); + let (accepted, accepting) = tokio::sync::oneshot::channel(); + let mutation = tokio::spawn(async move { + request + .run_mutation(async { + accepted.send(()).unwrap(); + std::future::pending::>().await + }) + .await + }); + accepting.await.unwrap(); + cancellation.cancel(); + assert_eq!( + mutation.await.unwrap(), + Err(Error::UncertainMutation { + request_id: 81, + reason: crowdb_sdk_native::error::MutationInterruption::Cancelled, + }) + ); + assert_eq!(resources.status().requests, 0); +} diff --git a/lib/crowdb-sdk-native/tests/runtime_test.rs b/lib/crowdb-sdk-native/tests/runtime_test.rs new file mode 100644 index 000000000..706914906 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/runtime_test.rs @@ -0,0 +1,76 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/metadata.rs"] +mod metadata; +#[path = "common/storage.rs"] +mod storage; + +use std::num::NonZeroU64; +use std::time::{Duration, Instant}; + +use crowdb_access_s3::metadata::TenantId; +use crowdb_chunk_client::SmallWritePolicy; +use crowdb_protocol::chunkdb::rpc::ChunkType; +use crowdb_sdk_native::s3::S3Client; +use crowdb_sdk_native::{Error, OperationContext}; + +fn context(id: u64) -> OperationContext { + OperationContext::new( + NonZeroU64::new(id).unwrap(), + Instant::now() + Duration::from_secs(1), + ) +} + +#[tokio::test] +async fn native_namespace_uses_shared_storage_without_a_listener_or_credentials() { + let runtime = storage::runtime().await; + let first = S3Client::new(runtime.clone(), TenantId::new(b"first".to_vec()).unwrap()).unwrap(); + let second = S3Client::new(runtime.clone(), TenantId::new(b"second".to_vec()).unwrap()).unwrap(); + let bucket = first.create_bucket(context(1), b"bucket").await.unwrap(); + assert_eq!(first.create_bucket(context(2), b"bucket").await.unwrap(), bucket); + assert_eq!(first.head_bucket(context(3), b"bucket").await.unwrap(), bucket); + assert!(matches!( + second.head_bucket(context(4), b"bucket").await, + Err(Error::S3(crowdb_access_s3::S3ErrorCode::NoSuchBucket)) + )); + assert_eq!(first.list_buckets(context(5), 100).await.unwrap().len(), 1); + drop(first); + assert!(second.list_buckets(context(6), 100).await.unwrap().is_empty()); + assert_eq!(runtime.resources().status().requests, 0); + runtime.shutdown().await.unwrap(); + assert!(matches!( + second.list_buckets(context(7), 100).await, + Err(Error::Shutdown) + )); +} + +#[tokio::test] +async fn writer_groups_keep_chunk_purposes_separate_and_share_admission() { + let runtime = storage::runtime().await; + let same = runtime + .with_writer_policy(&SmallWritePolicy::new(ChunkType::S3)) + .unwrap(); + assert!(std::sync::Arc::ptr_eq(runtime.resources(), same.resources())); + assert_eq!(runtime.writer_group_count(), 1); + let dataset = runtime + .with_writer_policy(&SmallWritePolicy::new(ChunkType::Dataset)) + .unwrap(); + assert!(std::sync::Arc::ptr_eq(runtime.resources(), dataset.resources())); + assert_eq!(runtime.writer_group_count(), 2); + assert!(matches!( + S3Client::new(dataset, TenantId::new(b"tenant".to_vec()).unwrap()), + Err(Error::Configuration(_)) + )); + let view = runtime.topology().unwrap(); + assert_eq!(view.catalog().generation(), 1); + assert_eq!(view.catalog().route(b"key").unwrap().owner.instance_id, 1); + assert!(view.disks.is_none()); + assert!(view.data_transport_json.is_none()); + runtime.shutdown().await.unwrap(); + assert_eq!(view.catalog().generation(), 1); + assert!(matches!( + same.with_writer_policy(&SmallWritePolicy::new(ChunkType::S3)), + Err(Error::Shutdown) + )); +} From 8537e2f4ed16df54ffd8d918efa078878a633539 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 12:42:52 +0800 Subject: [PATCH 059/258] Preserve verified object read failures across native stream polling --- doc/working/plan-rdma-fat-client-sdk.md | 10 ++ lib/crowdb-access-s3/src/retrieval.rs | 124 ++++++++++++++--- .../tests/common/retrieval_storage.rs | 54 ++++++++ lib/crowdb-access-s3/tests/retrieval_test.rs | 131 ++++++++++++++++++ 4 files changed, 298 insertions(+), 21 deletions(-) create mode 100644 lib/crowdb-access-s3/tests/common/retrieval_storage.rs create mode 100644 lib/crowdb-access-s3/tests/retrieval_test.rs diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index 0b48f38fd..8aa26af40 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -233,6 +233,16 @@ transport acceptance alone cannot establish SDK completion. `BufferLease` tests establish its own storage/credit behavior only; they do not establish end-to-end native RDMA receive ownership or actual-capacity propagation through DiskIO/ChunkIO. Phase 4 must complete that audit and path. +- Shared GET now offers typed half-open ranges and `next_verified`, with sticky + storage/length/integrity failures. Empty full reads also verify their checksum. + The legacy HTTP `next_chunk` remains fused after its first terminal result. + Three focused cases and the complete S3 library suite/clippy pass. Native + object metadata/body adaptation is still pending. +- Before completing native GET delivery, carry retained allocation identity, + actual capacity and source ownership from RPC through DiskIO/ChunkIO. Use an + owner-backed payload type with slices preserving one allocation owner; do + not rediscover receive ranges by scanning connection tables or adding a hot + lock, and do not substitute a guessed capacity for the actual native class. ## Subsequent Dataset integration under R220 diff --git a/lib/crowdb-access-s3/src/retrieval.rs b/lib/crowdb-access-s3/src/retrieval.rs index 8e89df232..8e34feaae 100644 --- a/lib/crowdb-access-s3/src/retrieval.rs +++ b/lib/crowdb-access-s3/src/retrieval.rs @@ -3,6 +3,7 @@ //! Metadata-consistent HEAD and pull-based GET preparation. +use std::sync::Arc; use std::time::{Duration, UNIX_EPOCH}; use crowdb_chunk_client::{ChunkIoClient, ChunkReadStream, ReadError}; @@ -32,18 +33,29 @@ pub struct PreparedGet { } pub struct VerifiedGetStream { - inner: ChunkReadStream, + inner: Option, integrity: Option, expected_checksum: Vec, terminal: bool, + failure: Option, + expected_length: u64, + delivered: u64, } -#[derive(Debug, thiserror::Error)] +#[derive(Clone, Debug, thiserror::Error)] pub enum GetStreamError { #[error("chunk read failed: {0}")] - Chunk(#[from] ReadError), + Chunk(Arc), #[error("object checksum does not match its published metadata")] Integrity, + #[error("object length mismatch: expected={expected} delivered={delivered}")] + Length { expected: u64, delivered: u64 }, +} + +impl From for GetStreamError { + fn from(error: ReadError) -> Self { + Self::Chunk(Arc::new(error)) + } } impl VerifiedGetStream { @@ -53,25 +65,65 @@ impl VerifiedGetStream { if self.terminal { return None; } - match self.inner.next_chunk().await { + self.next_verified().await.transpose() + } + + /// Successful EOF includes length and applicable object integrity checks. + /// A terminal failure remains visible on every subsequent pull. + /// # Errors + /// Returns storage, length or final checksum failure without successful EOF. + pub async fn next_verified(&mut self) -> Result, GetStreamError> { + if let Some(error) = &self.failure { + return Err(error.clone()); + } + if self.terminal { + return Ok(None); + } + let next = match &mut self.inner { + Some(inner) => inner.next_chunk().await, + None => None, + }; + match next { Some(Ok(bytes)) => { + self.delivered = self.delivered.saturating_add(bytes.len() as u64); + if self.delivered > self.expected_length { + return self.fail(GetStreamError::Length { + expected: self.expected_length, + delivered: self.delivered, + }); + } if let Some(integrity) = &mut self.integrity { integrity.update(&bytes); } - Some(Ok(bytes)) - } - Some(Err(error)) => { - self.terminal = true; - Some(Err(error.into())) + Ok(Some(bytes)) } + Some(Err(error)) => self.fail(error.into()), None => { self.terminal = true; - let integrity = self.integrity.take()?; - let (_, checksum) = integrity.finish(); - (checksum != self.expected_checksum).then_some(Err(GetStreamError::Integrity)) + self.inner = None; + if self.delivered != self.expected_length { + return self.fail(GetStreamError::Length { + expected: self.expected_length, + delivered: self.delivered, + }); + } + if let Some(integrity) = self.integrity.take() { + let (_, checksum) = integrity.finish(); + if checksum != self.expected_checksum { + return self.fail(GetStreamError::Integrity); + } + } + Ok(None) } } } + + fn fail(&mut self, error: GetStreamError) -> Result, GetStreamError> { + self.terminal = true; + self.inner = None; + self.failure = Some(error.clone()); + Err(error) + } } #[derive(Debug, thiserror::Error)] @@ -114,26 +166,56 @@ pub fn prepare_get( ) -> Result { check_conditions(record, conditions)?; let requested = resolve_range(range_header, record.logical_length).map_err(RetrievalError::Range)?; + build_get(client, record, requested) +} + +/// Prepares the same selected generation using a typed half-open range. +/// # Errors +/// Rejects conditions or intervals outside the published object length. +pub fn prepare_get_range( + client: &ChunkIoClient, + record: &ObjectRecord, + requested: Option, + conditions: &ObjectConditions<'_>, +) -> Result { + check_conditions(record, conditions)?; + if requested.is_some_and(|range| range.start >= range.end || range.end > record.logical_length) { + return Err(RetrievalError::Range(RangeError::Unsatisfiable)); + } + build_get(client, record, requested) +} + +fn build_get( + client: &ChunkIoClient, + record: &ObjectRecord, + requested: Option, +) -> Result { let interval = requested.unwrap_or(ByteRange { start: 0, end: record.logical_length, }); - let stream = if interval.start == interval.end { + let inner = if interval.start == interval.end { None } else { let locations: Vec = bincode::deserialize(&record.data_reference).map_err(|_| RetrievalError::DataReference)?; - Some(VerifiedGetStream { - inner: client + Some( + client .read_range_stream(&locations, interval.start, interval.end) .map_err(|_| RetrievalError::ChunkRead)?, - integrity: (requested.is_none() - && !crate::integrity::is_multipart_checksum(&record.checksum, &record.etag)) - .then(SinglePartIntegrity::default), - expected_checksum: record.checksum.clone(), - terminal: false, - }) + ) }; + let stream = Some(VerifiedGetStream { + inner, + integrity: (requested.is_none() + && !crate::integrity::is_multipart_checksum(&record.checksum, &record.etag)) + .then(SinglePartIntegrity::default), + expected_checksum: record.checksum.clone(), + terminal: false, + failure: None, + expected_length: interval.end - interval.start, + delivered: 0, + }); Ok(PreparedGet { headers: headers(record, requested), stream, diff --git a/lib/crowdb-access-s3/tests/common/retrieval_storage.rs b/lib/crowdb-access-s3/tests/common/retrieval_storage.rs new file mode 100644 index 000000000..860372af0 --- /dev/null +++ b/lib/crowdb-access-s3/tests/common/retrieval_storage.rs @@ -0,0 +1,54 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use async_trait::async_trait; +use crowdb_chunk_client::{ChunkAllocator, ChunkIoClient, DiskWriter, IoError, Result}; +use crowdb_protocol::chunkdb::rpc::*; +use crowdb_protocol::diskdb::rpc::Segment; +use hyper::body::Bytes; + +pub struct TestReadStorage { + pub queries: AtomicUsize, +} + +impl TestReadStorage { + pub fn client(self: &Arc) -> ChunkIoClient { + ChunkIoClient::from_parts(self.clone(), self.clone(), ChunkType::S3) + } +} + +#[async_trait] +impl ChunkAllocator for TestReadStorage { + async fn allocate_chunk(&self, _: AllocateChunkRequest) -> Result { + panic!("read allocated storage") + } + async fn append_chunk(&self, _: AppendChunkRequest) -> Result { + panic!("read appended storage") + } + async fn seal_chunk(&self, _: SealChunkRequest) -> Result { + panic!("read sealed storage") + } + async fn delete_chunk(&self, _: DeleteChunkRequest) -> Result { + panic!("read deleted storage") + } + async fn update_chunk_strip(&self, _: UpdateChunkStripRequest) -> Result { + panic!("read updated storage") + } + async fn query_chunk(&self, _: QueryChunkRequest) -> Result { + self.queries.fetch_add(1, Ordering::Relaxed); + Err(IoError::Topology("test owner unavailable".into())) + } +} + +#[async_trait] +impl DiskWriter for TestReadStorage { + async fn write(&self, _: &Segment, _: u64, _: Bytes) -> Result<()> { + panic!("read wrote storage") + } + async fn write_at_byte_offset(&self, _: &Segment, _: u64, _: u64, _: Bytes) -> Result<()> { + panic!("read wrote storage") + } +} diff --git a/lib/crowdb-access-s3/tests/retrieval_test.rs b/lib/crowdb-access-s3/tests/retrieval_test.rs new file mode 100644 index 000000000..57934acca --- /dev/null +++ b/lib/crowdb-access-s3/tests/retrieval_test.rs @@ -0,0 +1,131 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/retrieval_storage.rs"] +mod storage; + +use std::sync::atomic::Ordering; +use std::sync::Arc; + +use crowdb_access_s3::condition::ObjectConditions; +use crowdb_access_s3::metadata::{BucketId, ObjectRecord}; +use crowdb_access_s3::range::ByteRange; +use crowdb_access_s3::retrieval::{prepare_get_range, GetStreamError, RetrievalError}; +use crowdb_protocol::chunkdb::rpc::Location; +use crowdb_protocol::common::ChunkId; + +fn object(length: u64) -> ObjectRecord { + ObjectRecord { + bucket_id: BucketId::new([1; 16]), + key: b"key".to_vec(), + logical_length: length, + checksum: md5::compute(b"").0.to_vec(), + etag: format!("{:x}", md5::compute(b"")), + created_at_ms: 1, + modified_at_ms: 1, + content_type: "application/octet-stream".into(), + attributes: Vec::new(), + data_reference: bincode::serialize(&vec![Location { + chunk_id: Some(ChunkId { high: 1, low: 1 }), + offset: 0, + length, + logical_offset: 0, + logical_length: length, + }]) + .unwrap(), + data_length: length, + } +} + +fn backend() -> Arc { + Arc::new(storage::TestReadStorage { queries: 0.into() }) +} + +#[tokio::test] +async fn empty_full_read_checks_integrity_and_failure_never_turns_into_eof() { + let storage = backend(); + let client = storage.client(); + let mut record = object(0); + let mut valid = prepare_get_range(&client, &record, None, &ObjectConditions::default()) + .unwrap() + .stream + .unwrap(); + assert!(valid.next_verified().await.unwrap().is_none()); + assert!(valid.next_verified().await.unwrap().is_none()); + record.checksum = vec![0; 16]; + let mut invalid = prepare_get_range(&client, &record, None, &ObjectConditions::default()) + .unwrap() + .stream + .unwrap(); + assert!(matches!( + invalid.next_verified().await, + Err(GetStreamError::Integrity) + )); + assert!(matches!( + invalid.next_verified().await, + Err(GetStreamError::Integrity) + )); + assert!(invalid.next_chunk().await.is_none()); + assert_eq!(storage.queries.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn storage_failure_is_sticky_without_restarting_work_after_failed_pull() { + let storage = backend(); + let client = storage.client(); + let mut body = prepare_get_range(&client, &object(3), None, &ObjectConditions::default()) + .unwrap() + .stream + .unwrap(); + assert_eq!(storage.queries.load(Ordering::Relaxed), 0); + assert!(matches!( + body.next_verified().await, + Err(GetStreamError::Chunk(_)) + )); + let queries = storage.queries.load(Ordering::Relaxed); + assert!(queries > 0); + assert!(matches!( + body.next_verified().await, + Err(GetStreamError::Chunk(_)) + )); + assert_eq!(storage.queries.load(Ordering::Relaxed), queries); +} + +#[test] +fn typed_ranges_reject_invalid_intervals_and_keep_condition_precedence() { + let client = backend().client(); + let record = object(3); + for range in [ + ByteRange { start: 0, end: 4 }, + ByteRange { start: 3, end: 3 }, + ByteRange { start: 2, end: 1 }, + ] { + assert!(matches!( + prepare_get_range(&client, &record, Some(range), &ObjectConditions::default()), + Err(RetrievalError::Range(_)) + )); + } + let conditions = ObjectConditions { + if_none_match: Some(&record.etag), + ..ObjectConditions::default() + }; + assert!(matches!( + prepare_get_range( + &client, + &record, + Some(ByteRange { start: 3, end: 4 }), + &conditions + ), + Err(RetrievalError::NotModified) + )); + assert!( + prepare_get_range( + &client, + &record, + Some(ByteRange { start: 1, end: 3 }), + &ObjectConditions::default() + ) + .unwrap() + .partial + ); +} From de2805ceab3982c6acfa7984a6206c8c46cf01f8 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 12:57:37 +0800 Subject: [PATCH 060/258] Retain native receive allocations and expose owned payload views --- doc/working/plan-rdma-fat-client-sdk.md | 15 +- lib/crowdb-rpc/ffi/src/buffer.rs | 31 +++ lib/crowdb-rpc/ffi/src/client.rs | 29 +-- lib/crowdb-rpc/ffi/src/lib.rs | 7 +- lib/crowdb-rpc/ffi/src/payload.rs | 193 ++++++++++++++++++ lib/crowdb-rpc/ffi/src/sys.rs | 4 + lib/crowdb-rpc/ffi/tests/payload_test.rs | 56 +++++ .../ffi/tests/receive_lease_test.rs | 112 ++++++++++ lib/crowdb-rpc/include/crowdb-rpc/c_api.h | 21 +- lib/crowdb-rpc/src/c_api/buffer_lease.cpp | 70 +++++++ lib/crowdb-rpc/src/c_api/frame.cpp | 6 +- .../transport/rdma/rdma_direct_receive.cpp | 1 + .../transport/rdma/rdma_packed_receive.cpp | 1 + .../src/transport/rdma/rdma_receive.cpp | 1 + lib/crowdb-rpc/tests/buffer_lease_test.cpp | 62 ++++++ 15 files changed, 574 insertions(+), 35 deletions(-) create mode 100644 lib/crowdb-rpc/ffi/src/payload.rs create mode 100644 lib/crowdb-rpc/ffi/tests/payload_test.rs create mode 100644 lib/crowdb-rpc/ffi/tests/receive_lease_test.rs create mode 100644 lib/crowdb-rpc/src/c_api/buffer_lease.cpp create mode 100644 lib/crowdb-rpc/tests/buffer_lease_test.cpp diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index 8aa26af40..d2097e636 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -229,10 +229,17 @@ transport acceptance alone cannot establish SDK completion. - Basic bucket and point object lookup/delete orchestration is extracted into `S3Executor`, and HTTP delegates to it. Streaming GET/PUT, copy, multipart, typed native results and real HTTP/native parity are not complete. -- The existing FFI completion still has detach/copy branches. The initial - `BufferLease` tests establish its own storage/credit behavior only; they do - not establish end-to-end native RDMA receive ownership or actual-capacity - propagation through DiskIO/ChunkIO. Phase 4 must complete that audit and path. +- FFI completions now retain their owned native handles, including logical + control admission and receive recycling. Additive ABI queries expose the + canonical allocation identity, actual capacity, view offset and initialized + span; unknown external owners never substitute their view length for capacity. + `OwnedPayload` preserves this metadata across slices and logical credit owners. + DiskIO/ChunkIO propagation and SDK accounting remain outstanding. +- Explicit shared-HCA FFI acceptance with two posted receives retains both + response payloads, observes the third response remain pending, releases one + and completes the third. Held data stays readable after connection/server + destruction. The complete FFI suite/clippy and native RPC suite pass (211 + native cases, three explicitly unavailable or separately gated cases). - Shared GET now offers typed half-open ranges and `next_verified`, with sticky storage/length/integrity failures. Empty full reads also verify their checksum. The legacy HTTP `next_chunk` remains fused after its first terminal result. diff --git a/lib/crowdb-rpc/ffi/src/buffer.rs b/lib/crowdb-rpc/ffi/src/buffer.rs index 7f856055b..da001fca3 100644 --- a/lib/crowdb-rpc/ffi/src/buffer.rs +++ b/lib/crowdb-rpc/ffi/src/buffer.rs @@ -24,6 +24,13 @@ pub struct BufferLeaseMetadata { pub ownership: BufferOwnership, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct BufferAllocationMetadata { + pub identity: u64, + pub capacity: u32, + pub offset: u32, +} + #[repr(i32)] #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub enum BufferOwnership { @@ -223,6 +230,30 @@ impl Buffer { } } + /// Returns canonical allocation information when native ownership knows + /// the backing capacity. External opaque Bytes wrappers may return None. + #[must_use] + pub fn allocation_metadata(&self) -> Option { + // SAFETY: the borrowed Buffer retains its native handle and parents. + let capacity = unsafe { sys::crowdb_rpc_buffer_allocation_capacity(self.handle) }; + (capacity != 0).then(|| BufferAllocationMetadata { + // SAFETY: identity is read from that same retained allocation. + identity: unsafe { sys::crowdb_rpc_buffer_allocation_id(self.handle) }, + capacity, + // SAFETY: the same handle retains the initialized parent span. + offset: unsafe { sys::crowdb_rpc_buffer_allocation_offset(self.handle) }, + }) + } + + /// Retains the canonical initialized allocation span for shared accounting. + /// It can include framing; object data remains the original logical view. + #[must_use] + pub fn allocation_view(&self) -> Option { + // SAFETY: native code validates and retains the borrowed parent. + let handle = unsafe { sys::crowdb_rpc_buffer_allocation_ref(self.handle) }; + (!handle.is_null()).then(|| Self::from_raw(handle)) + } + pub fn ownership(&self) -> BufferOwnership { BufferOwnership::from_raw(unsafe { sys::crowdb_rpc_buffer_ownership(self.handle) }) } diff --git a/lib/crowdb-rpc/ffi/src/client.rs b/lib/crowdb-rpc/ffi/src/client.rs index ca464b570..0ee7afa6f 100644 --- a/lib/crowdb-rpc/ffi/src/client.rs +++ b/lib/crowdb-rpc/ffi/src/client.rs @@ -497,8 +497,8 @@ unsafe extern "C" fn on_complete_cb( let result = if status == sys::CROWDB_RPC_OK { Ok(Response { request_id, - control: standalone_buffer(control), - data: receive_data_buffer(data), + control: owned_completion_buffer(control), + data: owned_completion_buffer(data), }) } else { Err(RpcError::from_status(status)) @@ -507,26 +507,11 @@ unsafe extern "C" fn on_complete_cb( let _ = tx.send((result, io_thread_ts)); } -fn standalone_buffer(handle: sys::crowdb_rpc_buffer_t) -> Option { - if handle.is_null() { - return None; - } - let pooled = Buffer::from_raw(handle); - let bytes = pooled.bytes().to_vec(); - Some(Buffer::from_bytes(&bytes)) -} - -fn receive_data_buffer(handle: sys::crowdb_rpc_buffer_t) -> Option { - if handle.is_null() { - return None; - } - let mut buffer = Buffer::from_raw(handle); - if buffer.detach() { - Some(buffer) - } else { - let copy = Buffer::from_bytes(buffer.bytes()); - Some(copy) - } +fn owned_completion_buffer(handle: sys::crowdb_rpc_buffer_t) -> Option { + // frame_to_c_handles transfers owned references. System allocations retain + // their pool State; registered views retain their arena and final-release + // recycler. Control owners also retain logical assembly admission. + (!handle.is_null()).then(|| Buffer::from_raw(handle)) } // Rust-side histograms (registered with the Rust MetricsRegistry::global(), diff --git a/lib/crowdb-rpc/ffi/src/lib.rs b/lib/crowdb-rpc/ffi/src/lib.rs index ac8f0e059..333e60571 100644 --- a/lib/crowdb-rpc/ffi/src/lib.rs +++ b/lib/crowdb-rpc/ffi/src/lib.rs @@ -18,6 +18,7 @@ mod connection_pool; mod cpp_global_metrics; pub mod logging; mod owned_route; +mod payload; mod producer_buffer; mod server; pub mod sys; @@ -25,7 +26,10 @@ pub mod sys; mod test_support; mod transport; -pub use buffer::{Buffer, BufferChain, BufferChainError, BufferLeaseMetadata, BufferOwnership, BufferPool}; +pub use buffer::{ + Buffer, BufferAllocationMetadata, BufferChain, BufferChainError, BufferLeaseMetadata, BufferOwnership, + BufferPool, +}; pub use client::{noop_completion, CallFuture, ClientRequest, Response, RpcClient}; pub use connection_pool::{ConnectionPoolError, ConnectionPoolIndex, SelectedConnection}; pub use cpp_global_metrics::{cpp_global_metrics_max_name_len, flush_cpp_global_metrics}; @@ -34,6 +38,7 @@ pub use logging::{ shutdown_logging, }; pub use owned_route::{OwnedClientRoute, RpcClientHandle}; +pub use payload::{OwnedPayload, PayloadAllocation}; pub use producer_buffer::{ProducerBufferAllocator, WritableBuffer}; pub use server::{Connection, RpcError, RpcServer, ServerRequest}; pub use transport::{RdmaConfig, RdmaWireLayout}; diff --git a/lib/crowdb-rpc/ffi/src/payload.rs b/lib/crowdb-rpc/ffi/src/payload.rs new file mode 100644 index 000000000..e9e11ed31 --- /dev/null +++ b/lib/crowdb-rpc/ffi/src/payload.rs @@ -0,0 +1,193 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Host payload views retaining actual allocation metadata through storage layers. + +use std::ops::{Bound, Deref, RangeBounds}; +use std::sync::Arc; + +use bytes::Bytes; + +use crate::{Buffer, BufferLeaseMetadata}; + +pub struct PayloadAllocation { + bytes: Bytes, + capacity: usize, + native_identity: Option, + registration_id: u32, +} + +impl PayloadAllocation { + #[must_use] + pub fn bytes(&self) -> &Bytes { + &self.bytes + } + #[must_use] + pub fn capacity(&self) -> usize { + self.capacity + } + #[must_use] + pub fn native_identity(&self) -> Option { + self.native_identity + } + #[must_use] + pub fn registration_id(&self) -> u32 { + self.registration_id + } +} + +#[derive(Clone, Default)] +pub struct OwnedPayload { + bytes: Bytes, + allocation: Option>, + allocation_offset: usize, + native: Option, +} + +impl OwnedPayload { + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Retains a completed RPC buffer without detaching its receive recycler. + #[must_use] + pub fn from_buffer(buffer: Buffer) -> Self { + let metadata = buffer.lease_metadata(); + let allocation = buffer.allocation_metadata().zip(buffer.allocation_view()); + let bytes = Bytes::from_owner(buffer); + let allocation_offset = allocation + .as_ref() + .map_or(0, |(metadata, _)| metadata.offset as usize); + let allocation = allocation.map(|(allocation, owner)| { + Arc::new(PayloadAllocation { + bytes: Bytes::from_owner(owner), + capacity: allocation.capacity as usize, + native_identity: Some(allocation.identity), + registration_id: metadata.registration_id, + }) + }); + Self { + bytes, + allocation, + allocation_offset, + native: Some(metadata), + } + } + + #[must_use] + pub fn allocation(&self) -> Option<&Arc> { + self.allocation.as_ref() + } + + #[must_use] + pub fn allocation_offset(&self) -> usize { + self.allocation_offset + } + + #[must_use] + pub fn native_metadata(&self) -> Option { + self.native + } + + /// # Panics + /// Panics for ranges outside the current payload, matching Bytes::slice. + #[must_use] + pub fn slice(&self, range: impl RangeBounds) -> Self { + let start = match range.start_bound() { + Bound::Included(start) => *start, + Bound::Excluded(start) => start.checked_add(1).expect("payload slice overflow"), + Bound::Unbounded => 0, + }; + Self { + bytes: self.bytes.slice(range), + allocation: self.allocation.clone(), + allocation_offset: self.allocation_offset + start, + native: self.native, + } + } + + /// Keeps allocation metadata when another owner adds a lifetime dependency + /// around the exact same byte view, such as a logical stream credit. + /// # Panics + /// Panics if the wrapper changes the payload's address or length. + #[must_use] + pub fn with_owner(self, wrap: impl FnOnce(Bytes) -> Bytes) -> Self { + let pointer = self.bytes.as_ptr(); + let length = self.bytes.len(); + let bytes = wrap(self.bytes); + assert_eq!(bytes.as_ptr(), pointer, "payload owner changed its address"); + assert_eq!(bytes.len(), length, "payload owner changed its length"); + Self { + bytes, + allocation: self.allocation, + allocation_offset: self.allocation_offset, + native: self.native, + } + } + + #[must_use] + pub fn into_bytes(self) -> Bytes { + self.bytes + } +} + +impl From> for OwnedPayload { + fn from(data: Vec) -> Self { + let capacity = data.capacity(); + let bytes = Bytes::from(data); + Self { + allocation: Some(Arc::new(PayloadAllocation { + bytes: bytes.clone(), + capacity, + native_identity: None, + registration_id: 0, + })), + bytes, + allocation_offset: 0, + native: None, + } + } +} + +impl From for OwnedPayload { + fn from(bytes: Bytes) -> Self { + Self { + bytes, + allocation: None, + allocation_offset: 0, + native: None, + } + } +} + +impl Deref for OwnedPayload { + type Target = Bytes; + fn deref(&self) -> &Self::Target { + &self.bytes + } +} + +impl AsRef<[u8]> for OwnedPayload { + fn as_ref(&self) -> &[u8] { + &self.bytes + } +} + +impl + ?Sized> PartialEq for OwnedPayload { + fn eq(&self, other: &T) -> bool { + self.as_ref() == other.as_ref() + } +} +impl Eq for OwnedPayload {} + +impl std::fmt::Debug for OwnedPayload { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("OwnedPayload") + .field("length", &self.bytes.len()) + .field("allocation_offset", &self.allocation_offset) + .field("capacity", &self.allocation.as_ref().map(|owner| owner.capacity)) + .finish_non_exhaustive() + } +} diff --git a/lib/crowdb-rpc/ffi/src/sys.rs b/lib/crowdb-rpc/ffi/src/sys.rs index fc929cbb3..c837679f5 100644 --- a/lib/crowdb-rpc/ffi/src/sys.rs +++ b/lib/crowdb-rpc/ffi/src/sys.rs @@ -210,6 +210,10 @@ extern "C" { require_registered: c_int, ) -> crowdb_rpc_buffer_t; pub fn crowdb_rpc_buffer_capacity(buf: crowdb_rpc_buffer_t) -> u32; + pub fn crowdb_rpc_buffer_allocation_capacity(buf: crowdb_rpc_buffer_t) -> u32; + pub fn crowdb_rpc_buffer_allocation_id(buf: crowdb_rpc_buffer_t) -> u64; + pub fn crowdb_rpc_buffer_allocation_offset(buf: crowdb_rpc_buffer_t) -> u32; + pub fn crowdb_rpc_buffer_allocation_ref(buf: crowdb_rpc_buffer_t) -> crowdb_rpc_buffer_t; pub fn crowdb_rpc_buffer_request_id(buf: crowdb_rpc_buffer_t) -> u64; pub fn crowdb_rpc_buffer_stream_id(buf: crowdb_rpc_buffer_t) -> u64; pub fn crowdb_rpc_buffer_sequence(buf: crowdb_rpc_buffer_t) -> u64; diff --git a/lib/crowdb-rpc/ffi/tests/payload_test.rs b/lib/crowdb-rpc/ffi/tests/payload_test.rs new file mode 100644 index 000000000..0150b2343 --- /dev/null +++ b/lib/crowdb-rpc/ffi/tests/payload_test.rs @@ -0,0 +1,56 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_rpc_ffi::{Buffer, OwnedPayload}; + +#[test] +fn payload_slices_keep_one_exact_capacity_owner_and_original_addresses() { + let mut source = Vec::with_capacity(1024); + source.extend_from_slice(b"0123456789"); + let pointer = source.as_ptr(); + let payload = OwnedPayload::from(source); + assert_eq!(payload.as_ptr(), pointer); + assert_eq!(payload.allocation().unwrap().capacity(), 1024); + let first = payload.slice(2..8); + let nested = first.slice(1..4); + assert_eq!(nested.allocation_offset(), 3); + assert!(Arc::ptr_eq( + payload.allocation().unwrap(), + nested.allocation().unwrap() + )); + assert_eq!(nested.as_ptr(), payload.as_ref()[3..].as_ptr()); + drop(payload); + drop(first); + assert_eq!(nested.as_ref(), b"345"); + assert_eq!(nested.allocation().unwrap().bytes().as_ref(), b"0123456789"); +} + +#[test] +fn external_bytes_keep_storage_but_do_not_invent_an_allocation_capacity() { + let bytes = Bytes::from(vec![7; 16]); + let pointer = bytes.as_ptr(); + let payload = OwnedPayload::from_buffer(Buffer::from_owned_bytes(bytes)); + assert_eq!(payload.as_ptr(), pointer); + assert!(payload.allocation().is_none()); + assert!(payload.slice(4..).allocation().is_none()); + assert_eq!(payload.len(), 16); +} + +#[test] +fn adding_an_owner_keeps_the_allocation_and_its_view_offset() { + struct TestOwner(Bytes); + impl AsRef<[u8]> for TestOwner { + fn as_ref(&self) -> &[u8] { + &self.0 + } + } + let payload = OwnedPayload::from(vec![9; 16]).slice(4..12); + let allocation = Arc::clone(payload.allocation().unwrap()); + let payload = payload.with_owner(|bytes| Bytes::from_owner(TestOwner(bytes))); + assert_eq!(payload.allocation_offset(), 4); + assert!(Arc::ptr_eq(&allocation, payload.allocation().unwrap())); + assert_eq!(payload.into_bytes().as_ref(), &[9; 8]); +} diff --git a/lib/crowdb-rpc/ffi/tests/receive_lease_test.rs b/lib/crowdb-rpc/ffi/tests/receive_lease_test.rs new file mode 100644 index 000000000..b133d0255 --- /dev/null +++ b/lib/crowdb-rpc/ffi/tests/receive_lease_test.rs @@ -0,0 +1,112 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::time::Duration; + +use crowdb_protocol::fb::{ConnectionPingRequest, ConnectionPingRequestArgs}; +use crowdb_rpc_ffi::{Buffer, OwnedPayload, RdmaConfig, RpcClient, RpcServer}; +use flatbuffers::FlatBufferBuilder; + +fn control(id: u64) -> Buffer { + let mut builder = FlatBufferBuilder::new(); + let request = ConnectionPingRequest::create( + &mut builder, + &ConnectionPingRequestArgs { + id, + rpc_create_nano: 0, + }, + ); + builder.finish(request, None); + Buffer::from_bytes(builder.finished_data()) +} + +#[tokio::test] +async fn retained_rdma_payload_applies_receive_backpressure_and_survives_shutdown() { + if std::env::var_os("CROWDB_RDMA_E2E").is_none() { + return; + } + let address = std::env::var("CROWDB_RDMA_E2E_LISTEN_ADDR").unwrap(); + let config = RdmaConfig { + enabled: true, + allow_tcp_fallback: false, + listen_enabled: true, + device: std::env::var("CROWDB_RDMA_DEVICE").unwrap_or_default(), + source_address: address.clone(), + sq_depth: 4, + rq_depth: 2, + posted_receive_target: 2, + cq_depth: 16, + ..RdmaConfig::default() + } + .with_control_data_capacity(65_536) + .unwrap(); + let server = RpcServer::new(None); + server.configure_rdma(&config).unwrap(); + server.register_echo_handler(100); + server.listen(&address, 0).unwrap(); + server.start(); + let client = RpcServer::new(None); + client + .configure_rdma(&RdmaConfig { + listen_enabled: false, + ..config + }) + .unwrap(); + client.listen("127.0.0.1", 0).unwrap(); + client.start(); + let connection = client.connect(&address, server.port()).unwrap(); + assert!(connection.is_rdma()); + let caller = RpcClient::new(); + caller.attach(&connection); + let mut held = Vec::new(); + for id in 1..=2 { + let reply = tokio::time::timeout( + Duration::from_secs(2), + caller + .call( + &client, + &connection, + id, + control(id), + Some(Buffer::from_vec(vec![id as u8; 8192])), + 100, + ) + .unwrap(), + ) + .await + .unwrap() + .unwrap(); + let data = OwnedPayload::from_buffer(reply.data.unwrap()); + assert!(data.allocation().unwrap().capacity() >= 65_536); + assert_ne!(data.allocation().unwrap().native_identity(), Some(0)); + assert_eq!(data.native_metadata().unwrap().request_id, id); + held.push(data); + } + let mut pending = caller + .call( + &client, + &connection, + 3, + control(3), + Some(Buffer::from_vec(vec![3; 8192])), + 100, + ) + .unwrap(); + assert!(tokio::time::timeout(Duration::from_millis(30), &mut pending) + .await + .is_err()); + drop(held.remove(0)); + let reply = tokio::time::timeout(Duration::from_secs(2), pending) + .await + .unwrap() + .unwrap(); + let data = OwnedPayload::from_buffer(reply.data.unwrap()); + drop(connection); + client.stop(); + server.stop(); + drop(client); + drop(server); + assert_eq!(held[0].as_ref(), &[2; 8192]); + assert_eq!(data.as_ref(), &[3; 8192]); + assert!(!data.allocation().unwrap().bytes().is_empty()); +} diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h index f3e1d046a..560bf9474 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h @@ -42,12 +42,21 @@ uint32_t crowdb_rpc_buffer_len(crowdb_rpc_buffer_t buf); uint8_t *crowdb_rpc_buffer_mutable_data(crowdb_rpc_buffer_t handle); int crowdb_rpc_buffer_commit(crowdb_rpc_buffer_t handle, uint32_t initialized_length); uint32_t crowdb_rpc_buffer_capacity(crowdb_rpc_buffer_t buf); -uint64_t crowdb_rpc_buffer_request_id(crowdb_rpc_buffer_t buf); -uint64_t crowdb_rpc_buffer_stream_id(crowdb_rpc_buffer_t buf); -uint64_t crowdb_rpc_buffer_sequence(crowdb_rpc_buffer_t buf); -uint64_t crowdb_rpc_buffer_logical_offset(crowdb_rpc_buffer_t buf); -uint32_t crowdb_rpc_buffer_registration_id(crowdb_rpc_buffer_t buf); -int crowdb_rpc_buffer_ownership(crowdb_rpc_buffer_t buf); +// Canonical backing capacity, independent of a slice's logical view size. +// Zero means unknown external storage or an invalid/null handle. Allocation +// identity is opaque and meaningful only while a backing owner is retained. +uint32_t crowdb_rpc_buffer_allocation_capacity(crowdb_rpc_buffer_t handle); +uint64_t crowdb_rpc_buffer_allocation_id(crowdb_rpc_buffer_t handle); +uint32_t crowdb_rpc_buffer_allocation_offset(crowdb_rpc_buffer_t handle); +// Retains only the canonical allocation's initialized span, including any +// wire framing. Ordinary logical payload access remains on the original view. +crowdb_rpc_buffer_t crowdb_rpc_buffer_allocation_ref(crowdb_rpc_buffer_t handle); +uint64_t crowdb_rpc_buffer_request_id(crowdb_rpc_buffer_t buf); +uint64_t crowdb_rpc_buffer_stream_id(crowdb_rpc_buffer_t buf); +uint64_t crowdb_rpc_buffer_sequence(crowdb_rpc_buffer_t buf); +uint64_t crowdb_rpc_buffer_logical_offset(crowdb_rpc_buffer_t buf); +uint32_t crowdb_rpc_buffer_registration_id(crowdb_rpc_buffer_t buf); +int crowdb_rpc_buffer_ownership(crowdb_rpc_buffer_t buf); void crowdb_rpc_buffer_set_metadata(crowdb_rpc_buffer_t buf, uint64_t request_id, uint64_t stream_id, uint64_t sequence, uint64_t logical_offset, uint32_t registration_id); int crowdb_rpc_buffer_transition(crowdb_rpc_buffer_t buf, int from, int to); diff --git a/lib/crowdb-rpc/src/c_api/buffer_lease.cpp b/lib/crowdb-rpc/src/c_api/buffer_lease.cpp new file mode 100644 index 000000000..15cbf27c6 --- /dev/null +++ b/lib/crowdb-rpc/src/c_api/buffer_lease.cpp @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/c_api_internal.h" + +namespace +{ +crowdb::rpc::Buffer *allocation(crowdb_rpc_buffer_t handle) +{ + if (handle == nullptr || handle->buf == nullptr) { + return nullptr; + } + auto *buffer = handle->buf->allocation_owner == nullptr ? handle->buf : handle->buf->allocation_owner; + // External Bytes wrappers may retain a larger allocation than their view; + // only their owner can supply that capacity. Do not guess from view length. + if (buffer->allocation_capacity == 0 && buffer->free_cb != nullptr) { + return nullptr; + } + return buffer; +} +} // namespace + +extern "C" uint32_t crowdb_rpc_buffer_allocation_capacity(crowdb_rpc_buffer_t handle) +{ + const auto *buffer = allocation(handle); + if (buffer == nullptr) { + return 0; + } + return buffer->allocation_capacity == 0 ? buffer->capacity : buffer->allocation_capacity; +} + +extern "C" uint64_t crowdb_rpc_buffer_allocation_id(crowdb_rpc_buffer_t handle) +{ + const auto *buffer = allocation(handle); + if (buffer == nullptr) { + return 0; + } + return static_cast( + reinterpret_cast(buffer->allocation_data == nullptr ? buffer->data : buffer->allocation_data)); +} + +extern "C" uint32_t crowdb_rpc_buffer_allocation_offset(crowdb_rpc_buffer_t handle) +{ + const auto *buffer = allocation(handle); + if (buffer == nullptr) { + return 0; + } + const auto begin = reinterpret_cast(buffer->payload_data()); + const auto view = reinterpret_cast(handle->buf->payload_data()); + if (view < begin || view - begin > buffer->len) { + return 0; + } + return static_cast(view - begin); +} + +extern "C" crowdb_rpc_buffer_t crowdb_rpc_buffer_allocation_ref(crowdb_rpc_buffer_t handle) +{ + try { + auto *buffer = allocation(handle); + if (buffer == nullptr || buffer->len == 0) { + return nullptr; + } + auto result = std::make_unique(); + result->buf = buffer->slice(0, buffer->len); + return result->buf == nullptr ? nullptr : result.release(); + } + catch (...) { + return nullptr; + } +} diff --git a/lib/crowdb-rpc/src/c_api/frame.cpp b/lib/crowdb-rpc/src/c_api/frame.cpp index 0e4acda9d..075606a8c 100644 --- a/lib/crowdb-rpc/src/c_api/frame.cpp +++ b/lib/crowdb-rpc/src/c_api/frame.cpp @@ -31,10 +31,12 @@ crowdb_rpc_buffer_t wrap_control(Frame &frame) return handle; } -crowdb_rpc_buffer_t wrap_data(Buffer *buffer) +crowdb_rpc_buffer_t wrap_data(Buffer *buffer, uint64_t request_id) { if (buffer == nullptr || buffer->len == 0) return nullptr; + buffer->set_metadata(request_id, buffer->stream_id, buffer->sequence, buffer->logical_offset, + buffer->registration_id); auto handle = std::make_unique(); handle->buf = buffer->ref_clone(); return handle->buf == nullptr ? nullptr : handle.release(); @@ -51,7 +53,7 @@ void frame_to_c_handles(Frame *frame, crowdb_rpc_buffer_t *out_ctrl, crowdb_rpc_ // Budget-owned controls move their exact storage and admission into the // external owner. Ordinary controls preserve their existing copied handle. *out_ctrl = wrap_control(*incoming); - *out_data = wrap_data(incoming->data_buf); + *out_data = wrap_data(incoming->data_buf, incoming->request_id); } void invoke_c_handler(crowdb_rpc_handler_fn callback, void *user_data, Frame *request, Connection *conn) diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_direct_receive.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_receive.cpp index dce3bf96a..14a136fc8 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_direct_receive.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_direct_receive.cpp @@ -128,6 +128,7 @@ bool RdmaTransport::release_completed_direct_receive(WorkRequest *work, const ib return true; } queue.charge_receive(completion.byte_len); + slot->len = header.data_size; auto connection = std::move(work->conn); auto pools = std::move(work->pools); slot->last_release_cb = &RdmaTransport::repost_frame_release; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_packed_receive.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_packed_receive.cpp index e6eb4c14d..ff95dabed 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_packed_receive.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_packed_receive.cpp @@ -39,6 +39,7 @@ bool RdmaTransport::release_completed_packed_receive(WorkRequest *work, const ib return true; } queue.charge_receive(wc.byte_len, aggregate.count); + slot->len = wc.byte_len - RDMA_CONTROL_CAPACITY; auto connection = std::move(work->conn); auto pools = std::move(work->pools); slot->last_release_cb = &RdmaTransport::repost_frame_release; diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp index be4833ce7..ef506a845 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_receive.cpp @@ -46,6 +46,7 @@ bool RdmaTransport::release_completed_receive(WorkRequest *work, const ibv_wc &w return true; } queue.charge_receive(wc.byte_len, aggregate.count); + work->slot->len = wc.byte_len; auto connection = std::move(work->conn); auto pools = std::move(work->pools); diff --git a/lib/crowdb-rpc/tests/buffer_lease_test.cpp b/lib/crowdb-rpc/tests/buffer_lease_test.cpp new file mode 100644 index 000000000..2f8115ffc --- /dev/null +++ b/lib/crowdb-rpc/tests/buffer_lease_test.cpp @@ -0,0 +1,62 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/c_api_internal.h" + +#include + +#include +#include + +using namespace crowdb::rpc; + +TEST(BufferLeaseTest, NestedViewsRetainCanonicalInitializedSpanAfterPoolDestruction) +{ + crowdb_rpc_buffer_t handle = nullptr; + uint64_t identity = 0; + { + RegisteredBufferPool pool(65536, 2, 17); + auto *root = pool.alloc(128); + ASSERT_NE(root, nullptr); + std::array bytes{}; + bytes.fill(0x5a); + root->write(bytes.data(), bytes.size()); + auto *view = root->slice(8, 17); + auto *nested = view->slice(3, 6); + ASSERT_NE(nested, nullptr); + handle = new crowdb_rpc_buffer_s{nested}; + identity = crowdb_rpc_buffer_allocation_id(handle); + EXPECT_NE(identity, 0U); + EXPECT_EQ(crowdb_rpc_buffer_capacity(handle), 6U); + EXPECT_EQ(crowdb_rpc_buffer_allocation_capacity(handle), 1024U); + EXPECT_EQ(crowdb_rpc_buffer_allocation_offset(handle), 11U); + view->release(); + root->release(); + } + auto *allocation = crowdb_rpc_buffer_allocation_ref(handle); + ASSERT_NE(allocation, nullptr); + EXPECT_EQ(crowdb_rpc_buffer_len(allocation), 32U); + EXPECT_EQ(crowdb_rpc_buffer_allocation_id(allocation), identity); + EXPECT_EQ(crowdb_rpc_buffer_allocation_offset(allocation), 0U); + crowdb_rpc_buffer_release(handle); + for (uint32_t index = 0; index < 32; ++index) { + EXPECT_EQ(crowdb_rpc_buffer_data(allocation)[index], 0x5a); + } + crowdb_rpc_buffer_release(allocation); +} + +TEST(BufferLeaseTest, UnknownExternalCapacityIsNeverGuessedFromItsViewLength) +{ + std::array bytes{}; + uint32_t released = 0; + auto *handle = crowdb_rpc_buffer_create_external( + bytes.data(), bytes.size(), [](void *context) { ++*static_cast(context); }, &released); + ASSERT_NE(handle, nullptr); + EXPECT_EQ(crowdb_rpc_buffer_allocation_capacity(handle), 0U); + EXPECT_EQ(crowdb_rpc_buffer_allocation_id(handle), 0U); + EXPECT_EQ(crowdb_rpc_buffer_allocation_ref(handle), nullptr); + crowdb_rpc_buffer_release(handle); + EXPECT_EQ(released, 1U); + EXPECT_EQ(crowdb_rpc_buffer_allocation_capacity(nullptr), 0U); + EXPECT_EQ(crowdb_rpc_buffer_allocation_ref(nullptr), nullptr); +} From 2b3ea18be8d092bc498a2a99b2514c50637dd4d3 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 13:04:09 +0800 Subject: [PATCH 061/258] Preserve allocation ownership through storage read views --- doc/working/plan-rdma-fat-client-sdk.md | 10 + lib/crowdb-access-s3/src/retrieval.rs | 14 +- .../src/chunk/chunk_reader.rs | 450 +----------------- .../src/chunk/chunk_reader/frame_views.rs | 117 +++++ .../src/chunk/chunk_reader/framing.rs | 68 +-- .../src/chunk/chunk_reader/layout.rs | 88 +++- .../src/chunk/chunk_reader/location.rs | 243 ++++++++++ .../chunk_reader/{streaming.rs => stream.rs} | 96 +++- .../{verified.rs => verified_frame.rs} | 14 +- .../src/chunk/read_credit.rs | 5 +- .../src/chunk/strip_reader.rs | 32 +- .../src/disk_io/disk_writer.rs | 14 + .../src/disk_io/routing.rs | 16 +- .../tests/common/read_payload.rs | 60 +++ .../tests/read_payload_test.rs | 74 +++ lib/crowdb-diskio-client/src/client.rs | 16 +- lib/crowdb-diskio-client/src/semantic.rs | 25 +- 17 files changed, 806 insertions(+), 536 deletions(-) create mode 100644 lib/crowdb-chunk-client/src/chunk/chunk_reader/frame_views.rs create mode 100644 lib/crowdb-chunk-client/src/chunk/chunk_reader/location.rs rename lib/crowdb-chunk-client/src/chunk/chunk_reader/{streaming.rs => stream.rs} (72%) rename lib/crowdb-chunk-client/src/chunk/chunk_reader/{verified.rs => verified_frame.rs} (96%) create mode 100644 lib/crowdb-chunk-client/tests/common/read_payload.rs create mode 100644 lib/crowdb-chunk-client/tests/read_payload_test.rs diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index d2097e636..e000b3e24 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -250,6 +250,16 @@ transport acceptance alone cannot establish SDK completion. owner-backed payload type with slices preserving one allocation owner; do not rediscover receive ranges by scanning connection tables or adding a hot lock, and do not substitute a guessed capacity for the actual native class. +- DiskIO now exposes `read_payload`, and production ChunkIO mirror/direct + reads, frame slices, logical stream-credit wrappers and S3 `next_payload` + preserve `OwnedPayload`. Existing public `Bytes` adapters erase metadata only + at their compatibility boundary. Required EC/physical coalescing uses fresh + Vec owners with known capacity; metadata-less injected seams remain explicit. + The 1,500-line reader is split by layout, location, frame views, verified + frame reads and stream delivery. A focused framing test proves original + address/capacity retention and logical credit lifetime through final Bytes + release; all existing reader cases pass. Native SDK capacity admission and + S3 body adaptation are the next dependent tasks. ## Subsequent Dataset integration under R220 diff --git a/lib/crowdb-access-s3/src/retrieval.rs b/lib/crowdb-access-s3/src/retrieval.rs index 8e34feaae..4c13c5c9c 100644 --- a/lib/crowdb-access-s3/src/retrieval.rs +++ b/lib/crowdb-access-s3/src/retrieval.rs @@ -8,6 +8,7 @@ use std::time::{Duration, UNIX_EPOCH}; use crowdb_chunk_client::{ChunkIoClient, ChunkReadStream, ReadError}; use crowdb_protocol::chunkdb::rpc::Location; +use crowdb_rpc_ffi::OwnedPayload; use hyper::body::Bytes; use crate::condition::{evaluate, ConditionOutcome, ObjectConditions}; @@ -73,6 +74,15 @@ impl VerifiedGetStream { /// # Errors /// Returns storage, length or final checksum failure without successful EOF. pub async fn next_verified(&mut self) -> Result, GetStreamError> { + self.next_payload() + .await + .map(|data| data.map(OwnedPayload::into_bytes)) + } + + /// Pulls the same verified logical view with allocation ownership intact. + /// # Errors + /// Returns sticky storage, length or applicable final checksum failures. + pub async fn next_payload(&mut self) -> Result, GetStreamError> { if let Some(error) = &self.failure { return Err(error.clone()); } @@ -80,7 +90,7 @@ impl VerifiedGetStream { return Ok(None); } let next = match &mut self.inner { - Some(inner) => inner.next_chunk().await, + Some(inner) => inner.next_payload().await, None => None, }; match next { @@ -118,7 +128,7 @@ impl VerifiedGetStream { } } - fn fail(&mut self, error: GetStreamError) -> Result, GetStreamError> { + fn fail(&mut self, error: GetStreamError) -> Result, GetStreamError> { self.terminal = true; self.inner = None; self.failure = Some(error.clone()); diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader.rs index 33be7ab41..7135890f9 100644 --- a/lib/crowdb-chunk-client/src/chunk/chunk_reader.rs +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader.rs @@ -3,35 +3,33 @@ //! Object and range reads over current chunk layouts. -use std::collections::BTreeMap; -use std::ops::Range; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; -use bytes::{Bytes, BytesMut}; -use crowdb_protocol::chunkdb::rpc::{ - AdHocEcRecoveryDisposition, AdHocEcRecoveryRequest, Chunk, ChunkState, Location, QueryChunkRequest, - QueryChunkResponse, -}; +use bytes::Bytes; +use crowdb_protocol::chunkdb::rpc::{Chunk, Location, QueryChunkRequest, QueryChunkResponse}; use crowdb_protocol::common::ChunkId; -use crowdb_protocol::diskdb::rpc::Segment; +use crowdb_rpc_ffi::OwnedPayload; use tokio::sync::Semaphore; use tokio::task::JoinSet; use super::client_recovery::ClientRecovery; -use super::read_credit::{ReadBudget, ReadLease, StreamSlots}; +use super::read_credit::ReadBudget; use super::strip_reader::StripReader; use crate::{ChunkAllocator, DiskWriter, IoError, ReadError, ReadResult}; +mod frame_views; mod framing; -mod layout; mod layout_cache; -mod streaming; -mod verified; +mod layout; +mod location; +mod stream; +mod verified_frame; -use layout::{strip_end, strip_read_parts, validate_strip_layout}; +use layout::{normalize_locations, StripFailureObservation}; use layout_cache::FrameLayoutCache; +pub use stream::ChunkReadStream; const KIB: u64 = 1024; const DEFAULT_STREAM_WINDOW: usize = 1024 * 1024; @@ -55,7 +53,7 @@ pub struct ChunkReadPolicy { pub struct ReadRangeData { pub start: u64, pub end: u64, - pub data: Bytes, + pub data: OwnedPayload, } /// A logical sub-range that could not be reconstructed. @@ -317,376 +315,6 @@ impl ChunkReader { partial.failures = failures; Ok(partial) } - - pub fn read_stream(&self, locations: &[Location]) -> ReadResult { - self.flow_metrics.location_normalizations.inc(); - self.flow_metrics - .locations_examined - .inc_by(u64::try_from(locations.len()).unwrap_or(u64::MAX)); - let (locations, object_length) = normalize_locations(locations)?; - self.range_stream(locations, 0, object_length, object_length) - } - - /// Builds a pull-based stream for the exact logical half-open range. - pub fn read_range_stream( - &self, - locations: &[Location], - start: u64, - end: u64, - ) -> ReadResult { - self.flow_metrics.location_normalizations.inc(); - self.flow_metrics - .locations_examined - .inc_by(u64::try_from(locations.len()).unwrap_or(u64::MAX)); - let (locations, object_length) = normalize_locations(locations)?; - self.range_stream(locations, start, end, object_length) - } - - fn range_stream( - &self, - locations: Vec, - start: u64, - end: u64, - object_length: u64, - ) -> ReadResult { - if start > end || end > object_length { - return Err(ReadError::InvalidRange { - start, - end, - object_length, - }); - } - Ok(ChunkReadStream { - reader: self.clone(), - locations: Arc::from(locations), - cursor: start, - fetch_cursor: start, - delivery_cursor: start, - end, - window_bytes: self.policy.stream_window_bytes as u64, - slots: Arc::new(StreamSlots::new(self.policy.stream_slots)), - reads: JoinSet::new(), - completed: BTreeMap::new(), - layouts: Vec::new(), - pending_error: None, - pending_ranges: Vec::new().into_iter(), - }) - } - - async fn read_location_partial( - &self, - location: &Location, - local_start: u64, - length: u64, - logical_start: u64, - ) -> ReadResult { - self.read_location_partial_cached(location, local_start, length, logical_start, None) - .await - } - - async fn read_location_partial_cached( - &self, - location: &Location, - local_start: u64, - length: u64, - logical_start: u64, - mut first_layout: Option, - ) -> ReadResult { - if location.length != location.logical_length { - return self - .read_framed_location(location, local_start, length, logical_start, first_layout) - .await; - } - let chunk_id = location - .chunk_id - .ok_or_else(|| ReadError::InvalidLocations("location has no chunk ID".into()))?; - let physical_start = location - .offset - .checked_add(local_start) - .ok_or_else(|| ReadError::InvalidLocations("location physical offset overflows".into()))?; - for _ in 0..self.policy.max_layout_retries { - let layout = match first_layout.take().filter(LayoutSnapshot::usable) { - Some(layout) => layout, - None => self.query_layout(chunk_id).await?, - }; - let (partial, observations) = self - .read_chunk_range_partial(&layout.chunk, physical_start, length, logical_start) - .await?; - if Instant::now() >= layout.deadline { - layout.invalidate(); - continue; - } - if observations - .iter() - .any(|observation| !observation.failed_segments.is_empty()) - { - layout.invalidate(); - let mut chunk = (*layout.chunk).clone(); - if self - .mark_observed_failures(&mut chunk, observations) - .await - .is_err() - { - continue; - } - } - if Instant::now() < layout.deadline { - return Ok(partial); - } - } - Err(ReadError::LayoutExpired) - } - - async fn read_chunk_range_partial( - &self, - chunk: &Chunk, - start: u64, - length: u64, - logical_start: u64, - ) -> ReadResult<(PartialReadResult, Vec)> { - let started = Instant::now(); - let result = self - .read_chunk_range_partial_inner(chunk, start, length, logical_start) - .await; - self.flow_metrics - .chunk_read_wait_ns - .inc_by(u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX)); - result - } - - async fn read_chunk_range_partial_inner( - &self, - chunk: &Chunk, - start: u64, - length: u64, - logical_start: u64, - ) -> ReadResult<(PartialReadResult, Vec)> { - if chunk.state == ChunkState::Deleted as i32 { - let id = chunk.id.unwrap_or_default(); - return Err(ReadError::ChunkDeleted(format!("{}:{}", id.high, id.low))); - } - validate_strip_layout(chunk)?; - let end = start - .checked_add(length) - .ok_or_else(|| ReadError::InvalidLocations("chunk read end overflows".into()))?; - let first = chunk - .strips - .partition_point(|strip| strip_end(strip).is_ok_and(|strip_end| strip_end <= start)); - let mut cursor = start; - let mut partial = PartialReadResult::default(); - let mut observations = Vec::new(); - for strip in &chunk.strips[first..] { - let strip_start = u64::from(strip.chunk_offset) * KIB; - if strip_start >= end { - break; - } - if strip_start > cursor { - return Err(ReadError::InvalidLocations(format!( - "chunk layout has a gap at byte {cursor}" - ))); - } - let overlap_end = end.min(strip_end(strip)?); - if overlap_end <= cursor { - continue; - } - let acknowledged = chunk.acknowledged_cursor.saturating_sub(strip_start); - let durable_bytes = acknowledged.min(u64::from(strip.capacity) * KIB); - for (part_start, part_end) in strip_read_parts(strip, cursor, overlap_end)? { - let range_start = logical_start + (part_start - start); - let range_end = range_start + (part_end - part_start); - let read_started = Instant::now(); - let observed = self - .strip_reader - .read_observed( - strip, - chunk.modify_ts, - durable_bytes, - part_start - strip_start, - part_end - part_start, - ) - .await; - self.flow_metrics - .strip_read_wait_ns - .inc_by(u64::try_from(read_started.elapsed().as_nanos()).unwrap_or(u64::MAX)); - if !observed.failed_segments.is_empty() { - observations.push(StripFailureObservation { - strip_sequence: strip.strip_sequence, - failed_segments: observed.failed_segments, - served_segments: observed.served_segments, - }); - } else if !observed.served_segments.is_empty() { - observations.push(StripFailureObservation { - strip_sequence: strip.strip_sequence, - failed_segments: Vec::new(), - served_segments: observed.served_segments, - }); - } - match observed.result { - Ok(data) => partial.ranges.push(ReadRangeData { - start: range_start, - end: range_end, - data, - }), - Err(error) => partial.failures.push(FailedReadRange { - start: range_start, - end: range_end, - error, - }), - } - } - cursor = overlap_end; - } - if cursor != end { - return Err(ReadError::InvalidLocations(format!( - "chunk layout ends at byte {cursor}, requested {end}" - ))); - } - Ok((partial, observations)) - } - - async fn mark_observed_failures( - &self, - chunk: &mut Chunk, - observations: Vec, - ) -> ReadResult<()> { - let chunk_id = chunk - .id - .ok_or_else(|| ReadError::InvalidLocations("chunk has no ID".into()))?; - for observation in observations { - let Some(index) = chunk - .strips - .iter() - .position(|strip| strip.strip_sequence == observation.strip_sequence) - else { - return Err(ReadError::Metadata("failed strip disappeared".into())); - }; - let old = chunk.strips[index].clone(); - for segment in observation.failed_segments { - if old.unavailable_segments.contains(&segment) { - continue; - } - let mut replacement = chunk.strips[index].clone(); - replacement.unavailable_segments.push(segment); - replacement.unavailable_segments.sort_by_key(segment_identity); - let operation_id = failure_operation_id(chunk_id, &replacement); - let response = self - .chunkdb - .ad_hoc_ec_recovery(AdHocEcRecoveryRequest { - version: 1, - chunk_id: Some(chunk_id), - expected_modify_ts: chunk.modify_ts, - strip_sequence: observation.strip_sequence, - failed_segment: Some(segment), - operation_id: Some(operation_id), - request_full_block: false, - }) - .await - .map_err(|error| ReadError::Metadata(error.to_string()))?; - if response.disposition != AdHocEcRecoveryDisposition::Marked { - return Err(ReadError::Metadata(format!( - "corrupt segment report was {:?}", - response.disposition - ))); - } - *chunk = self - .query_chunk_timed(chunk_id) - .await - .map_err(|error| ReadError::Metadata(error.to_string()))? - .chunk - .ok_or_else(|| ReadError::Metadata("corrupt report returned no chunk".into()))?; - } - } - Ok(()) - } -} - -fn extract_physical_range(ranges: &[ReadRangeData], wanted: Range) -> ReadResult { - let first = ranges.partition_point(|range| range.end <= wanted.start); - if let Some(range) = ranges.get(first) { - if range.start <= wanted.start && range.end >= wanted.end { - let offset = usize::try_from(wanted.start - range.start) - .map_err(|_| ReadError::InvalidLocations("physical frame offset overflows".into()))?; - let length = usize::try_from(wanted.end - wanted.start) - .map_err(|_| ReadError::InvalidLocations("physical frame length overflows".into()))?; - return Ok(range.data.slice(offset..offset + length)); - } - } - let mut output = - BytesMut::with_capacity(usize::try_from(wanted.end - wanted.start).unwrap_or(usize::MAX)); - let mut cursor = wanted.start; - for range in &ranges[first..] { - if range.end <= cursor || range.start >= wanted.end { - continue; - } - let start = cursor.max(range.start); - if start != cursor { - return Err(ReadError::InvalidLocations( - "physical frame data has a gap".into(), - )); - } - let end = wanted.end.min(range.end); - let offset = usize::try_from(start - range.start) - .map_err(|_| ReadError::InvalidLocations("physical frame offset overflows".into()))?; - let length = usize::try_from(end - start) - .map_err(|_| ReadError::InvalidLocations("physical frame length overflows".into()))?; - output.extend_from_slice(&range.data[offset..offset + length]); - cursor = end; - if cursor == wanted.end { - break; - } - } - if cursor != wanted.end { - return Err(ReadError::InvalidLocations( - "physical frame data is incomplete".into(), - )); - } - Ok(output.freeze()) -} - -struct StripFailureObservation { - strip_sequence: u32, - failed_segments: Vec, - served_segments: Vec, -} - -fn mark_served_segments_corrupt( - mut observations: Vec, -) -> Vec { - let serving_count: usize = observations - .iter() - .map(|observation| observation.served_segments.len()) - .sum(); - if serving_count != 1 { - return Vec::new(); - } - for observation in &mut observations { - for segment in std::mem::take(&mut observation.served_segments) { - if !observation.failed_segments.contains(&segment) { - observation.failed_segments.push(segment); - } - } - } - observations.retain(|observation| !observation.failed_segments.is_empty()); - observations -} - -type StreamRead = (u64, u64, ReadResult, Arc); - -/// Ordered, bounded read pipeline whose frames retain their read credits. -pub struct ChunkReadStream { - reader: ChunkReader, - locations: Arc<[Location]>, - cursor: u64, - fetch_cursor: u64, - delivery_cursor: u64, - end: u64, - window_bytes: u64, - slots: Arc, - reads: JoinSet, - completed: BTreeMap, - layouts: Vec, - pending_error: Option, - pending_ranges: std::vec::IntoIter, } fn complete_read(partial: PartialReadResult, start: u64, end: u64) -> ReadResult> { @@ -706,7 +334,7 @@ fn complete_read(partial: PartialReadResult, start: u64, end: u64) -> ReadResult ))); } cursor = range.end; - buffers.push(range.data); + buffers.push(range.data.into_bytes()); } if cursor != end { return Err(ReadError::InvalidLocations(format!( @@ -716,58 +344,6 @@ fn complete_read(partial: PartialReadResult, start: u64, end: u64) -> ReadResult Ok(buffers) } -fn segment_identity(segment: &Segment) -> (u64, u64, u32, u64, u64) { - let disk = segment.disk_id.unwrap_or_default(); - ( - disk.high, - disk.low, - segment.zone_index, - segment.unit_offset, - segment.allocation_ts, - ) -} - -fn failure_operation_id(chunk_id: ChunkId, strip: &crowdb_protocol::chunkdb::rpc::ChunkStrip) -> ChunkId { - let mut high = chunk_id.high ^ 0x111f_a11e_d000_0001; - let mut low = chunk_id.low ^ u64::from(strip.strip_sequence); - for segment in &strip.unavailable_segments { - let disk = segment.disk_id.unwrap_or_default(); - high = high.rotate_left(13) ^ disk.high ^ segment.unit_offset; - low = low.rotate_left(17) ^ disk.low ^ segment.allocation_ts; - } - ChunkId { high, low } -} - -fn normalize_locations(locations: &[Location]) -> ReadResult<(Vec, u64)> { - let mut locations: Vec<_> = locations - .iter() - .filter(|location| location.logical_length != 0 && location.length != 0) - .cloned() - .collect(); - locations.sort_unstable_by_key(|location| location.logical_offset); - let mut cursor = 0u64; - for location in &locations { - if location.chunk_id.is_none() || location.logical_length > location.length { - return Err(ReadError::InvalidLocations( - "location is missing a chunk or logical length exceeds physical length".into(), - )); - } - if location.logical_offset != cursor { - return Err(ReadError::InvalidLocations(format!( - "logical locations are not contiguous at byte {cursor}" - ))); - } - location - .offset - .checked_add(location.length) - .ok_or_else(|| ReadError::InvalidLocations("physical location overflows".into()))?; - cursor = cursor - .checked_add(location.logical_length) - .ok_or_else(|| ReadError::InvalidLocations("object length overflows".into()))?; - } - Ok((locations, cursor)) -} - fn map_metadata_error(error: IoError) -> ReadError { match error { IoError::ChunkNotFound(message) => ReadError::ChunkDeleted(message), diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader/frame_views.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader/frame_views.rs new file mode 100644 index 000000000..c97936485 --- /dev/null +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader/frame_views.rs @@ -0,0 +1,117 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Physical payload views preserving allocation ownership. + +use super::ReadRangeData; +use crate::{ReadError, ReadResult}; +use crowdb_rpc_ffi::OwnedPayload; +use std::ops::Range; + +pub(super) fn extract_physical_range( + ranges: &[ReadRangeData], + wanted: Range, +) -> ReadResult { + let first = ranges.partition_point(|range| range.end <= wanted.start); + if let Some(range) = ranges.get(first) { + if range.start <= wanted.start && range.end >= wanted.end { + let offset = usize::try_from(wanted.start - range.start) + .map_err(|_| ReadError::InvalidLocations("physical frame offset overflows".into()))?; + let length = usize::try_from(wanted.end - wanted.start) + .map_err(|_| ReadError::InvalidLocations("physical frame length overflows".into()))?; + return Ok(range.data.slice(offset..offset + length)); + } + } + let mut output = Vec::with_capacity(usize::try_from(wanted.end - wanted.start).unwrap_or(usize::MAX)); + let mut cursor = wanted.start; + for range in &ranges[first..] { + if range.end <= cursor || range.start >= wanted.end { + continue; + } + let start = cursor.max(range.start); + if start != cursor { + return Err(ReadError::InvalidLocations( + "physical frame data has a gap".into(), + )); + } + let end = wanted.end.min(range.end); + let offset = usize::try_from(start - range.start) + .map_err(|_| ReadError::InvalidLocations("physical frame offset overflows".into()))?; + let length = usize::try_from(end - start) + .map_err(|_| ReadError::InvalidLocations("physical frame length overflows".into()))?; + output.extend_from_slice(&range.data[offset..offset + length]); + cursor = end; + if cursor == wanted.end { + break; + } + } + if cursor != wanted.end { + return Err(ReadError::InvalidLocations( + "physical frame data is incomplete".into(), + )); + } + Ok(OwnedPayload::from(output)) +} + +pub(super) fn extract_physical_views( + ranges: &[ReadRangeData], + wanted: Range, +) -> ReadResult> { + let first = ranges.partition_point(|range| range.end <= wanted.start); + let mut output = Vec::new(); + let mut cursor = wanted.start; + for range in &ranges[first..] { + if range.start >= wanted.end { + break; + } + if range.end <= cursor { + continue; + } + if range.start > cursor { + return Err(ReadError::InvalidLocations( + "physical frame data has a gap".into(), + )); + } + let end = wanted.end.min(range.end); + let offset = usize::try_from(cursor - range.start) + .map_err(|_| ReadError::InvalidLocations("physical frame offset overflows".into()))?; + let length = usize::try_from(end - cursor) + .map_err(|_| ReadError::InvalidLocations("physical frame length overflows".into()))?; + output.push(range.data.slice(offset..offset + length)); + cursor = end; + if cursor == wanted.end { + break; + } + } + if cursor != wanted.end { + return Err(ReadError::InvalidLocations( + "physical frame data is incomplete".into(), + )); + } + Ok(output) +} + +pub(super) fn slice_views(views: &[OwnedPayload], wanted: Range) -> ReadResult> { + let mut output = Vec::new(); + let mut cursor = 0_usize; + let mut covered = wanted.start; + for view in views { + let view_end = cursor.saturating_add(view.len()); + if view_end > covered && cursor < wanted.end { + let start = covered.saturating_sub(cursor); + let end = (wanted.end - cursor).min(view.len()); + output.push(view.slice(start..end)); + covered += end - start; + } + cursor = view_end; + if covered == wanted.end { + break; + } + } + if covered != wanted.end { + return Err(ReadError::InvalidLocations( + "frame payload views are incomplete".into(), + )); + } + Ok(output) +} diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader/framing.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader/framing.rs index 579331739..19110b056 100644 --- a/lib/crowdb-chunk-client/src/chunk/chunk_reader/framing.rs +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader/framing.rs @@ -2,12 +2,14 @@ // Licensed under the Apache License, Version 2.0. use super::{ - mark_served_segments_corrupt, ChunkReader, FailedReadRange, LayoutSnapshot, PartialReadResult, + ChunkReader, FailedReadRange, LayoutSnapshot, PartialReadResult, ReadRangeData, }; use crate::metrics::ReadFlowMetrics; use crate::{ReadError, ReadResult}; -use bytes::Bytes; +use crowdb_rpc_ffi::OwnedPayload; +use super::layout::mark_served_segments_corrupt; +use super::frame_views::{extract_physical_views, slice_views}; use crowdb_protocol::chunkdb::rpc::Location; use crowdb_protocol::frame::{ parse_frame, parse_frame_views, ChunkLocation, FrameLayout, FrameMagic, FRAME_FOOTER_BYTES, @@ -200,7 +202,7 @@ fn decode_framed_partial( parse_frame(&views[0], framed.location.chunk_id) .map(|frame| (frame.header, frame.physical_length)) } else { - let slices: Vec<_> = views.iter().map(Bytes::as_ref).collect(); + let slices: Vec<_> = views.iter().map(OwnedPayload::as_ref).collect(); parse_frame_views(&slices, framed.location.chunk_id) .map(|frame| (frame.header, frame.physical_length)) }; @@ -253,63 +255,3 @@ fn decode_framed_partial( } Ok(result) } - -fn extract_physical_views(ranges: &[ReadRangeData], wanted: Range) -> ReadResult> { - let first = ranges.partition_point(|range| range.end <= wanted.start); - let mut output = Vec::new(); - let mut cursor = wanted.start; - for range in &ranges[first..] { - if range.start >= wanted.end { - break; - } - if range.end <= cursor { - continue; - } - if range.start > cursor { - return Err(ReadError::InvalidLocations( - "physical frame data has a gap".into(), - )); - } - let end = wanted.end.min(range.end); - let offset = usize::try_from(cursor - range.start) - .map_err(|_| ReadError::InvalidLocations("physical frame offset overflows".into()))?; - let length = usize::try_from(end - cursor) - .map_err(|_| ReadError::InvalidLocations("physical frame length overflows".into()))?; - output.push(range.data.slice(offset..offset + length)); - cursor = end; - if cursor == wanted.end { - break; - } - } - if cursor != wanted.end { - return Err(ReadError::InvalidLocations( - "physical frame data is incomplete".into(), - )); - } - Ok(output) -} - -fn slice_views(views: &[Bytes], wanted: Range) -> ReadResult> { - let mut output = Vec::new(); - let mut cursor = 0_usize; - let mut covered = wanted.start; - for view in views { - let view_end = cursor.saturating_add(view.len()); - if view_end > covered && cursor < wanted.end { - let start = covered.saturating_sub(cursor); - let end = (wanted.end - cursor).min(view.len()); - output.push(view.slice(start..end)); - covered += end - start; - } - cursor = view_end; - if covered == wanted.end { - break; - } - } - if covered != wanted.end { - return Err(ReadError::InvalidLocations( - "frame payload views are incomplete".into(), - )); - } - Ok(output) -} diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader/layout.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader/layout.rs index 51d2b6aac..c78b0e5ec 100644 --- a/lib/crowdb-chunk-client/src/chunk/chunk_reader/layout.rs +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader/layout.rs @@ -1,9 +1,95 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +//! Layout geometry and failure observations. + use super::KIB; use crate::{ReadError, ReadResult}; -use crowdb_protocol::chunkdb::rpc::Chunk; +use crowdb_protocol::chunkdb::rpc::{Chunk, Location}; +use crowdb_protocol::common::ChunkId; +use crowdb_protocol::diskdb::rpc::Segment; + +pub(super) struct StripFailureObservation { + pub(super) strip_sequence: u32, + pub(super) failed_segments: Vec, + pub(super) served_segments: Vec, +} + +pub(super) fn mark_served_segments_corrupt( + mut observations: Vec, +) -> Vec { + let serving_count: usize = observations + .iter() + .map(|observation| observation.served_segments.len()) + .sum(); + if serving_count != 1 { + return Vec::new(); + } + for observation in &mut observations { + for segment in std::mem::take(&mut observation.served_segments) { + if !observation.failed_segments.contains(&segment) { + observation.failed_segments.push(segment); + } + } + } + observations.retain(|observation| !observation.failed_segments.is_empty()); + observations +} + +pub(super) fn segment_identity(segment: &Segment) -> (u64, u64, u32, u64, u64) { + let disk = segment.disk_id.unwrap_or_default(); + ( + disk.high, + disk.low, + segment.zone_index, + segment.unit_offset, + segment.allocation_ts, + ) +} + +pub(super) fn failure_operation_id( + chunk_id: ChunkId, + strip: &crowdb_protocol::chunkdb::rpc::ChunkStrip, +) -> ChunkId { + let mut high = chunk_id.high ^ 0x111f_a11e_d000_0001; + let mut low = chunk_id.low ^ u64::from(strip.strip_sequence); + for segment in &strip.unavailable_segments { + let disk = segment.disk_id.unwrap_or_default(); + high = high.rotate_left(13) ^ disk.high ^ segment.unit_offset; + low = low.rotate_left(17) ^ disk.low ^ segment.allocation_ts; + } + ChunkId { high, low } +} + +pub(super) fn normalize_locations(locations: &[Location]) -> ReadResult<(Vec, u64)> { + let mut locations: Vec<_> = locations + .iter() + .filter(|location| location.logical_length != 0 && location.length != 0) + .cloned() + .collect(); + locations.sort_unstable_by_key(|location| location.logical_offset); + let mut cursor = 0u64; + for location in &locations { + if location.chunk_id.is_none() || location.logical_length > location.length { + return Err(ReadError::InvalidLocations( + "location is missing a chunk or logical length exceeds physical length".into(), + )); + } + if location.logical_offset != cursor { + return Err(ReadError::InvalidLocations(format!( + "logical locations are not contiguous at byte {cursor}" + ))); + } + location + .offset + .checked_add(location.length) + .ok_or_else(|| ReadError::InvalidLocations("physical location overflows".into()))?; + cursor = cursor + .checked_add(location.logical_length) + .ok_or_else(|| ReadError::InvalidLocations("object length overflows".into()))?; + } + Ok((locations, cursor)) +} pub(super) fn validate_strip_layout(chunk: &Chunk) -> ReadResult<()> { let mut previous_end = 0; diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader/location.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader/location.rs new file mode 100644 index 000000000..ff261b2e2 --- /dev/null +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader/location.rs @@ -0,0 +1,243 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Logical location resolution and protected physical reads. + +use super::layout::{ + failure_operation_id, segment_identity, strip_end, strip_read_parts, + validate_strip_layout, StripFailureObservation, +}; +use super::{ChunkReader, FailedReadRange, LayoutSnapshot, PartialReadResult, ReadRangeData, KIB}; +use crate::{ChunkAllocator, ReadError, ReadResult}; +use crowdb_protocol::chunkdb::rpc::{ + AdHocEcRecoveryDisposition, AdHocEcRecoveryRequest, Chunk, ChunkState, Location, +}; +use std::time::Instant; + +impl ChunkReader { + pub(super) async fn read_location_partial( + &self, + location: &Location, + local_start: u64, + length: u64, + logical_start: u64, + ) -> ReadResult { + self.read_location_partial_cached(location, local_start, length, logical_start, None) + .await + } + + pub(super) async fn read_location_partial_cached( + &self, + location: &Location, + local_start: u64, + length: u64, + logical_start: u64, + mut first_layout: Option, + ) -> ReadResult { + if location.length != location.logical_length { + return self + .read_framed_location(location, local_start, length, logical_start, first_layout) + .await; + } + let chunk_id = location + .chunk_id + .ok_or_else(|| ReadError::InvalidLocations("location has no chunk ID".into()))?; + let physical_start = location + .offset + .checked_add(local_start) + .ok_or_else(|| ReadError::InvalidLocations("location physical offset overflows".into()))?; + for _ in 0..self.policy.max_layout_retries { + let layout = match first_layout.take().filter(LayoutSnapshot::usable) { + Some(layout) => layout, + None => self.query_layout(chunk_id).await?, + }; + let (partial, observations) = self + .read_chunk_range_partial(&layout.chunk, physical_start, length, logical_start) + .await?; + if Instant::now() >= layout.deadline { + layout.invalidate(); + continue; + } + if observations + .iter() + .any(|observation| !observation.failed_segments.is_empty()) + { + layout.invalidate(); + let mut chunk = (*layout.chunk).clone(); + if self + .mark_observed_failures(&mut chunk, observations) + .await + .is_err() + { + continue; + } + } + if Instant::now() < layout.deadline { + return Ok(partial); + } + } + Err(ReadError::LayoutExpired) + } + + pub(super) async fn read_chunk_range_partial( + &self, + chunk: &Chunk, + start: u64, + length: u64, + logical_start: u64, + ) -> ReadResult<(PartialReadResult, Vec)> { + let started = Instant::now(); + let result = self + .read_chunk_range_partial_inner(chunk, start, length, logical_start) + .await; + self.flow_metrics + .chunk_read_wait_ns + .inc_by(u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX)); + result + } + + pub(super) async fn read_chunk_range_partial_inner( + &self, + chunk: &Chunk, + start: u64, + length: u64, + logical_start: u64, + ) -> ReadResult<(PartialReadResult, Vec)> { + if chunk.state == ChunkState::Deleted as i32 { + let id = chunk.id.unwrap_or_default(); + return Err(ReadError::ChunkDeleted(format!("{}:{}", id.high, id.low))); + } + validate_strip_layout(chunk)?; + let end = start + .checked_add(length) + .ok_or_else(|| ReadError::InvalidLocations("chunk read end overflows".into()))?; + let first = chunk + .strips + .partition_point(|strip| strip_end(strip).is_ok_and(|strip_end| strip_end <= start)); + let mut cursor = start; + let mut partial = PartialReadResult::default(); + let mut observations = Vec::new(); + for strip in &chunk.strips[first..] { + let strip_start = u64::from(strip.chunk_offset) * KIB; + if strip_start >= end { + break; + } + if strip_start > cursor { + return Err(ReadError::InvalidLocations(format!( + "chunk layout has a gap at byte {cursor}" + ))); + } + let overlap_end = end.min(strip_end(strip)?); + if overlap_end <= cursor { + continue; + } + let acknowledged = chunk.acknowledged_cursor.saturating_sub(strip_start); + let durable_bytes = acknowledged.min(u64::from(strip.capacity) * KIB); + for (part_start, part_end) in strip_read_parts(strip, cursor, overlap_end)? { + let range_start = logical_start + (part_start - start); + let range_end = range_start + (part_end - part_start); + let read_started = Instant::now(); + let observed = self + .strip_reader + .read_observed( + strip, + chunk.modify_ts, + durable_bytes, + part_start - strip_start, + part_end - part_start, + ) + .await; + self.flow_metrics + .strip_read_wait_ns + .inc_by(u64::try_from(read_started.elapsed().as_nanos()).unwrap_or(u64::MAX)); + if !observed.failed_segments.is_empty() { + observations.push(StripFailureObservation { + strip_sequence: strip.strip_sequence, + failed_segments: observed.failed_segments, + served_segments: observed.served_segments, + }); + } else if !observed.served_segments.is_empty() { + observations.push(StripFailureObservation { + strip_sequence: strip.strip_sequence, + failed_segments: Vec::new(), + served_segments: observed.served_segments, + }); + } + match observed.result { + Ok(data) => partial.ranges.push(ReadRangeData { + start: range_start, + end: range_end, + data, + }), + Err(error) => partial.failures.push(FailedReadRange { + start: range_start, + end: range_end, + error, + }), + } + } + cursor = overlap_end; + } + if cursor != end { + return Err(ReadError::InvalidLocations(format!( + "chunk layout ends at byte {cursor}, requested {end}" + ))); + } + Ok((partial, observations)) + } + + pub(super) async fn mark_observed_failures( + &self, + chunk: &mut Chunk, + observations: Vec, + ) -> ReadResult<()> { + let chunk_id = chunk + .id + .ok_or_else(|| ReadError::InvalidLocations("chunk has no ID".into()))?; + for observation in observations { + let Some(index) = chunk + .strips + .iter() + .position(|strip| strip.strip_sequence == observation.strip_sequence) + else { + return Err(ReadError::Metadata("failed strip disappeared".into())); + }; + let old = chunk.strips[index].clone(); + for segment in observation.failed_segments { + if old.unavailable_segments.contains(&segment) { + continue; + } + let mut replacement = chunk.strips[index].clone(); + replacement.unavailable_segments.push(segment); + replacement.unavailable_segments.sort_by_key(segment_identity); + let operation_id = failure_operation_id(chunk_id, &replacement); + let response = self + .chunkdb + .ad_hoc_ec_recovery(AdHocEcRecoveryRequest { + version: 1, + chunk_id: Some(chunk_id), + expected_modify_ts: chunk.modify_ts, + strip_sequence: observation.strip_sequence, + failed_segment: Some(segment), + operation_id: Some(operation_id), + request_full_block: false, + }) + .await + .map_err(|error| ReadError::Metadata(error.to_string()))?; + if response.disposition != AdHocEcRecoveryDisposition::Marked { + return Err(ReadError::Metadata(format!( + "corrupt segment report was {:?}", + response.disposition + ))); + } + *chunk = self + .query_chunk_timed(chunk_id) + .await + .map_err(|error| ReadError::Metadata(error.to_string()))? + .chunk + .ok_or_else(|| ReadError::Metadata("corrupt report returned no chunk".into()))?; + } + } + Ok(()) + } +} diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader/streaming.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader/stream.rs similarity index 72% rename from lib/crowdb-chunk-client/src/chunk/chunk_reader/streaming.rs rename to lib/crowdb-chunk-client/src/chunk/chunk_reader/stream.rs index 7066ee425..853a8de34 100644 --- a/lib/crowdb-chunk-client/src/chunk/chunk_reader/streaming.rs +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader/stream.rs @@ -1,18 +1,108 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::super::read_credit::{retain, ReadLease}; +//! Ordered delivery and logical read-credit lifetime. + +use super::super::read_credit::{retain, ReadLease, StreamSlots}; +use super::layout::normalize_locations; use super::framing::object_layout; -use super::{ChunkReadStream, LayoutSnapshot, PartialReadResult, DEFAULT_STREAM_WINDOW}; +use super::{ChunkReader, LayoutSnapshot, PartialReadResult, ReadRangeData, DEFAULT_STREAM_WINDOW}; use crate::{ReadError, ReadResult}; use bytes::Bytes; +use crowdb_protocol::chunkdb::rpc::Location; use crowdb_protocol::common::ChunkId; use crowdb_protocol::frame::{ChunkLocation, MAX_FRAME_BYTES}; +use crowdb_rpc_ffi::OwnedPayload; +use std::collections::BTreeMap; use std::sync::Arc; use std::time::Instant; +use tokio::task::JoinSet; + +impl ChunkReader { + pub fn read_stream(&self, locations: &[Location]) -> ReadResult { + self.flow_metrics.location_normalizations.inc(); + self.flow_metrics + .locations_examined + .inc_by(u64::try_from(locations.len()).unwrap_or(u64::MAX)); + let (locations, object_length) = normalize_locations(locations)?; + self.range_stream(locations, 0, object_length, object_length) + } + + /// Builds a pull-based stream for the exact logical half-open range. + pub fn read_range_stream( + &self, + locations: &[Location], + start: u64, + end: u64, + ) -> ReadResult { + self.flow_metrics.location_normalizations.inc(); + self.flow_metrics + .locations_examined + .inc_by(u64::try_from(locations.len()).unwrap_or(u64::MAX)); + let (locations, object_length) = normalize_locations(locations)?; + self.range_stream(locations, start, end, object_length) + } + + pub(super) fn range_stream( + &self, + locations: Vec, + start: u64, + end: u64, + object_length: u64, + ) -> ReadResult { + if start > end || end > object_length { + return Err(ReadError::InvalidRange { + start, + end, + object_length, + }); + } + Ok(ChunkReadStream { + reader: self.clone(), + locations: Arc::from(locations), + cursor: start, + fetch_cursor: start, + delivery_cursor: start, + end, + window_bytes: self.policy.stream_window_bytes as u64, + slots: Arc::new(StreamSlots::new(self.policy.stream_slots)), + reads: JoinSet::new(), + completed: BTreeMap::new(), + layouts: Vec::new(), + pending_error: None, + pending_ranges: Vec::new().into_iter(), + }) + } +} + +type StreamRead = (u64, u64, ReadResult, Arc); + +/// Ordered, bounded read pipeline whose frames retain their read credits. +pub struct ChunkReadStream { + reader: ChunkReader, + locations: Arc<[Location]>, + cursor: u64, + fetch_cursor: u64, + delivery_cursor: u64, + end: u64, + window_bytes: u64, + slots: Arc, + reads: JoinSet, + completed: BTreeMap, + layouts: Vec, + pending_error: Option, + pending_ranges: std::vec::IntoIter, +} impl ChunkReadStream { pub async fn next_chunk(&mut self) -> Option> { + self.next_payload() + .await + .map(|result| result.map(OwnedPayload::into_bytes)) + } + + /// Pull a verified logical view retaining its native allocation metadata. + pub async fn next_payload(&mut self) -> Option> { loop { if let Some(pending) = self.next_pending() { return Some(pending); @@ -207,7 +297,7 @@ impl ChunkReadStream { self.pending_error = Some(error); } - fn next_pending(&mut self) -> Option> { + fn next_pending(&mut self) -> Option> { if let Some(range) = self.pending_ranges.next() { self.cursor = range.end; return Some(Ok(range.data)); diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader/verified.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader/verified_frame.rs similarity index 96% rename from lib/crowdb-chunk-client/src/chunk/chunk_reader/verified.rs rename to lib/crowdb-chunk-client/src/chunk/chunk_reader/verified_frame.rs index d9a717253..341529235 100644 --- a/lib/crowdb-chunk-client/src/chunk/chunk_reader/verified.rs +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader/verified_frame.rs @@ -3,10 +3,14 @@ //! Verified frame reads under a persisted or owner-synchronized prefix. -use super::{ - extract_physical_range, mark_served_segments_corrupt, Bytes, ChunkId, ChunkReader, ChunkState, Instant, - ReadError, ReadResult, Segment, -}; +use super::{ChunkReader, ChunkId}; +use super::frame_views::extract_physical_range; +use super::layout::mark_served_segments_corrupt; +use crate::{ReadError, ReadResult}; +use bytes::Bytes; +use crowdb_protocol::chunkdb::rpc::ChunkState; +use crowdb_protocol::diskdb::rpc::Segment; +use std::time::Instant; use crowdb_protocol::frame::{ parse_frame, FrameMagic, FRAME_FOOTER_BYTES, FRAME_HEADER_PREFIX_BYTES, MAX_FRAME_BYTES, }; @@ -199,7 +203,7 @@ impl ChunkReader { layout.invalidate(); self.mark_observed_failures(&mut chunk, observations).await?; } - return Ok(bytes); + return Ok(bytes.into_bytes()); } Ok(frame) => { layout.invalidate(); diff --git a/lib/crowdb-chunk-client/src/chunk/read_credit.rs b/lib/crowdb-chunk-client/src/chunk/read_credit.rs index 26b2b41c6..971ae4031 100644 --- a/lib/crowdb-chunk-client/src/chunk/read_credit.rs +++ b/lib/crowdb-chunk-client/src/chunk/read_credit.rs @@ -7,6 +7,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use bytes::Bytes; +use crowdb_rpc_ffi::OwnedPayload; use tokio::sync::Notify; use crate::metrics::ReadFlowMetrics; @@ -124,6 +125,6 @@ impl AsRef<[u8]> for LeasedView { } } -pub(super) fn retain(data: Bytes, lease: Arc) -> Bytes { - Bytes::from_owner(LeasedView { data, _lease: lease }) +pub(super) fn retain(data: OwnedPayload, lease: Arc) -> OwnedPayload { + data.with_owner(|data| Bytes::from_owner(LeasedView { data, _lease: lease })) } diff --git a/lib/crowdb-chunk-client/src/chunk/strip_reader.rs b/lib/crowdb-chunk-client/src/chunk/strip_reader.rs index 3ec82b639..ed1e85ac0 100644 --- a/lib/crowdb-chunk-client/src/chunk/strip_reader.rs +++ b/lib/crowdb-chunk-client/src/chunk/strip_reader.rs @@ -5,10 +5,11 @@ use std::sync::Arc; -use bytes::{Bytes, BytesMut}; +use bytes::Bytes; use crowdb_common::ec::{decode, EcScheme}; use crowdb_protocol::chunkdb::rpc::{ChunkStrip, EcState, Strip}; use crowdb_protocol::diskdb::rpc::Segment; +use crowdb_rpc_ffi::OwnedPayload; use tokio::sync::Semaphore; use tokio::task::JoinSet; @@ -18,7 +19,7 @@ use crate::{DiskWriter, ReadError, ReadResult}; /// Result of a strip read together with every segment that failed or was /// already marked unavailable while satisfying it. pub(crate) struct ObservedStripRead { - pub result: ReadResult, + pub result: ReadResult, pub failed_segments: Vec, /// Segments whose returned bytes have not yet passed frame validation. pub served_segments: Vec, @@ -62,6 +63,7 @@ impl StripReader { self.read_observed(strip, 0, durable_bytes, offset, length) .await .result + .map(OwnedPayload::into_bytes) } pub(crate) async fn read_observed( @@ -96,9 +98,9 @@ impl StripReader { durable_bytes: u64, offset: u64, length: u64, - ) -> Result<(Bytes, Vec, Vec), (ReadError, Vec)> { + ) -> Result<(OwnedPayload, Vec, Vec), (ReadError, Vec)> { if length == 0 { - return Ok((Bytes::new(), Vec::new(), Vec::new())); + return Ok((OwnedPayload::new(), Vec::new(), Vec::new())); } let sealed_bytes = kib_to_bytes(strip.sealed_length).map_err(|error| (error, Vec::new()))?; let available = if durable_bytes == 0 { @@ -139,7 +141,7 @@ impl StripReader { segments: &[Segment], offset: u64, length: u64, - ) -> Result<(Bytes, Vec, Vec), (ReadError, Vec)> { + ) -> Result<(OwnedPayload, Vec, Vec), (ReadError, Vec)> { let length = u32::try_from(length).map_err(|_| { ( ReadError::InvalidLocations("mirror read exceeds RPC size".into()), @@ -182,7 +184,7 @@ impl StripReader { sealed_bytes: u64, offset: u64, length: u64, - ) -> Result<(Bytes, Vec, Vec), (ReadError, Vec)> { + ) -> Result<(OwnedPayload, Vec, Vec), (ReadError, Vec)> { let geometry = validate_ec_read(strip, ec, offset, length).map_err(|error| (error, Vec::new()))?; let EcReadGeometry { scheme, @@ -218,7 +220,7 @@ impl StripReader { if let [(_, shard_index, _, _, Ok(data))] = pieces.as_slice() { return Ok((data.clone(), failed_segments, vec![ec.segments[*shard_index]])); } - let mut output = BytesMut::with_capacity(usize::try_from(length).unwrap_or(usize::MAX)); + let mut output = Vec::with_capacity(usize::try_from(length).unwrap_or(usize::MAX)); let mut served_segments = Vec::new(); for (_, shard_index, local_start, read_len, result) in pieces { if let Ok(data) = result { @@ -280,7 +282,7 @@ impl StripReader { output.extend_from_slice(&recovered.data); } } - Ok((output.freeze(), failed_segments, served_segments)) + Ok((OwnedPayload::from(output), failed_segments, served_segments)) } #[allow(clippy::too_many_arguments)] @@ -307,7 +309,7 @@ impl StripReader { Vec::new(), )); } - let mut output = BytesMut::with_capacity(length as usize); + let mut output = Vec::with_capacity(length as usize); let mut failed_segments = Vec::new(); let mut consumed = 0u32; while consumed < length { @@ -331,7 +333,7 @@ impl StripReader { consumed += part_len; } Ok(RecoveredSlice { - data: output.freeze(), + data: OwnedPayload::from(output), failed_segments, }) } @@ -432,7 +434,7 @@ impl StripReader { } let decoded = decode_recoverable(strip, scheme, shards, &failed_segments)?; Ok(RecoveredSlice { - data: Bytes::from(decoded[target].clone()), + data: OwnedPayload::from(decoded[target].clone()), failed_segments, }) } @@ -443,7 +445,7 @@ impl StripReader { unit_bytes: u64, offset: u64, length: u32, - ) -> crate::Result { + ) -> crate::Result { read_target(&*self.disk_io, segment, unit_bytes, offset, length).await } } @@ -454,10 +456,10 @@ async fn read_target( unit_bytes: u64, offset: u64, length: u32, -) -> crate::Result { +) -> crate::Result { let mut last_error = None; for attempt in 0..3 { - match disk_io.read(segment, unit_bytes, offset, length).await { + match disk_io.read_payload(segment, unit_bytes, offset, length).await { Ok(data) => return Ok(data), Err(error) => { let connection_failure = matches!(error, crate::IoError::Topology(_)); @@ -491,7 +493,7 @@ fn decode_recoverable( } struct RecoveredSlice { - data: Bytes, + data: OwnedPayload, failed_segments: Vec, } diff --git a/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs b/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs index 69303036e..43b46c0c5 100644 --- a/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs +++ b/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs @@ -6,6 +6,7 @@ use async_trait::async_trait; use bytes::Bytes; use crowdb_protocol::diskdb::rpc::Segment; +use crowdb_rpc_ffi::OwnedPayload; use crate::{IoError, Result}; @@ -180,6 +181,19 @@ pub trait DiskWriter: Send + Sync { )) } + /// Read while preserving backing metadata when the transport provides it. + async fn read_payload( + &self, + seg: &Segment, + unit_bytes: u64, + segment_offset: u64, + length: u32, + ) -> Result { + self.read(seg, unit_bytes, segment_offset, length) + .await + .map(OwnedPayload::from) + } + /// Write a unit-aligned range relative to the start of `seg`. async fn write_at(&self, seg: &Segment, unit_bytes: u64, segment_offset: u64, data: Bytes) -> Result<()> { validate_segment_write(seg, unit_bytes, segment_offset, data.len())?; diff --git a/lib/crowdb-chunk-client/src/disk_io/routing.rs b/lib/crowdb-chunk-client/src/disk_io/routing.rs index 0913cb380..67f0ee9f0 100644 --- a/lib/crowdb-chunk-client/src/disk_io/routing.rs +++ b/lib/crowdb-chunk-client/src/disk_io/routing.rs @@ -337,10 +337,22 @@ impl DiskWriter for RoutedDiskWriter { } async fn read(&self, seg: &Segment, unit_bytes: u64, segment_offset: u64, length: u32) -> Result { + self.read_payload(seg, unit_bytes, segment_offset, length) + .await + .map(crowdb_rpc_ffi::OwnedPayload::into_bytes) + } + + async fn read_payload( + &self, + seg: &Segment, + unit_bytes: u64, + segment_offset: u64, + length: u32, + ) -> Result { let target = Self::target(seg, unit_bytes).map_err(|error| IoError::ReadFailed(error.to_string()))?; let result = self .client - .read(target, segment_offset, length, self.client.normal_options()) + .read_payload(target, segment_offset, length, self.client.normal_options()) .await; match result { Err(DiskioError::TransportUnavailable(_) | DiskioError::TopologyUnavailable(_)) => { @@ -349,7 +361,7 @@ impl DiskWriter for RoutedDiskWriter { // Reads are safe to retry once after an authoritative refresh. self.client.refresh().await.map_err(map_read_error)?; self.client - .read(target, segment_offset, length, self.client.normal_options()) + .read_payload(target, segment_offset, length, self.client.normal_options()) .await .map_err(map_read_error) } diff --git a/lib/crowdb-chunk-client/tests/common/read_payload.rs b/lib/crowdb-chunk-client/tests/common/read_payload.rs new file mode 100644 index 000000000..afba83c58 --- /dev/null +++ b/lib/crowdb-chunk-client/tests/common/read_payload.rs @@ -0,0 +1,60 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_chunk_client::{ChunkAllocator, DiskWriter, Result}; +use crowdb_protocol::chunkdb::rpc::{ + AllocateChunkRequest, AllocateChunkResponse, AppendChunkRequest, AppendChunkResponse, Chunk, + DeleteChunkRequest, DeleteChunkResponse, QueryChunkRequest, QueryChunkResponse, SealChunkRequest, + SealChunkResponse, UpdateChunkStripRequest, UpdateChunkStripResponse, +}; +use crowdb_protocol::diskdb::rpc::Segment; +use crowdb_rpc_ffi::OwnedPayload; + +pub struct TestPayloadStorage { + pub chunk: Chunk, + pub data: OwnedPayload, +} + +#[async_trait] +impl ChunkAllocator for TestPayloadStorage { + async fn allocate_chunk(&self, _: AllocateChunkRequest) -> Result { + panic!("read allocated storage") + } + async fn append_chunk(&self, _: AppendChunkRequest) -> Result { + panic!("read appended storage") + } + async fn seal_chunk(&self, _: SealChunkRequest) -> Result { + panic!("read sealed storage") + } + async fn delete_chunk(&self, _: DeleteChunkRequest) -> Result { + panic!("read deleted storage") + } + async fn update_chunk_strip(&self, _: UpdateChunkStripRequest) -> Result { + panic!("read updated storage") + } + async fn query_chunk(&self, _: QueryChunkRequest) -> Result { + Ok(QueryChunkResponse { + chunk: Some(self.chunk.clone()), + layout_validity_ms: 60_000, + }) + } +} + +#[async_trait] +impl DiskWriter for TestPayloadStorage { + async fn write(&self, _: &Segment, _: u64, _: Bytes) -> Result<()> { + panic!("read wrote storage") + } + async fn write_at_byte_offset(&self, _: &Segment, _: u64, _: u64, _: Bytes) -> Result<()> { + panic!("read wrote storage") + } + async fn read(&self, _: &Segment, _: u64, _: u64, _: u32) -> Result { + panic!("reader erased allocation metadata before delivery") + } + async fn read_payload(&self, _: &Segment, _: u64, offset: u64, length: u32) -> Result { + let offset = usize::try_from(offset).unwrap(); + Ok(self.data.slice(offset..offset + length as usize)) + } +} diff --git a/lib/crowdb-chunk-client/tests/read_payload_test.rs b/lib/crowdb-chunk-client/tests/read_payload_test.rs new file mode 100644 index 000000000..1fa646a6b --- /dev/null +++ b/lib/crowdb-chunk-client/tests/read_payload_test.rs @@ -0,0 +1,74 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/read_payload.rs"] +mod storage; + +use crowdb_chunk_client::{ChunkReadPolicy, ChunkReader}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, ChunkStrip, Location, MirrorStrip, Strip}; +use crowdb_protocol::common::{ChunkId, DiskId}; +use crowdb_protocol::diskdb::rpc::Segment; +use crowdb_protocol::frame::{encode_frame, FrameMagic}; +use crowdb_rpc_ffi::OwnedPayload; +use std::sync::Arc; + +#[tokio::test] +async fn framing_and_stream_credits_preserve_original_allocation_metadata() { + let id = ChunkId { high: 13, low: 37 }; + let mut data = Vec::with_capacity(4096); + data.extend_from_slice(&encode_frame(FrameMagic::RepoSmallV1, id, b"payload", 1).unwrap()); + let frame_length = data.len(); + let original = OwnedPayload::from(data); + let backing = Arc::clone(original.allocation().unwrap()); + let segment = Segment { + disk_id: Some(DiskId { high: 1, low: 1 }), + unit_count: 1, + ..Segment::default() + }; + let storage = Arc::new(storage::TestPayloadStorage { + chunk: Chunk { + id: Some(id), + state: ChunkState::Sealed as i32, + capacity: 1, + sealed_length: 1, + strips: vec![ChunkStrip { + unit_kb: 1, + capacity: 1, + sealed_length: 1, + sealed_ts_ms: 1, + strip: Some(Strip::MirrorStrip(MirrorStrip { + segments: vec![segment], + })), + ..ChunkStrip::default() + }], + ..Chunk::default() + }, + data: original, + }); + let reader = ChunkReader::new(storage.clone(), storage, ChunkReadPolicy::default()).unwrap(); + let location = Location { + chunk_id: Some(id), + length: frame_length as u64, + logical_length: 7, + ..Location::default() + }; + let mut stream = reader.read_stream(&[location]).unwrap(); + let payload = stream.next_payload().await.unwrap().unwrap(); + assert_eq!(payload.as_ref(), b"payload"); + assert!(Arc::ptr_eq(payload.allocation().unwrap(), &backing)); + assert_eq!(payload.allocation().unwrap().capacity(), 4096); + assert_eq!( + payload.as_ptr(), + backing.bytes()[payload.allocation_offset()..].as_ptr() + ); + assert!(stream.next_payload().await.is_none()); + drop(stream); + assert_eq!(reader.flow_metrics_snapshot().stream_bytes_released, 0); + let bytes = payload.into_bytes(); + assert_eq!(reader.flow_metrics_snapshot().stream_bytes_released, 0); + drop(bytes); + assert_eq!( + reader.flow_metrics_snapshot().stream_bytes_released, + frame_length as u64 + ); +} diff --git a/lib/crowdb-diskio-client/src/client.rs b/lib/crowdb-diskio-client/src/client.rs index daa1614ff..fc8ea43be 100644 --- a/lib/crowdb-diskio-client/src/client.rs +++ b/lib/crowdb-diskio-client/src/client.rs @@ -3,6 +3,7 @@ //! Internal `DiskIO` wire transport. +#[cfg(feature = "test-util")] use bytes::Bytes; use crowdb_common::RequestIdGen; use crowdb_protocol::diskio_fb::{ @@ -12,7 +13,8 @@ use crowdb_protocol::diskio_fb::{ }; use crowdb_protocol::fb::FBMsgType; use crowdb_rpc_ffi::{ - Buffer, BufferChain, CallFuture, Connection, RpcClient, RpcClientHandle, RpcError, RpcServer, + Buffer, BufferChain, CallFuture, Connection, OwnedPayload, RpcClient, RpcClientHandle, RpcError, + RpcServer, }; use flatbuffers::FlatBufferBuilder; use std::time::{SystemTime, UNIX_EPOCH}; @@ -425,10 +427,20 @@ impl WireClient { /// # Errors /// /// Returns `DiskioError::Rpc` if the response is missing or invalid. + #[cfg(feature = "test-util")] pub async fn await_read_response(fut: CallFuture) -> WireResult<(DiskIoRetCode, Option)> { + let (code, data) = Self::await_read_payload(fut).await?; + Ok((code, data.map(OwnedPayload::into_bytes))) + } + + /// Preserve receive allocation and recycler ownership for native readers. + /// + /// # Errors + /// Returns a wire error for an invalid or unsuccessful response. + pub async fn await_read_payload(fut: CallFuture) -> WireResult<(DiskIoRetCode, Option)> { let resp = fut.await.map_err(WireError::from)?; let code = parse_ret_code(&resp)?; - let data = resp.data.map(Bytes::from_owner); + let data = resp.data.map(OwnedPayload::from_buffer); Ok((code, data)) } diff --git a/lib/crowdb-diskio-client/src/semantic.rs b/lib/crowdb-diskio-client/src/semantic.rs index 8a0c28b82..5c6206502 100644 --- a/lib/crowdb-diskio-client/src/semantic.rs +++ b/lib/crowdb-diskio-client/src/semantic.rs @@ -14,7 +14,8 @@ use arc_swap::ArcSwap; use bytes::Bytes; use crowdb_kv_client::{HardwareClient, ServiceRegistryClient}; use crowdb_rpc_ffi::{ - ConnectionPoolError, ConnectionPoolIndex, OwnedClientRoute, RpcError, RpcServer, SelectedConnection, + ConnectionPoolError, ConnectionPoolIndex, OwnedClientRoute, OwnedPayload, RpcError, RpcServer, + SelectedConnection, }; use crate::client::{DiskIoRetCode, WireClient, WireError}; @@ -523,10 +524,26 @@ impl DiskioClient { length: u32, options: OperationOptions, ) -> DiskioResult { + self.read_payload(target, offset, length, options) + .await + .map(OwnedPayload::into_bytes) + } + + /// Read a view retaining its backing allocation and native receive credit. + /// + /// # Errors + /// Returns the same typed failures and retry boundary as `read`. + pub async fn read_payload( + &self, + target: SegmentTarget, + offset: u64, + length: u32, + options: OperationOptions, + ) -> DiskioResult { self.refresh_topology_if_due().await; let (zone_offset, _) = target.checked_range(offset, length as usize)?; if length == 0 { - return Ok(Bytes::new()); + return Ok(OwnedPayload::new()); } let _inflight = self.begin_operation(OperationKind::Read, options.lane)?; let mut backoff = self.config.reconnect_initial_backoff; @@ -796,13 +813,13 @@ impl DiskioClient { future: crowdb_rpc_ffi::CallFuture, expected: u32, deadline: Instant, - ) -> DiskioResult { + ) -> DiskioResult { if Instant::now() >= deadline { return Err(DiskioError::DeadlineExceeded); } let (code, data) = tokio::time::timeout_at( tokio::time::Instant::from_std(deadline), - WireClient::await_read_response(future), + WireClient::await_read_payload(future), ) .await .map_err(|_| DiskioError::DeadlineExceeded)? From 30c2a261314611075c3ab3955eda32d132a0cc6b Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 13:10:04 +0800 Subject: [PATCH 062/258] Bound shared native allocation lifetimes with asynchronous admission --- Cargo.lock | 1 + doc/working/plan-rdma-fat-client-sdk.md | 9 + lib/crowdb-sdk-native/Cargo.toml | 1 + lib/crowdb-sdk-native/src/admission.rs | 29 +++ lib/crowdb-sdk-native/src/buffer.rs | 11 +- lib/crowdb-sdk-native/src/buffer/receive.rs | 193 ++++++++++++++++++ .../src/runtime/resources.rs | 10 +- .../tests/allocation_budget_test.rs | 142 +++++++++++++ 8 files changed, 394 insertions(+), 2 deletions(-) create mode 100644 lib/crowdb-sdk-native/src/buffer/receive.rs create mode 100644 lib/crowdb-sdk-native/tests/allocation_budget_test.rs diff --git a/Cargo.lock b/Cargo.lock index b18465fea..518d33774 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1282,6 +1282,7 @@ dependencies = [ "async-trait", "bincode", "bytes", + "crossbeam-skiplist", "crowdb-access-multipart", "crowdb-access-s3", "crowdb-chunk-client", diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index e000b3e24..190a75b4c 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -260,6 +260,15 @@ transport acceptance alone cannot establish SDK completion. address/capacity retention and logical credit lifetime through final Bytes release; all existing reader cases pass. Native SDK capacity admission and S3 body adaptation are the next dependent tasks. +- Runtime-wide held allocation accounting now uses the existing lock-free + skip-list dependency and weak entries removed at final backing release. + Per-view upstream logical read owners remain retained independently of the + shared physical charge. Admission waits register before checking capacity + and publication, honor request cancellation/deadline and wake on shutdown. + Unknown Bytes owners reserve capacity before a bounded copy, counted in + `ResourceStatus::compatibility_copy_bytes`. Five focused tests cover shared + capacity, 64 concurrent publication/release rounds, bounded copy pressure, + cancellation/shutdown and undersized views hiding oversized allocations. ## Subsequent Dataset integration under R220 diff --git a/lib/crowdb-sdk-native/Cargo.toml b/lib/crowdb-sdk-native/Cargo.toml index f32abbed0..f717d1d58 100644 --- a/lib/crowdb-sdk-native/Cargo.toml +++ b/lib/crowdb-sdk-native/Cargo.toml @@ -15,6 +15,7 @@ test-util = [] [dependencies] bytes = "1" +crossbeam-skiplist = "0.1" arc-swap = "1.9" bincode = "1" crowdb-access-s3 = { path = "../crowdb-access-s3" } diff --git a/lib/crowdb-sdk-native/src/admission.rs b/lib/crowdb-sdk-native/src/admission.rs index ea4478ea0..887df9ba3 100644 --- a/lib/crowdb-sdk-native/src/admission.rs +++ b/lib/crowdb-sdk-native/src/admission.rs @@ -5,6 +5,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; +use tokio::sync::Notify; use crate::{CapacityKind, Error, Result}; @@ -13,6 +14,8 @@ pub(crate) struct Budget { used: AtomicUsize, high_water: AtomicUsize, kind: CapacityKind, + waiters: AtomicUsize, + pub(crate) wake: Notify, } impl Budget { @@ -22,6 +25,8 @@ impl Budget { used: AtomicUsize::new(0), high_water: AtomicUsize::new(0), kind, + waiters: AtomicUsize::new(0), + wake: Notify::new(), }) } @@ -46,6 +51,29 @@ impl Budget { pub fn high_water(&self) -> usize { self.high_water.load(Ordering::Relaxed) } + + pub fn can_fit(&self, amount: usize) -> bool { + amount <= self.limit + } + + pub fn register(&self) -> WaitRegistration<'_> { + self.waiters.fetch_add(1, Ordering::AcqRel); + WaitRegistration(self) + } + + pub fn changed(&self) { + if self.waiters.load(Ordering::Acquire) != 0 { + self.wake.notify_waiters(); + } + } +} + +pub(crate) struct WaitRegistration<'a>(&'a Budget); + +impl Drop for WaitRegistration<'_> { + fn drop(&mut self) { + self.0.waiters.fetch_sub(1, Ordering::AcqRel); + } } pub(crate) struct Credit { @@ -56,5 +84,6 @@ pub(crate) struct Credit { impl Drop for Credit { fn drop(&mut self) { self.budget.used.fetch_sub(self.amount, Ordering::AcqRel); + self.budget.changed(); } } diff --git a/lib/crowdb-sdk-native/src/buffer.rs b/lib/crowdb-sdk-native/src/buffer.rs index 962b0944a..457827334 100644 --- a/lib/crowdb-sdk-native/src/buffer.rs +++ b/lib/crowdb-sdk-native/src/buffer.rs @@ -7,11 +7,14 @@ use std::ops::Range; use std::sync::Arc; use bytes::Bytes; -use crowdb_rpc_ffi::{Buffer, ProducerBufferAllocator, WritableBuffer}; +use crowdb_rpc_ffi::{Buffer, OwnedPayload, ProducerBufferAllocator, WritableBuffer}; use crate::admission::Credit; use crate::{CapacityKind, Error, Result, RuntimeResources}; +mod receive; +pub(crate) use receive::AllocationRegistry; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct LeaseMetadata { pub request_id: u64, @@ -26,11 +29,14 @@ pub struct LeaseMetadata { struct Backing { // Storage is dropped before admission is returned to another producer. bytes: Bytes, + registration: Option, _credit: Credit, } #[derive(Clone)] pub struct BufferLease { + // Keep each upstream logical read credit, even when physical storage is shared. + source: Option, backing: Arc, range: Range, metadata: LeaseMetadata, @@ -60,6 +66,7 @@ impl BufferLease { .ok_or(Error::BufferRange)?; metadata.length = range.len(); Ok(Self { + source: self.source.clone(), backing: Arc::clone(&self.backing), range: self.range.start + range.start..self.range.start + range.end, metadata, @@ -74,9 +81,11 @@ impl BufferLease { fn new(bytes: Bytes, metadata: LeaseMetadata, credit: Credit) -> Self { Self { + source: None, range: 0..bytes.len(), backing: Arc::new(Backing { bytes, + registration: None, _credit: credit, }), metadata, diff --git a/lib/crowdb-sdk-native/src/buffer/receive.rs b/lib/crowdb-sdk-native/src/buffer/receive.rs new file mode 100644 index 000000000..996998f5c --- /dev/null +++ b/lib/crowdb-sdk-native/src/buffer/receive.rs @@ -0,0 +1,193 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Runtime-wide allocation charges with lock-free publication and final release. + +use std::sync::atomic::Ordering; +use std::sync::{Arc, Weak}; + +use bytes::Bytes; +use crossbeam_skiplist::SkipMap; +use crowdb_rpc_ffi::{OwnedPayload, PayloadAllocation}; + +use super::{Backing, BufferLease, LeaseMetadata}; +use crate::{CapacityKind, Error, Result, RuntimeResources}; + +#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd)] +enum AllocationKey { + Native(u64), + Shared(usize), +} + +#[derive(Default)] +pub(crate) struct AllocationRegistry { + entries: SkipMap>, +} + +pub(super) struct Registration { + registry: Weak, + key: AllocationKey, + owner: Weak, +} + +impl Drop for Backing { + fn drop(&mut self) { + if let Some(registration) = &self.registration { + if let Some(registry) = registration.registry.upgrade() { + if let Some(entry) = registry.entries.get(®istration.key) { + if Weak::ptr_eq(entry.value(), ®istration.owner) { + entry.remove(); + } + } + } + } + // Fields then release backing bytes before returning capacity credit. + } +} + +impl RuntimeResources { + /// Retains a logical view, charging its actual allocation once across all + /// runtime handles. Unknown external Bytes owners use an explicit bounded copy. + /// Call under a request lease to apply its deadline and cancellation. + /// # Errors + /// Rejects invalid identities, closed admission or an allocation above the limit. + pub async fn retain_payload( + self: &Arc, + payload: OwnedPayload, + request_id: u64, + stream_id: u64, + sequence: u64, + logical_offset: u64, + ) -> Result { + if request_id == 0 || payload.is_empty() || logical_offset.checked_add(payload.len() as u64).is_none() + { + return Err(Error::BufferRange); + } + if payload.allocation().is_none() { + return tokio::select! { + biased; + () = self.closing.cancelled() => Err(Error::Shutdown), + result = self.copy_unknown(payload, request_id, stream_id, sequence, logical_offset) => result, + }; + } + let allocation = Arc::clone(payload.allocation().ok_or(Error::BufferRange)?); + let offset = payload.allocation_offset(); + if offset + .checked_add(payload.len()) + .map_or(true, |end| end > allocation.bytes().len()) + { + return Err(Error::BufferRange); + } + let metadata = LeaseMetadata { + request_id, + stream_id, + sequence, + logical_offset, + length: payload.len(), + allocation_capacity: allocation.capacity(), + registration_id: allocation.registration_id(), + }; + tokio::select! { + biased; + () = self.closing.cancelled() => Err(Error::Shutdown), + backing = self.retain_allocation(&allocation) => Ok(BufferLease { + range: offset..offset + payload.len(), + source: Some(payload), + backing: backing?, + metadata, + }), + } + } + + async fn copy_unknown( + &self, + payload: OwnedPayload, + request_id: u64, + stream_id: u64, + sequence: u64, + logical_offset: u64, + ) -> Result { + let amount = payload.len(); + if !self.held.can_fit(amount) { + return Err(Error::Capacity(CapacityKind::HeldBuffers)); + } + let _registration = self.held.register(); + let credit = loop { + let changed = self.held.wake.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + self.check_open()?; + if let Ok(credit) = self.held.charge(amount) { + break credit; + } + changed.await; + }; + let copied = payload.to_vec(); + if copied.capacity() != amount { + return Err(Error::Capacity(CapacityKind::HeldBuffers)); + } + self.compatibility_copy_bytes + .fetch_add(amount as u64, Ordering::Relaxed); + Ok(BufferLease::new( + Bytes::from(copied), + LeaseMetadata { + request_id, + stream_id, + sequence, + logical_offset, + length: amount, + allocation_capacity: amount, + registration_id: 0, + }, + credit, + )) + } + + async fn retain_allocation(&self, allocation: &Arc) -> Result> { + if !self.held.can_fit(allocation.capacity()) { + return Err(Error::Capacity(CapacityKind::HeldBuffers)); + } + let key = allocation.native_identity().map_or_else( + || AllocationKey::Shared(Arc::as_ptr(allocation) as usize), + AllocationKey::Native, + ); + let _registration = self.held.register(); + loop { + let changed = self.held.wake.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + self.check_open()?; + if let Some(entry) = self.allocations.entries.get(&key) { + if let Some(backing) = entry.value().upgrade() { + return Ok(backing); + } + entry.remove(); + } + if let Ok(credit) = self.held.charge(allocation.capacity()) { + let candidate = Arc::new_cyclic(|owner| Backing { + bytes: allocation.bytes().clone(), + registration: Some(Registration { + registry: Arc::downgrade(&self.allocations), + key, + owner: owner.clone(), + }), + _credit: credit, + }); + let entry = self + .allocations + .entries + .get_or_insert(key, Arc::downgrade(&candidate)); + let published = entry.value().upgrade(); + // A waiter that raced the first charge must observe publication, + // even if the new allocation consumes the entire held budget. + self.held.changed(); + if let Some(backing) = published { + return Ok(backing); + } + entry.remove(); + continue; + } + changed.await; + } + } +} diff --git a/lib/crowdb-sdk-native/src/runtime/resources.rs b/lib/crowdb-sdk-native/src/runtime/resources.rs index aaa1175d0..eaa17b72f 100644 --- a/lib/crowdb-sdk-native/src/runtime/resources.rs +++ b/lib/crowdb-sdk-native/src/runtime/resources.rs @@ -2,9 +2,11 @@ // Licensed under the Apache License, Version 2.0. use std::future::Future; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use crate::admission::{Budget, Credit}; +use crate::buffer::AllocationRegistry; use crate::{Cancellation, CapacityKind, Error, OperationContext, Result}; #[derive(Clone, Copy, Debug)] @@ -32,13 +34,16 @@ pub struct ResourceStatus { pub producer_bytes_high_water: usize, pub held_bytes: usize, pub held_bytes_high_water: usize, + pub compatibility_copy_bytes: u64, } pub struct RuntimeResources { pub(crate) producers: Arc, pub(crate) held: Arc, requests: Arc, - closing: Cancellation, + pub(crate) allocations: Arc, + pub(crate) compatibility_copy_bytes: AtomicU64, + pub(crate) closing: Cancellation, } impl RuntimeResources { @@ -60,6 +65,8 @@ impl RuntimeResources { producers: Budget::new(limits.producer_bytes, CapacityKind::Producers), held: Budget::new(limits.held_bytes, CapacityKind::HeldBuffers), closing: Cancellation::default(), + allocations: Arc::new(AllocationRegistry::default()), + compatibility_copy_bytes: AtomicU64::new(0), })) } @@ -98,6 +105,7 @@ impl RuntimeResources { producer_bytes_high_water: self.producers.high_water(), held_bytes: self.held.used(), held_bytes_high_water: self.held.high_water(), + compatibility_copy_bytes: self.compatibility_copy_bytes.load(Ordering::Relaxed), } } } diff --git a/lib/crowdb-sdk-native/tests/allocation_budget_test.rs b/lib/crowdb-sdk-native/tests/allocation_budget_test.rs new file mode 100644 index 000000000..34e8a55c5 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/allocation_budget_test.rs @@ -0,0 +1,142 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::num::NonZeroU64; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use bytes::Bytes; +use crowdb_rpc_ffi::OwnedPayload; +use crowdb_sdk_native::{CapacityKind, Error, OperationContext, RuntimeLimits, RuntimeResources}; + +fn resources(bytes: usize) -> Arc { + RuntimeResources::new(RuntimeLimits { + held_bytes: bytes, + ..RuntimeLimits::default() + }) + .unwrap() +} + +fn payload(capacity: usize) -> OwnedPayload { + let mut data = Vec::with_capacity(capacity); + data.extend_from_slice(b"0123456789"); + OwnedPayload::from(data) +} + +#[tokio::test] +async fn separate_deliveries_of_one_allocation_share_capacity_until_final_release() { + let resources = resources(4096); + let source = payload(4096); + let original = source.as_ptr(); + let first = resources + .retain_payload(source.slice(..4), 1, 10, 0, 0) + .await + .unwrap(); + let second = resources + .retain_payload(source.slice(4..), 2, 11, 0, 200) + .await + .unwrap(); + assert_eq!(first.as_slice().as_ptr(), original); + assert_eq!(second.as_slice().as_ptr(), source[4..].as_ptr()); + assert_eq!(second.metadata().logical_offset, 200); + assert_eq!(second.metadata().allocation_capacity, 4096); + assert_eq!(resources.status().held_bytes, 4096); + drop(source); + drop(first); + assert_eq!(resources.status().held_bytes, 4096); + resources.shutdown(); + let bytes = second.into_bytes(); + assert_eq!(bytes, b"456789"[..]); + assert_eq!(resources.status().held_bytes, 4096); + drop(bytes); + assert_eq!(resources.status().held_bytes, 0); + assert_eq!(resources.status().compatibility_copy_bytes, 0); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_publication_and_repeated_final_release_do_not_double_charge_or_miss_wakes() { + let resources = resources(1024); + let source = payload(1024); + for _ in 0..64 { + let mut deliveries = tokio::task::JoinSet::new(); + let barrier = Arc::new(tokio::sync::Barrier::new(16)); + for id in 1..=16 { + let resources = resources.clone(); + let data = source.slice(2..5); + let barrier = barrier.clone(); + deliveries.spawn(async move { + barrier.wait().await; + resources.retain_payload(data, id, id, 0, 0).await + }); + } + let mut held = Vec::new(); + while let Some(result) = tokio::time::timeout(Duration::from_secs(1), deliveries.join_next()) + .await + .unwrap() + { + held.push(result.unwrap().unwrap()); + } + assert_eq!(held.len(), 16); + assert_eq!(resources.status().held_bytes, 1024); + drop(held); + assert_eq!(resources.status().held_bytes, 0); + } + assert_eq!(resources.status().held_bytes_high_water, 1024); +} + +#[tokio::test] +async fn unknown_owner_waits_before_copying_and_resumes_after_capacity_release() { + let resources = resources(16); + let held = resources.retain_vec(vec![7; 16], 1, 1, 0, 0).unwrap(); + let data = OwnedPayload::from(Bytes::from_static(b"unknown")); + let mut pending = Box::pin(resources.retain_payload(data, 2, 2, 0, 0)); + assert!(tokio::time::timeout(Duration::from_millis(10), &mut pending) + .await + .is_err()); + assert_eq!(resources.status().compatibility_copy_bytes, 0); + drop(held); + let copied = tokio::time::timeout(Duration::from_secs(1), pending) + .await + .unwrap() + .unwrap(); + assert_eq!(copied.as_slice(), b"unknown"); + assert_eq!(copied.metadata().allocation_capacity, 7); + assert_eq!(resources.status().compatibility_copy_bytes, 7); + drop(copied); + assert_eq!(resources.status().held_bytes, 0); +} + +#[tokio::test] +async fn capacity_wait_obeys_request_cancellation_and_runtime_shutdown() { + let resources = resources(16); + let held = resources.retain_vec(vec![7; 16], 1, 1, 0, 0).unwrap(); + let context = OperationContext::new( + NonZeroU64::new(2).unwrap(), + Instant::now() + Duration::from_secs(1), + ); + let request = resources.admit(context.clone()).unwrap(); + let mut pending = Box::pin(request.run(resources.retain_payload(payload(16), 2, 2, 0, 0))); + assert!(tokio::time::timeout(Duration::from_millis(10), &mut pending) + .await + .is_err()); + context.cancellation.cancel(); + assert!(matches!(pending.await, Err(Error::Cancelled))); + resources.shutdown(); + assert!(matches!( + resources.retain_payload(payload(16), 3, 3, 0, 0).await, + Err(Error::Shutdown) + )); + assert_eq!(held.as_slice(), &[7; 16]); + drop(held); + assert_eq!(resources.status().held_bytes, 0); +} + +#[tokio::test] +async fn a_short_view_cannot_hide_an_oversized_backing_allocation() { + let resources = resources(32); + let result = resources + .retain_payload(payload(4096).slice(..2), 1, 1, 0, 0) + .await; + assert!(matches!(result, Err(Error::Capacity(CapacityKind::HeldBuffers)))); + assert_eq!(resources.status().held_bytes, 0); +} From 48b0ad3047ca88aeb8c9232eae15434418b370a2 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 13:23:36 +0800 Subject: [PATCH 063/258] Add generation-bound native S3 reads with verified leased delivery --- doc/working/plan-rdma-fat-client-sdk.md | 28 +- lib/crowdb-access-s3/src/condition.rs | 92 ++++-- lib/crowdb-access-s3/src/execution.rs | 14 +- lib/crowdb-access-s3/src/metadata/user.rs | 14 + lib/crowdb-access-s3/src/object.rs | 34 ++- lib/crowdb-access-s3/src/retrieval.rs | 15 +- lib/crowdb-access-s3/tests/condition_test.rs | 30 +- lib/crowdb-sdk-native/src/error.rs | 22 ++ lib/crowdb-sdk-native/src/s3.rs | 36 ++- lib/crowdb-sdk-native/src/s3/body.rs | 155 ++++++++++ lib/crowdb-sdk-native/src/s3/config.rs | 20 ++ lib/crowdb-sdk-native/src/s3/object.rs | 84 +++++ lib/crowdb-sdk-native/src/s3/read.rs | 108 +++++++ .../tests/allocation_budget_test.rs | 26 ++ .../tests/common/payload_owner.rs | 23 ++ .../tests/common/read_storage.rs | 208 +++++++++++++ lib/crowdb-sdk-native/tests/s3_read_test.rs | 288 ++++++++++++++++++ 17 files changed, 1154 insertions(+), 43 deletions(-) create mode 100644 lib/crowdb-sdk-native/src/s3/body.rs create mode 100644 lib/crowdb-sdk-native/src/s3/config.rs create mode 100644 lib/crowdb-sdk-native/src/s3/object.rs create mode 100644 lib/crowdb-sdk-native/src/s3/read.rs create mode 100644 lib/crowdb-sdk-native/tests/common/payload_owner.rs create mode 100644 lib/crowdb-sdk-native/tests/common/read_storage.rs create mode 100644 lib/crowdb-sdk-native/tests/s3_read_test.rs diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index 190a75b4c..2f4eb2043 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -53,13 +53,13 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. shared clients instead of reconnecting for each facade. Files: `lib/crowdb-sdk-native/src/runtime.rs`, `lib/crowdb-access-s3/src/storage.rs`, `lib/crowdb-chunk-client/src/client.rs`. -- [ ] **Define business context**: expose `S3Client::new(runtime, config)` with +- [~] **Define business context**: expose `S3Client::new(runtime, config)` with tenant/write policy and typed per-operation request identity, deadline, cancellation, conditions, and range. Preserve S3 errors and uncertain outcomes without HTTP headers, XML, or `CredentialProvider`. Files: `lib/crowdb-sdk-native/src/s3/`, `lib/crowdb-access-s3/src/error.rs`, `lib/crowdb-access-s3/src/condition.rs`, `lib/crowdb-access-s3/src/range.rs`. -- [ ] **Define common delivery**: reads return metadata plus a pull-based +- [~] **Define common delivery**: reads return metadata plus a pull-based `next() -> Result, _>` body; `Ok(None)` means applicable length/integrity checks passed. Preserve terminal failure across subsequent polling; early drop is cancellation, not successful EOF. Define unique @@ -107,7 +107,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. `app/crowdb-access-server/src/s3/operations.rs`, `app/crowdb-access-server/src/s3/operations/delete.rs`, `lib/crowdb-sdk-native/src/s3/`. -- [ ] **Extract streaming GET**: adapt `PreparedGet`/`VerifiedGetStream` to +- [~] **Extract streaming GET**: adapt `PreparedGet`/`VerifiedGetStream` to typed metadata and a lazy native body with logical offsets. Preserve selected generation, range checks, storage checksums, and final checksum errors. Do not expose HTTP strings or RPC fragments as native object data. @@ -269,6 +269,28 @@ transport acceptance alone cannot establish SDK completion. `ResourceStatus::compatibility_copy_bytes`. Five focused tests cover shared capacity, 64 concurrent publication/release rounds, bounded copy pressure, cancellation/shutdown and undersized views hiding oversized allocations. +- Native S3 now has typed configuration, entity-tag/date conditions, HEAD, + lazy GET and half-open ranges. Shared condition evaluation serves HTTP and + typed inputs. `ResolvedObject` captures the metadata revision in the same + point read, retains its immutable record, and rejects another tenant or + metadata authority; identical overwrites have distinct revisions. Native + metadata exposes validated user values without HTTP headers. +- `ObjectBody::next` returns retained `BufferLease` views and verifies final + length/integrity before successful EOF. Terminal errors remain sticky; + cancellation/early drop return request admission independently of delivered + buffers. A cancelled pending pull retains its not-yet-delivered payload, so + resuming after capacity release cannot skip it. Seven native read cases + cover original pointer/capacity, range offsets, overwrite/authority binding, + conditions, empty/corrupt checksum, capacity deadline, cancellation/shutdown + and pending-pull cancellation. The native suite now has 22 passing cases; + six allocation cases separately verify retention of each upstream logical + owner while sharing one physical charge. Shared S3 tests and affected HTTP + service/inspection tests, all-target clippy and Rust formatting pass. +- Still outstanding: native S3 PUT/copy/multipart/listing completion and HTTP + extraction, explicit data transport configuration, object locality/stale-view + acceptance, owned transport shutdown, and the complete native/OCI S3 corpus. + Current read fixtures use real storage reader/framing with injected metadata + and disk seams; they are not the required real-service TCP/RDMA acceptance. ## Subsequent Dataset integration under R220 diff --git a/lib/crowdb-access-s3/src/condition.rs b/lib/crowdb-access-s3/src/condition.rs index cb2013335..a5b1f055f 100644 --- a/lib/crowdb-access-s3/src/condition.rs +++ b/lib/crowdb-access-s3/src/condition.rs @@ -22,35 +22,87 @@ pub enum ConditionOutcome { PreconditionFailed, } +/// Typed entity-tag selection for trusted callers without HTTP headers. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum EtagSelection { + Any, + Values(Vec), +} + +impl EtagSelection { + fn matches(&self, etag: &str) -> bool { + match self { + Self::Any => true, + Self::Values(values) => values.iter().any(|value| value == etag), + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct ReadPreconditions { + pub if_match: Option, + pub if_none_match: Option, + pub if_modified_since_seconds: Option, + pub if_unmodified_since_seconds: Option, +} + +pub trait ReadCondition { + fn evaluate(&self, record: &ObjectRecord) -> ConditionOutcome; +} + +impl ReadCondition for ReadPreconditions { + fn evaluate(&self, record: &ObjectRecord) -> ConditionOutcome { + evaluate_values( + record, + self.if_match.as_ref().map(|value| value.matches(&record.etag)), + self.if_none_match + .as_ref() + .map(|value| value.matches(&record.etag)), + self.if_modified_since_seconds, + self.if_unmodified_since_seconds, + ) + } +} + +impl ReadCondition for ObjectConditions<'_> { + fn evaluate(&self, record: &ObjectRecord) -> ConditionOutcome { + evaluate(record, self) + } +} + /// Evaluates standard GET/HEAD conditions using fields from one KV value. #[must_use] pub fn evaluate(record: &ObjectRecord, conditions: &ObjectConditions<'_>) -> ConditionOutcome { - if conditions - .if_match - .is_some_and(|value| !etag_list_matches(value, &record.etag)) - { + evaluate_values( + record, + conditions + .if_match + .map(|value| etag_list_matches(value, &record.etag)), + conditions + .if_none_match + .map(|value| etag_list_matches(value, &record.etag)), + conditions.if_modified_since.and_then(parse_http_seconds), + conditions.if_unmodified_since.and_then(parse_http_seconds), + ) +} + +fn evaluate_values( + record: &ObjectRecord, + if_match: Option, + if_none_match: Option, + modified_since: Option, + unmodified_since: Option, +) -> ConditionOutcome { + if if_match == Some(false) { return ConditionOutcome::PreconditionFailed; } - if conditions.if_match.is_none() - && conditions - .if_unmodified_since - .and_then(parse_http_seconds) - .is_some_and(|limit| record.modified_at_ms / 1000 > limit) - { + if if_match.is_none() && unmodified_since.is_some_and(|limit| record.modified_at_ms / 1000 > limit) { return ConditionOutcome::PreconditionFailed; } - if conditions - .if_none_match - .is_some_and(|value| etag_list_matches(value, &record.etag)) - { + if if_none_match == Some(true) { return ConditionOutcome::NotModified; } - if conditions.if_none_match.is_none() - && conditions - .if_modified_since - .and_then(parse_http_seconds) - .is_some_and(|limit| record.modified_at_ms / 1000 <= limit) - { + if if_none_match.is_none() && modified_since.is_some_and(|limit| record.modified_at_ms / 1000 <= limit) { return ConditionOutcome::NotModified; } ConditionOutcome::Proceed diff --git a/lib/crowdb-access-s3/src/execution.rs b/lib/crowdb-access-s3/src/execution.rs index 565f553b1..6c0c296a5 100644 --- a/lib/crowdb-access-s3/src/execution.rs +++ b/lib/crowdb-access-s3/src/execution.rs @@ -6,10 +6,10 @@ use std::sync::Arc; use crate::bucket::{self, BucketError, DeleteBucketResult, RandomBucketIdGenerator}; -use crate::condition::ObjectConditions; +use crate::condition::{ObjectConditions, ReadCondition}; use crate::error::S3ErrorCode; use crate::metadata::{BucketId, BucketNameRecord, ObjectRecord, TenantId}; -use crate::object::{self, ObjectMetadataError}; +use crate::object::{self, ObjectMetadataError, SelectedObject}; use crate::retrieval::{self, ObjectHeaders, PreparedGet, RetrievalError}; use crate::storage::S3StorageClients; @@ -88,8 +88,14 @@ impl S3Executor { /// # Errors /// Returns missing bucket/key, invalid metadata or routed storage failure. pub async fn object(&self, bucket: &[u8], key: &[u8]) -> Result { + Ok(self.selected_object(bucket, key).await?.record) + } + + /// # Errors + /// Resolves one record and revision without an additional metadata read. + pub async fn selected_object(&self, bucket: &[u8], key: &[u8]) -> Result { let bucket_id = self.resolve_bucket(bucket).await?; - object::head(&self.storage.metadata, &self.tenant, bucket_id, key) + object::head_versioned(&self.storage.metadata, &self.tenant, bucket_id, key) .await .map_err(|error| object_error(&error))? .ok_or(S3ErrorCode::NoSuchKey) @@ -109,7 +115,7 @@ impl S3Executor { pub fn prepare_head( &self, record: &ObjectRecord, - conditions: &ObjectConditions<'_>, + conditions: &(impl ReadCondition + ?Sized), ) -> Result { retrieval::prepare_head(record, conditions) } diff --git a/lib/crowdb-access-s3/src/metadata/user.rs b/lib/crowdb-access-s3/src/metadata/user.rs index 03b9c78e2..8da2a1416 100644 --- a/lib/crowdb-access-s3/src/metadata/user.rs +++ b/lib/crowdb-access-s3/src/metadata/user.rs @@ -18,6 +18,20 @@ const MAX_ENCODED_BYTES: u64 = 64 * 1024; pub struct UserMetadata(BTreeMap); impl UserMetadata { + /// Validates normalized names and values supplied by a native caller. + /// # Errors + /// Rejects invalid names, non-ASCII values and combined metadata above 2 KiB. + pub fn from_values(values: BTreeMap) -> Result { + let metadata = Self(values); + metadata.validate()?; + Ok(metadata) + } + + #[must_use] + pub fn values(&self) -> &BTreeMap { + &self.0 + } + /// Captures normalized names and opaque printable ASCII values. /// /// # Errors diff --git a/lib/crowdb-access-s3/src/object.rs b/lib/crowdb-access-s3/src/object.rs index c2651c8e1..ede32bac4 100644 --- a/lib/crowdb-access-s3/src/object.rs +++ b/lib/crowdb-access-s3/src/object.rs @@ -15,6 +15,12 @@ pub struct ObjectListPage { pub objects: Vec, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SelectedObject { + pub record: ObjectRecord, + pub revision: u64, +} + /// Bounded, normalized `ListObjectsV2` inputs. pub struct ListObjectsV2Request<'a> { pub tenant: &'a TenantId, @@ -63,11 +69,35 @@ pub async fn head( bucket: BucketId, key: &[u8], ) -> Result, ObjectMetadataError> { - let key = MetadataKey::object(tenant, bucket, key)?; + Ok(head_versioned(store, tenant, bucket, key) + .await? + .map(|selected| selected.record)) +} + +/// Reads the record and revision in the same point operation, preserving even +/// an identical overwrite as a distinct selected generation. +/// # Errors +/// Returns invalid namespace/record identity or routed storage failure. +pub async fn head_versioned( + store: &ChunkKvMetadataStore, + tenant: &TenantId, + bucket: BucketId, + object_key: &[u8], +) -> Result, ObjectMetadataError> { + let key = MetadataKey::object(tenant, bucket, object_key)?; store .get(key) .await? - .map(|value| ObjectRecord::decode(&value.value)) + .map(|value| { + let record = ObjectRecord::decode(&value.value)?; + if record.bucket_id != bucket || record.key != object_key { + return Err(MetadataRecordError::Invalid); + } + Ok(SelectedObject { + record, + revision: value.revision, + }) + }) .transpose() .map_err(Into::into) } diff --git a/lib/crowdb-access-s3/src/retrieval.rs b/lib/crowdb-access-s3/src/retrieval.rs index 4c13c5c9c..c7f88d85b 100644 --- a/lib/crowdb-access-s3/src/retrieval.rs +++ b/lib/crowdb-access-s3/src/retrieval.rs @@ -11,7 +11,7 @@ use crowdb_protocol::chunkdb::rpc::Location; use crowdb_rpc_ffi::OwnedPayload; use hyper::body::Bytes; -use crate::condition::{evaluate, ConditionOutcome, ObjectConditions}; +use crate::condition::{ConditionOutcome, ReadCondition}; use crate::integrity::SinglePartIntegrity; use crate::metadata::ObjectRecord; use crate::range::{resolve_range, ByteRange, RangeError}; @@ -157,7 +157,7 @@ pub enum RetrievalError { /// Returns a conditional outcome; all returned headers come from `record`. pub fn prepare_head( record: &ObjectRecord, - conditions: &ObjectConditions<'_>, + conditions: &(impl ReadCondition + ?Sized), ) -> Result { check_conditions(record, conditions)?; Ok(headers(record, None)) @@ -172,7 +172,7 @@ pub fn prepare_get( client: &ChunkIoClient, record: &ObjectRecord, range_header: Option<&str>, - conditions: &ObjectConditions<'_>, + conditions: &(impl ReadCondition + ?Sized), ) -> Result { check_conditions(record, conditions)?; let requested = resolve_range(range_header, record.logical_length).map_err(RetrievalError::Range)?; @@ -186,7 +186,7 @@ pub fn prepare_get_range( client: &ChunkIoClient, record: &ObjectRecord, requested: Option, - conditions: &ObjectConditions<'_>, + conditions: &(impl ReadCondition + ?Sized), ) -> Result { check_conditions(record, conditions)?; if requested.is_some_and(|range| range.start >= range.end || range.end > record.logical_length) { @@ -233,8 +233,11 @@ fn build_get( }) } -fn check_conditions(record: &ObjectRecord, conditions: &ObjectConditions<'_>) -> Result<(), RetrievalError> { - match evaluate(record, conditions) { +fn check_conditions( + record: &ObjectRecord, + conditions: &(impl ReadCondition + ?Sized), +) -> Result<(), RetrievalError> { + match conditions.evaluate(record) { ConditionOutcome::Proceed => Ok(()), ConditionOutcome::NotModified => Err(RetrievalError::NotModified), ConditionOutcome::PreconditionFailed => Err(RetrievalError::PreconditionFailed), diff --git a/lib/crowdb-access-s3/tests/condition_test.rs b/lib/crowdb-access-s3/tests/condition_test.rs index 98334cb82..234d25ae7 100644 --- a/lib/crowdb-access-s3/tests/condition_test.rs +++ b/lib/crowdb-access-s3/tests/condition_test.rs @@ -1,7 +1,9 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use crowdb_access_s3::condition::{evaluate, ConditionOutcome, ObjectConditions}; +use crowdb_access_s3::condition::{ + evaluate, ConditionOutcome, EtagSelection, ObjectConditions, ReadCondition, ReadPreconditions, +}; use crowdb_access_s3::metadata::{BucketId, ObjectRecord}; fn record() -> ObjectRecord { @@ -45,3 +47,29 @@ fn etag_conditions_take_precedence_over_dates() { ConditionOutcome::NotModified ); } + +#[test] +fn typed_conditions_match_http_condition_precedence() { + let value = record(); + for if_match in [None, Some("abc"), Some("other")] { + for if_none_match in [None, Some("abc"), Some("other")] { + for seconds in [1, 3] { + let date = + httpdate::fmt_http_date(std::time::UNIX_EPOCH + std::time::Duration::from_secs(seconds)); + let http = ObjectConditions { + if_match, + if_none_match, + if_modified_since: Some(&date), + if_unmodified_since: Some(&date), + }; + let typed = ReadPreconditions { + if_match: if_match.map(|value| EtagSelection::Values(vec![value.into()])), + if_none_match: if_none_match.map(|value| EtagSelection::Values(vec![value.into()])), + if_modified_since_seconds: Some(seconds), + if_unmodified_since_seconds: Some(seconds), + }; + assert_eq!(typed.evaluate(&value), evaluate(&value, &http)); + } + } + } +} diff --git a/lib/crowdb-sdk-native/src/error.rs b/lib/crowdb-sdk-native/src/error.rs index 82a7f15fd..43f70dac7 100644 --- a/lib/crowdb-sdk-native/src/error.rs +++ b/lib/crowdb-sdk-native/src/error.rs @@ -17,6 +17,20 @@ pub enum MutationInterruption { Shutdown, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ReadFailureKind { + InvalidLocations, + InvalidRange, + Deleted, + NotAvailable, + LayoutExpired, + DataLoss, + Metadata, + DiskIo, + EcDecode, + FailedRange, +} + #[derive(Clone, Debug, thiserror::Error, Eq, PartialEq)] pub enum Error { #[error("invalid native client configuration: {0}")] @@ -35,6 +49,14 @@ pub enum Error { BufferRange, #[error("S3 operation failed: {0:?}")] S3(crowdb_access_s3::S3ErrorCode), + #[error("object was not modified")] + NotModified, + #[error("object checksum does not match its published metadata")] + Integrity, + #[error("object length mismatch: expected={expected} delivered={delivered}")] + Length { expected: u64, delivered: u64 }, + #[error("object read failed: {kind:?}: {detail}")] + Read { kind: ReadFailureKind, detail: String }, #[error("mutation outcome is uncertain: request={request_id} reason={reason:?}")] UncertainMutation { request_id: u64, diff --git a/lib/crowdb-sdk-native/src/s3.rs b/lib/crowdb-sdk-native/src/s3.rs index eb878edb5..546f36523 100644 --- a/lib/crowdb-sdk-native/src/s3.rs +++ b/lib/crowdb-sdk-native/src/s3.rs @@ -4,13 +4,25 @@ //! S3 business handles over the shared trusted runtime, without HTTP authentication. use crowdb_access_s3::execution::S3Executor; -use crowdb_access_s3::metadata::{BucketId, BucketNameRecord, ChunkKvMetadataStore, ObjectRecord, TenantId}; +use crowdb_access_s3::metadata::{BucketId, BucketNameRecord, ChunkKvMetadataStore}; use crowdb_access_s3::storage::S3StorageClients; use crowdb_protocol::chunkdb::rpc::ChunkType; use crate::{ClientRuntime, Error, OperationContext, Result}; use std::sync::Arc; +mod body; +mod config; +mod object; +mod read; + +pub use body::ObjectBody; +pub use config::S3ClientConfig; +pub use crowdb_access_s3::condition::{EtagSelection, ReadPreconditions}; +pub use crowdb_access_s3::range::ByteRange; +pub use object::{ObjectMetadata, ResolvedObject}; +pub use read::{GetObject, GetOptions}; + #[derive(Clone)] pub struct S3Client { runtime: ClientRuntime, @@ -21,7 +33,19 @@ impl S3Client { /// Creates a tenant handle using the runtime's existing storage owners. /// # Errors /// Rejects closed runtime or a writer group for another concrete chunk type. - pub fn new(runtime: ClientRuntime, tenant: TenantId) -> Result { + pub fn new(runtime: ClientRuntime, config: impl Into) -> Result { + let config = config.into(); + if config + .small_write + .as_ref() + .is_some_and(|policy| policy.chunk_type != ChunkType::S3) + { + return Err(Error::Configuration("S3 requires an S3 writer policy".into())); + } + let runtime = match &config.small_write { + Some(policy) => runtime.with_writer_policy(policy)?, + None => runtime, + }; runtime.resources.check_open()?; if runtime.storage.chunk_type != ChunkType::S3 { return Err(Error::Configuration("S3 requires an S3 writer group".into())); @@ -32,7 +56,7 @@ impl S3Client { chunks: Arc::clone(&runtime.storage.chunks), }; Ok(Self { - execution: S3Executor::new(storage, tenant), + execution: S3Executor::new(storage, config.tenant), runtime, }) } @@ -85,11 +109,9 @@ impl S3Client { context: OperationContext, bucket: &[u8], key: &[u8], - ) -> Result { + ) -> Result { let request = self.runtime.resources.admit(context)?; - request - .run(async { self.execution.object(bucket, key).await.map_err(Error::S3) }) - .await + request.run(self.select_object(bucket, key)).await } /// # Errors diff --git a/lib/crowdb-sdk-native/src/s3/body.rs b/lib/crowdb-sdk-native/src/s3/body.rs new file mode 100644 index 000000000..9206dde15 --- /dev/null +++ b/lib/crowdb-sdk-native/src/s3/body.rs @@ -0,0 +1,155 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_access_s3::retrieval::{GetStreamError, VerifiedGetStream}; +use crowdb_chunk_client::ReadError; +use crowdb_rpc_ffi::OwnedPayload; + +use super::ByteRange; +use crate::buffer::BufferLease; +use crate::error::ReadFailureKind; +use crate::runtime::RequestLease; +use crate::{Error, Result, RuntimeResources}; + +/// Pull-based logical object delivery. Dropping a pending pull retains its +/// pending payload for a subsequent pull. Dropping the body cancels production. +pub struct ObjectBody { + stream: Option, + pending: Option, + request: Option, + resources: Arc, + range: ByteRange, + offset: u64, + sequence: u64, + complete: bool, + failure: Option, +} + +impl ObjectBody { + pub(super) fn new( + stream: Option, + request: RequestLease, + resources: Arc, + range: ByteRange, + ) -> Self { + Self { + stream, + pending: None, + request: Some(request), + resources, + range, + offset: range.start, + sequence: 0, + complete: false, + failure: None, + } + } + + /// Successful EOF means all applicable length and integrity checks passed. + /// Failures remain visible on every later pull. + /// # Errors + /// Returns cancellation, deadline, capacity or typed verified-read failures. + pub async fn next(&mut self) -> Result> { + if let Some(error) = &self.failure { + return Err(error.clone()); + } + if self.complete { + return Ok(None); + } + match self.pull().await { + Ok(Some(lease)) => Ok(Some(lease)), + Ok(None) => { + self.complete = true; + self.stream = None; + self.request = None; + Ok(None) + } + Err(error) => { + self.fail(error.clone()); + Err(error) + } + } + } + + #[must_use] + pub fn is_complete(&self) -> bool { + self.complete + } + + pub fn cancel(&mut self) { + if !self.complete && self.failure.is_none() { + self.fail(Error::Cancelled); + } + } + + fn fail(&mut self, error: Error) { + self.stream = None; + self.pending = None; + self.request = None; + self.failure = Some(error); + } + + async fn pull(&mut self) -> Result> { + let request = self.request.as_ref().ok_or(Error::Cancelled)?; + if self.pending.is_none() { + let stream = self + .stream + .as_mut() + .ok_or(Error::S3(crowdb_access_s3::S3ErrorCode::InternalError))?; + self.pending = request + .run(async { stream.next_payload().await.map_err(read_error) }) + .await?; + if self.pending.is_none() { + if self.offset != self.range.end { + return Err(Error::Length { + expected: self.range.end - self.range.start, + delivered: self.offset - self.range.start, + }); + } + return Ok(None); + } + } + let payload = self.pending.as_ref().ok_or(Error::BufferRange)?.clone(); + let id = request.context().request_id.get(); + let lease = request + .run( + self.resources + .retain_payload(payload, id, id, self.sequence, self.offset), + ) + .await?; + self.pending = None; + self.offset = self + .offset + .checked_add(lease.metadata().length as u64) + .ok_or(Error::BufferRange)?; + self.sequence = self.sequence.checked_add(1).ok_or(Error::BufferRange)?; + Ok(Some(lease)) + } +} + +fn read_error(error: GetStreamError) -> Error { + match error { + GetStreamError::Integrity => Error::Integrity, + GetStreamError::Length { expected, delivered } => Error::Length { expected, delivered }, + GetStreamError::Chunk(error) => { + let kind = match &*error { + ReadError::InvalidLocations(_) => ReadFailureKind::InvalidLocations, + ReadError::InvalidRange { .. } => ReadFailureKind::InvalidRange, + ReadError::ChunkDeleted(_) => ReadFailureKind::Deleted, + ReadError::NotYetAvailable(_) => ReadFailureKind::NotAvailable, + ReadError::LayoutExpired => ReadFailureKind::LayoutExpired, + ReadError::DataLoss(_) => ReadFailureKind::DataLoss, + ReadError::Metadata(_) => ReadFailureKind::Metadata, + ReadError::DiskIo(_) => ReadFailureKind::DiskIo, + ReadError::EcDecode(_) => ReadFailureKind::EcDecode, + ReadError::FailedRange { .. } => ReadFailureKind::FailedRange, + }; + Error::Read { + kind, + detail: error.to_string(), + } + } + } +} diff --git a/lib/crowdb-sdk-native/src/s3/config.rs b/lib/crowdb-sdk-native/src/s3/config.rs new file mode 100644 index 000000000..5661970d9 --- /dev/null +++ b/lib/crowdb-sdk-native/src/s3/config.rs @@ -0,0 +1,20 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_access_s3::metadata::TenantId; +use crowdb_chunk_client::SmallWritePolicy; + +pub struct S3ClientConfig { + pub tenant: TenantId, + /// None reuses the runtime's current compatible S3 writer group. + pub small_write: Option, +} + +impl From for S3ClientConfig { + fn from(tenant: TenantId) -> Self { + Self { + tenant, + small_write: None, + } + } +} diff --git a/lib/crowdb-sdk-native/src/s3/object.rs b/lib/crowdb-sdk-native/src/s3/object.rs new file mode 100644 index 000000000..14c0e6e61 --- /dev/null +++ b/lib/crowdb-sdk-native/src/s3/object.rs @@ -0,0 +1,84 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_access_s3::metadata::{BucketId, ObjectRecord, TenantId, UserMetadata}; +use crowdb_access_s3::S3ErrorCode; +use crowdb_chunk_kv_client::ChunkKvClient; + +use super::S3Client; +use crate::{Error, Result}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ObjectMetadata { + pub bucket_id: BucketId, + pub key: Vec, + pub revision: u64, + pub logical_length: u64, + pub etag: String, + pub checksum: Vec, + pub created_at_ms: u64, + pub modified_at_ms: u64, + pub content_type: String, + pub user_metadata: UserMetadata, +} + +/// An immutable metadata selection bound to its tenant and storage authority. +/// Execution does not look the mutable object name up again. +#[derive(Clone)] +pub struct ResolvedObject { + pub(super) record: Arc, + metadata: ObjectMetadata, + tenant: TenantId, + authority: Arc, +} + +impl ResolvedObject { + #[must_use] + pub fn metadata(&self) -> &ObjectMetadata { + &self.metadata + } +} + +impl S3Client { + pub(super) async fn select_object(&self, bucket: &[u8], key: &[u8]) -> Result { + let selected = self + .execution + .selected_object(bucket, key) + .await + .map_err(Error::S3)?; + let record = selected.record; + let user_metadata = + UserMetadata::decode(&record.attributes).map_err(|_| Error::S3(S3ErrorCode::InternalError))?; + let metadata = ObjectMetadata { + bucket_id: record.bucket_id, + key: record.key.clone(), + revision: selected.revision, + logical_length: record.logical_length, + etag: record.etag.clone(), + checksum: record.checksum.clone(), + created_at_ms: record.created_at_ms, + modified_at_ms: record.modified_at_ms, + content_type: record.content_type.clone(), + user_metadata, + }; + Ok(ResolvedObject { + record: Arc::new(record), + metadata, + tenant: self.execution.tenant().clone(), + authority: Arc::clone(&self.runtime.storage.metadata), + }) + } + + pub(super) fn validate_selection(&self, selected: &ResolvedObject) -> Result<()> { + if selected.tenant != *self.execution.tenant() + || !Arc::ptr_eq(&selected.authority, &self.runtime.storage.metadata) + { + return Err(Error::Configuration( + "object selection belongs to another tenant or storage authority".into(), + )); + } + Ok(()) + } +} diff --git a/lib/crowdb-sdk-native/src/s3/read.rs b/lib/crowdb-sdk-native/src/s3/read.rs new file mode 100644 index 000000000..1d647f1a0 --- /dev/null +++ b/lib/crowdb-sdk-native/src/s3/read.rs @@ -0,0 +1,108 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_access_s3::retrieval::{prepare_get_range, RetrievalError}; +use crowdb_access_s3::S3ErrorCode; + +use super::{ByteRange, ObjectBody, ObjectMetadata, ReadPreconditions, ResolvedObject, S3Client}; +use crate::runtime::RequestLease; +use crate::{Error, OperationContext, Result}; + +#[derive(Clone, Debug, Default)] +pub struct GetOptions { + pub range: Option, + pub conditions: ReadPreconditions, +} + +pub struct GetObject { + pub metadata: ObjectMetadata, + pub range: ByteRange, + pub body: ObjectBody, +} + +impl S3Client { + /// Reads metadata and conditions without requesting payload storage. + /// # Errors + /// Returns admission, missing object, conditional or metadata failures. + pub async fn head_object( + &self, + context: OperationContext, + bucket: &[u8], + key: &[u8], + conditions: &ReadPreconditions, + ) -> Result { + let request = self.runtime.resources.admit(context)?; + request + .run(async { + let selected = self.select_object(bucket, key).await?; + self.execution + .prepare_head(&selected.record, conditions) + .map_err(|error| retrieval_error(&error))?; + Ok(selected.metadata().clone()) + }) + .await + } + + /// Selects one published generation and returns a lazy verified body. + /// # Errors + /// Returns admission, namespace, conditional or range failures before delivery. + pub async fn get_object( + &self, + context: OperationContext, + bucket: &[u8], + key: &[u8], + options: &GetOptions, + ) -> Result { + let request = self.runtime.resources.admit(context)?; + let selected = request.run(self.select_object(bucket, key)).await?; + self.prepare_read(request, &selected, options) + } + + /// Reads exactly the selected record, even after its name is overwritten. + /// # Errors + /// Also rejects selections from a different tenant or storage authority. + pub fn get_resolved( + &self, + context: OperationContext, + selected: &ResolvedObject, + options: &GetOptions, + ) -> Result { + let request = self.runtime.resources.admit(context)?; + self.prepare_read(request, selected, options) + } + + fn prepare_read( + &self, + request: RequestLease, + selected: &ResolvedObject, + options: &GetOptions, + ) -> Result { + self.validate_selection(selected)?; + request.context().check()?; + let prepared = prepare_get_range( + self.execution.chunks(), + &selected.record, + options.range, + &options.conditions, + ) + .map_err(|error| retrieval_error(&error))?; + let range = options.range.unwrap_or(ByteRange { + start: 0, + end: selected.record.logical_length, + }); + Ok(GetObject { + metadata: selected.metadata().clone(), + range, + body: ObjectBody::new(prepared.stream, request, self.runtime.resources.clone(), range), + }) + } +} + +fn retrieval_error(error: &RetrievalError) -> Error { + match error { + RetrievalError::NotModified => Error::NotModified, + RetrievalError::PreconditionFailed => Error::S3(S3ErrorCode::PreconditionFailed), + RetrievalError::Range(_) => Error::S3(S3ErrorCode::InvalidRange), + RetrievalError::DataReference | RetrievalError::ChunkRead => Error::S3(S3ErrorCode::InternalError), + } +} diff --git a/lib/crowdb-sdk-native/tests/allocation_budget_test.rs b/lib/crowdb-sdk-native/tests/allocation_budget_test.rs index 34e8a55c5..cbf1ee15f 100644 --- a/lib/crowdb-sdk-native/tests/allocation_budget_test.rs +++ b/lib/crowdb-sdk-native/tests/allocation_budget_test.rs @@ -1,6 +1,9 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#[path = "common/payload_owner.rs"] +mod owner; + use std::num::NonZeroU64; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -140,3 +143,26 @@ async fn a_short_view_cannot_hide_an_oversized_backing_allocation() { assert!(matches!(result, Err(Error::Capacity(CapacityKind::HeldBuffers)))); assert_eq!(resources.status().held_bytes, 0); } + +#[tokio::test] +async fn a_shared_physical_charge_keeps_each_upstream_logical_owner() { + let resources = resources(1024); + let released = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let base = payload(1024); + let mut leases = Vec::new(); + for id in 1..=2 { + let released = released.clone(); + let logical = base + .slice(..3) + .with_owner(|bytes| Bytes::from_owner(owner::TestPayloadOwner { bytes, released })); + leases.push(resources.retain_payload(logical, id, id, 0, 0).await.unwrap()); + } + assert_eq!(released.load(std::sync::atomic::Ordering::Acquire), 0); + assert_eq!(resources.status().held_bytes, 1024); + drop(leases.pop()); + assert_eq!(released.load(std::sync::atomic::Ordering::Acquire), 1); + assert_eq!(resources.status().held_bytes, 1024); + drop(leases); + assert_eq!(released.load(std::sync::atomic::Ordering::Acquire), 2); + assert_eq!(resources.status().held_bytes, 0); +} diff --git a/lib/crowdb-sdk-native/tests/common/payload_owner.rs b/lib/crowdb-sdk-native/tests/common/payload_owner.rs new file mode 100644 index 000000000..8d0cea6e8 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/common/payload_owner.rs @@ -0,0 +1,23 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +pub struct TestPayloadOwner { + pub bytes: Bytes, + pub released: Arc, +} + +impl AsRef<[u8]> for TestPayloadOwner { + fn as_ref(&self) -> &[u8] { + &self.bytes + } +} + +impl Drop for TestPayloadOwner { + fn drop(&mut self) { + self.released.fetch_add(1, Ordering::Release); + } +} diff --git a/lib/crowdb-sdk-native/tests/common/read_storage.rs b/lib/crowdb-sdk-native/tests/common/read_storage.rs new file mode 100644 index 000000000..f1269a259 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/common/read_storage.rs @@ -0,0 +1,208 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::num::NonZeroU64; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_access_s3::integrity::SinglePartIntegrity; +use crowdb_access_s3::metadata::{BucketId, ChunkKvMetadataStore, MetadataKey, ObjectRecord, TenantId}; +use crowdb_chunk_client::{ChunkAllocator, ChunkIoClient, DiskWriter, Result}; +use crowdb_chunk_kv_client::{ChunkKvRpcTransport, ChunkKvRpcTransportConfig}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_protocol::chunkdb::rpc::*; +use crowdb_protocol::common::{ChunkId, DiskId}; +use crowdb_protocol::diskdb::rpc::Segment; +use crowdb_protocol::frame::{encode_frame, FrameMagic}; +use crowdb_rpc_ffi::OwnedPayload; +use crowdb_sdk_native::runtime::RuntimeClients; +use crowdb_sdk_native::s3::S3Client; +use crowdb_sdk_native::{ClientRuntime, OperationContext, RuntimeLimits}; + +pub struct TestReadStorage { + pub data: Vec, + pub reads: AtomicUsize, +} + +fn chunk_id(index: usize) -> ChunkId { + ChunkId { + high: 1, + low: index as u64 + 1, + } +} + +#[async_trait] +impl ChunkAllocator for TestReadStorage { + async fn allocate_chunk(&self, _: AllocateChunkRequest) -> Result { + panic!("read allocated") + } + async fn append_chunk(&self, _: AppendChunkRequest) -> Result { + panic!("read appended") + } + async fn seal_chunk(&self, _: SealChunkRequest) -> Result { + panic!("read sealed") + } + async fn delete_chunk(&self, _: DeleteChunkRequest) -> Result { + panic!("read deleted") + } + async fn update_chunk_strip(&self, _: UpdateChunkStripRequest) -> Result { + panic!("read updated") + } + async fn query_chunk(&self, request: QueryChunkRequest) -> Result { + let id = request.chunk_id.unwrap(); + Ok(QueryChunkResponse { + chunk: Some(Chunk { + id: Some(id), + state: ChunkState::Sealed as i32, + capacity: 1, + sealed_length: 1, + strips: vec![ChunkStrip { + unit_kb: 1, + capacity: 1, + sealed_length: 1, + sealed_ts_ms: 1, + strip: Some(Strip::MirrorStrip(MirrorStrip { + segments: vec![Segment { + disk_id: Some(DiskId { + high: id.high, + low: id.low, + }), + unit_count: 1, + ..Segment::default() + }], + })), + ..ChunkStrip::default() + }], + ..Chunk::default() + }), + layout_validity_ms: 60_000, + }) + } +} + +#[async_trait] +impl DiskWriter for TestReadStorage { + async fn write(&self, _: &Segment, _: u64, _: Bytes) -> Result<()> { + panic!("read wrote") + } + async fn write_at_byte_offset(&self, _: &Segment, _: u64, _: u64, _: Bytes) -> Result<()> { + panic!("read wrote") + } + async fn read_payload( + &self, + segment: &Segment, + _: u64, + offset: u64, + length: u32, + ) -> Result { + self.reads.fetch_add(1, Ordering::Relaxed); + let index = usize::try_from(segment.disk_id.unwrap().low - 1).unwrap(); + let offset = usize::try_from(offset).unwrap(); + Ok(self.data[index].slice(offset..offset + length as usize)) + } +} + +pub fn context(id: u64) -> OperationContext { + OperationContext::new( + NonZeroU64::new(id).unwrap(), + Instant::now() + Duration::from_secs(1), + ) +} + +pub struct TestFixture { + pub runtime: ClientRuntime, + pub client: S3Client, + pub backend: Arc, + pub metadata: ChunkKvMetadataStore, + pub tenant: TenantId, + pub bucket: BucketId, +} + +impl TestFixture { + pub async fn new(held_bytes: usize) -> Self { + let backend = Arc::new(TestReadStorage { + data: [b"original".as_slice(), b"replacement".as_slice()] + .into_iter() + .enumerate() + .map(|(index, payload)| { + let mut frame = Vec::with_capacity(4096); + frame.extend_from_slice( + &encode_frame(FrameMagic::RepoSmallV1, chunk_id(index), payload, 1).unwrap(), + ); + OwnedPayload::from(frame) + }) + .collect(), + reads: AtomicUsize::new(0), + }); + let metadata = super::metadata::client().await; + let runtime = ClientRuntime::from_clients( + RuntimeClients { + control: Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))), + metadata: metadata.clone(), + metadata_transport: Arc::new( + ChunkKvRpcTransport::with_config(8, 1, 1, &ChunkKvRpcTransportConfig::default()).unwrap(), + ), + chunks: Arc::new(ChunkIoClient::from_parts( + backend.clone(), + backend.clone(), + ChunkType::S3, + )), + }, + RuntimeLimits { + held_bytes, + ..RuntimeLimits::default() + }, + ) + .unwrap(); + let tenant = TenantId::new(b"tenant".to_vec()).unwrap(); + let client = S3Client::new(runtime.clone(), tenant.clone()).unwrap(); + let bucket = client.create_bucket(context(1), b"bucket").await.unwrap(); + Self { + runtime, + client, + backend, + metadata: ChunkKvMetadataStore::new(metadata), + tenant, + bucket, + } + } + + pub fn record(&self, index: usize) -> ObjectRecord { + let payload = [b"original".as_slice(), b"replacement".as_slice()][index]; + let mut integrity = SinglePartIntegrity::default(); + integrity.update(&Bytes::from_static(payload)); + let (etag, checksum) = integrity.finish(); + ObjectRecord { + bucket_id: self.bucket, + key: b"key".to_vec(), + logical_length: payload.len() as u64, + checksum, + etag, + created_at_ms: 1000, + modified_at_ms: 2000, + content_type: "application/octet-stream".into(), + attributes: Vec::new(), + data_reference: bincode::serialize(&vec![Location { + chunk_id: Some(chunk_id(index)), + length: self.backend.data[index].len() as u64, + logical_length: payload.len() as u64, + ..Location::default() + }]) + .unwrap(), + data_length: payload.len() as u64, + } + } + + pub async fn publish(&self, record: &ObjectRecord) -> u64 { + self.metadata + .put( + MetadataKey::object(&self.tenant, self.bucket, &record.key).unwrap(), + record.encode().unwrap(), + ) + .await + .unwrap() + } +} diff --git a/lib/crowdb-sdk-native/tests/s3_read_test.rs b/lib/crowdb-sdk-native/tests/s3_read_test.rs new file mode 100644 index 000000000..f75f82c0f --- /dev/null +++ b/lib/crowdb-sdk-native/tests/s3_read_test.rs @@ -0,0 +1,288 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/metadata.rs"] +mod metadata; +#[path = "common/read_storage.rs"] +mod storage; + +use std::sync::atomic::Ordering; +use std::time::Duration; + +use crowdb_access_s3::metadata::TenantId; +use crowdb_access_s3::S3ErrorCode; +use crowdb_sdk_native::s3::{ByteRange, EtagSelection, GetOptions, ReadPreconditions, S3Client}; +use crowdb_sdk_native::Error; +use storage::{context, TestFixture}; + +#[tokio::test] +async fn native_head_and_ranged_get_retain_original_storage_without_http_or_credentials() { + let fixture = TestFixture::new(4096).await; + let mut record = fixture.record(0); + record.attributes = crowdb_access_s3::metadata::UserMetadata::from_values( + [("sample".to_owned(), "opaque value".to_owned())] + .into_iter() + .collect(), + ) + .unwrap() + .encode() + .unwrap(); + let revision = fixture.publish(&record).await; + let head = fixture + .client + .head_object(context(2), b"bucket", b"key", &ReadPreconditions::default()) + .await + .unwrap(); + assert_eq!(head.revision, revision); + assert_eq!(head.user_metadata.values()["sample"], "opaque value"); + assert_eq!(fixture.backend.reads.load(Ordering::Relaxed), 0); + let options = GetOptions { + range: Some(ByteRange { start: 2, end: 6 }), + ..GetOptions::default() + }; + let mut read = fixture + .client + .get_object(context(3), b"bucket", b"key", &options) + .await + .unwrap(); + assert_eq!(read.metadata, head); + assert_eq!(fixture.backend.reads.load(Ordering::Relaxed), 0); + let lease = read.body.next().await.unwrap().unwrap(); + assert_eq!(lease.as_slice(), b"igin"); + assert_eq!(lease.metadata().logical_offset, 2); + assert_eq!(lease.metadata().request_id, 3); + assert_eq!(lease.metadata().sequence, 0); + assert_eq!(lease.metadata().allocation_capacity, 4096); + let frame = &fixture.backend.data[0]; + let payload_offset = + crowdb_protocol::frame::parse_frame(frame, crowdb_protocol::common::ChunkId { high: 1, low: 1 }) + .unwrap() + .header + .payload_offset; + assert_eq!( + lease.as_slice().as_ptr(), + frame[usize::from(payload_offset) + 2..].as_ptr() + ); + assert!(read.body.next().await.unwrap().is_none()); + assert!(read.body.is_complete()); + assert_eq!(fixture.runtime.resources().status().requests, 0); + assert_eq!(fixture.runtime.resources().status().held_bytes, 4096); + assert_eq!(fixture.runtime.resources().status().compatibility_copy_bytes, 0); + fixture.runtime.shutdown().await.unwrap(); + assert_eq!(lease.as_slice(), b"igin"); + drop(lease); + assert_eq!(fixture.runtime.resources().status().held_bytes, 0); +} + +#[tokio::test] +async fn resolved_generation_survives_overwrite_and_rejects_other_tenants() { + let fixture = TestFixture::new(8192).await; + let original = fixture.record(0); + let first_revision = fixture.publish(&original).await; + let selected = fixture + .client + .resolve_object(context(2), b"bucket", b"key") + .await + .unwrap(); + let identical_revision = fixture.publish(&original).await; + let identical = fixture + .client + .resolve_object(context(3), b"bucket", b"key") + .await + .unwrap(); + assert_ne!(first_revision, identical_revision); + assert_eq!(identical.metadata().revision, identical_revision); + fixture.publish(&fixture.record(1)).await; + let mut old = fixture + .client + .get_resolved(context(4), &selected, &GetOptions::default()) + .unwrap(); + assert_eq!(old.body.next().await.unwrap().unwrap().as_slice(), b"original"); + assert!(old.body.next().await.unwrap().is_none()); + let mut current = fixture + .client + .get_object(context(5), b"bucket", b"key", &GetOptions::default()) + .await + .unwrap(); + assert_eq!( + current.body.next().await.unwrap().unwrap().as_slice(), + b"replacement" + ); + assert!(current.body.next().await.unwrap().is_none()); + let other = S3Client::new(fixture.runtime.clone(), TenantId::new(b"other".to_vec()).unwrap()).unwrap(); + assert!(matches!( + other.get_resolved(context(6), &selected, &GetOptions::default()), + Err(Error::Configuration(_)) + )); + let another_authority = TestFixture::new(8192).await; + assert!(matches!( + another_authority + .client + .get_resolved(context(7), &selected, &GetOptions::default()), + Err(Error::Configuration(_)) + )); +} + +#[tokio::test] +async fn dropping_a_pending_pull_does_not_lose_bytes_waiting_for_capacity() { + let fixture = TestFixture::new(4096).await; + fixture.publish(&fixture.record(0)).await; + let mut first = fixture + .client + .get_object(context(2), b"bucket", b"key", &GetOptions::default()) + .await + .unwrap(); + let held = first.body.next().await.unwrap().unwrap(); + assert!(first.body.next().await.unwrap().is_none()); + fixture.publish(&fixture.record(1)).await; + let mut second = fixture + .client + .get_object(context(3), b"bucket", b"key", &GetOptions::default()) + .await + .unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(10), second.body.next()) + .await + .is_err() + ); + assert_eq!(fixture.backend.reads.load(Ordering::Relaxed), 2); + drop(held); + let next = second.body.next().await.unwrap().unwrap(); + assert_eq!(next.as_slice(), b"replacement"); + assert_eq!(next.metadata().sequence, 0); + assert_eq!(next.metadata().logical_offset, 0); + assert!(second.body.next().await.unwrap().is_none()); + assert_eq!(fixture.backend.reads.load(Ordering::Relaxed), 2); +} + +#[tokio::test] +async fn typed_conditions_share_precedence_and_integrity_failure_stays_terminal() { + let fixture = TestFixture::new(4096).await; + let mut record = fixture.record(0); + fixture.publish(&record).await; + let conditions = ReadPreconditions { + if_match: Some(EtagSelection::Values(vec!["wrong".into()])), + if_unmodified_since_seconds: Some(u64::MAX), + ..ReadPreconditions::default() + }; + assert!(matches!( + fixture + .client + .head_object(context(2), b"bucket", b"key", &conditions) + .await, + Err(Error::S3(S3ErrorCode::PreconditionFailed)) + )); + let conditions = ReadPreconditions { + if_none_match: Some(EtagSelection::Any), + ..ReadPreconditions::default() + }; + assert!(matches!( + fixture + .client + .head_object(context(3), b"bucket", b"key", &conditions) + .await, + Err(Error::NotModified) + )); + assert_eq!(fixture.backend.reads.load(Ordering::Relaxed), 0); + record.checksum[0] ^= 0xff; + fixture.publish(&record).await; + let mut read = fixture + .client + .get_object(context(4), b"bucket", b"key", &GetOptions::default()) + .await + .unwrap(); + let held = read.body.next().await.unwrap().unwrap(); + assert!(matches!(read.body.next().await, Err(Error::Integrity))); + read.body.cancel(); + assert!(matches!(read.body.next().await, Err(Error::Integrity))); + assert!(!read.body.is_complete()); + assert_eq!(fixture.runtime.resources().status().requests, 0); + assert_eq!(held.as_slice(), b"original"); + assert_eq!(fixture.runtime.resources().status().held_bytes, 4096); +} + +#[tokio::test] +async fn cancellation_and_shutdown_leave_delivered_leases_valid() { + let fixture = TestFixture::new(4096).await; + fixture.publish(&fixture.record(0)).await; + let request = context(2); + let mut read = fixture + .client + .get_object(request.clone(), b"bucket", b"key", &GetOptions::default()) + .await + .unwrap(); + let held = read.body.next().await.unwrap().unwrap(); + request.cancellation.cancel(); + assert!(matches!(read.body.next().await, Err(Error::Cancelled))); + assert!(matches!(read.body.next().await, Err(Error::Cancelled))); + assert_eq!(held.as_slice(), b"original"); + let mut second = fixture + .client + .get_object(context(3), b"bucket", b"key", &GetOptions::default()) + .await + .unwrap(); + fixture.runtime.shutdown().await.unwrap(); + assert!(matches!(second.body.next().await, Err(Error::Shutdown))); + assert_eq!(held.as_slice(), b"original"); +} + +#[tokio::test] +async fn empty_native_reads_check_integrity_before_successful_eof() { + let fixture = TestFixture::new(4096).await; + let mut record = fixture.record(0); + record.logical_length = 0; + record.data_length = 0; + record.data_reference = + bincode::serialize(&Vec::::new()).unwrap(); + let (etag, checksum) = crowdb_access_s3::integrity::SinglePartIntegrity::default().finish(); + record.etag = etag; + record.checksum = checksum; + fixture.publish(&record).await; + let mut valid = fixture + .client + .get_object(context(2), b"bucket", b"key", &GetOptions::default()) + .await + .unwrap(); + assert!(!valid.body.is_complete()); + assert!(valid.body.next().await.unwrap().is_none()); + assert!(valid.body.is_complete()); + record.checksum[0] ^= 0xff; + fixture.publish(&record).await; + let mut invalid = fixture + .client + .get_object(context(3), b"bucket", b"key", &GetOptions::default()) + .await + .unwrap(); + assert!(matches!(invalid.body.next().await, Err(Error::Integrity))); + assert!(matches!(invalid.body.next().await, Err(Error::Integrity))); + assert_eq!(fixture.backend.reads.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn native_body_capacity_wait_expires_without_releasing_delivered_data() { + let fixture = TestFixture::new(4096).await; + fixture.publish(&fixture.record(0)).await; + let mut first = fixture + .client + .get_object(context(2), b"bucket", b"key", &GetOptions::default()) + .await + .unwrap(); + let held = first.body.next().await.unwrap().unwrap(); + drop(first); + assert_eq!(fixture.runtime.resources().status().requests, 0); + fixture.publish(&fixture.record(1)).await; + let mut deadline = context(3); + deadline.deadline = std::time::Instant::now() + Duration::from_millis(20); + let mut second = fixture + .client + .get_object(deadline, b"bucket", b"key", &GetOptions::default()) + .await + .unwrap(); + assert!(matches!(second.body.next().await, Err(Error::Deadline))); + assert!(matches!(second.body.next().await, Err(Error::Deadline))); + assert_eq!(fixture.runtime.resources().status().requests, 0); + assert_eq!(fixture.runtime.resources().status().held_bytes, 4096); + assert_eq!(held.as_slice(), b"original"); + drop(held); + assert_eq!(fixture.runtime.resources().status().held_bytes, 0); +} From 32724b67fccbea2ebdbbd0ed51d2564259902466 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 13:30:52 +0800 Subject: [PATCH 064/258] Configure native data transport and verify leased RPC reads --- Cargo.lock | 1 + doc/working/plan-rdma-fat-client-sdk.md | 22 +++- lib/crowdb-chunk-client/src/client.rs | 76 +---------- .../src/client/connection.rs | 98 ++++++++++++++ .../src/disk_io/routing.rs | 9 ++ lib/crowdb-rpc/ffi/src/server.rs | 38 ++++++ lib/crowdb-rpc/ffi/tests/ffi_handler_test.rs | 18 +-- lib/crowdb-sdk-native/Cargo.toml | 2 + lib/crowdb-sdk-native/src/runtime.rs | 3 + lib/crowdb-sdk-native/src/runtime/storage.rs | 3 +- .../tests/common/read_storage.rs | 49 ++++--- .../tests/common/rpc_read.rs | 122 ++++++++++++++++++ .../tests/s3_rpc_read_test.rs | 86 ++++++++++++ 13 files changed, 419 insertions(+), 108 deletions(-) create mode 100644 lib/crowdb-chunk-client/src/client/connection.rs create mode 100644 lib/crowdb-sdk-native/tests/common/rpc_read.rs create mode 100644 lib/crowdb-sdk-native/tests/s3_rpc_read_test.rs diff --git a/Cargo.lock b/Cargo.lock index 518d33774..492f3a2c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1292,6 +1292,7 @@ dependencies = [ "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-sdk-native", + "flatbuffers", "futures", "serde_json", "thiserror 2.0.18", diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index 2f4eb2043..baf9bdc6f 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -140,7 +140,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. ## Phase 4: retained leases, RDMA, and bounded lifecycle -- [ ] **Complete lease ABI**: reuse opaque `crowdb_rpc_buffer_t`, +- [x] **Complete lease ABI**: reuse opaque `crowdb_rpc_buffer_t`, `BufferLeaseMetadata`, and actual buffer length for native `BufferLease`. Preserve request/stream identity, sequence, offset, capacity class, and single final release. Rust borrowing prevents access after release; @@ -148,7 +148,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. handles is safe. Files: `lib/crowdb-rpc/ffi/src/buffer.rs`, `lib/crowdb-rpc/ffi/src/sys.rs`, `lib/crowdb-rpc/include/crowdb-rpc/c_api.h`, `lib/crowdb-sdk-native/src/buffer.rs`. -- [ ] **Preserve completion ownership**: audit C++ completion, +- [x] **Preserve completion ownership**: audit C++ completion, `on_complete_cb`, `receive_data_buffer`, DiskIO conversion, ChunkIO views, and S3 adaptation. Retain eligible receive storage through caller release rather than a copy/detach shortcut that loses credits. Keep control @@ -165,7 +165,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. `lib/crowdb-rpc/ffi/src/producer_buffer.rs`, `lib/crowdb-chunk-client/src/io.rs`, `lib/crowdb-sdk-native/src/buffer.rs`, `lib/crowdb-access-s3/src/streaming.rs`. -- [ ] **Enforce held-capacity budgets**: charge actual allocation capacity +- [x] **Enforce held-capacity budgets**: charge actual allocation capacity once across shared views until final release. Keep request/producer/held limits explicit and lock-free; waits honor deadlines/cancellation without blocking RPC workers. Release backing storage before waking waiters. Files: @@ -287,10 +287,24 @@ transport acceptance alone cannot establish SDK completion. owner while sharing one physical charge. Shared S3 tests and affected HTTP service/inspection tests, all-target clippy and Rust formatting pass. - Still outstanding: native S3 PUT/copy/multipart/listing completion and HTTP - extraction, explicit data transport configuration, object locality/stale-view + extraction, object locality/stale-view acceptance, owned transport shutdown, and the complete native/OCI S3 corpus. Current read fixtures use real storage reader/framing with injected metadata and disk seams; they are not the required real-service TCP/RDMA acceptance. +- Runtime data transport now accepts an explicit `RdmaConfig`, forwarded + through shared ChunkIO discovery before DiskIO connections open. Legacy + constructors keep environment defaults. Connection construction is split + out of the nearly 1,000-line ChunkIO client. +- Native GET now also traverses real TCP and shared-HCA RC RPC through the + production semantic DiskIO adapter, ChunkIO framing and S3 verification. + The RDMA case charges an 8-byte object to its actual 64 KiB receive class, + blocks the next delivery until final release and keeps the delivered lease + readable after all client/server owners are destroyed. Both profiles report + zero SDK compatibility-copy bytes and no read retries. `ServerRequest::reply` + retains the original connection/server via a safe owned-handler registration; + new native test code adds no unsafe exception. FFI/ChunkIO/native all-target + tests, clippy and formatting pass. These focused RPC tests still inject + metadata/layout and disk contents; the final corpus must use real services. ## Subsequent Dataset integration under R220 diff --git a/lib/crowdb-chunk-client/src/client.rs b/lib/crowdb-chunk-client/src/client.rs index 5bd8dfd1a..75d9fa331 100644 --- a/lib/crowdb-chunk-client/src/client.rs +++ b/lib/crowdb-chunk-client/src/client.rs @@ -9,6 +9,7 @@ use disk_metrics::MetricsDiskWriter; mod readiness; #[cfg(feature = "test-util")] pub use readiness::observe_readiness_phase_for_tests; +mod connection; mod observation; mod writer_policy; @@ -18,11 +19,9 @@ use std::time::{Duration, Instant}; use async_trait::async_trait; use bytes::Bytes; -use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; +use crowdb_chunkdb_client::ChunkdbClient; use crowdb_common::ec::EcScheme; -use crowdb_kv_client::{ - ClientConfig, CrowdbKvClient, HardwareClient, RangeBindingClient, ServiceRegistryClient, -}; +use crowdb_kv_client::{CrowdbKvClient, HardwareClient, RangeBindingClient, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{ AdvanceChunkWriteRequest, AdvanceChunkWriteResponse, AllocateChunkRequest, AllocateChunkResponse, AllocateReplacementSegmentRequest, AllocateReplacementSegmentResponse, AppendChunkRequest, @@ -111,75 +110,6 @@ impl ChunkIoClient { self.disk_writer.producer_allocator() } - /// Discover services and build lock-free DiskIO routing. - pub async fn connect(config: ChunkIoClientConfig) -> Result { - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( - config.management_seeds.clone(), - ))); - Self::connect_with_kv(config, kv).await - } - - /// Discover services using an existing KV topology client shared with the - /// embedding process. - pub async fn connect_with_kv(config: ChunkIoClientConfig, kv: Arc) -> Result { - Self::connect_with_kv_read_policy(config, kv, ChunkReadPolicy::default()).await - } - - /// Connect with explicit shared read resource limits. - pub async fn connect_with_kv_read_policy( - config: ChunkIoClientConfig, - kv: Arc, - read_policy: ChunkReadPolicy, - ) -> Result { - let service = ServiceRegistryClient::from_shared(kv.clone()); - let hardware = HardwareClient::from_shared(kv.clone()); - let range_binding = discover_current_range_bindings(&service, kv.clone()).await?; - let mut chunkdb = ChunkdbClient::new(service.clone(), Arc::new(ChunkdbRpcTransport::new())); - if let Some(range_binding) = range_binding { - chunkdb = chunkdb.with_range_binding(range_binding); - } - let chunkdb = Arc::new(chunkdb); - chunkdb.refresh_endpoints().await?; - let disk_writer = Arc::new( - RoutedDiskWriter::connect_with_connections_and_workers( - &service, - &hardware, - config.diskio_connections_per_endpoint, - config.diskio_rpc_workers, - ) - .await?, - ); - let failed_disks = Arc::new(FailedDiskList::new(config.small_write.failed_disk_ttl)); - let large_write_repair = Arc::new(crate::metrics::LargeWriteRepairMetrics::default()); - let large_write_buffer = Arc::new(crate::metrics::LargeWriteBufferMetrics::default()); - let small_pool = SmallWritePool::new( - chunkdb.clone(), - disk_writer.clone(), - config.small_write, - Arc::new(SmallWriteMetrics::default()), - Arc::clone(&failed_disks), - )?; - let reader = ChunkReader::new(chunkdb.clone(), disk_writer.clone(), read_policy) - .map_err(|error| crate::IoError::Internal(error.to_string()))?; - Ok(Self { - allocator: chunkdb.clone(), - disk_writer: disk_writer.clone(), - topology: Some(Arc::new(ClientTopology { - service, - hardware, - chunkdb, - disk_writer, - })), - metrics: None, - readiness: Arc::default(), - small_pool, - reader, - failed_disks, - large_write_repair, - large_write_buffer, - }) - } - /// Construct from low-level seams. Intended for focused tests and embedded fixtures. pub fn from_parts( allocator: Arc, diff --git a/lib/crowdb-chunk-client/src/client/connection.rs b/lib/crowdb-chunk-client/src/client/connection.rs new file mode 100644 index 000000000..94bf01829 --- /dev/null +++ b/lib/crowdb-chunk-client/src/client/connection.rs @@ -0,0 +1,98 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Shared discovery and explicit data-transport construction. + +use super::{discover_current_range_bindings, ClientTopology}; +use crate::metrics::SmallWriteMetrics; +use crate::negative_list::FailedDiskList; +use crate::writer::small_pool::SmallWritePool; +use crate::{ChunkIoClient, ChunkIoClientConfig, ChunkReadPolicy, ChunkReader, Result, RoutedDiskWriter}; +use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient, ServiceRegistryClient}; +use std::sync::Arc; + +impl ChunkIoClient { + /// Discover services and build lock-free DiskIO routing. + pub async fn connect(config: ChunkIoClientConfig) -> Result { + let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + config.management_seeds.clone(), + ))); + Self::connect_with_kv(config, kv).await + } + + /// Discover services using an existing KV topology client shared with the + /// embedding process. + pub async fn connect_with_kv(config: ChunkIoClientConfig, kv: Arc) -> Result { + Self::connect_with_kv_read_policy(config, kv, ChunkReadPolicy::default()).await + } + + /// Connect with explicit shared read resource limits. + pub async fn connect_with_kv_read_policy( + config: ChunkIoClientConfig, + kv: Arc, + read_policy: ChunkReadPolicy, + ) -> Result { + Self::connect_with_kv_transport(config, kv, read_policy, None).await + } + + /// Connect using explicit data transport settings; None retains environment defaults. + pub async fn connect_with_kv_transport( + config: ChunkIoClientConfig, + kv: Arc, + read_policy: ChunkReadPolicy, + rdma: Option, + ) -> Result { + let service = ServiceRegistryClient::from_shared(kv.clone()); + let hardware = HardwareClient::from_shared(kv.clone()); + let range_binding = discover_current_range_bindings(&service, kv.clone()).await?; + let mut chunkdb = ChunkdbClient::new(service.clone(), Arc::new(ChunkdbRpcTransport::new())); + if let Some(range_binding) = range_binding { + chunkdb = chunkdb.with_range_binding(range_binding); + } + let chunkdb = Arc::new(chunkdb); + chunkdb.refresh_endpoints().await?; + let disk_writer = Arc::new( + RoutedDiskWriter::connect_with_config( + &service, + &hardware, + crowdb_diskio_client::DiskioClientConfig { + normal_connections_per_endpoint: config.diskio_connections_per_endpoint, + rpc_workers: config.diskio_rpc_workers, + rdma, + ..crowdb_diskio_client::DiskioClientConfig::default() + }, + ) + .await?, + ); + let failed_disks = Arc::new(FailedDiskList::new(config.small_write.failed_disk_ttl)); + let large_write_repair = Arc::new(crate::metrics::LargeWriteRepairMetrics::default()); + let large_write_buffer = Arc::new(crate::metrics::LargeWriteBufferMetrics::default()); + let small_pool = SmallWritePool::new( + chunkdb.clone(), + disk_writer.clone(), + config.small_write, + Arc::new(SmallWriteMetrics::default()), + Arc::clone(&failed_disks), + )?; + let reader = ChunkReader::new(chunkdb.clone(), disk_writer.clone(), read_policy) + .map_err(|error| crate::IoError::Internal(error.to_string()))?; + Ok(Self { + allocator: chunkdb.clone(), + disk_writer: disk_writer.clone(), + topology: Some(Arc::new(ClientTopology { + service, + hardware, + chunkdb, + disk_writer, + })), + metrics: None, + readiness: Arc::default(), + small_pool, + reader, + failed_disks, + large_write_repair, + large_write_buffer, + }) + } +} diff --git a/lib/crowdb-chunk-client/src/disk_io/routing.rs b/lib/crowdb-chunk-client/src/disk_io/routing.rs index 67f0ee9f0..a9cdd81f2 100644 --- a/lib/crowdb-chunk-client/src/disk_io/routing.rs +++ b/lib/crowdb-chunk-client/src/disk_io/routing.rs @@ -62,6 +62,15 @@ impl RoutedDiskWriter { rpc_workers, ..DiskioClientConfig::default() }; + Self::connect_with_config(service, hardware, config).await + } + + /// Discover owners with typed transport, admission and connection settings. + pub async fn connect_with_config( + service: &ServiceRegistryClient, + hardware: &HardwareClient, + config: DiskioClientConfig, + ) -> Result { let client = DiskioClient::connect_with_clients(service.clone(), hardware.clone(), config) .await .map_err(map_topology_error)?; diff --git a/lib/crowdb-rpc/ffi/src/server.rs b/lib/crowdb-rpc/ffi/src/server.rs index 5856fb889..d095761ab 100644 --- a/lib/crowdb-rpc/ffi/src/server.rs +++ b/lib/crowdb-rpc/ffi/src/server.rs @@ -242,6 +242,22 @@ impl RpcServer { } } + /// Registers a handler whose requests retain their original server and can + /// use `ServerRequest::reply` without exposing raw connection lifetimes. + /// The registry keeps a weak server owner to avoid a reference cycle. + pub fn register_owned_handler(self: &Arc, msg_type: u16, handler: F) + where + F: Fn(ServerRequest) + Send + 'static, + { + let server = Arc::downgrade(self); + self.register_handler(msg_type, move |mut request| { + if let Some(server) = server.upgrade() { + request.response_server = Some(server); + handler(request); + } + }); + } + /// Submit a response on a server-side connection. Thread-safe — may /// be called from any thread (e.g. a Rust thread pool worker). /// conn_handle is the raw pointer passed to the dispatch callback. @@ -529,6 +545,8 @@ pub struct ServerRequest { pub rpc_create_nano: u64, pub msg_type: u16, pub conn_handle: *mut std::ffi::c_void, + response_connection: *mut std::ffi::c_void, + response_server: Option>, frame_handle: *mut std::ffi::c_void, control_ptr: *const u8, control_len: u32, @@ -537,6 +555,24 @@ pub struct ServerRequest { } impl ServerRequest { + /// Consumes a request from `register_owned_handler` and transfers immutable + /// response buffers to its retained original server/connection. + /// # Errors + /// Rejects legacy unowned handlers, a closed connection or a full send queue. + pub fn reply( + self, + control: crate::Buffer, + data: Option, + msg_type: u16, + ) -> Result<(), RpcError> { + let server = self.response_server.as_ref().ok_or(RpcError::InvalidArg)?; + // SAFETY: the private connection was captured by the original callback; + // this request retains its Frame connection owner and originating server. + unsafe { + server.submit_response_buffer(self.response_connection, control, data, msg_type, self.request_id) + } + } + /// Fail this logical connection when a response cannot be submitted. /// The native Frame retains the connection through this request's release. pub fn close_connection(&self) { @@ -613,6 +649,8 @@ unsafe extern "C" fn rust_handler_trampoline( rpc_create_nano, msg_type, conn_handle, + response_connection: conn_handle, + response_server: None, frame_handle, control_ptr: control, control_len, diff --git a/lib/crowdb-rpc/ffi/tests/ffi_handler_test.rs b/lib/crowdb-rpc/ffi/tests/ffi_handler_test.rs index 0c509de1f..4150e84f1 100644 --- a/lib/crowdb-rpc/ffi/tests/ffi_handler_test.rs +++ b/lib/crowdb-rpc/ffi/tests/ffi_handler_test.rs @@ -106,8 +106,7 @@ async fn custom_rust_handler_with_data() { let port = server.port(); assert!(port > 0); - let server_for_handler = server.clone(); - server.register_handler(HANDLER_MSG_TYPE, move |req| { + server.register_owned_handler(HANDLER_MSG_TYPE, move |req| { // Echo the request data back in the response. let mut fbb = FlatBufferBuilder::new(); let resp = ConnectionPingResponse::create( @@ -123,15 +122,12 @@ async fn custom_rust_handler_with_data() { let ctrl_bytes = fbb.finished_data().to_vec(); let data_bytes = req.data().map(|d| d.to_vec()); let resp_msg_type = FBMsgType::EConnectionPingResponse.0 as u16; - unsafe { - let _ = server_for_handler.submit_response( - req.conn_handle, - &ctrl_bytes, - data_bytes.as_deref(), - resp_msg_type, - req.request_id, - ); - } + req.reply( + crowdb_rpc_ffi::Buffer::from_vec(ctrl_bytes), + data_bytes.map(crowdb_rpc_ffi::Buffer::from_vec), + resp_msg_type, + ) + .unwrap(); }); server.start(); diff --git a/lib/crowdb-sdk-native/Cargo.toml b/lib/crowdb-sdk-native/Cargo.toml index f717d1d58..f4f528e80 100644 --- a/lib/crowdb-sdk-native/Cargo.toml +++ b/lib/crowdb-sdk-native/Cargo.toml @@ -33,6 +33,8 @@ tracing = { workspace = true } [dev-dependencies] async-trait = "0.1" +flatbuffers = { workspace = true } +crowdb-diskio-client = { path = "../crowdb-diskio-client", features = ["test-util"] } crowdb-sdk-native = { path = ".", features = ["test-util"] } crowdb-chunk-client = { path = "../crowdb-chunk-client", features = ["test-util"] } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi", features = ["test-util"] } diff --git a/lib/crowdb-sdk-native/src/runtime.rs b/lib/crowdb-sdk-native/src/runtime.rs index 39b8b7e7f..a407ba907 100644 --- a/lib/crowdb-sdk-native/src/runtime.rs +++ b/lib/crowdb-sdk-native/src/runtime.rs @@ -30,6 +30,8 @@ pub struct RuntimeConfig { pub small_write: SmallWritePolicy, pub read_policy: ChunkReadPolicy, pub chunk_kv_transport: ChunkKvRpcTransportConfig, + /// None retains service environment defaults; Some applies before data connections open. + pub data_rdma: Option, pub limits: RuntimeLimits, pub connect_timeout: Duration, } @@ -44,6 +46,7 @@ impl RuntimeConfig { small_write: SmallWritePolicy::new(ChunkType::S3), read_policy: ChunkReadPolicy::default(), chunk_kv_transport: ChunkKvRpcTransportConfig::default(), + data_rdma: None, limits: RuntimeLimits::default(), connect_timeout: Duration::from_secs(10), } diff --git a/lib/crowdb-sdk-native/src/runtime/storage.rs b/lib/crowdb-sdk-native/src/runtime/storage.rs index 2febb23fc..9e6012b26 100644 --- a/lib/crowdb-sdk-native/src/runtime/storage.rs +++ b/lib/crowdb-sdk-native/src/runtime/storage.rs @@ -55,7 +55,7 @@ impl Storage { .await .map_err(|error| Error::Storage(error.to_string()))?; let catalog_observed_at = Instant::now(); - let chunks = ChunkIoClient::connect_with_kv_read_policy( + let chunks = ChunkIoClient::connect_with_kv_transport( ChunkIoClientConfig { management_seeds: config.management_seeds, diskio_connections_per_endpoint: config.diskio_connections_per_endpoint, @@ -64,6 +64,7 @@ impl Storage { }, Arc::clone(&control), config.read_policy, + config.data_rdma, ) .await .map_err(|error| Error::Storage(error.to_string()))?; diff --git a/lib/crowdb-sdk-native/tests/common/read_storage.rs b/lib/crowdb-sdk-native/tests/common/read_storage.rs index f1269a259..741581a50 100644 --- a/lib/crowdb-sdk-native/tests/common/read_storage.rs +++ b/lib/crowdb-sdk-native/tests/common/read_storage.rs @@ -27,6 +27,25 @@ pub struct TestReadStorage { pub reads: AtomicUsize, } +impl TestReadStorage { + pub fn new() -> Self { + Self { + data: [b"original".as_slice(), b"replacement".as_slice()] + .into_iter() + .enumerate() + .map(|(index, payload)| { + let mut frame = Vec::with_capacity(4096); + frame.extend_from_slice( + &encode_frame(FrameMagic::RepoSmallV1, chunk_id(index), payload, 1).unwrap(), + ); + OwnedPayload::from(frame) + }) + .collect(), + reads: AtomicUsize::new(0), + } + } +} + fn chunk_id(index: usize) -> ChunkId { ChunkId { high: 1, @@ -122,21 +141,17 @@ pub struct TestFixture { } impl TestFixture { + #[allow(dead_code)] // Shared fixture also supports the RPC-only test target. pub async fn new(held_bytes: usize) -> Self { - let backend = Arc::new(TestReadStorage { - data: [b"original".as_slice(), b"replacement".as_slice()] - .into_iter() - .enumerate() - .map(|(index, payload)| { - let mut frame = Vec::with_capacity(4096); - frame.extend_from_slice( - &encode_frame(FrameMagic::RepoSmallV1, chunk_id(index), payload, 1).unwrap(), - ); - OwnedPayload::from(frame) - }) - .collect(), - reads: AtomicUsize::new(0), - }); + let backend = Arc::new(TestReadStorage::new()); + Self::with_disk(held_bytes, backend.clone(), backend).await + } + + pub async fn with_disk( + held_bytes: usize, + backend: Arc, + disk: Arc, + ) -> Self { let metadata = super::metadata::client().await; let runtime = ClientRuntime::from_clients( RuntimeClients { @@ -145,11 +160,7 @@ impl TestFixture { metadata_transport: Arc::new( ChunkKvRpcTransport::with_config(8, 1, 1, &ChunkKvRpcTransportConfig::default()).unwrap(), ), - chunks: Arc::new(ChunkIoClient::from_parts( - backend.clone(), - backend.clone(), - ChunkType::S3, - )), + chunks: Arc::new(ChunkIoClient::from_parts(backend.clone(), disk, ChunkType::S3)), }, RuntimeLimits { held_bytes, diff --git a/lib/crowdb-sdk-native/tests/common/rpc_read.rs b/lib/crowdb-sdk-native/tests/common/rpc_read.rs new file mode 100644 index 000000000..2ffdb3cd5 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/common/rpc_read.rs @@ -0,0 +1,122 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use crowdb_diskio_client::{DiskId, DiskioClient, DiskioClientConfig, TestDiskRoute}; +use crowdb_protocol::diskio_fb::{ + FBDiskIoRetCode, FBDiskReadRequest, FBDiskReadResponse, FBDiskReadResponseArgs, +}; +use crowdb_protocol::fb::FBMsgType; +use crowdb_rpc_ffi::{Buffer, RdmaConfig, RpcServer}; +use flatbuffers::FlatBufferBuilder; + +use super::storage::TestReadStorage; + +pub struct TestRpcReadServer { + pub server: Arc, + pub replies: Arc, + address: String, + config: RdmaConfig, +} + +impl TestRpcReadServer { + pub fn start(backend: &TestReadStorage, rdma: bool) -> Self { + let address = if rdma { + std::env::var("CROWDB_RDMA_E2E_LISTEN_ADDR").unwrap() + } else { + "127.0.0.1".into() + }; + let config = RdmaConfig { + enabled: rdma, + allow_tcp_fallback: !rdma, + listen_enabled: rdma, + source_address: if rdma { address.clone() } else { String::new() }, + device: if rdma { + std::env::var("CROWDB_RDMA_DEVICE").unwrap_or_default() + } else { + String::new() + }, + sq_depth: 4, + rq_depth: 4, + posted_receive_target: 4, + cq_depth: 16, + ..RdmaConfig::default() + } + .with_control_data_capacity(65_536) + .unwrap(); + let server = Arc::new(RpcServer::new(None)); + server.configure_rdma(&config).unwrap(); + server.listen(&address, 0).unwrap(); + let replies = Arc::new(AtomicUsize::new(0)); + let count = replies.clone(); + let data = backend.data[0].clone(); + server.register_owned_handler( + u16::try_from(FBMsgType::EDiskReadRequest.0).unwrap(), + move |request| { + let read = flatbuffers::root::>(request.control()).unwrap(); + assert_eq!(read.zone_offset(), 0); + assert_eq!(read.size() as usize, data.len()); + let mut builder = FlatBufferBuilder::new(); + let response = FBDiskReadResponse::create( + &mut builder, + &FBDiskReadResponseArgs { + id: request.request_id, + rpc_create_nano: 0, + ret_code: FBDiskIoRetCode::Success, + }, + ); + builder.finish(response, None); + request + .reply( + Buffer::from_bytes(builder.finished_data()), + Some(Buffer::from_vec(data.to_vec())), + u16::try_from(FBMsgType::EDiskReadResponse.0).unwrap(), + ) + .unwrap(); + count.fetch_add(1, Ordering::Release); + }, + ); + server.start(); + Self { + server, + replies, + address, + config, + } + } + + pub fn client(&self) -> Arc { + Arc::new( + DiskioClient::connect_for_tests( + vec![TestDiskRoute { + disk_id: DiskId::new(1, 1), + rack_id: 1, + node_id: 1, + disk_group_id: 1, + instance_id: 1, + endpoint: format!("{}:{}", self.address, self.server.port()), + }], + DiskioClientConfig { + rdma: Some(RdmaConfig { + listen_enabled: false, + ..self.config.clone() + }), + normal_connections_per_endpoint: 1, + priority_connections_per_endpoint: 1, + rpc_workers: 1, + retry_attempts: 1, + ..DiskioClientConfig::default() + }, + ) + .unwrap(), + ) + } +} + +impl Drop for TestRpcReadServer { + fn drop(&mut self) { + self.server.stop(); + } +} diff --git a/lib/crowdb-sdk-native/tests/s3_rpc_read_test.rs b/lib/crowdb-sdk-native/tests/s3_rpc_read_test.rs new file mode 100644 index 000000000..262e0d0a6 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/s3_rpc_read_test.rs @@ -0,0 +1,86 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/metadata.rs"] +mod metadata; +#[path = "common/rpc_read.rs"] +mod rpc; +#[path = "common/read_storage.rs"] +mod storage; + +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::Duration; + +use crowdb_chunk_client::RoutedDiskWriter; +use crowdb_sdk_native::s3::GetOptions; +use storage::{context, TestFixture, TestReadStorage}; + +async fn assert_rpc_delivery(rdma: bool) { + let backend = Arc::new(TestReadStorage::new()); + let server = rpc::TestRpcReadServer::start(&backend, rdma); + let disk = server.client(); + let fixture = TestFixture::with_disk( + 65_536, + backend, + Arc::new(RoutedDiskWriter::from_client(disk.clone())), + ) + .await; + fixture.publish(&fixture.record(0)).await; + let mut first = fixture + .client + .get_object(context(2), b"bucket", b"key", &GetOptions::default()) + .await + .unwrap(); + let held = first.body.next().await.unwrap().unwrap(); + assert_eq!(held.as_slice(), b"original"); + assert_eq!(held.metadata().registration_id != 0, rdma); + let capacity = held.metadata().allocation_capacity; + if rdma { + assert_eq!(capacity, 65_536); + } + assert!(first.body.next().await.unwrap().is_none()); + let mut second = fixture + .client + .get_object(context(3), b"bucket", b"key", &GetOptions::default()) + .await + .unwrap(); + if rdma { + assert!( + tokio::time::timeout(Duration::from_millis(10), second.body.next()) + .await + .is_err() + ); + assert_eq!(server.replies.load(Ordering::Acquire), 2); + } + drop(held); + let lease = second.body.next().await.unwrap().unwrap(); + assert!(second.body.next().await.unwrap().is_none()); + let resources = fixture.runtime.resources().clone(); + assert_eq!(resources.status().compatibility_copy_bytes, 0); + assert_eq!(resources.status().held_bytes, capacity); + assert_eq!(disk.status().read_operations, 2); + assert_eq!(disk.status().retries, 0); + fixture.runtime.shutdown().await.unwrap(); + drop(first); + drop(second); + drop(fixture); + drop(disk); + drop(server); + assert_eq!(lease.as_slice(), b"original"); + assert_eq!(resources.status().held_bytes, capacity); + drop(lease); + assert_eq!(resources.status().held_bytes, 0); +} + +#[tokio::test] +async fn tcp_rpc_get_preserves_receive_allocation_through_final_sdk_release() { + assert_rpc_delivery(false).await; +} + +#[tokio::test] +async fn hardware_rdma_get_preserves_registered_receive_capacity_and_shutdown_lifetime() { + if std::env::var_os("CROWDB_RDMA_E2E").is_some() { + assert_rpc_delivery(true).await; + } +} From 00397bd6fb4b1eaa34220d9f5a60d81c754a28cd Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 13:47:30 +0800 Subject: [PATCH 065/258] Expose bounded object locality and independent topology observations --- doc/working/plan-rdma-fat-client-sdk.md | 25 +- lib/crowdb-access-s3/src/retrieval.rs | 24 +- lib/crowdb-access-s3/tests/retrieval_test.rs | 18 ++ .../src/client/observation.rs | 12 +- .../src/disk_io/disk_writer.rs | 9 + .../src/disk_io/routing.rs | 8 + lib/crowdb-sdk-native/Cargo.toml | 2 +- lib/crowdb-sdk-native/src/error.rs | 25 ++ lib/crowdb-sdk-native/src/s3.rs | 5 + lib/crowdb-sdk-native/src/s3/locality.rs | 262 ++++++++++++++++++ .../src/s3/locality/placement.rs | 159 +++++++++++ lib/crowdb-sdk-native/src/topology.rs | 94 +++++-- .../src/topology/capabilities.rs | 122 ++++++++ .../tests/common/metadata.rs | 62 +++-- .../tests/common/read_storage.rs | 20 +- lib/crowdb-sdk-native/tests/common/storage.rs | 6 +- .../tests/common/topology.rs | 25 ++ .../tests/s3_locality_test.rs | 158 +++++++++++ .../tests/s3_rpc_read_test.rs | 30 +- lib/crowdb-sdk-native/tests/topology_test.rs | 69 +++++ 20 files changed, 1072 insertions(+), 63 deletions(-) create mode 100644 lib/crowdb-sdk-native/src/s3/locality.rs create mode 100644 lib/crowdb-sdk-native/src/s3/locality/placement.rs create mode 100644 lib/crowdb-sdk-native/src/topology/capabilities.rs create mode 100644 lib/crowdb-sdk-native/tests/common/topology.rs create mode 100644 lib/crowdb-sdk-native/tests/s3_locality_test.rs create mode 100644 lib/crowdb-sdk-native/tests/topology_test.rs diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index baf9bdc6f..387658d85 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -70,7 +70,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. ## Phase 2: supported topology and scheduling surface -- [ ] **Expose retained topology views**: provide inspection and explicit +- [x] **Expose retained topology views**: provide inspection and explicit refresh of service endpoints, rack/node/disk membership, Chunk-KV range owners, transport capabilities, and source generation/age. Reuse existing catalog and route owners; independently observed generations must not @@ -80,7 +80,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. `lib/crowdb-diskio-client/src/topology.rs`, `lib/crowdb-diskio-client/src/status.rs`, `lib/crowdb-diskio-client/src/semantic/native_routes.rs`. -- [ ] **Expose object locality**: resolve a logical object generation/range to +- [x] **Expose object locality**: resolve a logical object generation/range to chunk placement and route observations for caller scheduling/prefetch. Preserve the metadata-selected identity during execution instead of re-resolving a mutable key to a different generation. Observation cannot @@ -89,7 +89,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. `lib/crowdb-sdk-native/src/topology.rs`, `lib/crowdb-access-s3/src/metadata/record.rs`, `lib/crowdb-chunk-client/src/client.rs`. -- [ ] **Handle stale observations**: retain catalog invalidation, endpoint +- [~] **Handle stale observations**: retain catalog invalidation, endpoint refresh, ownership epochs, and generation fencing. Old views stay safe to inspect but cannot force execution past validation. Return typed stale/fenced/deadline outcomes when refresh fails; never replay ambiguous @@ -305,6 +305,25 @@ transport acceptance alone cannot establish SDK completion. new native test code adds no unsafe exception. FFI/ChunkIO/native all-target tests, clippy and formatting pass. These focused RPC tests still inject metadata/layout and disk contents; the final corpus must use real services. +- Retained topology now follows newer catalog generations installed by normal + operations, with unknown original observation age rather than a fabricated + timestamp. Explicit catalog-only refresh reports typed invalid/unavailable + failures and refuses generation regression. Endpoint, disk and catalog + sources remain independent. Typed transport observations expose readiness, + wire classes, TCP fallback policy and configured queue/registration bounds; + configured values do not pretend to be provider connection limits. +- `object_locality` inspects one tenant/authority-bound `ResolvedObject`, + range and bounded 1..128 extent page without fetching payload. Each chunk + observation retains its own modification revision and layout validity; + mirror/EC roles, unavailable fragments and observed disk/rack/node owners + remain explicit. Shared chunk references query once per page. Pagination + stays on the selected generation after name overwrite. Old views remain + inspectable, report typed stale outcomes at expiry and cannot force a deleted + layout into execution. General injected disk writers can expose actual + native route/transport observations; real TCP/RDMA cases verify these owners. + Three new catalog/locality cases pass; the native suite now has 27 cases. + Native/S3/ChunkIO all-target suites and clippy pass. Full service restart, + stale-catalog/endpoint fault acceptance remains part of the final corpus. ## Subsequent Dataset integration under R220 diff --git a/lib/crowdb-access-s3/src/retrieval.rs b/lib/crowdb-access-s3/src/retrieval.rs index c7f88d85b..4c76db2c4 100644 --- a/lib/crowdb-access-s3/src/retrieval.rs +++ b/lib/crowdb-access-s3/src/retrieval.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use std::time::{Duration, UNIX_EPOCH}; +use bincode::Options; use crowdb_chunk_client::{ChunkIoClient, ChunkReadStream, ReadError}; use crowdb_protocol::chunkdb::rpc::Location; use crowdb_rpc_ffi::OwnedPayload; @@ -207,8 +208,7 @@ fn build_get( let inner = if interval.start == interval.end { None } else { - let locations: Vec = - bincode::deserialize(&record.data_reference).map_err(|_| RetrievalError::DataReference)?; + let locations = object_locations(record)?; Some( client .read_range_stream(&locations, interval.start, interval.end) @@ -233,6 +233,26 @@ fn build_get( }) } +/// Decodes the selected generation's bounded physical references for reading +/// or scheduling. It never resolves the object name or reads payload storage. +/// # Errors +/// Rejects oversized or malformed references before constructing a stream. +pub fn object_locations(record: &ObjectRecord) -> Result, RetrievalError> { + const LIMIT: usize = crate::inspection::MAX_REFERENCE_BYTES; + if record.data_reference.len() > LIMIT { + return Err(RetrievalError::DataReference); + } + if record.data_reference.is_empty() && record.logical_length == 0 { + return Ok(Vec::new()); + } + bincode::DefaultOptions::new() + .with_fixint_encoding() + .with_limit(LIMIT as u64) + .reject_trailing_bytes() + .deserialize(&record.data_reference) + .map_err(|_| RetrievalError::DataReference) +} + fn check_conditions( record: &ObjectRecord, conditions: &(impl ReadCondition + ?Sized), diff --git a/lib/crowdb-access-s3/tests/retrieval_test.rs b/lib/crowdb-access-s3/tests/retrieval_test.rs index 57934acca..54b57db12 100644 --- a/lib/crowdb-access-s3/tests/retrieval_test.rs +++ b/lib/crowdb-access-s3/tests/retrieval_test.rs @@ -129,3 +129,21 @@ fn typed_ranges_reject_invalid_intervals_and_keep_condition_precedence() { .partial ); } + +#[test] +fn invalid_references_are_bounded_before_payload_access() { + let storage = backend(); + let client = storage.client(); + let mut record = object(1); + record.data_reference.push(0); + assert!(matches!( + prepare_get_range(&client, &record, None, &ObjectConditions::default()), + Err(RetrievalError::DataReference) + )); + record.data_reference = vec![0; crowdb_access_s3::inspection::MAX_REFERENCE_BYTES + 1]; + assert!(matches!( + prepare_get_range(&client, &record, None, &ObjectConditions::default()), + Err(RetrievalError::DataReference) + )); + assert_eq!(storage.queries.load(Ordering::Relaxed), 0); +} diff --git a/lib/crowdb-chunk-client/src/client/observation.rs b/lib/crowdb-chunk-client/src/client/observation.rs index 89aa94079..23f927e2a 100644 --- a/lib/crowdb-chunk-client/src/client/observation.rs +++ b/lib/crowdb-chunk-client/src/client/observation.rs @@ -7,19 +7,15 @@ impl ChunkIoClient { /// Retains existing disk routes without opening any new connection group. #[must_use] pub fn disk_topology(&self) -> Option { - self.topology - .as_ref() - .map(|topology| topology.disk_writer.topology()) + self.disk_writer.disk_topology() } /// Actual native transport configuration and counters for these routes. /// # Errors /// Returns unavailable topology for clients constructed from low-level seams. pub fn transport_status_json(&self) -> crate::Result> { - self.topology - .as_ref() - .ok_or_else(|| crate::IoError::Topology("client has no discovered production topology".into()))? - .disk_writer - .transport_status_json() + self.disk_writer + .transport_observation()? + .ok_or_else(|| crate::IoError::Topology("disk writer has no native transport observation".into())) } } diff --git a/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs b/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs index 43b46c0c5..6b16f6720 100644 --- a/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs +++ b/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs @@ -13,6 +13,15 @@ use crate::{IoError, Result}; /// Block-IO seam. Production routing is owned by `crowdb-diskio-client`. #[async_trait] pub trait DiskWriter: Send + Sync { + /// Retained route inspection never opens a connection or publishes placement. + fn disk_topology(&self) -> Option { + None + } + + fn transport_observation(&self) -> Result>> { + Ok(None) + } + /// Optional native storage for an owner-backed upstream producer. fn producer_allocator(&self) -> Option { None diff --git a/lib/crowdb-chunk-client/src/disk_io/routing.rs b/lib/crowdb-chunk-client/src/disk_io/routing.rs index a9cdd81f2..8e88fc0f5 100644 --- a/lib/crowdb-chunk-client/src/disk_io/routing.rs +++ b/lib/crowdb-chunk-client/src/disk_io/routing.rs @@ -128,6 +128,14 @@ impl RoutedDiskWriter { #[async_trait] impl DiskWriter for RoutedDiskWriter { + fn disk_topology(&self) -> Option { + Some(self.topology()) + } + + fn transport_observation(&self) -> Result>> { + self.transport_status_json().map(Some) + } + fn producer_allocator(&self) -> Option { Some(self.client.producer_allocator()) } diff --git a/lib/crowdb-sdk-native/Cargo.toml b/lib/crowdb-sdk-native/Cargo.toml index f4f528e80..a33d4be11 100644 --- a/lib/crowdb-sdk-native/Cargo.toml +++ b/lib/crowdb-sdk-native/Cargo.toml @@ -30,6 +30,7 @@ futures = "0.3" thiserror = { workspace = true } tokio = { workspace = true, features = ["time", "sync", "macros"] } tracing = { workspace = true } +serde_json = "1" [dev-dependencies] async-trait = "0.1" @@ -39,4 +40,3 @@ crowdb-sdk-native = { path = ".", features = ["test-util"] } crowdb-chunk-client = { path = "../crowdb-chunk-client", features = ["test-util"] } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi", features = ["test-util"] } tokio = { workspace = true, features = ["rt-multi-thread", "test-util"] } -serde_json = "1" diff --git a/lib/crowdb-sdk-native/src/error.rs b/lib/crowdb-sdk-native/src/error.rs index 43f70dac7..f01ae50f7 100644 --- a/lib/crowdb-sdk-native/src/error.rs +++ b/lib/crowdb-sdk-native/src/error.rs @@ -31,6 +31,25 @@ pub enum ReadFailureKind { FailedRange, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TopologySource { + Catalog, + ChunkLayout, + ChunkRoutes, + DiskRoutes, + Services, + MetadataTransport, + DataTransport, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TopologyFailureKind { + Unavailable, + Invalid, + Stale, + Fenced, +} + #[derive(Clone, Debug, thiserror::Error, Eq, PartialEq)] pub enum Error { #[error("invalid native client configuration: {0}")] @@ -57,6 +76,12 @@ pub enum Error { Length { expected: u64, delivered: u64 }, #[error("object read failed: {kind:?}: {detail}")] Read { kind: ReadFailureKind, detail: String }, + #[error("topology {origin:?} failed: {kind:?}: {detail}")] + Topology { + origin: TopologySource, + kind: TopologyFailureKind, + detail: String, + }, #[error("mutation outcome is uncertain: request={request_id} reason={reason:?}")] UncertainMutation { request_id: u64, diff --git a/lib/crowdb-sdk-native/src/s3.rs b/lib/crowdb-sdk-native/src/s3.rs index 546f36523..b4e796aff 100644 --- a/lib/crowdb-sdk-native/src/s3.rs +++ b/lib/crowdb-sdk-native/src/s3.rs @@ -13,6 +13,7 @@ use std::sync::Arc; mod body; mod config; +mod locality; mod object; mod read; @@ -20,6 +21,10 @@ pub use body::ObjectBody; pub use config::S3ClientConfig; pub use crowdb_access_s3::condition::{EtagSelection, ReadPreconditions}; pub use crowdb_access_s3::range::ByteRange; +pub use locality::{ + ChunkLayoutObservation, FragmentPlacement, FragmentRole, LocalityOptions, ObjectExtent, ObjectLocality, + StripPlacement, +}; pub use object::{ObjectMetadata, ResolvedObject}; pub use read::{GetObject, GetOptions}; diff --git a/lib/crowdb-sdk-native/src/s3/locality.rs b/lib/crowdb-sdk-native/src/s3/locality.rs new file mode 100644 index 000000000..a54e798d5 --- /dev/null +++ b/lib/crowdb-sdk-native/src/s3/locality.rs @@ -0,0 +1,262 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Bounded scheduling observations for one immutable object selection. + +mod placement; +pub use placement::{FragmentPlacement, FragmentRole, StripPlacement}; + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crowdb_access_s3::retrieval::object_locations; +use crowdb_access_s3::S3ErrorCode; +use crowdb_chunk_client::{ChunkAllocator, IoError}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, Location, QueryChunkRequest}; +use crowdb_protocol::common::ChunkId; + +use super::{ByteRange, ObjectMetadata, ResolvedObject, S3Client}; +use crate::error::{TopologyFailureKind, TopologySource}; +use crate::topology::TopologyView; +use crate::{Error, OperationContext, Result}; + +#[derive(Clone, Debug)] +pub struct LocalityOptions { + pub range: Option, + /// Index in the selected generation's logically ordered extent list. + pub start_index: usize, + pub limit: usize, +} + +impl Default for LocalityOptions { + fn default() -> Self { + Self { + range: None, + start_index: 0, + limit: 64, + } + } +} + +#[derive(Clone, Debug)] +pub struct ChunkLayoutObservation { + pub chunk_id: ChunkId, + pub revision: u64, + pub state: i32, + pub observed_at: Instant, + pub valid_until: Instant, + chunk: Arc, +} + +impl ChunkLayoutObservation { + #[must_use] + pub fn is_expired(&self) -> bool { + Instant::now() >= self.valid_until + } +} + +#[derive(Clone, Debug)] +pub struct ObjectExtent { + pub index: usize, + pub logical: ByteRange, + /// Includes complete integrity frames when a logical range is framed. + pub physical: ByteRange, + pub layout: Arc, + pub strips: Vec, +} + +pub struct ObjectLocality { + pub object: ObjectMetadata, + pub range: ByteRange, + pub extents: Vec, + pub next_index: Option, + pub topology: TopologyView, +} + +impl ObjectLocality { + /// Old observations remain readable after expiry; this checks only the + /// source layout validity, without inventing a global topology epoch. + /// # Errors + /// Returns a typed stale-layout outcome; execution must obtain a fresh layout. + pub fn check_layout_freshness(&self) -> Result<()> { + if self.extents.iter().any(|extent| extent.layout.is_expired()) { + Err(failure( + TopologyFailureKind::Stale, + "observed chunk layout expired", + )) + } else { + Ok(()) + } + } +} + +impl S3Client { + /// Resolves bounded placement observations without reading object payload. + /// These views cannot be supplied as execution routing or publish placement. + /// # Errors + /// Returns selection, range, admission, deadline or typed layout failures. + pub async fn object_locality( + &self, + context: OperationContext, + selected: &ResolvedObject, + options: &LocalityOptions, + ) -> Result { + let request = self.runtime.resources.admit(context)?; + request + .run(async { + self.validate_selection(selected)?; + let locations = normalized_locations(selected)?; + let range = locality_range(selected, options, locations.len())?; + let topology = self.runtime.topology()?; + let placements = placement::disk_index(&topology); + let (allocator, _) = self.execution.chunks().storage_parts(); + let mut layouts = HashMap::new(); + let mut extents = Vec::new(); + let mut next_index = None; + for (index, location) in locations.iter().enumerate().skip(options.start_index) { + let end = location.logical_offset + location.logical_length; + if end <= range.start || location.logical_offset >= range.end { + continue; + } + if extents.len() == options.limit { + next_index = Some(index); + break; + } + let logical = ByteRange { + start: range.start.max(location.logical_offset), + end: range.end.min(end), + }; + let physical = placement::physical_range(location, logical)?; + let id = location + .chunk_id + .ok_or_else(|| failure(TopologyFailureKind::Invalid, "extent has no chunk"))?; + let key = (id.high, id.low); + let layout = if let Some(layout) = layouts.get(&key) { + Arc::clone(layout) + } else { + let layout = query_layout(&*allocator, id).await?; + layouts.insert(key, Arc::clone(&layout)); + layout + }; + let strips = placement::strips(&layout.chunk, physical, &placements)?; + extents.push(ObjectExtent { + index, + logical, + physical, + layout, + strips, + }); + } + let view = ObjectLocality { + object: selected.metadata().clone(), + range, + extents, + next_index, + topology, + }; + view.check_layout_freshness()?; + Ok(view) + }) + .await + } +} + +fn locality_range( + selected: &ResolvedObject, + options: &LocalityOptions, + locations: usize, +) -> Result { + if !(1..=128).contains(&options.limit) || options.start_index > locations { + return Err(Error::S3(S3ErrorCode::InvalidRequest)); + } + let range = options.range.unwrap_or(ByteRange { + start: 0, + end: selected.record.logical_length, + }); + if range.start > range.end + || range.end > selected.record.logical_length + || (options.range.is_some() && range.start == range.end) + { + return Err(Error::S3(S3ErrorCode::InvalidRange)); + } + Ok(range) +} + +async fn query_layout( + allocator: &dyn ChunkAllocator, + chunk_id: ChunkId, +) -> Result> { + let observed_at = Instant::now(); + let response = allocator + .query_chunk(QueryChunkRequest { + chunk_id: Some(chunk_id), + }) + .await + .map_err(|error| { + let kind = match error { + IoError::ChunkNotFound(_) => TopologyFailureKind::Stale, + IoError::MetadataConflict(_) => TopologyFailureKind::Fenced, + _ => TopologyFailureKind::Unavailable, + }; + failure(kind, &error.to_string()) + })?; + let chunk = response + .chunk + .ok_or_else(|| failure(TopologyFailureKind::Stale, "chunk no longer exists"))?; + if chunk.id != Some(chunk_id) { + return Err(failure(TopologyFailureKind::Invalid, "chunk identity mismatch")); + } + if chunk.state == ChunkState::Deleted as i32 { + return Err(failure(TopologyFailureKind::Stale, "chunk was deleted")); + } + let valid_until = observed_at + .checked_add(Duration::from_millis(response.layout_validity_ms)) + .ok_or_else(|| failure(TopologyFailureKind::Invalid, "layout validity overflows"))?; + Ok(Arc::new(ChunkLayoutObservation { + chunk_id, + revision: chunk.modify_ts, + state: chunk.state, + observed_at, + valid_until, + chunk: Arc::new(chunk), + })) +} + +fn normalized_locations(selected: &ResolvedObject) -> Result> { + let mut locations = + object_locations(&selected.record).map_err(|_| Error::S3(S3ErrorCode::InternalError))?; + locations.retain(|location| location.length != 0 && location.logical_length != 0); + locations.sort_unstable_by_key(|location| location.logical_offset); + let mut cursor = 0u64; + for location in &locations { + if location.chunk_id.is_none() + || location.logical_offset != cursor + || location.logical_length > location.length + || location.offset.checked_add(location.length).is_none() + { + return Err(failure( + TopologyFailureKind::Invalid, + "object extent geometry is inconsistent", + )); + } + cursor = cursor + .checked_add(location.logical_length) + .ok_or_else(|| failure(TopologyFailureKind::Invalid, "logical extent overflows"))?; + } + if cursor != selected.record.logical_length { + return Err(failure( + TopologyFailureKind::Invalid, + "object extents are incomplete", + )); + } + Ok(locations) +} + +fn failure(kind: TopologyFailureKind, detail: &str) -> Error { + Error::Topology { + origin: TopologySource::ChunkLayout, + kind, + detail: detail.into(), + } +} diff --git a/lib/crowdb-sdk-native/src/s3/locality/placement.rs b/lib/crowdb-sdk-native/src/s3/locality/placement.rs new file mode 100644 index 000000000..6f4570807 --- /dev/null +++ b/lib/crowdb-sdk-native/src/s3/locality/placement.rs @@ -0,0 +1,159 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::collections::HashMap; + +use crowdb_diskio_client::{DiskId, DiskPlacement}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkStrip, Location, Strip}; +use crowdb_protocol::diskdb::rpc::Segment; +use crowdb_protocol::frame::ChunkLocation; + +use super::failure; +use crate::error::TopologyFailureKind; +use crate::s3::ByteRange; +use crate::topology::TopologyView; +use crate::Result; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FragmentRole { + MirrorReplica(usize), + DataShard(usize), + ParityShard(usize), +} + +#[derive(Clone, Debug)] +pub struct FragmentPlacement { + pub role: FragmentRole, + pub segment: Segment, + pub unavailable: bool, + /// None means no disk route was observed, without guessing locality. + pub disk: Option, +} + +#[derive(Clone, Debug)] +pub struct StripPlacement { + pub sequence: u32, + pub physical: ByteRange, + pub unit_bytes: u64, + pub fragments: Vec, +} + +pub(super) fn disk_index(topology: &TopologyView) -> HashMap { + topology + .disks + .as_ref() + .map(|view| view.disks().map(|disk| (disk.disk_id, disk)).collect()) + .unwrap_or_default() +} + +pub(super) fn physical_range(location: &Location, logical: ByteRange) -> Result { + let start = logical.start - location.logical_offset; + let end = logical.end - location.logical_offset; + if location.length == location.logical_length { + return Ok(ByteRange { + start: location.offset + start, + end: location.offset + end, + }); + } + let framed = ChunkLocation { + chunk_id: location + .chunk_id + .ok_or_else(|| failure(TopologyFailureKind::Invalid, "extent has no chunk"))?, + frame_offset: location.offset, + logical_length: location.logical_length, + }; + let range = framed + .physical_range_for_subrange(start..end) + .map_err(|_| failure(TopologyFailureKind::Invalid, "framed extent range is invalid"))?; + if range.end > location.offset + location.length { + return Err(failure( + TopologyFailureKind::Invalid, + "framing exceeds extent length", + )); + } + Ok(ByteRange { + start: range.start, + end: range.end, + }) +} + +pub(super) fn strips( + chunk: &Chunk, + physical: ByteRange, + disks: &HashMap, +) -> Result> { + let mut result = Vec::new(); + let mut cursor = physical.start; + for strip in &chunk.strips { + let start = u64::from(strip.chunk_offset) * 1024; + let end = start + u64::from(strip.capacity) * 1024; + if end <= cursor || start >= physical.end { + continue; + } + if start > cursor || strip.unit_kb == 0 { + return Err(failure( + TopologyFailureKind::Invalid, + "strip geometry has a gap or zero unit", + )); + } + let selected = ByteRange { + start: cursor, + end: end.min(physical.end), + }; + let fragments = fragments(strip, disks)?; + result.push(StripPlacement { + sequence: strip.strip_sequence, + physical: selected, + unit_bytes: u64::from(strip.unit_kb) * 1024, + fragments, + }); + cursor = selected.end; + if cursor == physical.end { + break; + } + } + if cursor != physical.end { + return Err(failure( + TopologyFailureKind::Invalid, + "strip geometry is incomplete", + )); + } + Ok(result) +} + +fn fragments(strip: &ChunkStrip, disks: &HashMap) -> Result> { + let (segments, data_shards) = match &strip.strip { + Some(Strip::MirrorStrip(mirror)) => (&mirror.segments, None), + Some(Strip::EcStrip(ec)) => { + if ec.data_num == 0 || ec.segments.len() != ec.data_num as usize + ec.code_num as usize { + return Err(failure( + TopologyFailureKind::Invalid, + "EC geometry is inconsistent", + )); + } + (&ec.segments, Some(ec.data_num as usize)) + } + None => return Err(failure(TopologyFailureKind::Invalid, "strip has no fragments")), + }; + if segments.is_empty() { + return Err(failure(TopologyFailureKind::Invalid, "strip has no fragments")); + } + Ok(segments + .iter() + .enumerate() + .map(|(index, segment)| FragmentPlacement { + role: data_shards.map_or(FragmentRole::MirrorReplica(index), |data| { + if index < data { + FragmentRole::DataShard(index) + } else { + FragmentRole::ParityShard(index - data) + } + }), + segment: *segment, + unavailable: strip.unavailable_segments.contains(segment), + disk: segment + .disk_id + .and_then(|disk| disks.get(&DiskId::new(disk.high, disk.low)).cloned()), + }) + .collect()) +} diff --git a/lib/crowdb-sdk-native/src/topology.rs b/lib/crowdb-sdk-native/src/topology.rs index 9468c9994..bc1625bb0 100644 --- a/lib/crowdb-sdk-native/src/topology.rs +++ b/lib/crowdb-sdk-native/src/topology.rs @@ -3,6 +3,9 @@ //! Retained independent routing observations for caller scheduling. +mod capabilities; +pub use capabilities::{RdmaCapabilities, TransportCapabilities, TransportGroup}; + use std::sync::Arc; use std::time::{Duration, Instant}; @@ -11,6 +14,7 @@ use crowdb_diskio_client::DiskTopologyView; use crowdb_kv_client::ServiceRegistryClient; use crowdb_protocol::common::InstanceValue; +use crate::error::{TopologyFailureKind, TopologySource}; use crate::{ClientRuntime, Error, OperationContext, Result}; pub(crate) struct CatalogObservation { @@ -34,6 +38,7 @@ pub struct TopologyView { pub services: Arc>, pub metadata_transport_json: Vec, pub data_transport_json: Option>, + pub transports: Vec, } impl TopologyView { @@ -60,24 +65,40 @@ impl ClientRuntime { self.storage .chunks .transport_status_json() - .map_err(|error| Error::Storage(error.to_string()))?, + .map_err(|error| unavailable(TopologySource::DataTransport, &error))?, ) } else { None }; + let metadata_transport_json = self + .storage + .metadata_transport + .transport_status_json() + .map_err(|error| unavailable(TopologySource::MetadataTransport, &error))?; + let transports = capabilities::observe(&metadata_transport_json, data_transport_json.as_deref())?; Ok(TopologyView { - catalog: self.storage.catalog.load_full(), + catalog: self.catalog_observation(), disks, services: self.storage.services.load_full(), - metadata_transport_json: self - .storage - .metadata_transport - .transport_status_json() - .map_err(|error| Error::Storage(error.to_string()))?, + metadata_transport_json, data_transport_json, + transports, }) } + /// Refresh only the catalog authority, without dialing data services. + /// # Errors + /// Returns admission, cancellation/deadline or a typed catalog failure. + pub async fn refresh_catalog(&self, context: OperationContext) -> Result { + let request = self.resources.admit(context)?; + request + .run(async { + self.refresh_catalog_observation().await?; + self.topology() + }) + .await + } + /// Refreshes each authority independently; old views remain safe to inspect. /// This never uses a caller-supplied view to override current placement. /// # Errors @@ -86,30 +107,24 @@ impl ClientRuntime { let request = self.resources.admit(context)?; request .run(async { - let catalog = self - .storage - .metadata - .refresh_catalog() - .await - .map_err(|error| Error::Storage(error.to_string()))?; - self.record_catalog(&catalog); + self.refresh_catalog_observation().await?; self.storage .chunks .refresh_chunkdb_routes() .await - .map_err(|error| Error::Storage(error.to_string()))?; + .map_err(|error| unavailable(TopologySource::ChunkRoutes, &error))?; self.storage .chunks .refresh_diskio_routes() .await - .map_err(|error| Error::Storage(error.to_string()))?; + .map_err(|error| unavailable(TopologySource::DiskRoutes, &error))?; let registry = ServiceRegistryClient::from_shared(Arc::clone(&self.storage.control)); let mut services = Vec::with_capacity(3); for service in ["chunkdb", "chunk-kv", "diskio"] { let instances = registry .read_all_instances(service) .await - .map_err(|error| Error::Storage(error.to_string()))?; + .map_err(|error| unavailable(TopologySource::Services, &error))?; services.push(ServiceObservation { service, observed_at: Instant::now(), @@ -122,15 +137,48 @@ impl ClientRuntime { .await } - fn record_catalog(&self, catalog: &Arc) { + async fn refresh_catalog_observation(&self) -> Result<()> { + let catalog = self.storage.metadata.refresh_catalog().await.map_err(|error| { + if error == crowdb_chunk_kv_client::ClientError::Deadline { + return Error::Deadline; + } + Error::Topology { + origin: TopologySource::Catalog, + kind: if matches!(error, crowdb_chunk_kv_client::ClientError::InvalidCatalog(_)) { + TopologyFailureKind::Invalid + } else { + TopologyFailureKind::Unavailable + }, + detail: error.to_string(), + } + })?; + self.record_catalog(&catalog, Some(Instant::now())); + Ok(()) + } + + fn catalog_observation(&self) -> Arc { + if let Some(catalog) = self.storage.metadata.cached_catalog() { + if catalog.generation() > self.storage.catalog.load().catalog.generation() { + // A different client operation refreshed the routing cache. Its + // original observation time is unknown to this runtime facade. + self.record_catalog(&catalog, None); + } + } + self.storage.catalog.load_full() + } + + fn record_catalog(&self, catalog: &Arc, observed_at: Option) { loop { let observed = self.storage.catalog.load_full(); if catalog.generation() < observed.catalog.generation() { return; } + if catalog.generation() == observed.catalog.generation() && observed_at.is_none() { + return; + } let next = Arc::new(CatalogObservation { catalog: Arc::clone(catalog), - observed_at: Some(Instant::now()), + observed_at, }); if Arc::ptr_eq(&self.storage.catalog.compare_and_swap(&observed, next), &observed) { return; @@ -138,3 +186,11 @@ impl ClientRuntime { } } } + +pub(crate) fn unavailable(origin: TopologySource, error: &impl std::fmt::Display) -> Error { + Error::Topology { + origin, + kind: TopologyFailureKind::Unavailable, + detail: error.to_string(), + } +} diff --git a/lib/crowdb-sdk-native/src/topology/capabilities.rs b/lib/crowdb-sdk-native/src/topology/capabilities.rs new file mode 100644 index 000000000..a53ccb488 --- /dev/null +++ b/lib/crowdb-sdk-native/src/topology/capabilities.rs @@ -0,0 +1,122 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::error::{TopologyFailureKind, TopologySource}; +use crate::{Error, Result}; +use crowdb_rpc_ffi::RdmaWireLayout; +use serde_json::Value; +use std::time::Instant; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TransportGroup { + Metadata, + Aggregate, + Data, +} + +#[derive(Clone, Debug)] +pub struct RdmaCapabilities { + pub ready: bool, + pub layout: RdmaWireLayout, + pub data_slot_capacity: u32, + pub control_slot_capacity: u32, + pub max_control_message_bytes: u32, + /// Configured SQ bound; each provider connection reports its actual bound separately. + pub configured_sq_depth: u32, + pub configured_rq_depth: u32, + pub registered_memory_budget_bytes: u64, +} + +#[derive(Clone, Debug)] +pub struct TransportCapabilities { + pub group: TransportGroup, + pub observed_at: Instant, + pub tcp_fallback_allowed: bool, + pub rdma: Option, +} + +pub(super) fn observe(metadata: &[u8], data: Option<&[u8]>) -> Result> { + let metadata = parse(metadata, TopologySource::MetadataTransport)?; + let mut result = Vec::with_capacity(3); + for (name, group) in [ + ("metadata", TransportGroup::Metadata), + ("aggregate", TransportGroup::Aggregate), + ] { + result.push(native( + &metadata[name]["transport"], + group, + TopologySource::MetadataTransport, + )?); + } + if let Some(data) = data { + let data = parse(data, TopologySource::DataTransport)?; + result.push(native( + &data, + TransportGroup::Data, + TopologySource::DataTransport, + )?); + } + Ok(result) +} + +fn parse(bytes: &[u8], origin: TopologySource) -> Result { + let value: Value = serde_json::from_slice(bytes).map_err(|error| invalid(origin, &error.to_string()))?; + if value["schema_version"].as_u64() != Some(1) { + return Err(invalid(origin, "unsupported transport status schema")); + } + Ok(value) +} + +fn native(value: &Value, group: TransportGroup, origin: TopologySource) -> Result { + if value["schema_version"].as_u64() != Some(1) { + return Err(invalid(origin, "unsupported native status schema")); + } + let tcp_fallback_allowed = value["allow_tcp_fallback"] + .as_bool() + .ok_or_else(|| invalid(origin, "missing TCP policy"))?; + let rdma = if value["rdma"].is_null() { + None + } else { + let rdma = &value["rdma"]; + let layout = match rdma["wire_layout"].as_str() { + Some("control_data") => RdmaWireLayout::ControlData, + Some("contiguous") => RdmaWireLayout::Contiguous, + _ => return Err(invalid(origin, "unknown RDMA layout")), + }; + Some(RdmaCapabilities { + ready: rdma["ready"] + .as_bool() + .ok_or_else(|| invalid(origin, "missing RDMA readiness"))?, + layout, + data_slot_capacity: number(rdma, "data_slot_capacity", origin)?, + control_slot_capacity: number(rdma, "control_slot_capacity", origin)?, + max_control_message_bytes: number(rdma, "max_control_message_bytes", origin)?, + configured_sq_depth: number(rdma, "sq_depth", origin)?, + configured_rq_depth: number(rdma, "rq_depth", origin)?, + registered_memory_budget_bytes: rdma["registered_memory_budget_bytes"] + .as_u64() + .ok_or_else(|| invalid(origin, "missing registration budget"))?, + }) + }; + Ok(TransportCapabilities { + group, + observed_at: Instant::now(), + tcp_fallback_allowed, + rdma, + }) +} + +fn number(value: &Value, field: &str, origin: TopologySource) -> Result { + value[field] + .as_u64() + .and_then(|number| u32::try_from(number).ok()) + .ok_or_else(|| invalid(origin, field)) +} + +fn invalid(origin: TopologySource, detail: &str) -> Error { + Error::Topology { + origin, + kind: TopologyFailureKind::Invalid, + detail: detail.into(), + } +} diff --git a/lib/crowdb-sdk-native/tests/common/metadata.rs b/lib/crowdb-sdk-native/tests/common/metadata.rs index b929f1171..7fcbfd8b2 100644 --- a/lib/crowdb-sdk-native/tests/common/metadata.rs +++ b/lib/crowdb-sdk-native/tests/common/metadata.rs @@ -132,26 +132,16 @@ fn scan(request: &ScanRequest, values: &BTreeMap, RpcValue>) -> Operatio } pub async fn client() -> Arc { - let (sender, mut receiver) = mpsc::channel(64); - tokio::spawn(async move { - let mut values = BTreeMap::new(); - let mut revision = 0; - while let Some(command) = receiver.recv().await { - let (result, response) = match command { - TestCommand::Point(operation, response) => { - (point(operation, &mut values, &mut revision), response) - } - TestCommand::Scan(request, response) => (scan(&request, &values), response), - }; - let _ = response.send(ChunkKvResponse { - map_revision: 1, - journal_position: None, - result: Ok(result), - }); - } - }); + let (head, pages) = catalog(1, 1); + client_with_source(Arc::new(TestCatalog(head, pages))).await +} + +pub fn catalog( + generation: u64, + owner_instance: u64, +) -> (ChunkKvRangeCatalogHead, Vec) { let mut page = ChunkKvRangeCatalogPage { - generation: 1, + generation, page_index: 0, entries: vec![ChunkKvRangeCatalogEntry { partition_id: Id128 { high: 1, low: 1 }, @@ -160,10 +150,10 @@ pub async fn client() -> Arc { end: None, }, owner: OwnerDescriptor { - instance_id: 1, - rpc_endpoint: "test-owner".into(), + instance_id: owner_instance, + rpc_endpoint: format!("test-owner-{owner_instance}"), }, - owner_epoch: 1, + owner_epoch: generation, state: ChunkKvRangeCatalogPartitionState::Serving, artifact: PartitionArtifact { tree_id: 1, @@ -176,10 +166,10 @@ pub async fn client() -> Arc { }; page.seal().unwrap(); let mut head = ChunkKvRangeCatalogHead { - generation: 1, + generation, previous_generation: None, pages: vec![ChunkKvRangeCatalogPageRef { - page_generation: 1, + page_generation: generation, page_index: 0, first_key: Vec::new(), page_checksum: page.checksum, @@ -187,10 +177,32 @@ pub async fn client() -> Arc { checksum: [0; 32], }; head.seal().unwrap(); + (head, vec![page]) +} + +pub async fn client_with_source(source: Arc) -> Arc { + let (sender, mut receiver) = mpsc::channel(64); + tokio::spawn(async move { + let mut values = BTreeMap::new(); + let mut revision = 0; + while let Some(command) = receiver.recv().await { + let (result, response) = match command { + TestCommand::Point(operation, response) => { + (point(operation, &mut values, &mut revision), response) + } + TestCommand::Scan(request, response) => (scan(&request, &values), response), + }; + let _ = response.send(ChunkKvResponse { + map_revision: 1, + journal_position: None, + result: Ok(result), + }); + } + }); let client = Arc::new( ChunkKvClient::new( ClientConfig::default(), - Arc::new(TestCatalog(head, vec![page])), + source, Arc::new(TestMetadataTransport(sender)), ) .unwrap(), diff --git a/lib/crowdb-sdk-native/tests/common/read_storage.rs b/lib/crowdb-sdk-native/tests/common/read_storage.rs index 741581a50..deb996b7c 100644 --- a/lib/crowdb-sdk-native/tests/common/read_storage.rs +++ b/lib/crowdb-sdk-native/tests/common/read_storage.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use std::num::NonZeroU64; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -25,6 +25,10 @@ use crowdb_sdk_native::{ClientRuntime, OperationContext, RuntimeLimits}; pub struct TestReadStorage { pub data: Vec, pub reads: AtomicUsize, + pub queries: AtomicUsize, + pub validity_ms: AtomicU64, + pub revision: AtomicU64, + pub deleted: AtomicBool, } impl TestReadStorage { @@ -42,6 +46,10 @@ impl TestReadStorage { }) .collect(), reads: AtomicUsize::new(0), + queries: AtomicUsize::new(0), + validity_ms: AtomicU64::new(60_000), + revision: AtomicU64::new(1), + deleted: AtomicBool::new(false), } } } @@ -71,11 +79,17 @@ impl ChunkAllocator for TestReadStorage { panic!("read updated") } async fn query_chunk(&self, request: QueryChunkRequest) -> Result { + self.queries.fetch_add(1, Ordering::Relaxed); let id = request.chunk_id.unwrap(); Ok(QueryChunkResponse { chunk: Some(Chunk { id: Some(id), - state: ChunkState::Sealed as i32, + state: if self.deleted.load(Ordering::Acquire) { + ChunkState::Deleted as i32 + } else { + ChunkState::Sealed as i32 + }, + modify_ts: self.revision.load(Ordering::Acquire), capacity: 1, sealed_length: 1, strips: vec![ChunkStrip { @@ -97,7 +111,7 @@ impl ChunkAllocator for TestReadStorage { }], ..Chunk::default() }), - layout_validity_ms: 60_000, + layout_validity_ms: self.validity_ms.load(Ordering::Acquire), }) } } diff --git a/lib/crowdb-sdk-native/tests/common/storage.rs b/lib/crowdb-sdk-native/tests/common/storage.rs index 3a01d7063..6400e7dfe 100644 --- a/lib/crowdb-sdk-native/tests/common/storage.rs +++ b/lib/crowdb-sdk-native/tests/common/storage.rs @@ -47,8 +47,12 @@ impl DiskWriter for TestNoPayload { } } +#[allow(dead_code)] // Shared fixture has a custom catalog entry point in topology tests. pub async fn runtime() -> ClientRuntime { - let metadata = super::metadata::client().await; + runtime_with_metadata(super::metadata::client().await) +} + +pub fn runtime_with_metadata(metadata: Arc) -> ClientRuntime { let metadata_transport = Arc::new(ChunkKvRpcTransport::with_config(8, 1, 1, &ChunkKvRpcTransportConfig::default()).unwrap()); ClientRuntime::from_clients( diff --git a/lib/crowdb-sdk-native/tests/common/topology.rs b/lib/crowdb-sdk-native/tests/common/topology.rs new file mode 100644 index 000000000..e5a25c4dd --- /dev/null +++ b/lib/crowdb-sdk-native/tests/common/topology.rs @@ -0,0 +1,25 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use crowdb_chunk_kv_client::{ChunkKvRangeCatalogSource, ClientError, Result}; +use crowdb_protocol::chunk_kv::{ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + +pub struct TestCatalogSource { + pub generation: AtomicU64, + pub unavailable: AtomicBool, +} + +#[async_trait] +impl ChunkKvRangeCatalogSource for TestCatalogSource { + async fn load(&self) -> Result<(ChunkKvRangeCatalogHead, Vec)> { + if self.unavailable.load(Ordering::Acquire) { + return Err(ClientError::CatalogUnavailable( + "test authority unavailable".into(), + )); + } + let generation = self.generation.load(Ordering::Acquire); + Ok(super::metadata::catalog(generation, generation)) + } +} diff --git a/lib/crowdb-sdk-native/tests/s3_locality_test.rs b/lib/crowdb-sdk-native/tests/s3_locality_test.rs new file mode 100644 index 000000000..bab53eead --- /dev/null +++ b/lib/crowdb-sdk-native/tests/s3_locality_test.rs @@ -0,0 +1,158 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/metadata.rs"] +mod metadata; +#[path = "common/read_storage.rs"] +mod storage; + +use crowdb_sdk_native::error::{TopologyFailureKind, TopologySource}; +use crowdb_sdk_native::s3::{ByteRange, FragmentRole, GetOptions, LocalityOptions}; +use crowdb_sdk_native::Error; +use std::sync::atomic::Ordering; +use std::time::Duration; +use storage::{context, TestFixture}; + +#[tokio::test] +async fn locality_is_generation_bound_bounded_and_keeps_layout_revisions_independent() { + let fixture = TestFixture::new(8192).await; + let mut record = fixture.record(0); + let first: Vec = + bincode::deserialize(&record.data_reference).unwrap(); + let mut second = first[0].clone(); + second.logical_offset = 8; + record.logical_length = 16; + record.data_length = 16; + let mut integrity = crowdb_access_s3::integrity::SinglePartIntegrity::default(); + integrity.update(&bytes::Bytes::from_static(b"originaloriginal")); + (record.etag, record.checksum) = integrity.finish(); + record.data_reference = bincode::serialize(&vec![first[0].clone(), second]).unwrap(); + let object_revision = fixture.publish(&record).await; + let selected = fixture + .client + .resolve_object(context(2), b"bucket", b"key") + .await + .unwrap(); + for options in [ + LocalityOptions { + limit: 0, + ..LocalityOptions::default() + }, + LocalityOptions { + range: Some(ByteRange { start: 0, end: 17 }), + ..LocalityOptions::default() + }, + ] { + assert!(matches!( + fixture + .client + .object_locality(context(20), &selected, &options) + .await, + Err(Error::S3(_)) + )); + } + assert_eq!(fixture.backend.queries.load(Ordering::Relaxed), 0); + fixture.backend.revision.store(7, Ordering::Release); + let options = LocalityOptions { + range: Some(ByteRange { start: 2, end: 12 }), + limit: 1, + ..LocalityOptions::default() + }; + let first = fixture + .client + .object_locality(context(3), &selected, &options) + .await + .unwrap(); + assert_eq!(first.object.revision, object_revision); + assert_eq!(first.extents.len(), 1); + assert_eq!(first.extents[0].logical, ByteRange { start: 2, end: 8 }); + assert_eq!(first.extents[0].layout.revision, 7); + assert_eq!( + first.extents[0].strips[0].fragments[0].role, + FragmentRole::MirrorReplica(0) + ); + assert!(first.extents[0].strips[0].fragments[0].disk.is_none()); + assert_eq!(first.next_index, Some(1)); + assert_eq!(first.topology.catalog().generation(), 1); + fixture.publish(&fixture.record(1)).await; + fixture.backend.revision.store(8, Ordering::Release); + let next = fixture + .client + .object_locality( + context(4), + &selected, + &LocalityOptions { + start_index: first.next_index.unwrap(), + ..options + }, + ) + .await + .unwrap(); + assert_eq!(next.object.revision, object_revision); + assert_eq!(next.extents[0].logical, ByteRange { start: 8, end: 12 }); + assert_eq!(next.extents[0].layout.revision, 8); + assert_eq!(first.extents[0].layout.revision, 7); + assert_eq!(next.next_index, None); + let before = fixture.backend.queries.load(Ordering::Relaxed); + let together = fixture + .client + .object_locality(context(5), &selected, &LocalityOptions::default()) + .await + .unwrap(); + assert_eq!(together.extents.len(), 2); + assert!(std::sync::Arc::ptr_eq( + &together.extents[0].layout, + &together.extents[1].layout + )); + assert_eq!(fixture.backend.queries.load(Ordering::Relaxed), before + 1); + assert_eq!(fixture.backend.reads.load(Ordering::Relaxed), 0); + fixture.runtime.shutdown().await.unwrap(); + assert_eq!(first.extents[0].layout.revision, 7); +} + +#[tokio::test] +async fn expired_views_remain_inspectable_and_cannot_force_a_deleted_layout_into_execution() { + let fixture = TestFixture::new(4096).await; + fixture.publish(&fixture.record(0)).await; + let selected = fixture + .client + .resolve_object(context(2), b"bucket", b"key") + .await + .unwrap(); + fixture.backend.validity_ms.store(100, Ordering::Release); + let old = fixture + .client + .object_locality(context(3), &selected, &LocalityOptions::default()) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(110)).await; + assert!(matches!( + old.check_layout_freshness(), + Err(Error::Topology { + origin: TopologySource::ChunkLayout, + kind: TopologyFailureKind::Stale, + .. + }) + )); + fixture.backend.deleted.store(true, Ordering::Release); + assert!(matches!( + fixture + .client + .object_locality(context(4), &selected, &LocalityOptions::default()) + .await, + Err(Error::Topology { + kind: TopologyFailureKind::Stale, + .. + }) + )); + let mut read = fixture + .client + .get_resolved(context(5), &selected, &GetOptions::default()) + .unwrap(); + assert!(matches!(read.body.next().await, Err(Error::Read { .. }))); + assert_eq!(fixture.backend.reads.load(Ordering::Relaxed), 0); + assert_eq!( + old.extents[0].layout.state, + crowdb_protocol::chunkdb::rpc::ChunkState::Sealed as i32 + ); +} diff --git a/lib/crowdb-sdk-native/tests/s3_rpc_read_test.rs b/lib/crowdb-sdk-native/tests/s3_rpc_read_test.rs index 262e0d0a6..a72b63a10 100644 --- a/lib/crowdb-sdk-native/tests/s3_rpc_read_test.rs +++ b/lib/crowdb-sdk-native/tests/s3_rpc_read_test.rs @@ -13,7 +13,7 @@ use std::sync::Arc; use std::time::Duration; use crowdb_chunk_client::RoutedDiskWriter; -use crowdb_sdk_native::s3::GetOptions; +use crowdb_sdk_native::s3::{GetOptions, LocalityOptions}; use storage::{context, TestFixture, TestReadStorage}; async fn assert_rpc_delivery(rdma: bool) { @@ -27,6 +27,34 @@ async fn assert_rpc_delivery(rdma: bool) { ) .await; fixture.publish(&fixture.record(0)).await; + let selected = fixture + .client + .resolve_object(context(101), b"bucket", b"key") + .await + .unwrap(); + let locality = fixture + .client + .object_locality(context(102), &selected, &LocalityOptions::default()) + .await + .unwrap(); + assert!(locality.topology.disks.is_some()); + assert!(locality.topology.data_transport_json.is_some()); + let capability = locality + .topology + .transports + .iter() + .find(|group| group.group == crowdb_sdk_native::topology::TransportGroup::Data) + .unwrap(); + assert_eq!(capability.rdma.is_some(), rdma); + if let Some(capability) = &capability.rdma { + assert!(capability.ready); + assert_eq!(capability.data_slot_capacity, 65_536); + } + let placement = locality.extents[0].strips[0].fragments[0].disk.as_ref().unwrap(); + assert_eq!(placement.rack_id, 1); + assert_eq!(placement.node_id, 1); + assert_eq!(placement.disk_id, crowdb_diskio_client::DiskId::new(1, 1)); + assert_eq!(disk.status().read_operations, 0); let mut first = fixture .client .get_object(context(2), b"bucket", b"key", &GetOptions::default()) diff --git a/lib/crowdb-sdk-native/tests/topology_test.rs b/lib/crowdb-sdk-native/tests/topology_test.rs new file mode 100644 index 000000000..e813db20c --- /dev/null +++ b/lib/crowdb-sdk-native/tests/topology_test.rs @@ -0,0 +1,69 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/metadata.rs"] +mod metadata; +#[path = "common/storage.rs"] +mod storage; +#[path = "common/topology.rs"] +mod topology; + +use std::num::NonZeroU64; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crowdb_sdk_native::error::{TopologyFailureKind, TopologySource}; +use crowdb_sdk_native::{Error, OperationContext}; + +fn context(id: u64) -> OperationContext { + OperationContext::new( + NonZeroU64::new(id).unwrap(), + Instant::now() + Duration::from_secs(1), + ) +} + +#[tokio::test] +async fn retained_catalog_views_do_not_regress_and_independent_refresh_age_is_unknown() { + let source = Arc::new(topology::TestCatalogSource { + generation: 1.into(), + unavailable: false.into(), + }); + let metadata = metadata::client_with_source(source.clone()).await; + let runtime = storage::runtime_with_metadata(metadata.clone()); + let old = runtime.topology().unwrap(); + assert_eq!(old.catalog().generation(), 1); + assert!(old.catalog_age().is_none()); + source.generation.store(2, Ordering::Release); + metadata.refresh_catalog().await.unwrap(); + let observed = runtime.topology().unwrap(); + assert_eq!(observed.catalog().generation(), 2); + assert_eq!(observed.catalog().route(b"key").unwrap().owner.instance_id, 2); + assert!(observed.catalog_age().is_none()); + let refreshed = runtime.refresh_catalog(context(1)).await.unwrap(); + assert!(refreshed.catalog_age().is_some()); + source.generation.store(1, Ordering::Release); + assert!(matches!( + runtime.refresh_catalog(context(2)).await, + Err(Error::Topology { + origin: TopologySource::Catalog, + kind: TopologyFailureKind::Invalid, + .. + }) + )); + assert_eq!(runtime.topology().unwrap().catalog().generation(), 2); + assert_eq!(old.catalog().generation(), 1); + assert_eq!(observed.catalog().generation(), 2); + source.unavailable.store(true, Ordering::Release); + assert!(matches!( + runtime.refresh_catalog(context(3)).await, + Err(Error::Topology { + origin: TopologySource::Catalog, + kind: TopologyFailureKind::Unavailable, + .. + }) + )); + runtime.shutdown().await.unwrap(); + assert_eq!(old.catalog().route(b"key").unwrap().owner.instance_id, 1); + assert_eq!(refreshed.catalog().generation(), 2); +} From e768051603f8207dde9af18883f94e22eeb04834 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 14:02:42 +0800 Subject: [PATCH 066/258] Share native S3 listing and ordered deletion with uncertain outcomes --- app/crowdb-access-server/src/s3/operations.rs | 50 ++--- .../src/s3/operations/delete.rs | 19 +- doc/working/plan-rdma-fat-client-sdk.md | 33 ++- lib/crowdb-access-s3/src/delete/selection.rs | 14 ++ lib/crowdb-access-s3/src/execution.rs | 18 +- .../src/execution/namespace.rs | 109 +++++++++ lib/crowdb-sdk-native/src/error.rs | 1 + .../src/runtime/resources.rs | 6 + lib/crowdb-sdk-native/src/s3.rs | 30 ++- lib/crowdb-sdk-native/src/s3/config.rs | 3 + lib/crowdb-sdk-native/src/s3/namespace.rs | 209 ++++++++++++++++++ .../tests/common/metadata.rs | 49 +++- .../tests/s3_namespace_test.rs | 186 ++++++++++++++++ 13 files changed, 671 insertions(+), 56 deletions(-) create mode 100644 lib/crowdb-access-s3/src/execution/namespace.rs create mode 100644 lib/crowdb-sdk-native/src/s3/namespace.rs create mode 100644 lib/crowdb-sdk-native/tests/s3_namespace_test.rs diff --git a/app/crowdb-access-server/src/s3/operations.rs b/app/crowdb-access-server/src/s3/operations.rs index 3da8a01d8..4dd31e8f5 100644 --- a/app/crowdb-access-server/src/s3/operations.rs +++ b/app/crowdb-access-server/src/s3/operations.rs @@ -10,9 +10,8 @@ use base64::Engine as _; use crowdb_access_s3::condition::ObjectConditions; use crowdb_access_s3::continuation::ContinuationTokenSigner; use crowdb_access_s3::error::{S3Error, S3ErrorCode}; -use crowdb_access_s3::execution::S3Executor; +use crowdb_access_s3::execution::{ObjectListRequest, S3Executor}; use crowdb_access_s3::metadata::{ObjectRecord, TenantId}; -use crowdb_access_s3::object::{self, ListObjectsV2Request, ObjectMetadataError}; use crowdb_access_s3::publication::PublicationRequest; use crowdb_access_s3::retrieval::{ObjectHeaders, RetrievalError}; use crowdb_access_s3::route::{S3Operation, S3Route}; @@ -356,25 +355,24 @@ impl ProductionS3Operations { .map_or(Ok(DEFAULT_LIST_LIMIT), |value| value.parse::()) .map_err(|_| S3ErrorCode::InvalidRequest)? .min(DEFAULT_LIST_LIMIT); - let page = object::list_v2( - &self.storage.metadata, - &self.signer, - &ListObjectsV2Request { - tenant: &self.config.tenant, - bucket: bucket_id, - prefix: &prefix, - delimiter: delimiter.as_deref(), - start_after: start_after.as_deref(), - continuation_token: continuation.as_deref(), - max_keys, - max_scan_items: self.config.list_scan_items, - max_scan_bytes: self.config.list_scan_bytes, - now_unix_seconds: unix_seconds(), - token_ttl_seconds: self.config.continuation_ttl_seconds, - }, - ) - .await - .map_err(|error| map_object_error(&error))?; + let page = self + .execution + .list_objects( + &self.signer, + &ObjectListRequest { + bucket: bucket_id, + prefix: &prefix, + delimiter: delimiter.as_deref(), + start_after: start_after.as_deref(), + continuation_token: continuation.as_deref(), + max_keys, + max_scan_items: self.config.list_scan_items, + max_scan_bytes: self.config.list_scan_bytes, + now_unix_seconds: unix_seconds(), + token_ttl_seconds: self.config.continuation_ttl_seconds, + }, + ) + .await?; xml_response(wire::list_objects( bucket_name, &prefix, @@ -628,16 +626,6 @@ fn map_retrieval(error: &RetrievalError) -> S3ErrorCode { } } -fn map_object_error(error: &ObjectMetadataError) -> S3ErrorCode { - match error { - ObjectMetadataError::Key(_) - | ObjectMetadataError::Continuation(_) - | ObjectMetadataError::InvalidListLimit => S3ErrorCode::InvalidRequest, - ObjectMetadataError::Record(_) => S3ErrorCode::InternalError, - ObjectMetadataError::Store(_) => S3ErrorCode::ServiceUnavailable, - } -} - fn map_put_outcome(outcome: &PutOutcome) -> S3ErrorCode { if matches!( outcome, diff --git a/app/crowdb-access-server/src/s3/operations/delete.rs b/app/crowdb-access-server/src/s3/operations/delete.rs index df29c62be..fa1041b66 100644 --- a/app/crowdb-access-server/src/s3/operations/delete.rs +++ b/app/crowdb-access-server/src/s3/operations/delete.rs @@ -5,7 +5,7 @@ use crowdb_access_s3::delete::{DeleteSelection, MAX_DELETE_BODY}; use crowdb_access_s3::error::S3ErrorCode; -use crowdb_access_s3::{object, route::S3Route, wire}; +use crowdb_access_s3::{route::S3Route, wire}; use futures::{stream, StreamExt}; use http_body_util::BodyExt as _; use http_body_util::StreamBody; @@ -15,9 +15,7 @@ use hyper::{ Request, Response, StatusCode, }; -use super::{ - map_object_error, required_bucket, strict_header, BoxError, ProductionS3Operations, ResponseBody, -}; +use super::{required_bucket, strict_header, BoxError, ProductionS3Operations, ResponseBody}; impl ProductionS3Operations { pub(super) async fn delete_objects( @@ -58,21 +56,12 @@ impl ProductionS3Operations { let bytes = Bytes::from(bytes); crowdb_access_s3::delete::validate_integrity(&headers, &bytes)?; let selection = DeleteSelection::parse(&bytes)?; - let metadata = self.storage.metadata.clone(); - let tenant = self.config.tenant.clone(); let quiet = selection.quiet; - let results = selection.stream(move |key| { - let metadata = metadata.clone(); - let tenant = tenant.clone(); - async move { - object::delete(&metadata, &tenant, bucket, &key) - .await - .map_err(|error| map_object_error(&error)) - } - })?; + let results = self.execution.delete_objects(bucket, &selection)?; // Send progress while preserving one ordered mutation at a time. // Quiet successes emit XML whitespace so idle-read deadlines still see progress. let results = results.map(move |(key, result)| { + let result = result.map_err(|error| error.code()); let xml = wire::delete_object_result(&key, &result, quiet); let bytes = if xml.is_empty() { Bytes::from_static(b"\n") diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index 387658d85..d8185f182 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -99,7 +99,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. ## Phase 3: shared S3 execution and first native frontend -- [ ] **Extract bucket and metadata operations**: move typed bucket, HEAD, +- [x] **Extract bucket and metadata operations**: move typed bucket, HEAD, list, and delete orchestration into the S3 library. Preserve bounded listing, continuation scope, conditions, and shared-range cleanup. HTTP parses/authenticates, calls shared execution, and encodes. Files: @@ -324,6 +324,37 @@ transport acceptance alone cannot establish SDK completion. Three new catalog/locality cases pass; the native suite now has 27 cases. Native/S3/ChunkIO all-target suites and clippy pass. Full service restart, stale-catalog/endpoint fault acceptance remains part of the final corpus. +- HTTP and native listing/batch deletion now use shared `S3Executor` execution. + Native listing returns bounded typed entries and scoped continuation; handles + share a runtime-local random signer key unless configured for stable cursors. + This key is not a user credential and requires no credential database. + Lazy native batch deletion validates the entire selection, returns each + independent ordered result, keeps duplicate semantics and stops its tail on + drop/cancellation. Shared typed deletion preserves transport/deadline reply + loss as uncertainty; HTTP renders the same stable ServiceUnavailable code. + Four focused native cases cover paging/scope, lazy ordered partial progress, + cancellation and an applied mutation with a deliberately lost reply. The + native suite now has 31 cases. Native/S3 all-target tests, affected HTTP + service/inspection/multipart tests, three-crate clippy and formatting pass. + +## Immediate dependent implementation + +- Extract HTTP `ObjectWriter` selection/finalization and publication into + shared S3 write sessions before native PUT/copy/multipart. Preserve the + existing digest pipeline, native framed owners, small-writer pools, large + EC policy, and `PutOutcome::Timeout`; frontend parsing/encoding stays HTTP. + Ordinary native input and frozen registered input must use these sessions, + with reserved frame space for in-place preparation when eligible. +- Complete owned runtime shutdown without prematurely destroying held leases. + Distinguish borrowed attached clients from runtime-owned transports and + newly created writer groups. Shared shutdown must run once, drain owned + producers, close RPC workers/reapers and let all shutdown callers observe + the same terminal result. Do not stop externally borrowed clients implicitly. +- Audit accepted RDMA read failures above semantic DiskIO: the routed adapter + and strip recovery currently have read retry paths. Ensure a native accepted + failure cannot turn into a TCP suffix across these layers, while retaining + qualified ordinary mirror/EC recovery. Add native GET fault coverage before + claiming the final fallback boundary. ## Subsequent Dataset integration under R220 diff --git a/lib/crowdb-access-s3/src/delete/selection.rs b/lib/crowdb-access-s3/src/delete/selection.rs index 431f46df1..70467170b 100644 --- a/lib/crowdb-access-s3/src/delete/selection.rs +++ b/lib/crowdb-access-s3/src/delete/selection.rs @@ -60,6 +60,20 @@ impl DeleteSelection { where F: FnMut(Vec) -> Fut, Fut: std::future::Future>, + { + self.stream_typed(delete) + } + + /// Preserves typed mutation uncertainty for trusted execution frontends. + /// # Errors + /// Validates the entire selection before invoking any mutation. + pub fn stream_typed( + &self, + delete: F, + ) -> Result, Result<(), E>)>, S3ErrorCode> + where + F: FnMut(Vec) -> Fut, + Fut: std::future::Future>, { if self.keys.is_empty() || self.keys.len() > 1000 { return Err(S3ErrorCode::InvalidRequest); diff --git a/lib/crowdb-access-s3/src/execution.rs b/lib/crowdb-access-s3/src/execution.rs index 6c0c296a5..627c04f45 100644 --- a/lib/crowdb-access-s3/src/execution.rs +++ b/lib/crowdb-access-s3/src/execution.rs @@ -3,6 +3,9 @@ //! Trusted S3 execution shared by HTTP parsing and native business handles. +mod namespace; +pub use namespace::{DeleteObjectFailure, DeleteObjectResult, DeleteObjectStream, ObjectListRequest}; + use std::sync::Arc; use crate::bucket::{self, BucketError, DeleteBucketResult, RandomBucketIdGenerator}; @@ -104,10 +107,21 @@ impl S3Executor { /// # Errors /// Returns namespace or metadata mutation failure. Physical reclamation is deferred. pub async fn delete_object(&self, bucket: &[u8], key: &[u8]) -> Result<(), S3ErrorCode> { - let bucket_id = self.resolve_bucket(bucket).await?; + self.delete_object_typed(bucket, key) + .await + .map_err(|error| error.code()) + } + + /// # Errors + /// Preserves a possibly applied metadata mutation after a lost transport reply. + pub async fn delete_object_typed(&self, bucket: &[u8], key: &[u8]) -> Result<(), DeleteObjectFailure> { + let bucket_id = self + .resolve_bucket(bucket) + .await + .map_err(DeleteObjectFailure::Error)?; object::delete(&self.storage.metadata, &self.tenant, bucket_id, key) .await - .map_err(|error| object_error(&error)) + .map_err(|error| namespace::delete_failure(&error)) } /// # Errors diff --git a/lib/crowdb-access-s3/src/execution/namespace.rs b/lib/crowdb-access-s3/src/execution/namespace.rs new file mode 100644 index 000000000..ffae1fc95 --- /dev/null +++ b/lib/crowdb-access-s3/src/execution/namespace.rs @@ -0,0 +1,109 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::pin::Pin; + +use super::{object_error, S3Executor}; +use crate::continuation::ContinuationTokenSigner; +use crate::delete::DeleteSelection; +use crate::error::S3ErrorCode; +use crate::metadata::{BucketId, MetadataStoreError}; +use crate::object::{self, ListObjectsV2Page, ListObjectsV2Request, ObjectMetadataError}; + +pub struct ObjectListRequest<'a> { + pub bucket: BucketId, + pub prefix: &'a [u8], + pub delimiter: Option<&'a [u8]>, + pub start_after: Option<&'a [u8]>, + pub continuation_token: Option<&'a str>, + pub max_keys: usize, + pub max_scan_items: usize, + pub max_scan_bytes: usize, + pub now_unix_seconds: u64, + pub token_ttl_seconds: u64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DeleteObjectFailure { + Error(S3ErrorCode), + Uncertain, +} + +impl DeleteObjectFailure { + #[must_use] + pub fn code(&self) -> S3ErrorCode { + match self { + Self::Error(code) => *code, + Self::Uncertain => S3ErrorCode::ServiceUnavailable, + } + } +} + +pub type DeleteObjectResult = (Vec, Result<(), DeleteObjectFailure>); +pub type DeleteObjectStream = Pin + Send>>; + +pub(super) fn delete_failure(error: &ObjectMetadataError) -> DeleteObjectFailure { + if matches!( + error, + ObjectMetadataError::Store(MetadataStoreError::Client( + crowdb_chunk_kv_client::ClientError::Deadline | crowdb_chunk_kv_client::ClientError::Transport(_) + )) + ) { + DeleteObjectFailure::Uncertain + } else { + DeleteObjectFailure::Error(object_error(error)) + } +} + +impl S3Executor { + /// # Errors + /// Returns bounded scan, continuation-scope or metadata validation failures. + pub async fn list_objects( + &self, + signer: &ContinuationTokenSigner, + request: &ObjectListRequest<'_>, + ) -> Result { + object::list_v2( + &self.storage.metadata, + signer, + &ListObjectsV2Request { + tenant: &self.tenant, + bucket: request.bucket, + prefix: request.prefix, + delimiter: request.delimiter, + start_after: request.start_after, + continuation_token: request.continuation_token, + max_keys: request.max_keys, + max_scan_items: request.max_scan_items, + max_scan_bytes: request.max_scan_bytes, + now_unix_seconds: request.now_unix_seconds, + token_ttl_seconds: request.token_ttl_seconds, + }, + ) + .await + .map_err(|error| object_error(&error)) + } + + /// Validates the complete selection before any mutation, then yields one + /// independently completed deletion at a time in input order. + /// # Errors + /// Rejects invalid selection bounds before polling any deletion. + pub fn delete_objects( + &self, + bucket: BucketId, + selection: &DeleteSelection, + ) -> Result { + let metadata = self.storage.metadata.clone(); + let tenant = self.tenant.clone(); + let stream = selection.stream_typed(move |key| { + let metadata = metadata.clone(); + let tenant = tenant.clone(); + async move { + object::delete(&metadata, &tenant, bucket, &key) + .await + .map_err(|error| delete_failure(&error)) + } + })?; + Ok(Box::pin(stream)) + } +} diff --git a/lib/crowdb-sdk-native/src/error.rs b/lib/crowdb-sdk-native/src/error.rs index f01ae50f7..d85cf41c7 100644 --- a/lib/crowdb-sdk-native/src/error.rs +++ b/lib/crowdb-sdk-native/src/error.rs @@ -15,6 +15,7 @@ pub enum MutationInterruption { Cancelled, Deadline, Shutdown, + Transport, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] diff --git a/lib/crowdb-sdk-native/src/runtime/resources.rs b/lib/crowdb-sdk-native/src/runtime/resources.rs index eaa17b72f..c3e8113b2 100644 --- a/lib/crowdb-sdk-native/src/runtime/resources.rs +++ b/lib/crowdb-sdk-native/src/runtime/resources.rs @@ -44,6 +44,7 @@ pub struct RuntimeResources { pub(crate) allocations: Arc, pub(crate) compatibility_copy_bytes: AtomicU64, pub(crate) closing: Cancellation, + pub(crate) continuation_key: Vec, } impl RuntimeResources { @@ -60,6 +61,10 @@ impl RuntimeResources { { return Err(Error::Configuration("invalid request or memory limits".into())); } + let identity = crowdb_chunk_kv_client::RequestIdentityAllocator::new().client_instance_id(); + let mut continuation_key = Vec::with_capacity(16); + continuation_key.extend_from_slice(&identity.high.to_le_bytes()); + continuation_key.extend_from_slice(&identity.low.to_le_bytes()); Ok(Arc::new(Self { requests: Budget::new(limits.requests, CapacityKind::Requests), producers: Budget::new(limits.producer_bytes, CapacityKind::Producers), @@ -67,6 +72,7 @@ impl RuntimeResources { closing: Cancellation::default(), allocations: Arc::new(AllocationRegistry::default()), compatibility_copy_bytes: AtomicU64::new(0), + continuation_key, })) } diff --git a/lib/crowdb-sdk-native/src/s3.rs b/lib/crowdb-sdk-native/src/s3.rs index b4e796aff..8e4806614 100644 --- a/lib/crowdb-sdk-native/src/s3.rs +++ b/lib/crowdb-sdk-native/src/s3.rs @@ -14,8 +14,11 @@ use std::sync::Arc; mod body; mod config; mod locality; +mod namespace; mod object; mod read; +pub use crowdb_access_s3::execution::{DeleteObjectFailure, DeleteObjectResult}; +pub use namespace::{DeleteObjects, ListObject, ListObjects, ListOptions}; pub use body::ObjectBody; pub use config::S3ClientConfig; @@ -32,6 +35,7 @@ pub use read::{GetObject, GetOptions}; pub struct S3Client { runtime: ClientRuntime, execution: S3Executor, + signer: Arc, } impl S3Client { @@ -40,6 +44,11 @@ impl S3Client { /// Rejects closed runtime or a writer group for another concrete chunk type. pub fn new(runtime: ClientRuntime, config: impl Into) -> Result { let config = config.into(); + if config.continuation_key.as_ref().is_some_and(Vec::is_empty) { + return Err(Error::Configuration( + "S3 continuation key must be nonempty".into(), + )); + } if config .small_write .as_ref() @@ -60,8 +69,15 @@ impl S3Client { metadata: Arc::new(ChunkKvMetadataStore::new(Arc::clone(&runtime.storage.metadata))), chunks: Arc::clone(&runtime.storage.chunks), }; + let signer = crowdb_access_s3::continuation::ContinuationTokenSigner::new( + config + .continuation_key + .unwrap_or_else(|| runtime.resources.continuation_key.clone()), + ) + .map_err(|_| Error::Configuration("S3 continuation key must be nonempty".into()))?; Ok(Self { execution: S3Executor::new(storage, config.tenant), + signer: Arc::new(signer), runtime, }) } @@ -122,9 +138,21 @@ impl S3Client { /// # Errors /// Returns namespace, admission or routed metadata mutation failure. pub async fn delete_object(&self, context: OperationContext, bucket: &[u8], key: &[u8]) -> Result<()> { + let id = context.request_id.get(); let request = self.runtime.resources.admit(context)?; request - .run_mutation(async { self.execution.delete_object(bucket, key).await.map_err(Error::S3) }) + .run_mutation(async { + self.execution + .delete_object_typed(bucket, key) + .await + .map_err(|error| match error { + DeleteObjectFailure::Error(code) => Error::S3(code), + DeleteObjectFailure::Uncertain => Error::UncertainMutation { + request_id: id, + reason: crate::error::MutationInterruption::Transport, + }, + }) + }) .await } } diff --git a/lib/crowdb-sdk-native/src/s3/config.rs b/lib/crowdb-sdk-native/src/s3/config.rs index 5661970d9..69235c48d 100644 --- a/lib/crowdb-sdk-native/src/s3/config.rs +++ b/lib/crowdb-sdk-native/src/s3/config.rs @@ -8,6 +8,8 @@ pub struct S3ClientConfig { pub tenant: TenantId, /// None reuses the runtime's current compatible S3 writer group. pub small_write: Option, + /// None shares a runtime-local key. Supply a stable key for cross-runtime cursors. + pub continuation_key: Option>, } impl From for S3ClientConfig { @@ -15,6 +17,7 @@ impl From for S3ClientConfig { Self { tenant, small_write: None, + continuation_key: None, } } } diff --git a/lib/crowdb-sdk-native/src/s3/namespace.rs b/lib/crowdb-sdk-native/src/s3/namespace.rs new file mode 100644 index 000000000..3b13b9ecf --- /dev/null +++ b/lib/crowdb-sdk-native/src/s3/namespace.rs @@ -0,0 +1,209 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use crowdb_access_s3::delete::DeleteSelection; +use crowdb_access_s3::execution::{DeleteObjectResult, DeleteObjectStream, ObjectListRequest}; +use crowdb_access_s3::S3ErrorCode; +use futures::StreamExt; + +use super::S3Client; +use crate::error::MutationInterruption; +use crate::runtime::RequestLease; +use crate::{Error, OperationContext, Result}; + +#[derive(Clone, Debug)] +pub struct ListOptions { + pub prefix: Vec, + pub delimiter: Option>, + pub start_after: Option>, + pub continuation: Option, + pub limit: usize, +} + +impl Default for ListOptions { + fn default() -> Self { + Self { + prefix: Vec::new(), + delimiter: None, + start_after: None, + continuation: None, + limit: 1000, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ListObject { + pub key: Vec, + pub logical_length: u64, + pub etag: String, + pub modified_at_ms: u64, +} + +pub struct ListObjects { + pub objects: Vec, + pub common_prefixes: Vec>, + pub next_continuation: Option, +} + +impl S3Client { + /// Reads one bounded, non-snapshot page with scoped continuation. + /// # Errors + /// Returns namespace, admission, deadline or invalid continuation failures. + pub async fn list_objects( + &self, + context: OperationContext, + bucket: &[u8], + options: &ListOptions, + ) -> Result { + let request = self.runtime.resources.admit(context)?; + request + .run(async { + if options.limit > 1000 { + return Err(Error::S3(S3ErrorCode::InvalidRequest)); + } + let bucket = self.execution.resolve_bucket(bucket).await.map_err(Error::S3)?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| Error::Configuration(error.to_string()))? + .as_secs(); + let page = self + .execution + .list_objects( + &self.signer, + &ObjectListRequest { + bucket, + prefix: &options.prefix, + delimiter: options.delimiter.as_deref(), + start_after: options.start_after.as_deref(), + continuation_token: options.continuation.as_deref(), + max_keys: options.limit, + max_scan_items: 1001, + max_scan_bytes: 4 << 20, + now_unix_seconds: now, + token_ttl_seconds: 900, + }, + ) + .await + .map_err(Error::S3)?; + Ok(ListObjects { + objects: page + .objects + .into_iter() + .map(|record| ListObject { + key: record.key, + logical_length: record.logical_length, + etag: record.etag, + modified_at_ms: record.modified_at_ms, + }) + .collect(), + common_prefixes: page.common_prefixes, + next_continuation: page.next_continuation_token, + }) + }) + .await + } + + /// Validates every key before starting ordered independent deletions. + /// Dropping the returned body stops the remaining selection. + /// # Errors + /// Returns invalid selection, namespace or operation admission failures. + pub async fn delete_objects( + &self, + context: OperationContext, + bucket: &[u8], + keys: Vec>, + ) -> Result { + let request = self.runtime.resources.admit(context)?; + let bucket = request + .run(async { self.execution.resolve_bucket(bucket).await.map_err(Error::S3) }) + .await?; + let selection = DeleteSelection { keys, quiet: false }; + let stream = self + .execution + .delete_objects(bucket, &selection) + .map_err(Error::S3)?; + Ok(DeleteObjects { + stream: Some(stream), + request: Some(request), + keys: selection.keys, + cursor: 0, + pending: false, + failure: None, + }) + } +} + +pub struct DeleteObjects { + stream: Option, + request: Option, + keys: Vec>, + cursor: usize, + pending: bool, + failure: Option, +} + +impl DeleteObjects { + #[must_use] + pub fn pending_key(&self) -> Option<&[u8]> { + self.pending + .then(|| self.keys.get(self.cursor)) + .flatten() + .map(Vec::as_slice) + } + + /// Produces one independent result. Cancellation while a mutation is pending + /// is reported as uncertain; completed results are never replayed or rolled back. + /// # Errors + /// Returns sticky operation cancellation/deadline/uncertainty or stream failure. + pub async fn next(&mut self) -> Result> { + if let Some(error) = &self.failure { + return Err(error.clone()); + } + if self.cursor == self.keys.len() { + return Ok(None); + } + let request = self.request.as_ref().ok_or(Error::Cancelled)?; + let stream = self.stream.as_mut().ok_or(Error::Cancelled)?; + let already_pending = self.pending; + self.pending = true; + let result = request.run_mutation(async { Ok(stream.next().await) }).await; + match result { + Ok(Some(result)) => { + self.cursor += 1; + self.pending = false; + if self.cursor == self.keys.len() { + self.stream = None; + self.request = None; + } + Ok(Some(result)) + } + result => { + let mut error = result.err().unwrap_or(Error::S3(S3ErrorCode::InternalError)); + if already_pending { + let reason = match &error { + Error::Cancelled => Some(MutationInterruption::Cancelled), + Error::Deadline => Some(MutationInterruption::Deadline), + Error::Shutdown => Some(MutationInterruption::Shutdown), + _ => None, + }; + if let Some(reason) = reason { + error = Error::UncertainMutation { + request_id: request.context().request_id.get(), + reason, + }; + } + } + self.failure = Some(error.clone()); + if matches!(error, Error::Cancelled | Error::Deadline | Error::Shutdown) { + self.pending = false; + } + self.stream = None; + self.request = None; + Err(error) + } + } + } +} diff --git a/lib/crowdb-sdk-native/tests/common/metadata.rs b/lib/crowdb-sdk-native/tests/common/metadata.rs index 7fcbfd8b2..f3b3e0e92 100644 --- a/lib/crowdb-sdk-native/tests/common/metadata.rs +++ b/lib/crowdb-sdk-native/tests/common/metadata.rs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use async_trait::async_trait; @@ -26,17 +27,28 @@ enum TestCommand { Scan(ScanRequest, oneshot::Sender), } -pub struct TestMetadataTransport(mpsc::Sender); +pub struct TestMetadataTransport { + sender: mpsc::Sender, + lose_mutation_reply: Arc, +} #[async_trait] impl ChunkKvTransport for TestMetadataTransport { async fn point(&self, _: &str, request: &PointRequest) -> Result { let (sender, receiver) = oneshot::channel(); - self.0 + self.sender .send(TestCommand::Point(request.operation.clone(), sender)) .await .unwrap(); - Ok(receiver.await.unwrap()) + let response = receiver.await.unwrap(); + if !matches!(request.operation, PointOperation::Get { .. }) + && self.lose_mutation_reply.load(Ordering::Acquire) + { + return Err(crowdb_chunk_kv_client::ClientError::Transport( + "test lost an applied mutation reply".into(), + )); + } + Ok(response) } async fn seek(&self, _: &str, _: &SeekRequest) -> Result { @@ -45,7 +57,7 @@ impl ChunkKvTransport for TestMetadataTransport { async fn scan(&self, _: &str, request: &ScanRequest) -> Result { let (sender, receiver) = oneshot::channel(); - self.0 + self.sender .send(TestCommand::Scan(request.clone(), sender)) .await .unwrap(); @@ -181,6 +193,28 @@ pub fn catalog( } pub async fn client_with_source(source: Arc) -> Arc { + client_with_options(source, Arc::new(AtomicBool::new(false)), ClientConfig::default()).await +} + +#[allow(dead_code)] // Used by namespace uncertainty tests only. +pub async fn client_with_loss(loss: Arc) -> Arc { + let (head, pages) = catalog(1, 1); + client_with_options( + Arc::new(TestCatalog(head, pages)), + loss, + ClientConfig { + max_attempts: 1, + ..ClientConfig::default() + }, + ) + .await +} + +async fn client_with_options( + source: Arc, + loss: Arc, + config: ClientConfig, +) -> Arc { let (sender, mut receiver) = mpsc::channel(64); tokio::spawn(async move { let mut values = BTreeMap::new(); @@ -201,9 +235,12 @@ pub async fn client_with_source(source: Arc) -> A }); let client = Arc::new( ChunkKvClient::new( - ClientConfig::default(), + config, source, - Arc::new(TestMetadataTransport(sender)), + Arc::new(TestMetadataTransport { + sender, + lose_mutation_reply: loss, + }), ) .unwrap(), ); diff --git a/lib/crowdb-sdk-native/tests/s3_namespace_test.rs b/lib/crowdb-sdk-native/tests/s3_namespace_test.rs new file mode 100644 index 000000000..a3af66c44 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/s3_namespace_test.rs @@ -0,0 +1,186 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/metadata.rs"] +mod metadata; +#[path = "common/storage.rs"] +mod runtime_storage; +#[path = "common/read_storage.rs"] +mod storage; + +use crowdb_access_s3::metadata::TenantId; +use crowdb_access_s3::S3ErrorCode; +use crowdb_sdk_native::s3::{DeleteObjectFailure, ListOptions, S3Client}; +use crowdb_sdk_native::Error; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use storage::{context, TestFixture}; + +async fn populate(fixture: &TestFixture, keys: &[&[u8]]) { + for key in keys { + let mut record = fixture.record(0); + record.key = key.to_vec(); + fixture.publish(&record).await; + } +} + +#[tokio::test] +async fn native_listing_pages_share_scoped_cursors_across_runtime_handles() { + let fixture = TestFixture::new(8192).await; + populate(&fixture, &[b"a/one", b"a/two", b"z"]).await; + let options = ListOptions { + limit: 1, + ..ListOptions::default() + }; + let first = fixture + .client + .list_objects(context(2), b"bucket", &options) + .await + .unwrap(); + assert_eq!(first.objects[0].key, b"a/one"); + let continuation = first.next_continuation.unwrap(); + let sibling = S3Client::new(fixture.runtime.clone(), fixture.tenant.clone()).unwrap(); + let second = sibling + .list_objects( + context(3), + b"bucket", + &ListOptions { + continuation: Some(continuation.clone()), + ..options.clone() + }, + ) + .await + .unwrap(); + assert_eq!(second.objects[0].key, b"a/two"); + assert!(matches!( + sibling + .list_objects( + context(4), + b"bucket", + &ListOptions { + prefix: b"z".to_vec(), + continuation: Some(continuation), + ..options + } + ) + .await, + Err(Error::S3(S3ErrorCode::InvalidRequest)) + )); + let grouped = sibling + .list_objects( + context(5), + b"bucket", + &ListOptions { + delimiter: Some(b"/".to_vec()), + ..ListOptions::default() + }, + ) + .await + .unwrap(); + assert_eq!(grouped.common_prefixes, vec![b"a/".to_vec()]); + assert_eq!(grouped.objects[0].key, b"z"); + assert_eq!(fixture.backend.reads.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn native_batch_delete_is_lazy_ordered_and_dropping_it_keeps_the_tail() { + let fixture = TestFixture::new(8192).await; + populate(&fixture, &[b"a", b"b", b"c"]).await; + let mut deletion = fixture + .client + .delete_objects( + context(2), + b"bucket", + vec![b"a".to_vec(), b"b".to_vec(), b"c".to_vec()], + ) + .await + .unwrap(); + fixture + .client + .resolve_object(context(3), b"bucket", b"a") + .await + .unwrap(); + let (key, result) = deletion.next().await.unwrap().unwrap(); + assert_eq!(key, b"a"); + assert!(result.is_ok()); + drop(deletion); + fixture + .client + .resolve_object(context(4), b"bucket", b"b") + .await + .unwrap(); + let mut duplicate = fixture + .client + .delete_objects( + context(5), + b"bucket", + vec![b"a".to_vec(), b"a".to_vec(), b"c".to_vec()], + ) + .await + .unwrap(); + for key in [b"a", b"a", b"c"] { + let result = duplicate.next().await.unwrap().unwrap(); + assert_eq!(result.0, key); + assert!(result.1.is_ok()); + } + assert!(duplicate.next().await.unwrap().is_none()); + assert!(matches!( + fixture + .client + .delete_objects(context(6), b"bucket", vec![b"b".to_vec(), Vec::new()]) + .await, + Err(Error::S3(S3ErrorCode::InvalidRequest)) + )); + fixture + .client + .resolve_object(context(7), b"bucket", b"b") + .await + .unwrap(); + assert_eq!(fixture.runtime.resources().status().requests, 0); +} + +#[tokio::test] +async fn native_batch_cancellation_stops_before_the_next_key_and_remains_terminal() { + let fixture = TestFixture::new(8192).await; + populate(&fixture, &[b"a", b"b"]).await; + let request = context(2); + let mut deletion = fixture + .client + .delete_objects(request.clone(), b"bucket", vec![b"a".to_vec(), b"b".to_vec()]) + .await + .unwrap(); + deletion.next().await.unwrap().unwrap().1.unwrap(); + request.cancellation.cancel(); + assert!(matches!(deletion.next().await, Err(Error::Cancelled))); + assert!(matches!(deletion.next().await, Err(Error::Cancelled))); + assert!(deletion.pending_key().is_none()); + fixture + .client + .resolve_object(context(3), b"bucket", b"b") + .await + .unwrap(); +} + +#[tokio::test] +async fn lost_delete_replies_remain_uncertain_in_single_and_batch_native_results() { + let loss = Arc::new(AtomicBool::new(false)); + let metadata = metadata::client_with_loss(loss.clone()).await; + let runtime = runtime_storage::runtime_with_metadata(metadata); + let client = S3Client::new(runtime, TenantId::new(b"tenant".to_vec()).unwrap()).unwrap(); + client.create_bucket(context(1), b"bucket").await.unwrap(); + loss.store(true, Ordering::Release); + assert!(matches!( + client.delete_object(context(2), b"bucket", b"missing").await, + Err(Error::UncertainMutation { + reason: crowdb_sdk_native::error::MutationInterruption::Transport, + .. + }) + )); + let mut batch = client + .delete_objects(context(3), b"bucket", vec![b"missing".to_vec()]) + .await + .unwrap(); + let result = batch.next().await.unwrap().unwrap(); + assert_eq!(result.1, Err(DeleteObjectFailure::Uncertain)); + assert!(batch.next().await.unwrap().is_none()); +} From 440e9c81632493fcb14ac5cad5102b3e71ca3480 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 14:40:08 +0800 Subject: [PATCH 067/258] Share durable S3 PUT completion and freeze published native frames --- app/crowdb-access-server/src/s3/operations.rs | 194 ++++-------------- doc/working/plan-rdma-fat-client-sdk.md | 9 + lib/crowdb-access-s3/src/execution.rs | 4 + lib/crowdb-access-s3/src/execution/put.rs | 152 ++++++++++++++ lib/crowdb-access-s3/src/execution/write.rs | 143 +++++++++++++ lib/crowdb-access-s3/src/native_buffer.rs | 114 +--------- .../src/native_buffer/framed.rs | 140 +++++++++++++ lib/crowdb-access-s3/src/publication.rs | 13 +- lib/crowdb-access-s3/src/streaming.rs | 2 +- .../tests/native_buffer_test.rs | 34 +++ 10 files changed, 543 insertions(+), 262 deletions(-) create mode 100644 lib/crowdb-access-s3/src/execution/put.rs create mode 100644 lib/crowdb-access-s3/src/execution/write.rs create mode 100644 lib/crowdb-access-s3/src/native_buffer/framed.rs diff --git a/app/crowdb-access-server/src/s3/operations.rs b/app/crowdb-access-server/src/s3/operations.rs index 4dd31e8f5..137a845d3 100644 --- a/app/crowdb-access-server/src/s3/operations.rs +++ b/app/crowdb-access-server/src/s3/operations.rs @@ -10,15 +10,14 @@ use base64::Engine as _; use crowdb_access_s3::condition::ObjectConditions; use crowdb_access_s3::continuation::ContinuationTokenSigner; use crowdb_access_s3::error::{S3Error, S3ErrorCode}; -use crowdb_access_s3::execution::{ObjectListRequest, S3Executor}; +use crowdb_access_s3::execution::{ + ObjectListRequest, ObjectWriter, PutObjectInfo, S3Executor, S3WritePolicy, +}; use crowdb_access_s3::metadata::{ObjectRecord, TenantId}; -use crowdb_access_s3::publication::PublicationRequest; use crowdb_access_s3::retrieval::{ObjectHeaders, RetrievalError}; use crowdb_access_s3::route::{S3Operation, S3Route}; -use crowdb_access_s3::streaming::{publish_completed_locations, PutErrorCode, PutOutcome}; -use crowdb_chunk_client::{ - ChunkClientConfig, ChunkIoWriter, IoError, LargeWritePolicy, PreparedLargeWrite, SharedObjectWriter, -}; +use crowdb_access_s3::streaming::{PutErrorCode, PutOutcome}; +use crowdb_chunk_client::{ChunkClientConfig, LargeWritePolicy}; use crowdb_common::ec::EcScheme; use futures::stream; use http_body_util::{BodyExt, StreamBody}; @@ -192,25 +191,33 @@ impl ProductionS3Operations { ) -> Result, S3ErrorCode> { let bucket_name = required_bucket(&route)?; let key = required_key(&route)?.to_vec(); - let bucket_id = self.resolve_bucket(bucket_name).await?; let streaming = take_streaming(&mut request)?; let content_length = upload_length(&request, streaming.is_some())?; let content_type = header(&request, CONTENT_TYPE) .unwrap_or("application/octet-stream") .to_owned(); - let attributes = - crowdb_access_s3::metadata::UserMetadata::from_headers(request.headers())?.encode()?; + let user_metadata = crowdb_access_s3::metadata::UserMetadata::from_headers(request.headers())?; let content_md5 = content_md5_header(&request)?.map(str::to_owned); let payload_sha256 = strict_header(&request, "x-amz-content-sha256", S3ErrorCode::InvalidRequest)? .filter(|value| *value != "UNSIGNED-PAYLOAD" && streaming.is_none()) .map(str::to_owned); - let mut route_key = - Vec::with_capacity(self.config.tenant.as_bytes().len() + bucket_id.as_bytes().len() + key.len()); - route_key.extend_from_slice(self.config.tenant.as_bytes()); - route_key.extend_from_slice(bucket_id.as_bytes()); - route_key.extend_from_slice(&key); - let mut writer = self.prepare_writer(content_length, &route_key).await?; - let native_receiver = if writer.is_large() && content_length.is_some() && streaming.is_none() { + let mut put = self + .execution + .prepare_put( + bucket_name, + &key, + PutObjectInfo { + content_length, + content_type, + user_metadata, + }, + &S3WritePolicy { + small_object_limit: self.config.small_object_limit, + large_write: self.config.large_write.clone(), + }, + ) + .await?; + let native_receiver = if put.is_large() && content_length.is_some() && streaming.is_none() { install_body_receive_provider(&mut request) } else { None @@ -224,7 +231,7 @@ impl ProductionS3Operations { body.defer_md5(); let write_result = write_object_body( &mut body, - &mut writer, + put.writer_mut(), native_receiver.as_deref(), content_length, content_md5.as_deref(), @@ -236,45 +243,20 @@ impl ProductionS3Operations { let (etag, checksum) = match write_result { Ok(result) => result, Err(outcome) => { - let _ = writer.on_error().await; + put.abort().await; return Err(body .failure() .map_or_else(|| map_put_outcome(&outcome), encoding_error)); } }; - let locations = writer.on_finish().await.map_err(|error| { - tracing::warn!(%error, "S3 PUT chunk finalization failed"); - S3ErrorCode::ServiceUnavailable - })?; - let logical_length = locations.iter().map(|location| location.logical_length).sum(); - if content_length.is_some_and(|expected| expected != logical_length) { - return Err(S3ErrorCode::InvalidRequest); - } - let now = unix_millis(); - let mut publication = PublicationRequest { - tenant: self.config.tenant.clone(), - object: ObjectRecord { - bucket_id, - key, - logical_length, - checksum, - etag: etag.clone(), - created_at_ms: now, - modified_at_ms: now, - content_type, - attributes, - data_reference: vec![0], - data_length: logical_length, - }, - }; - match publish_completed_locations(&self.storage.metadata, &mut publication, &locations).await { - PutOutcome::Success => Response::builder() - .status(StatusCode::OK) - .header(ETAG, format!("\"{etag}\"")) - .body(full_body(Vec::new().into())) - .map_err(|_| S3ErrorCode::InternalError), - outcome => Err(map_put_outcome(&outcome)), - } + put.finish(etag.clone(), checksum) + .await + .map_err(|outcome| map_put_outcome(&outcome))?; + Response::builder() + .status(StatusCode::OK) + .header(ETAG, format!("\"{etag}\"")) + .body(full_body(Vec::new().into())) + .map_err(|_| S3ErrorCode::InternalError) } async fn head_object( @@ -406,35 +388,16 @@ impl ProductionS3Operations { content_length: Option, key: &[u8], ) -> Result { - if let Some(length) = content_length - .and_then(|length| usize::try_from(length).ok()) - .filter(|length| *length <= self.config.small_object_limit) - { - if length > crowdb_protocol::frame::MAX_FRAME_V2_PAYLOAD_BYTES { - return self - .storage - .chunks - .prepare_shared_object_write_for_key(length, key) - .await - .map(ObjectWriter::Small) - .map_err(|_| S3ErrorCode::SlowDown); - } - match self.storage.chunks.prepare_small_write_for_key(length, key).await { - Ok(writer) => return Ok(ObjectWriter::Small(writer)), - Err(IoError::ObjectTooLarge { .. }) => {} - Err(IoError::MemoryBudgetExhausted) => return Err(S3ErrorCode::SlowDown), - Err(_) => return Err(S3ErrorCode::ServiceUnavailable), - } - } - let mut prepared = self - .storage - .chunks - .prepare_large_write(content_length, self.config.large_write.clone()); - prepared.wait_until_prepared().await.map_err(|error| { - tracing::warn!(%error, "S3 large write preparation failed"); - S3ErrorCode::ServiceUnavailable - })?; - Ok(ObjectWriter::Large(Box::new(prepared))) + self.execution + .prepare_writer( + content_length, + key, + &S3WritePolicy { + small_object_limit: self.config.small_object_limit, + large_write: self.config.large_write.clone(), + }, + ) + .await } } @@ -454,77 +417,6 @@ impl S3Operations for ProductionS3Operations { } } -enum ObjectWriter { - Small(SharedObjectWriter), - Large(Box), -} - -impl ObjectWriter { - fn is_large(&self) -> bool { - matches!(self, Self::Large(_)) - } -} - -#[async_trait::async_trait] -impl ChunkIoWriter for ObjectWriter { - async fn on_data( - &mut self, - buffer: hyper::body::Bytes, - ) -> crowdb_chunk_client::Result { - match self { - Self::Small(writer) => writer.on_data(buffer).await, - Self::Large(writer) => writer.on_data(buffer).await, - } - } - - async fn on_framed_data( - &mut self, - buffer: Box, - ) -> crowdb_chunk_client::Result { - match self { - Self::Small(_) => Err(IoError::WriteFailed( - "small object writer does not accept framed owners".into(), - )), - Self::Large(writer) => writer.on_framed_data(buffer).await, - } - } - - async fn on_finish(&mut self) -> crowdb_chunk_client::Result> { - match self { - Self::Small(writer) => writer.finish_durable().await, - Self::Large(writer) => writer.on_finish().await, - } - } - - async fn on_error(&mut self) -> crowdb_chunk_client::Result> { - match self { - Self::Small(writer) => writer.on_error().await, - Self::Large(writer) => writer.on_error().await, - } - } - - fn require_data(&self) -> bool { - match self { - Self::Small(writer) => writer.require_data(), - Self::Large(writer) => writer.require_data(), - } - } - - fn input_complete(&self) -> bool { - match self { - Self::Small(writer) => writer.input_complete(), - Self::Large(writer) => writer.input_complete(), - } - } - - async fn wait_for_capacity(&mut self) { - match self { - Self::Small(writer) => writer.wait_for_capacity().await, - Self::Large(writer) => writer.wait_for_capacity().await, - } - } -} - fn required_bucket(route: &S3Route) -> Result<&[u8], S3ErrorCode> { route.bucket.as_deref().ok_or(S3ErrorCode::InvalidRequest) } diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index d8185f182..6c802211b 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -339,6 +339,15 @@ transport acceptance alone cannot establish SDK completion. ## Immediate dependent implementation +- Shared writer selection and `PreparedPut` now serve HTTP PUT. Durable + completion precedes one versioned publication, retaining the same metadata + revision and uncertain timeout classification without a subsequent HEAD. + Native physical views require finalization and reject later header/footer + rewrites, including a changed chunk destination. The new ownership test, + full S3 library suite, affected HTTP tests, three-crate clippy and formatting + pass. Explicit HCA native producer framing still sends 1 MiB directly with + zero staging-copy bytes. Native PUT attachment is the next task. + - Extract HTTP `ObjectWriter` selection/finalization and publication into shared S3 write sessions before native PUT/copy/multipart. Preserve the existing digest pipeline, native framed owners, small-writer pools, large diff --git a/lib/crowdb-access-s3/src/execution.rs b/lib/crowdb-access-s3/src/execution.rs index 627c04f45..76c5bd49a 100644 --- a/lib/crowdb-access-s3/src/execution.rs +++ b/lib/crowdb-access-s3/src/execution.rs @@ -4,7 +4,11 @@ //! Trusted S3 execution shared by HTTP parsing and native business handles. mod namespace; +mod put; +mod write; pub use namespace::{DeleteObjectFailure, DeleteObjectResult, DeleteObjectStream, ObjectListRequest}; +pub use put::{PreparedPut, PutObjectInfo}; +pub use write::{ObjectWriter, S3WritePolicy}; use std::sync::Arc; diff --git a/lib/crowdb-access-s3/src/execution/put.rs b/lib/crowdb-access-s3/src/execution/put.rs new file mode 100644 index 000000000..6f2560e65 --- /dev/null +++ b/lib/crowdb-access-s3/src/execution/put.rs @@ -0,0 +1,152 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Durable object completion followed by exactly one metadata publication. + +use super::{ObjectWriter, S3Executor, S3WritePolicy}; +use crate::metadata::{ChunkKvMetadataStore, MetadataKey, ObjectRecord, UserMetadata}; +use crate::object::SelectedObject; +use crate::publication::{publish_versioned, PublicationRequest}; +use crate::streaming::{attach_completed_locations, classify_publication_error, PutErrorCode, PutOutcome}; +use crate::S3ErrorCode; +use crowdb_chunk_client::ChunkIoWriter; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +pub struct PutObjectInfo { + pub content_length: Option, + pub content_type: String, + pub user_metadata: UserMetadata, +} + +impl Default for PutObjectInfo { + fn default() -> Self { + Self { + content_length: None, + content_type: "application/octet-stream".into(), + user_metadata: UserMetadata::default(), + } + } +} + +pub struct PreparedPut { + writer: ObjectWriter, + store: Arc, + publication: PublicationRequest, + declared_length: Option, +} + +impl S3Executor { + /// Resolves the destination and reserves its writer before accepting input. + /// # Errors + /// Rejects invalid metadata, missing bucket or exhausted writer capacity. + pub async fn prepare_put( + &self, + bucket: &[u8], + key: &[u8], + info: PutObjectInfo, + policy: &S3WritePolicy, + ) -> Result { + if info.content_type.is_empty() || info.content_length.is_some_and(|length| length > 5 << 40) { + return Err(S3ErrorCode::InvalidRequest); + } + let bucket_id = self.resolve_bucket(bucket).await?; + MetadataKey::object(&self.tenant, bucket_id, key).map_err(|_| S3ErrorCode::InvalidRequest)?; + let mut route_key = Vec::with_capacity(self.tenant.as_bytes().len() + 16 + key.len()); + route_key.extend_from_slice(self.tenant.as_bytes()); + route_key.extend_from_slice(bucket_id.as_bytes()); + route_key.extend_from_slice(key); + let attributes = info.user_metadata.encode()?; + let writer = self + .prepare_writer(info.content_length, &route_key, policy) + .await?; + Ok(PreparedPut { + writer, + store: Arc::clone(&self.storage.metadata), + declared_length: info.content_length, + publication: PublicationRequest { + tenant: self.tenant.clone(), + object: ObjectRecord { + bucket_id, + key: key.to_vec(), + logical_length: 0, + checksum: Vec::new(), + etag: String::new(), + created_at_ms: 0, + modified_at_ms: 0, + content_type: info.content_type, + attributes, + data_reference: Vec::new(), + data_length: 0, + }, + }, + }) + } +} + +impl PreparedPut { + pub fn writer_mut(&mut self) -> &mut ObjectWriter { + &mut self.writer + } + + #[must_use] + pub fn is_large(&self) -> bool { + self.writer.is_large() + } + + pub async fn abort(&mut self) { + let _ = self.writer.on_error().await; + } + + /// Seals data before publication. Uncertain publication retains its locations + /// for subsequent reconciliation; physical reclamation remains deferred. + /// # Errors + /// Returns definite write/length/encoding failures or an uncertain mutation. + pub async fn finish(mut self, etag: String, checksum: Vec) -> Result { + let locations = match self.writer.on_finish().await { + Ok(locations) => locations, + Err(error) => { + self.abort().await; + return Err(failure(PutErrorCode::ChunkWrite, error)); + } + }; + let length = locations.iter().try_fold(0u64, |length, location| { + length.checked_add(location.logical_length) + }); + let length = length.ok_or_else(|| failure(PutErrorCode::ChunkWrite, "object length overflow"))?; + if self.declared_length.is_some_and(|expected| expected != length) { + return Err(failure( + PutErrorCode::BodyRead, + "body differs from declared length", + )); + } + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) + }); + let object = &mut self.publication.object; + object.logical_length = length; + object.data_length = length; + object.etag = etag; + object.checksum = checksum; + object.created_at_ms = now; + object.modified_at_ms = now; + attach_completed_locations(object, &locations) + .map_err(|error| failure(PutErrorCode::LocationEncoding, error))?; + let revision = publish_versioned(&self.store, &self.publication) + .await + .map_err(classify_publication_error)?; + Ok(SelectedObject { + record: self.publication.object, + revision, + }) + } +} + +fn failure(code: PutErrorCode, error: impl std::fmt::Display) -> PutOutcome { + PutOutcome::Error { + code, + message: error.to_string(), + } +} diff --git a/lib/crowdb-access-s3/src/execution/write.rs b/lib/crowdb-access-s3/src/execution/write.rs new file mode 100644 index 000000000..3645f412a --- /dev/null +++ b/lib/crowdb-access-s3/src/execution/write.rs @@ -0,0 +1,143 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Shared bounded S3 writer selection and durable completion. + +use super::S3Executor; +use crate::error::S3ErrorCode; +use crowdb_chunk_client::{ + ChunkClientConfig, ChunkIoWriter, IoError, LargeWritePolicy, PreparedLargeWrite, SharedObjectWriter, +}; +use crowdb_common::ec::EcScheme; +use crowdb_protocol::chunkdb::rpc::ChunkType; +use std::sync::Arc; + +#[derive(Clone)] +pub struct S3WritePolicy { + pub small_object_limit: usize, + pub large_write: LargeWritePolicy, +} + +impl S3WritePolicy { + #[must_use] + pub fn basic(small_object_limit: usize) -> Self { + Self { + small_object_limit, + large_write: LargeWritePolicy { + ec_scheme: EcScheme::new(8, 4), + client: Arc::new(ChunkClientConfig::new(ChunkType::S3)), + }, + } + } +} + +impl S3Executor { + /// # Errors + /// Returns bounded admission, discovery or chunk preparation failure. + pub async fn prepare_writer( + &self, + content_length: Option, + key: &[u8], + policy: &S3WritePolicy, + ) -> Result { + if let Some(length) = content_length + .and_then(|length| usize::try_from(length).ok()) + .filter(|length| *length <= policy.small_object_limit) + { + if length > crowdb_protocol::frame::MAX_FRAME_V2_PAYLOAD_BYTES { + return self + .storage + .chunks + .prepare_shared_object_write_for_key(length, key) + .await + .map(ObjectWriter::Small) + .map_err(|_| S3ErrorCode::SlowDown); + } + match self.storage.chunks.prepare_small_write_for_key(length, key).await { + Ok(writer) => return Ok(ObjectWriter::Small(writer)), + Err(IoError::ObjectTooLarge { .. }) => {} + Err(IoError::MemoryBudgetExhausted) => return Err(S3ErrorCode::SlowDown), + Err(_) => return Err(S3ErrorCode::ServiceUnavailable), + } + } + let mut prepared = self + .storage + .chunks + .prepare_large_write(content_length, policy.large_write.clone()); + prepared.wait_until_prepared().await.map_err(|error| { + tracing::warn!(%error, "S3 large write preparation failed"); + S3ErrorCode::ServiceUnavailable + })?; + Ok(ObjectWriter::Large(Box::new(prepared))) + } +} + +pub enum ObjectWriter { + Small(SharedObjectWriter), + Large(Box), +} + +impl ObjectWriter { + #[must_use] + pub fn is_large(&self) -> bool { + matches!(self, Self::Large(_)) + } +} + +#[async_trait::async_trait] +impl ChunkIoWriter for ObjectWriter { + async fn on_data( + &mut self, + buffer: hyper::body::Bytes, + ) -> crowdb_chunk_client::Result { + match self { + Self::Small(writer) => writer.on_data(buffer).await, + Self::Large(writer) => writer.on_data(buffer).await, + } + } + + async fn on_framed_data( + &mut self, + buffer: Box, + ) -> crowdb_chunk_client::Result { + match self { + Self::Small(writer) => writer.on_framed_data(buffer).await, + Self::Large(writer) => writer.on_framed_data(buffer).await, + } + } + + async fn on_finish(&mut self) -> crowdb_chunk_client::Result> { + match self { + Self::Small(writer) => writer.finish_durable().await, + Self::Large(writer) => writer.on_finish().await, + } + } + + async fn on_error(&mut self) -> crowdb_chunk_client::Result> { + match self { + Self::Small(writer) => writer.on_error().await, + Self::Large(writer) => writer.on_error().await, + } + } + + fn require_data(&self) -> bool { + match self { + Self::Small(writer) => writer.require_data(), + Self::Large(writer) => writer.require_data(), + } + } + + fn input_complete(&self) -> bool { + match self { + Self::Small(writer) => writer.input_complete(), + Self::Large(writer) => writer.input_complete(), + } + } + + async fn wait_for_capacity(&mut self) { + match self { + Self::Small(writer) => writer.wait_for_capacity().await, + Self::Large(writer) => writer.wait_for_capacity().await, + } + } +} diff --git a/lib/crowdb-access-s3/src/native_buffer.rs b/lib/crowdb-access-s3/src/native_buffer.rs index d5a44ce27..5a2cf4417 100644 --- a/lib/crowdb-access-s3/src/native_buffer.rs +++ b/lib/crowdb-access-s3/src/native_buffer.rs @@ -9,7 +9,6 @@ use std::cell::UnsafeCell; use std::future::{poll_fn, Future}; use std::io; use std::mem::MaybeUninit; -use std::ops::Range; use std::pin::Pin; use std::ptr::NonNull; use std::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; @@ -19,12 +18,11 @@ use std::time::{Instant, SystemTime, UNIX_EPOCH}; use arc_swap::ArcSwap; use atomic_waker::AtomicWaker; -use crowdb_chunk_client::FramedWriteBuffer; use crowdb_common::metrics::{Bandwidth, MetricName}; use crowdb_protocol::common::ChunkId; use crowdb_protocol::frame::{ - encode_frame_regions, prepare_frame_regions, set_frame_chunk_id, FrameError, FrameMagic, - FRAME_FOOTER_BYTES, FRAME_V2_HEADER_BYTES, MAX_FRAME_BYTES, MAX_FRAME_V2_PAYLOAD_BYTES, + prepare_frame_regions, FrameMagic, FRAME_FOOTER_BYTES, FRAME_V2_HEADER_BYTES, MAX_FRAME_BYTES, + MAX_FRAME_V2_PAYLOAD_BYTES, }; use crowdb_rpc_ffi::{ProducerBufferAllocator, WritableBuffer}; use hyper::body::{Bytes, Http1BodyReceiveBuffer, Http1BodyReceiveProvider}; @@ -52,6 +50,7 @@ pub struct NativeFramedOwner { prepared_slots: usize, prepared: bool, magic: FrameMagic, + finalized_chunks: Box<[Option]>, } // SAFETY: Hyper invokes one provider serially for one Incoming body. The @@ -116,6 +115,7 @@ pub enum NativeBufferConfigError { } mod allocator; +mod framed; impl AllocatorState { fn wake_credit_waiters(&self) { @@ -426,8 +426,10 @@ fn completed_owner(state: &mut ReceiverState, owner: Arc) -> io::Re let prepared = prepared_slots == lengths.len(); state.append_slot = None; state.payload_lengths.fill(0); + let finalized_chunks = vec![None; lengths.len()].into_boxed_slice(); Ok(NativeFramedOwner { owner, + finalized_chunks, payload_lengths: lengths, logical_len, physical_len, @@ -447,110 +449,6 @@ impl Drop for NativeBodyReceiver { } } -impl FramedWriteBuffer for NativeFramedOwner { - fn logical_len(&self) -> u64 { - self.logical_len - } - - fn frame_count(&self) -> usize { - self.payload_lengths.len() - } - - fn frame_payload_len(&self, index: usize) -> Option { - self.payload_lengths.get(index).copied().map(usize::from) - } - - fn prepare_frames(&mut self, magic: FrameMagic, write_time_ms: u64) -> Result<(), FrameError> { - if !matches!(magic, FrameMagic::RepoSmallV2 | FrameMagic::RepoLargeV2) { - return Err(FrameError::InvalidRole); - } - if self.magic != magic { - self.prepared = false; - self.prepared_slots = 0; - self.magic = magic; - } - if self.prepared { - return Ok(()); - } - for index in self.prepared_slots..self.payload_lengths.len() { - let payload_len = usize::from(self.payload_lengths[index]); - let frame_offset = index - .checked_mul(MAX_FRAME_BYTES) - .ok_or(FrameError::LengthOverflow)?; - let payload_offset = frame_offset + FRAME_V2_HEADER_BYTES; - // SAFETY: socket receive has completed this slot. Header and - // footer regions are disjoint from immutable payload views. - unsafe { - let header = std::slice::from_raw_parts_mut( - self.owner.pointer.as_ptr().add(frame_offset), - FRAME_V2_HEADER_BYTES, - ); - let payload = - std::slice::from_raw_parts(self.owner.pointer.as_ptr().add(payload_offset), payload_len); - let footer = std::slice::from_raw_parts_mut( - self.owner.pointer.as_ptr().add(payload_offset + payload_len), - FRAME_FOOTER_BYTES, - ); - prepare_frame_regions(magic, payload, write_time_ms, header, footer)?; - } - } - self.prepared = true; - Ok(()) - } - - fn finalize_frame( - &mut self, - index: usize, - magic: FrameMagic, - chunk_id: ChunkId, - write_time_ms: u64, - ) -> Result, FrameError> { - if !matches!(magic, FrameMagic::RepoSmallV2 | FrameMagic::RepoLargeV2) { - return Err(FrameError::InvalidRole); - } - let payload_len = self - .frame_payload_len(index) - .ok_or(FrameError::InvalidLocationRange)?; - let frame_offset = index - .checked_mul(MAX_FRAME_BYTES) - .ok_or(FrameError::LengthOverflow)?; - let payload_offset = frame_offset + FRAME_V2_HEADER_BYTES; - let frame_len = FRAME_V2_HEADER_BYTES + payload_len + FRAME_FOOTER_BYTES; - // SAFETY: the provider never exposes the reserved header/footer bytes. - // This consumed slot is their only writer; payload views alias only - // the disjoint initialized payload range. - unsafe { - let header = std::slice::from_raw_parts_mut( - self.owner.pointer.as_ptr().add(frame_offset), - FRAME_V2_HEADER_BYTES, - ); - let payload = - std::slice::from_raw_parts(self.owner.pointer.as_ptr().add(payload_offset), payload_len); - let footer = std::slice::from_raw_parts_mut( - self.owner.pointer.as_ptr().add(payload_offset + payload_len), - FRAME_FOOTER_BYTES, - ); - if self.prepared && self.magic == magic { - set_frame_chunk_id(chunk_id, footer)?; - } else { - encode_frame_regions(magic, chunk_id, payload, write_time_ms, header, footer)?; - } - } - Ok(frame_offset..frame_offset + frame_len) - } - - fn views(&self, range: Range) -> Result, FrameError> { - if range.start >= range.end || range.end > self.physical_len { - return Err(FrameError::InvalidLocationRange); - } - Ok(vec![Bytes::from_owner(NativePhysicalFrameView { - owner: Arc::clone(&self.owner), - frame_offset: range.start, - frame_len: range.len(), - })]) - } -} - struct ReceiverStateGuard<'a> { receiver: &'a NativeBodyReceiver, } diff --git a/lib/crowdb-access-s3/src/native_buffer/framed.rs b/lib/crowdb-access-s3/src/native_buffer/framed.rs new file mode 100644 index 000000000..56e320eb1 --- /dev/null +++ b/lib/crowdb-access-s3/src/native_buffer/framed.rs @@ -0,0 +1,140 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! One-time frame publication over immutable native payload storage. + +use super::{NativeFramedOwner, NativePhysicalFrameView}; +use crowdb_chunk_client::FramedWriteBuffer; +use crowdb_protocol::common::ChunkId; +use crowdb_protocol::frame::{ + encode_frame_regions, prepare_frame_regions, set_frame_chunk_id, FrameError, FrameMagic, + FRAME_FOOTER_BYTES, FRAME_V2_HEADER_BYTES, MAX_FRAME_BYTES, +}; +use hyper::body::Bytes; +use std::ops::Range; +use std::sync::Arc; + +impl FramedWriteBuffer for NativeFramedOwner { + fn logical_len(&self) -> u64 { + self.logical_len + } + + fn frame_count(&self) -> usize { + self.payload_lengths.len() + } + + fn frame_payload_len(&self, index: usize) -> Option { + self.payload_lengths.get(index).copied().map(usize::from) + } + + fn prepare_frames(&mut self, magic: FrameMagic, write_time_ms: u64) -> Result<(), FrameError> { + if !matches!(magic, FrameMagic::RepoSmallV2 | FrameMagic::RepoLargeV2) { + return Err(FrameError::InvalidRole); + } + if self.magic != magic && self.finalized_chunks.iter().any(Option::is_some) { + return Err(FrameError::InvalidLocationRange); + } + if self.magic != magic { + self.prepared = false; + self.prepared_slots = 0; + self.magic = magic; + } + if self.prepared { + return Ok(()); + } + for index in self.prepared_slots..self.payload_lengths.len() { + if self.finalized_chunks[index].is_some() { + continue; + } + let payload_len = usize::from(self.payload_lengths[index]); + let frame_offset = index + .checked_mul(MAX_FRAME_BYTES) + .ok_or(FrameError::LengthOverflow)?; + let payload_offset = frame_offset + FRAME_V2_HEADER_BYTES; + // SAFETY: socket receive has completed this slot. Header and + // footer regions are disjoint from immutable payload views. + unsafe { + let header = std::slice::from_raw_parts_mut( + self.owner.pointer.as_ptr().add(frame_offset), + FRAME_V2_HEADER_BYTES, + ); + let payload = + std::slice::from_raw_parts(self.owner.pointer.as_ptr().add(payload_offset), payload_len); + let footer = std::slice::from_raw_parts_mut( + self.owner.pointer.as_ptr().add(payload_offset + payload_len), + FRAME_FOOTER_BYTES, + ); + prepare_frame_regions(magic, payload, write_time_ms, header, footer)?; + } + } + self.prepared = true; + Ok(()) + } + + fn finalize_frame( + &mut self, + index: usize, + magic: FrameMagic, + chunk_id: ChunkId, + write_time_ms: u64, + ) -> Result, FrameError> { + if !matches!(magic, FrameMagic::RepoSmallV2 | FrameMagic::RepoLargeV2) { + return Err(FrameError::InvalidRole); + } + let payload_len = self + .frame_payload_len(index) + .ok_or(FrameError::InvalidLocationRange)?; + let frame_offset = index + .checked_mul(MAX_FRAME_BYTES) + .ok_or(FrameError::LengthOverflow)?; + let payload_offset = frame_offset + FRAME_V2_HEADER_BYTES; + let frame_len = FRAME_V2_HEADER_BYTES + payload_len + FRAME_FOOTER_BYTES; + if let Some(previous) = self.finalized_chunks[index] { + if previous == chunk_id && self.magic == magic { + return Ok(frame_offset..frame_offset + frame_len); + } + return Err(FrameError::InvalidLocationRange); + } + if self.magic != magic { + self.prepare_frames(magic, write_time_ms)?; + } + // SAFETY: the provider never exposes the reserved header/footer bytes. + // This consumed slot is their only writer; payload views alias only + // the disjoint initialized payload range. + unsafe { + let header = std::slice::from_raw_parts_mut( + self.owner.pointer.as_ptr().add(frame_offset), + FRAME_V2_HEADER_BYTES, + ); + let payload = + std::slice::from_raw_parts(self.owner.pointer.as_ptr().add(payload_offset), payload_len); + let footer = std::slice::from_raw_parts_mut( + self.owner.pointer.as_ptr().add(payload_offset + payload_len), + FRAME_FOOTER_BYTES, + ); + if self.prepared { + set_frame_chunk_id(chunk_id, footer)?; + } else { + encode_frame_regions(magic, chunk_id, payload, write_time_ms, header, footer)?; + } + } + self.finalized_chunks[index] = Some(chunk_id); + Ok(frame_offset..frame_offset + frame_len) + } + + fn views(&self, range: Range) -> Result, FrameError> { + if range.start >= range.end || range.end > self.physical_len { + return Err(FrameError::InvalidLocationRange); + } + let first = range.start / MAX_FRAME_BYTES; + let last = (range.end - 1) / MAX_FRAME_BYTES; + if self.finalized_chunks[first..=last].iter().any(Option::is_none) { + return Err(FrameError::InvalidLocationRange); + } + Ok(vec![Bytes::from_owner(NativePhysicalFrameView { + owner: Arc::clone(&self.owner), + frame_offset: range.start, + frame_len: range.len(), + })]) + } +} diff --git a/lib/crowdb-access-s3/src/publication.rs b/lib/crowdb-access-s3/src/publication.rs index bd5cf76f5..5bd1efa22 100644 --- a/lib/crowdb-access-s3/src/publication.rs +++ b/lib/crowdb-access-s3/src/publication.rs @@ -31,7 +31,16 @@ pub async fn publish( store: &ChunkKvMetadataStore, request: &mut PublicationRequest, ) -> Result<(), PublicationError> { + publish_versioned(store, request).await.map(|_| ()) +} + +/// Publishes once and returns the revision from that same mutation. +/// # Errors +/// Returns encoding or routed storage failure without replaying the publication. +pub async fn publish_versioned( + store: &ChunkKvMetadataStore, + request: &PublicationRequest, +) -> Result { let object_key = MetadataKey::object(&request.tenant, request.object.bucket_id, &request.object.key)?; - store.put(object_key, request.object.encode()?).await?; - Ok(()) + Ok(store.put(object_key, request.object.encode()?).await?) } diff --git a/lib/crowdb-access-s3/src/streaming.rs b/lib/crowdb-access-s3/src/streaming.rs index 2c6d81733..84ce42493 100644 --- a/lib/crowdb-access-s3/src/streaming.rs +++ b/lib/crowdb-access-s3/src/streaming.rs @@ -487,7 +487,7 @@ pub async fn cleanup_after_definite_error( outcome } -fn classify_publication_error(error: PublicationError) -> PutOutcome { +pub(crate) fn classify_publication_error(error: PublicationError) -> PutOutcome { match error { timeout @ PublicationError::Store(MetadataStoreError::Client( ClientError::Deadline | ClientError::Transport(_), diff --git a/lib/crowdb-access-s3/tests/native_buffer_test.rs b/lib/crowdb-access-s3/tests/native_buffer_test.rs index b314b4cc2..7ae64efc4 100644 --- a/lib/crowdb-access-s3/tests/native_buffer_test.rs +++ b/lib/crowdb-access-s3/tests/native_buffer_test.rs @@ -50,6 +50,40 @@ async fn native_owner_rejects_wrong_purposes_before_touching_reserved_regions() assert_eq!(frame.payload, b"abc"); } +#[tokio::test] +async fn native_physical_views_require_finalization_and_cannot_be_rewritten_after_publication() { + let allocator = NativeBodyAllocator::new(MAX_FRAME_BYTES, MAX_FRAME_BYTES).unwrap(); + let provider = allocator.object_receiver(); + provider.enable_owner_handoff(); + let mut allocation = poll_fn(|cx| provider.poll_next_buffer(cx, 7)).await.unwrap(); + for (slot, byte) in allocation.spare_capacity_mut()[..7].iter_mut().zip(*b"payload") { + slot.write(byte); + } + allocation.advance(7).unwrap(); + drop(provider.on_data_ready(allocation).unwrap()); + let mut owner = provider.finish_owner().unwrap().unwrap(); + assert!(owner.views(0..1).is_err()); + let chunk = ChunkId { high: 17, low: 29 }; + let range = owner + .finalize_frame(0, FrameMagic::RepoLargeV2, chunk, 123) + .unwrap(); + let view = owner.views(range.clone()).unwrap().pop().unwrap(); + let expected = view.to_vec(); + assert_eq!( + owner + .finalize_frame(0, FrameMagic::RepoLargeV2, chunk, 999) + .unwrap(), + range + ); + assert!(owner + .finalize_frame(0, FrameMagic::RepoLargeV2, ChunkId { high: 1, low: 2 }, 999) + .is_err()); + assert!(owner.prepare_frames(FrameMagic::RepoSmallV2, 999).is_err()); + owner.prepare_frames(FrameMagic::RepoLargeV2, 999).unwrap(); + assert_eq!(view.as_ref(), expected); + assert_eq!(parse_frame(&view, chunk).unwrap().payload, b"payload"); +} + #[tokio::test] async fn native_frame_retains_and_releases_allocator_credit() { let allocator = NativeBodyAllocator::new(MAX_FRAME_BYTES, MAX_FRAME_BYTES).unwrap(); From 49031352d058f129b44632ae4fb3594695a16baf Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 14:54:30 +0800 Subject: [PATCH 068/258] Add bounded native S3 PUT with retained framed producers --- doc/working/plan-rdma-fat-client-sdk.md | 19 +- lib/crowdb-access-s3/src/native_buffer.rs | 21 +- .../src/native_buffer/producer.rs | 108 +++++++ lib/crowdb-sdk-native/src/admission.rs | 6 + lib/crowdb-sdk-native/src/buffer.rs | 13 +- lib/crowdb-sdk-native/src/buffer/producer.rs | 70 +++++ lib/crowdb-sdk-native/src/buffer/receive.rs | 2 +- lib/crowdb-sdk-native/src/runtime.rs | 12 + lib/crowdb-sdk-native/src/s3.rs | 27 ++ lib/crowdb-sdk-native/src/s3/config.rs | 3 + lib/crowdb-sdk-native/src/s3/object.rs | 7 + lib/crowdb-sdk-native/src/s3/write.rs | 268 ++++++++++++++++++ .../tests/common/write_storage.rs | 257 +++++++++++++++++ .../tests/framed_producer_test.rs | 73 +++++ lib/crowdb-sdk-native/tests/s3_write_test.rs | 262 +++++++++++++++++ 15 files changed, 1135 insertions(+), 13 deletions(-) create mode 100644 lib/crowdb-access-s3/src/native_buffer/producer.rs create mode 100644 lib/crowdb-sdk-native/src/buffer/producer.rs create mode 100644 lib/crowdb-sdk-native/src/s3/write.rs create mode 100644 lib/crowdb-sdk-native/tests/common/write_storage.rs create mode 100644 lib/crowdb-sdk-native/tests/framed_producer_test.rs create mode 100644 lib/crowdb-sdk-native/tests/s3_write_test.rs diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index 6c802211b..51651561c 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -115,7 +115,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. `app/crowdb-access-server/src/s3/operations.rs`, `lib/crowdb-sdk-native/src/s3/`, `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`. -- [ ] **Extract PUT and copy**: separate HTTP decoding from bounded input, +- [~] **Extract PUT and copy**: separate HTTP decoding from bounded input, checksums, ChunkIO writes, publication, and definite-failure cleanup. Add ordinary-input and frozen native-buffer entry points. Preserve `PutOutcome::Timeout` uncertainty; final success requires durable data and @@ -158,7 +158,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. `lib/crowdb-diskio-client/src/semantic.rs`, `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`, `lib/crowdb-access-s3/src/retrieval.rs`. -- [ ] **Wire producer buffers**: reuse `ProducerBufferAllocator` and +- [~] **Wire producer buffers**: reuse `ProducerBufferAllocator` and `WritableBuffer`, expose only initialized frozen prefixes, and retain submitted owners through transport/disk completion. Ordinary input adapters stay bounded and account for copies. Files: @@ -339,6 +339,21 @@ transport acceptance alone cannot establish SDK completion. ## Immediate dependent implementation +- Native `begin_put`/`put_vec` now use the shared durable PUT session and return + the publication's exact revision. Input calls accept at most 1 MiB and retain + actual allocation capacity; foreign-runtime owners are rejected. Registered + producers initialize payload slots directly with reserved frame space and + freeze into `FrozenObjectBuffer`; all physical/payload views retain their + producer credit, including initializer failure/panic and shutdown. + Large framed streams require aligned interior frames; one small framed owner + must hold the complete declared object. Native writer waits are cancellable; + dropping a pending input terminates that writer and returns request admission. + Seven new cases cover mixed sizes through multiple chunks, original payload + addresses, length/checksum/cancellation, pending-input drop and an applied + publication whose reply is lost. Native/S3 all-target suites pass (38 native + cases), with focused three-crate clippy and formatting. Native RPC write/HCA + acceptance, copy and multipart remain outstanding. + - Shared writer selection and `PreparedPut` now serve HTTP PUT. Durable completion precedes one versioned publication, retaining the same metadata revision and uncertain timeout classification without a subsequent HEAD. diff --git a/lib/crowdb-access-s3/src/native_buffer.rs b/lib/crowdb-access-s3/src/native_buffer.rs index 5a2cf4417..16229ff60 100644 --- a/lib/crowdb-access-s3/src/native_buffer.rs +++ b/lib/crowdb-access-s3/src/native_buffer.rs @@ -116,6 +116,7 @@ pub enum NativeBufferConfigError { mod allocator; mod framed; +mod producer; impl AllocatorState { fn wake_credit_waiters(&self) { @@ -656,8 +657,10 @@ impl Http1BodyReceiveProvider for NativeBodyReceiver { struct NativeOwner { pointer: NonNull, capacity: usize, - allocator: Arc, + allocator: Option>, native: Option, + // Released after the native allocation, including foreign runtime credits. + _lifetime: Option>, } // SAFETY: provider-issued regions are disjoint physical-frame payload slots. @@ -678,8 +681,9 @@ impl NativeOwner { return Ok(Self { pointer, capacity, - allocator, + allocator: Some(allocator), native: Some(native), + _lifetime: None, }); } // SAFETY: malloc returns either a suitably aligned allocation of at @@ -689,8 +693,9 @@ impl NativeOwner { Ok(Self { pointer, capacity, - allocator, + allocator: Some(allocator), native: None, + _lifetime: None, }) } } @@ -705,10 +710,12 @@ impl Drop for NativeOwner { } // Return native admission before waking credit waiters. drop(native); - self.allocator - .retained_bytes - .fetch_sub(self.capacity, Ordering::AcqRel); - self.allocator.wake_credit_waiters(); + if let Some(allocator) = &self.allocator { + allocator + .retained_bytes + .fetch_sub(self.capacity, Ordering::AcqRel); + allocator.wake_credit_waiters(); + } } } diff --git a/lib/crowdb-access-s3/src/native_buffer/producer.rs b/lib/crowdb-access-s3/src/native_buffer/producer.rs new file mode 100644 index 000000000..27f515da7 --- /dev/null +++ b/lib/crowdb-access-s3/src/native_buffer/producer.rs @@ -0,0 +1,108 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Safe producer initialization into payload slots with reserved frame space. + +use super::{NativeFramedOwner, NativeOwner, NativePayloadView}; +use crowdb_protocol::frame::{ + FrameMagic, FRAME_FOOTER_BYTES, FRAME_HEADER_PREFIX_BYTES, MAX_FRAME_BYTES, MAX_FRAME_PAYLOAD_BYTES, +}; +use crowdb_rpc_ffi::WritableBuffer; +use hyper::body::Bytes; +use std::io; +use std::ptr::NonNull; +use std::sync::Arc; + +impl NativeFramedOwner { + /// Initializes disjoint payload slots in the producer's final allocation. + /// The callback receives the logical offset; it cannot retain mutable views. + /// `lifetime` remains owned until the final payload/physical view is released. + /// # Errors + /// Rejects empty input or a logical length exceeding complete physical slots. + /// # Panics + /// Propagates initializer panics, or a violated foreign allocation invariant. + pub fn initialize_producer( + mut buffer: WritableBuffer, + length: usize, + initial: u8, + lifetime: Box, + mut initialize: impl FnMut(usize, &mut [u8]), + ) -> io::Result { + let capacity = buffer.capacity(); + let pointer = NonNull::new(buffer.uninit_bytes_mut().as_mut_ptr().cast::()) + .expect("native producer allocation is non-null"); + let owner = NativeOwner { + pointer, + capacity, + allocator: None, + native: Some(buffer), + _lifetime: Some(lifetime), + }; + if length == 0 || length > capacity / MAX_FRAME_BYTES * MAX_FRAME_PAYLOAD_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid framed producer length", + )); + } + let mut lengths = Vec::with_capacity(length.div_ceil(MAX_FRAME_PAYLOAD_BYTES)); + for (index, offset) in (0..length).step_by(MAX_FRAME_PAYLOAD_BYTES).enumerate() { + let count = (length - offset).min(MAX_FRAME_PAYLOAD_BYTES); + // SAFETY: this unique producer owns all slots. Initialize each + // payload before exposing its mutable callback slice; reserved + // headers/footers stay inaccessible until one-time finalization. + let payload = unsafe { + let pointer = owner + .pointer + .as_ptr() + .add(index * MAX_FRAME_BYTES + FRAME_HEADER_PREFIX_BYTES); + pointer.write_bytes(initial, count); + std::slice::from_raw_parts_mut(pointer, count) + }; + initialize(offset, payload); + lengths.push(u16::try_from(count).expect("frame payload fits u16")); + } + let last = usize::from(*lengths.last().expect("nonempty payload")); + Ok(Self { + owner: Arc::new(owner), + logical_len: length as u64, + physical_len: (lengths.len() - 1) * MAX_FRAME_BYTES + + FRAME_HEADER_PREFIX_BYTES + + last + + FRAME_FOOTER_BYTES, + prepared_slots: 0, + prepared: false, + magic: FrameMagic::RepoLargeV1, + finalized_chunks: vec![None; lengths.len()].into_boxed_slice(), + payload_lengths: lengths.into_boxed_slice(), + }) + } + + #[must_use] + pub fn allocation_capacity(&self) -> usize { + self.owner.capacity + } + + #[must_use] + pub fn registration_id(&self) -> u32 { + self.owner + .native + .as_ref() + .map_or(0, WritableBuffer::registration_id) + } + + /// Immutable logical views for checksumming without materializing a copy. + #[must_use] + pub fn payload_views(&self) -> Vec { + self.payload_lengths + .iter() + .enumerate() + .map(|(index, length)| { + Bytes::from_owner(NativePayloadView { + owner: Arc::clone(&self.owner), + payload_offset: index * MAX_FRAME_BYTES + FRAME_HEADER_PREFIX_BYTES, + initialized: usize::from(*length), + }) + }) + .collect() + } +} diff --git a/lib/crowdb-sdk-native/src/admission.rs b/lib/crowdb-sdk-native/src/admission.rs index 887df9ba3..e3a86c456 100644 --- a/lib/crowdb-sdk-native/src/admission.rs +++ b/lib/crowdb-sdk-native/src/admission.rs @@ -81,6 +81,12 @@ pub(crate) struct Credit { amount: usize, } +impl Credit { + pub fn budget(&self) -> &Arc { + &self.budget + } +} + impl Drop for Credit { fn drop(&mut self) { self.budget.used.fetch_sub(self.amount, Ordering::AcqRel); diff --git a/lib/crowdb-sdk-native/src/buffer.rs b/lib/crowdb-sdk-native/src/buffer.rs index 457827334..e21a465b2 100644 --- a/lib/crowdb-sdk-native/src/buffer.rs +++ b/lib/crowdb-sdk-native/src/buffer.rs @@ -12,7 +12,9 @@ use crowdb_rpc_ffi::{Buffer, OwnedPayload, ProducerBufferAllocator, WritableBuff use crate::admission::Credit; use crate::{CapacityKind, Error, Result, RuntimeResources}; +mod producer; mod receive; +pub use producer::FrozenObjectBuffer; pub(crate) use receive::AllocationRegistry; #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -30,7 +32,7 @@ struct Backing { // Storage is dropped before admission is returned to another producer. bytes: Bytes, registration: Option, - _credit: Credit, + credit: Credit, } #[derive(Clone)] @@ -43,6 +45,10 @@ pub struct BufferLease { } impl BufferLease { + pub(crate) fn belongs_to(&self, resources: &RuntimeResources) -> bool { + let budget = self.backing.credit.budget(); + Arc::ptr_eq(budget, &resources.held) || Arc::ptr_eq(budget, &resources.producers) + } #[must_use] pub fn as_slice(&self) -> &[u8] { &self.backing.bytes[self.range.clone()] @@ -86,7 +92,7 @@ impl BufferLease { backing: Arc::new(Backing { bytes, registration: None, - _credit: credit, + credit, }), metadata, } @@ -190,7 +196,8 @@ impl RuntimeResources { .allocate(capacity) .ok_or(Error::Capacity(CapacityKind::Producers))?; let credit = self.producers.charge(buffer.capacity())?; + let lease = WritableLease { buffer, credit }; self.check_open()?; - Ok(WritableLease { buffer, credit }) + Ok(lease) } } diff --git a/lib/crowdb-sdk-native/src/buffer/producer.rs b/lib/crowdb-sdk-native/src/buffer/producer.rs new file mode 100644 index 000000000..5ae8aedb5 --- /dev/null +++ b/lib/crowdb-sdk-native/src/buffer/producer.rs @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Immutable framed producers retain runtime credits through physical completion. + +use super::WritableLease; +use crate::{Error, Result}; +use crowdb_access_s3::native_buffer::NativeFramedOwner; +use crowdb_chunk_client::FramedWriteBuffer; +use crowdb_protocol::frame::{MAX_FRAME_BYTES, MAX_FRAME_PAYLOAD_BYTES}; + +pub struct FrozenObjectBuffer { + pub(crate) owner: NativeFramedOwner, + budget: std::sync::Arc, +} + +impl FrozenObjectBuffer { + pub(crate) fn belongs_to(&self, resources: &crate::RuntimeResources) -> bool { + std::sync::Arc::ptr_eq(&self.budget, &resources.producers) + } + #[must_use] + pub fn length(&self) -> u64 { + self.owner.logical_len() + } + + #[must_use] + pub fn allocation_capacity(&self) -> usize { + self.owner.allocation_capacity() + } + + #[must_use] + pub fn registration_id(&self) -> u32 { + self.owner.registration_id() + } + + #[must_use] + pub fn payload_views(&self) -> Vec { + self.owner.payload_views() + } +} + +impl WritableLease { + /// Maximum logical payload with the storage headers and footers reserved. + #[must_use] + pub fn framed_capacity(&self) -> usize { + self.capacity() / MAX_FRAME_BYTES * MAX_FRAME_PAYLOAD_BYTES + } + + /// Prepares data directly in its final registered storage frame slots. + /// # Errors + /// Rejects empty or oversized input. The callback runs once per frame and + /// receives its logical offset, with only initialized payload bytes exposed. + pub fn initialize_framed( + self, + length: usize, + initial: u8, + initialize: impl FnMut(usize, &mut [u8]), + ) -> Result { + let budget = std::sync::Arc::clone(self.credit.budget()); + let owner = NativeFramedOwner::initialize_producer( + self.buffer, + length, + initial, + Box::new(self.credit), + initialize, + ) + .map_err(|_| Error::BufferRange)?; + Ok(FrozenObjectBuffer { owner, budget }) + } +} diff --git a/lib/crowdb-sdk-native/src/buffer/receive.rs b/lib/crowdb-sdk-native/src/buffer/receive.rs index 996998f5c..86961fd6a 100644 --- a/lib/crowdb-sdk-native/src/buffer/receive.rs +++ b/lib/crowdb-sdk-native/src/buffer/receive.rs @@ -171,7 +171,7 @@ impl RuntimeResources { key, owner: owner.clone(), }), - _credit: credit, + credit, }); let entry = self .allocations diff --git a/lib/crowdb-sdk-native/src/runtime.rs b/lib/crowdb-sdk-native/src/runtime.rs index a407ba907..adb0208b8 100644 --- a/lib/crowdb-sdk-native/src/runtime.rs +++ b/lib/crowdb-sdk-native/src/runtime.rs @@ -91,6 +91,18 @@ impl ClientRuntime { &self.resources } + /// Allocates unique producer storage from this runtime's data transport. + /// # Errors + /// Returns unsupported allocation, exhausted actual-capacity budget or shutdown. + pub fn allocate_producer(&self, capacity: u32) -> Result { + self.resources.check_open()?; + let allocator = + self.storage.chunks.producer_allocator().ok_or_else(|| { + Error::Configuration("data transport has no native producer allocator".into()) + })?; + self.resources.allocate(&allocator, capacity) + } + #[must_use] pub fn writer_group_count(&self) -> usize { self.groups.load().len() diff --git a/lib/crowdb-sdk-native/src/s3.rs b/lib/crowdb-sdk-native/src/s3.rs index 8e4806614..77865a8d3 100644 --- a/lib/crowdb-sdk-native/src/s3.rs +++ b/lib/crowdb-sdk-native/src/s3.rs @@ -17,12 +17,15 @@ mod locality; mod namespace; mod object; mod read; +mod write; pub use crowdb_access_s3::execution::{DeleteObjectFailure, DeleteObjectResult}; pub use namespace::{DeleteObjects, ListObject, ListObjects, ListOptions}; pub use body::ObjectBody; pub use config::S3ClientConfig; pub use crowdb_access_s3::condition::{EtagSelection, ReadPreconditions}; +pub use crowdb_access_s3::execution::{PutObjectInfo, S3WritePolicy}; +pub use crowdb_access_s3::metadata::UserMetadata; pub use crowdb_access_s3::range::ByteRange; pub use locality::{ ChunkLayoutObservation, FragmentPlacement, FragmentRole, LocalityOptions, ObjectExtent, ObjectLocality, @@ -30,12 +33,14 @@ pub use locality::{ }; pub use object::{ObjectMetadata, ResolvedObject}; pub use read::{GetObject, GetOptions}; +pub use write::{PutObject, PutOptions}; #[derive(Clone)] pub struct S3Client { runtime: ClientRuntime, execution: S3Executor, signer: Arc, + write_policy: S3WritePolicy, } impl S3Client { @@ -64,6 +69,27 @@ impl S3Client { if runtime.storage.chunk_type != ChunkType::S3 { return Err(Error::Configuration("S3 requires an S3 writer group".into())); } + let write_policy = config.write_policy.unwrap_or_else(|| { + S3WritePolicy::basic( + runtime + .storage + .policy + .object_limit + .min(16 * crowdb_protocol::frame::MAX_FRAME_PAYLOAD_BYTES), + ) + }); + if write_policy.small_object_limit > runtime.storage.policy.object_limit + || write_policy.large_write.client.chunk_type != ChunkType::S3 + { + return Err(Error::Configuration( + "S3 write policy exceeds its writer group".into(), + )); + } + write_policy + .large_write + .client + .validate() + .map_err(|error| Error::Configuration(error.to_string()))?; let storage = S3StorageClients { control: Arc::clone(&runtime.storage.control), metadata: Arc::new(ChunkKvMetadataStore::new(Arc::clone(&runtime.storage.metadata))), @@ -79,6 +105,7 @@ impl S3Client { execution: S3Executor::new(storage, config.tenant), signer: Arc::new(signer), runtime, + write_policy, }) } diff --git a/lib/crowdb-sdk-native/src/s3/config.rs b/lib/crowdb-sdk-native/src/s3/config.rs index 69235c48d..7d16df285 100644 --- a/lib/crowdb-sdk-native/src/s3/config.rs +++ b/lib/crowdb-sdk-native/src/s3/config.rs @@ -10,6 +10,8 @@ pub struct S3ClientConfig { pub small_write: Option, /// None shares a runtime-local key. Supply a stable key for cross-runtime cursors. pub continuation_key: Option>, + /// None uses the current S3 small writer and the standard large-write policy. + pub write_policy: Option, } impl From for S3ClientConfig { @@ -18,6 +20,7 @@ impl From for S3ClientConfig { tenant, small_write: None, continuation_key: None, + write_policy: None, } } } diff --git a/lib/crowdb-sdk-native/src/s3/object.rs b/lib/crowdb-sdk-native/src/s3/object.rs index 14c0e6e61..bd0c33ddf 100644 --- a/lib/crowdb-sdk-native/src/s3/object.rs +++ b/lib/crowdb-sdk-native/src/s3/object.rs @@ -48,6 +48,13 @@ impl S3Client { .selected_object(bucket, key) .await .map_err(Error::S3)?; + self.bind_object(selected) + } + + pub(super) fn bind_object( + &self, + selected: crowdb_access_s3::object::SelectedObject, + ) -> Result { let record = selected.record; let user_metadata = UserMetadata::decode(&record.attributes).map_err(|_| Error::S3(S3ErrorCode::InternalError))?; diff --git a/lib/crowdb-sdk-native/src/s3/write.rs b/lib/crowdb-sdk-native/src/s3/write.rs new file mode 100644 index 000000000..7c6462d98 --- /dev/null +++ b/lib/crowdb-sdk-native/src/s3/write.rs @@ -0,0 +1,268 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Bounded native PUT input, with publication only after durable completion. + +use super::{ObjectMetadata, PutObjectInfo, S3Client}; +use crate::buffer::{BufferLease, FrozenObjectBuffer}; +use crate::runtime::RequestLease; +use crate::{Error, OperationContext, Result}; +use crowdb_access_s3::execution::PreparedPut; +use crowdb_access_s3::integrity::SinglePartIntegrity; +use crowdb_access_s3::streaming::{PutErrorCode, PutOutcome}; +use crowdb_access_s3::S3ErrorCode; +use crowdb_chunk_client::ChunkIoWriter; +use crowdb_protocol::frame::MAX_FRAME_PAYLOAD_BYTES; + +#[derive(Default)] +pub struct PutOptions { + pub object: PutObjectInfo, + pub expected_md5: Option<[u8; 16]>, +} + +pub struct PutObject { + client: S3Client, + request: Option, + session: Option, + integrity: SinglePartIntegrity, + expected_md5: Option<[u8; 16]>, + expected_length: Option, + length: u64, + final_frame: bool, + failure: Option, +} + +impl S3Client { + /// Creates a pull-driven writer; no future message is needed to flush it. + /// # Errors + /// Returns admission, namespace or writer preparation failure. + pub async fn begin_put( + &self, + context: OperationContext, + bucket: &[u8], + key: &[u8], + options: PutOptions, + ) -> Result { + let request = self.runtime.resources.admit(context)?; + let expected_length = options.object.content_length; + let session = request + .run(async { + self.execution + .prepare_put(bucket, key, options.object, &self.write_policy) + .await + .map_err(Error::S3) + }) + .await?; + Ok(PutObject { + client: self.clone(), + request: Some(request), + session: Some(session), + integrity: SinglePartIntegrity::default(), + expected_md5: options.expected_md5, + expected_length, + length: 0, + final_frame: false, + failure: None, + }) + } + + /// Retains ordinary input using its actual allocation capacity. + /// # Errors + /// Returns capacity, write, integrity or uncertain publication failure. + pub async fn put_vec( + &self, + context: OperationContext, + bucket: &[u8], + key: &[u8], + data: Vec, + mut options: PutOptions, + ) -> Result { + let id = context.request_id.get(); + options.object.content_length.get_or_insert(data.len() as u64); + let mut put = self.begin_put(context, bucket, key, options).await?; + let lease = self.runtime.resources.retain_vec(data, id, id, 0, 0)?; + for start in (0..lease.as_slice().len()).step_by(1 << 20) { + put.write(lease.slice(start..(start + (1 << 20)).min(lease.as_slice().len()))?) + .await?; + } + drop(lease); + put.finish().await + } +} + +impl PutObject { + #[must_use] + pub fn accepted_bytes(&self) -> u64 { + self.length + } + + /// Accepts one owned input and applies the existing bounded storage adapter. + /// Cancellation or dropping this pending future terminates the writer. + /// Each call accepts at most 1 MiB; larger ordinary owners can be sliced. + /// # Errors + /// Returns length, cancellation or durable storage failure. + pub async fn write(&mut self, buffer: BufferLease) -> Result<()> { + if !buffer.belongs_to(&self.client.runtime.resources) { + return Err(Error::Configuration( + "input lease belongs to another runtime".into(), + )); + } + if buffer.as_slice().len() > 1 << 20 { + return Err(Error::BufferRange); + } + let next = self.check_input(buffer.as_slice().len() as u64)?; + if buffer.as_slice().is_empty() { + return Ok(()); + } + let bytes = buffer.into_bytes(); + let (request, mut session) = self.take_active()?; + let result = request + .run(async { + let writer = session.writer_mut(); + while !writer.require_data() && !writer.input_complete() { + writer.wait_for_capacity().await; + } + writer + .on_data(bytes.clone()) + .await + .map_err(|error| Error::Storage(error.to_string()))?; + Ok(()) + }) + .await; + self.restore(result, request, session)?; + self.integrity.update(&bytes); + self.length = next; + Ok(()) + } + + /// Submits final registered frame storage without copying its payload. + /// A small object must arrive in one frozen owner. Large inputs use full + /// payload frames except for their final owner; raw/framed boundaries align. + /// # Errors + /// Rejects an interior partial frame or returns cancellation/storage failure. + pub async fn write_framed(&mut self, buffer: FrozenObjectBuffer) -> Result<()> { + if !buffer.belongs_to(&self.client.runtime.resources) { + return Err(Error::Configuration( + "input producer belongs to another runtime".into(), + )); + } + let length = buffer.length(); + let next = self.check_input(length)?; + if self.length % MAX_FRAME_PAYLOAD_BYTES as u64 != 0 + || self.session.as_ref().is_some_and(|session| !session.is_large()) + && (self.length != 0 || self.expected_length != Some(length)) + { + return Err(Error::BufferRange); + } + let payload = buffer.payload_views(); + let (request, mut session) = self.take_active()?; + let result = request + .run(async { + let writer = session.writer_mut(); + while !writer.require_data() && !writer.input_complete() { + writer.wait_for_capacity().await; + } + writer + .on_framed_data(Box::new(buffer.owner)) + .await + .map_err(|error| Error::Storage(error.to_string()))?; + Ok(()) + }) + .await; + self.restore(result, request, session)?; + for bytes in &payload { + self.integrity.update(bytes); + } + self.final_frame = length % MAX_FRAME_PAYLOAD_BYTES as u64 != 0; + self.length = next; + Ok(()) + } + + /// Returns success only after data durability and object publication. + /// # Errors + /// Returns incomplete input, checksum failure or an uncertain publication. + pub async fn finish(mut self) -> Result { + let (request, mut session) = self.take_active()?; + if let Some(expected) = self.expected_length.filter(|expected| *expected != self.length) { + session.abort().await; + return Err(Error::Length { + expected, + delivered: self.length, + }); + } + let (etag, checksum) = self.integrity.finish(); + if self + .expected_md5 + .is_some_and(|expected| expected.as_slice() != checksum) + { + session.abort().await; + return Err(Error::Integrity); + } + let id = request.context().request_id.get(); + let selected = request + .run_mutation(async { + session + .finish(etag, checksum) + .await + .map_err(|outcome| map_outcome(outcome, id)) + }) + .await?; + Ok(self.client.bind_object(selected)?.metadata().clone()) + } + + fn check_input(&self, count: u64) -> Result { + if let Some(error) = &self.failure { + return Err(error.clone()); + } + let next = self.length.checked_add(count).ok_or(Error::BufferRange)?; + if self.final_frame || next > 5 << 40 || self.expected_length.is_some_and(|expected| next > expected) + { + return Err(Error::BufferRange); + } + Ok(next) + } + + fn take_active(&mut self) -> Result<(RequestLease, PreparedPut)> { + if let Some(error) = &self.failure { + return Err(error.clone()); + } + self.failure = Some(Error::Cancelled); + Ok(( + self.request.take().ok_or(Error::Cancelled)?, + self.session.take().ok_or(Error::Cancelled)?, + )) + } + + fn restore(&mut self, result: Result<()>, request: RequestLease, session: PreparedPut) -> Result<()> { + match result { + Ok(()) => { + self.request = Some(request); + self.session = Some(session); + self.failure = None; + Ok(()) + } + Err(error) => { + self.failure = Some(error.clone()); + Err(error) + } + } + } +} + +fn map_outcome(outcome: PutOutcome, request_id: u64) -> Error { + match outcome { + PutOutcome::Timeout => Error::UncertainMutation { + request_id, + reason: crate::error::MutationInterruption::Transport, + }, + PutOutcome::Error { code, message } => match code { + PutErrorCode::BodyRead => Error::S3(S3ErrorCode::InvalidRequest), + PutErrorCode::InvalidDigest + | PutErrorCode::BadDigest + | PutErrorCode::InvalidPayloadDigest + | PutErrorCode::PayloadMismatch => Error::Integrity, + _ => Error::Storage(message), + }, + PutOutcome::Success => Error::Storage("unexpected successful publication failure".into()), + } +} diff --git a/lib/crowdb-sdk-native/tests/common/write_storage.rs b/lib/crowdb-sdk-native/tests/common/write_storage.rs new file mode 100644 index 000000000..c2975bd70 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/common/write_storage.rs @@ -0,0 +1,257 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; + +use arc_swap::ArcSwap; +use async_trait::async_trait; +use bytes::Bytes; +use crossbeam_skiplist::SkipMap; +use crowdb_chunk_client::{ChunkAllocator, ChunkIoClient, DiskWriter, Result}; +use crowdb_chunk_kv_client::{ChunkKvClient, ChunkKvRpcTransport, ChunkKvRpcTransportConfig}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_protocol::chunkdb::rpc::*; +use crowdb_protocol::common::{ChunkId, DiskId}; +use crowdb_protocol::diskdb::rpc::Segment; +use crowdb_rpc_ffi::{ProducerBufferAllocator, RpcServer}; +use crowdb_sdk_native::runtime::RuntimeClients; +use crowdb_sdk_native::{ClientRuntime, RuntimeLimits}; + +type TestDiskAddress = (u64, u64, u64); + +pub struct TestWriteStorage { + chunks: SkipMap<(u64, u64), ArcSwap>, + pub data: SkipMap, + pub writes: AtomicUsize, + pub deletes: AtomicUsize, + pub fail_write: AtomicBool, + pub block_write: AtomicBool, + pub write_started: tokio::sync::Notify, + pub write_allowed: tokio::sync::Notify, + producer: ProducerBufferAllocator, +} + +impl TestWriteStorage { + pub fn new() -> Arc { + Arc::new(Self { + chunks: SkipMap::new(), + data: SkipMap::new(), + writes: AtomicUsize::new(0), + deletes: AtomicUsize::new(0), + fail_write: AtomicBool::new(false), + block_write: AtomicBool::new(false), + write_started: tokio::sync::Notify::new(), + write_allowed: tokio::sync::Notify::new(), + producer: ProducerBufferAllocator::new(Arc::new(RpcServer::new(None)), false), + }) + } + + fn chunk(&self, id: ChunkId) -> Chunk { + self.chunks + .get(&(id.high, id.low)) + .unwrap() + .value() + .load_full() + .as_ref() + .clone() + } + + fn update(&self, id: ChunkId, mutate: impl Fn(&mut Chunk)) -> Chunk { + let entry = self.chunks.get(&(id.high, id.low)).unwrap(); + entry.value().rcu(|previous| { + let mut next = previous.as_ref().clone(); + mutate(&mut next); + next.modify_ts += 1; + next + }); + self.chunk(id) + } + + pub fn runtime(self: &Arc, metadata: Arc) -> ClientRuntime { + ClientRuntime::from_clients( + RuntimeClients { + control: Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))), + metadata, + metadata_transport: Arc::new( + ChunkKvRpcTransport::with_config(8, 1, 1, &ChunkKvRpcTransportConfig::default()).unwrap(), + ), + chunks: Arc::new(ChunkIoClient::from_parts( + self.clone(), + self.clone(), + ChunkType::S3, + )), + }, + RuntimeLimits::default(), + ) + .unwrap() + } +} + +fn strip(id: ChunkId, index: u32, size: u32, copies: u32) -> ChunkStrip { + ChunkStrip { + unit_kb: size, + capacity: size, + chunk_offset: index * size, + strip_sequence: index, + strip_type: StripType::Mirror as i32, + strip: Some(Strip::MirrorStrip(MirrorStrip { + segments: (0..copies) + .map(|copy| Segment { + disk_id: Some(DiskId { + high: id.high ^ u64::from(copy), + low: id.low, + }), + unit_offset: u64::from(index), + unit_count: 1, + ..Segment::default() + }) + .collect(), + })), + ..ChunkStrip::default() + } +} + +#[async_trait] +impl ChunkAllocator for TestWriteStorage { + async fn allocate_chunk(&self, request: AllocateChunkRequest) -> Result { + assert_eq!(request.strip_type, StripType::Mirror as i32); + let id = request.chunk_id.unwrap(); + let chunk = Chunk { + id: Some(id), + modify_ts: 1, + state: ChunkState::Active as i32, + capacity: request.write_granularity * request.strip_count, + chunk_type: request.chunk_type, + strips: (0..request.strip_count) + .map(|index| strip(id, index, request.write_granularity, request.copy_count)) + .collect(), + ..Chunk::default() + }; + self.chunks + .insert((id.high, id.low), ArcSwap::from_pointee(chunk.clone())); + Ok(AllocateChunkResponse { chunk: Some(chunk) }) + } + async fn append_chunk(&self, request: AppendChunkRequest) -> Result { + let id = request.chunk_id.unwrap(); + let previous = self.chunk(id); + if previous.modify_ts != request.modify_ts { + return Ok(AppendChunkResponse { + modify_ts: previous.modify_ts, + chunk: Some(previous), + strips: vec![], + }); + } + let start = u32::try_from(previous.strips.len()).unwrap(); + let size = previous.strips[0].unit_kb * request.strip_size; + let strips: Vec<_> = (0..request.strip_count) + .map(|index| strip(id, start + index, size, request.copy_count)) + .collect(); + let chunk = self.update(id, |chunk| { + chunk.capacity += size * request.strip_count; + chunk.strips.extend(strips.clone()); + }); + Ok(AppendChunkResponse { + modify_ts: chunk.modify_ts, + chunk: None, + strips, + }) + } + async fn seal_chunk(&self, request: SealChunkRequest) -> Result { + let chunk = self.update(request.chunk_id.unwrap(), |chunk| { + chunk.state = ChunkState::Sealed as i32; + chunk.sealed_length = request.seal_length; + chunk.acknowledged_cursor = request.seal_bytes; + }); + Ok(SealChunkResponse { chunk: Some(chunk) }) + } + async fn delete_chunk(&self, request: DeleteChunkRequest) -> Result { + self.deletes.fetch_add(1, Ordering::AcqRel); + let chunk = self.update(request.chunk_id.unwrap(), |chunk| { + chunk.state = ChunkState::Deleted as i32; + }); + Ok(DeleteChunkResponse { chunk: Some(chunk) }) + } + async fn update_chunk_strip(&self, request: UpdateChunkStripRequest) -> Result { + let chunk = self.update(request.chunk_id.unwrap(), |chunk| { + chunk.strips[request.strip_index as usize] = request.strip.clone().unwrap(); + }); + Ok(UpdateChunkStripResponse { chunk: Some(chunk) }) + } + async fn query_chunk(&self, request: QueryChunkRequest) -> Result { + Ok(QueryChunkResponse { + chunk: Some(self.chunk(request.chunk_id.unwrap())), + layout_validity_ms: 60_000, + }) + } +} + +#[async_trait] +impl DiskWriter for TestWriteStorage { + fn producer_allocator(&self) -> Option { + Some(self.producer.clone()) + } + + async fn write(&self, segment: &Segment, unit_bytes: u64, data: Bytes) -> Result<()> { + self.write_views_at_byte_offset(segment, unit_bytes, 0, vec![data]) + .await + } + async fn write_at_byte_offset( + &self, + segment: &Segment, + unit_bytes: u64, + offset: u64, + data: Bytes, + ) -> Result<()> { + self.write_views_at_byte_offset(segment, unit_bytes, offset, vec![data]) + .await + } + async fn write_views_at_byte_offset( + &self, + segment: &Segment, + unit_bytes: u64, + offset: u64, + data: Vec, + ) -> Result<()> { + self.writes.fetch_add(1, Ordering::AcqRel); + self.write_started.notify_one(); + if self.block_write.load(Ordering::Acquire) { + self.write_allowed.notified().await; + } + if self.fail_write.load(Ordering::Acquire) { + return Err(crowdb_chunk_client::IoError::WriteFailed( + "injected disk failure".into(), + )); + } + let disk = segment.disk_id.unwrap(); + let mut offset = segment.unit_offset * unit_bytes + offset; + for view in data { + let count = view.len() as u64; + self.data.insert((disk.high, disk.low, offset), view); + offset += count; + } + Ok(()) + } + async fn read(&self, segment: &Segment, unit_bytes: u64, offset: u64, length: u32) -> Result { + let disk = segment.disk_id.unwrap(); + let start = segment.unit_offset * unit_bytes + offset; + let end = start + u64::from(length); + let mut output = vec![0; length as usize]; + for entry in &self.data { + let (high, low, offset) = *entry.key(); + if (high, low) != (disk.high, disk.low) { + continue; + } + let from = start.max(offset); + let to = end.min(offset + entry.value().len() as u64); + if from < to { + output[usize::try_from(from - start).unwrap()..usize::try_from(to - start).unwrap()] + .copy_from_slice( + &entry.value() + [usize::try_from(from - offset).unwrap()..usize::try_from(to - offset).unwrap()], + ); + } + } + Ok(Bytes::from(output)) + } +} diff --git a/lib/crowdb-sdk-native/tests/framed_producer_test.rs b/lib/crowdb-sdk-native/tests/framed_producer_test.rs new file mode 100644 index 000000000..89c4c3eec --- /dev/null +++ b/lib/crowdb-sdk-native/tests/framed_producer_test.rs @@ -0,0 +1,73 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_protocol::frame::{MAX_FRAME_BYTES, MAX_FRAME_PAYLOAD_BYTES}; +use crowdb_rpc_ffi::{ProducerBufferAllocator, RpcServer}; +use crowdb_sdk_native::{Error, RuntimeLimits, RuntimeResources}; + +fn resources() -> Arc { + RuntimeResources::new(RuntimeLimits { + producer_bytes: 2 * MAX_FRAME_BYTES, + ..RuntimeLimits::default() + }) + .unwrap() +} + +#[test] +fn immutable_framed_payloads_keep_actual_capacity_until_the_final_external_view() { + let resources = resources(); + let allocator = ProducerBufferAllocator::new(Arc::new(RpcServer::new(None)), false); + let writable = resources + .allocate(&allocator, u32::try_from(2 * MAX_FRAME_BYTES).unwrap()) + .unwrap(); + assert_eq!(writable.framed_capacity(), 2 * MAX_FRAME_PAYLOAD_BYTES); + let mut addresses = Vec::new(); + let frozen = writable + .initialize_framed(MAX_FRAME_PAYLOAD_BYTES + 7, 0x32, |offset, bytes| { + addresses.push(bytes.as_ptr() as usize); + bytes[0] = u8::from(offset != 0); + }) + .unwrap(); + let mut views = frozen.payload_views(); + assert_eq!(views[0].as_ptr() as usize, addresses[0]); + assert_eq!(views[1].as_ptr() as usize, addresses[1]); + assert_eq!(views[0][0], 0); + assert_eq!(views[1].as_ref(), &[1, 0x32, 0x32, 0x32, 0x32, 0x32, 0x32]); + assert_eq!(resources.status().producer_bytes, 2 * MAX_FRAME_BYTES); + resources.shutdown(); + drop(allocator); + drop(frozen); + assert_eq!(views.pop().unwrap()[0], 1); + assert_eq!(resources.status().producer_bytes, 2 * MAX_FRAME_BYTES); + drop(views); + assert_eq!(resources.status().producer_bytes, 0); +} + +#[test] +fn invalid_lengths_and_initializer_panics_release_native_storage_before_returning_credit() { + let resources = resources(); + let allocator = ProducerBufferAllocator::new(Arc::new(RpcServer::new(None)), false); + for length in [0, MAX_FRAME_PAYLOAD_BYTES + 1] { + let writable = resources + .allocate(&allocator, u32::try_from(MAX_FRAME_BYTES).unwrap()) + .unwrap(); + assert!(matches!( + writable.initialize_framed(length, 0, |_, _| {}), + Err(Error::BufferRange) + )); + assert_eq!(resources.status().producer_bytes, 0); + } + let writable = resources + .allocate(&allocator, u32::try_from(MAX_FRAME_BYTES).unwrap()) + .unwrap(); + let caught = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _ = writable.initialize_framed(7, 0, |_, _| panic!("test initializer panic")); + })); + assert!(caught.is_err()); + assert_eq!(resources.status().producer_bytes, 0); + assert!(resources + .allocate(&allocator, u32::try_from(MAX_FRAME_BYTES).unwrap()) + .is_ok()); +} diff --git a/lib/crowdb-sdk-native/tests/s3_write_test.rs b/lib/crowdb-sdk-native/tests/s3_write_test.rs new file mode 100644 index 000000000..6864a964e --- /dev/null +++ b/lib/crowdb-sdk-native/tests/s3_write_test.rs @@ -0,0 +1,262 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/metadata.rs"] +mod metadata; +#[path = "common/write_storage.rs"] +mod write_storage; + +use std::num::NonZeroU64; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crowdb_access_s3::metadata::TenantId; +use crowdb_protocol::frame::{MAX_FRAME_BYTES, MAX_FRAME_PAYLOAD_BYTES}; +use crowdb_sdk_native::s3::{GetOptions, PutObjectInfo, PutOptions, S3Client, S3ClientConfig, S3WritePolicy}; +use crowdb_sdk_native::{ClientRuntime, Error, OperationContext}; +use write_storage::TestWriteStorage; + +fn context(id: u64) -> OperationContext { + OperationContext::new( + NonZeroU64::new(id).unwrap(), + Instant::now() + Duration::from_secs(5), + ) +} + +async fn fixture(loss: Option>) -> (ClientRuntime, S3Client, Arc) { + let backend = TestWriteStorage::new(); + let metadata = match loss { + Some(loss) => metadata::client_with_loss(loss).await, + None => metadata::client().await, + }; + let runtime = backend.runtime(metadata); + let mut config = S3ClientConfig::from(TenantId::new(b"tenant".to_vec()).unwrap()); + let mut policy = S3WritePolicy::basic(0); + let large = Arc::make_mut(&mut policy.large_write.client); + large.large_mirror_copies = Some(1); + large.max_chunk_size = 2 << 20; + large.large_parallel_strip_writes = 1; + config.write_policy = Some(policy); + let client = S3Client::new(runtime.clone(), config).unwrap(); + client.create_bucket(context(1), b"bucket").await.unwrap(); + (runtime, client, backend) +} + +fn options(length: usize) -> PutOptions { + PutOptions { + object: PutObjectInfo { + content_length: Some(length as u64), + ..PutObjectInfo::default() + }, + ..PutOptions::default() + } +} + +async fn read(client: &S3Client, key: &[u8]) -> Vec { + let mut get = client + .get_object(context(900), b"bucket", key, &GetOptions::default()) + .await + .unwrap(); + let mut output = Vec::new(); + while let Some(lease) = get.body.next().await.unwrap() { + output.extend_from_slice(lease.as_slice()); + } + output +} + +#[tokio::test] +async fn ordinary_put_streams_mixed_sizes_and_returns_the_published_revision() { + let (runtime, client, _) = fixture(None).await; + for (index, length) in [ + 0, + 7, + MAX_FRAME_PAYLOAD_BYTES, + MAX_FRAME_PAYLOAD_BYTES + 1, + (1 << 20) + 13, + (2 << 20) + 31, + ] + .into_iter() + .enumerate() + { + let key = format!("size-{index}").into_bytes(); + let data = vec![u8::try_from(index).unwrap(); length]; + let result = client + .put_vec( + context(index as u64 + 10), + b"bucket", + &key, + data.clone(), + options(length), + ) + .await + .unwrap(); + let selected = client + .resolve_object(context(100), b"bucket", &key) + .await + .unwrap(); + assert_eq!(&result, selected.metadata()); + assert_eq!(result.logical_length, length as u64); + assert_eq!(read(&client, &key).await, data); + } + assert_eq!(runtime.resources().status().requests, 0); + assert_eq!(runtime.resources().status().held_bytes, 0); + runtime.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn framed_put_keeps_original_payload_addresses_through_disk_completion() { + let (runtime, client, backend) = fixture(None).await; + let length = 16 * MAX_FRAME_PAYLOAD_BYTES + 19; + let mut put = client + .begin_put(context(10), b"bucket", b"native", options(length)) + .await + .unwrap(); + let first = runtime + .allocate_producer(1 << 20) + .unwrap() + .initialize_framed(16 * MAX_FRAME_PAYLOAD_BYTES, 0x53, |offset, bytes| { + bytes[0] = u8::try_from(offset / MAX_FRAME_PAYLOAD_BYTES).unwrap(); + }) + .unwrap(); + let first_payload = first.payload_views()[0].as_ptr() as usize; + assert_eq!(first.allocation_capacity(), 1 << 20); + put.write_framed(first).await.unwrap(); + let last = runtime + .allocate_producer(u32::try_from(MAX_FRAME_BYTES).unwrap()) + .unwrap() + .initialize_framed(19, 0x61, |_, _| {}) + .unwrap(); + put.write_framed(last).await.unwrap(); + assert_eq!(put.accepted_bytes(), length as u64); + put.finish().await.unwrap(); + assert!(backend + .data + .iter() + .any(|entry| entry.value().as_ptr() as usize + 14 == first_payload)); + let data = read(&client, b"native").await; + assert_eq!(data.len(), length); + for index in 0..16 { + assert_eq!( + data[index * MAX_FRAME_PAYLOAD_BYTES], + u8::try_from(index).unwrap() + ); + assert!( + data[index * MAX_FRAME_PAYLOAD_BYTES + 1..(index + 1) * MAX_FRAME_PAYLOAD_BYTES] + .iter() + .all(|byte| *byte == 0x53) + ); + } + assert_eq!(&data[16 * MAX_FRAME_PAYLOAD_BYTES..], &[0x61; 19]); + runtime.shutdown().await.unwrap(); + assert!(runtime.resources().status().producer_bytes > 0); +} + +#[tokio::test] +async fn invalid_length_checksum_and_cancel_never_publish_an_object() { + let (runtime, client, _) = fixture(None).await; + let mut incomplete = client + .begin_put(context(10), b"bucket", b"short", options(7)) + .await + .unwrap(); + incomplete + .write(runtime.resources().retain_vec(vec![1; 6], 10, 10, 0, 0).unwrap()) + .await + .unwrap(); + assert_eq!( + incomplete.finish().await, + Err(Error::Length { + expected: 7, + delivered: 6 + }) + ); + let mut bad = options(7); + bad.expected_md5 = Some([0; 16]); + assert_eq!( + client + .put_vec(context(11), b"bucket", b"digest", vec![1; 7], bad) + .await, + Err(Error::Integrity) + ); + let context = context(12); + let mut cancelled = client + .begin_put(context.clone(), b"bucket", b"cancel", options(7)) + .await + .unwrap(); + context.cancellation.cancel(); + let input = runtime.resources().retain_vec(vec![1; 7], 12, 12, 0, 0).unwrap(); + assert_eq!(cancelled.write(input).await, Err(Error::Cancelled)); + assert_eq!(cancelled.finish().await, Err(Error::Cancelled)); + for key in [b"short".as_slice(), b"digest", b"cancel"] { + assert!(matches!( + client.resolve_object(self::context(30), b"bucket", key).await, + Err(Error::S3(crowdb_access_s3::S3ErrorCode::NoSuchKey)) + )); + } + assert_eq!(runtime.resources().status().requests, 0); + runtime.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn lost_publication_reply_preserves_uncertainty_and_durable_data() { + let loss = Arc::new(AtomicBool::new(false)); + let (runtime, client, backend) = fixture(Some(loss.clone())).await; + loss.store(true, Ordering::Release); + assert_eq!( + client + .put_vec( + context(41), + b"bucket", + b"uncertain", + b"applied".to_vec(), + options(7) + ) + .await, + Err(Error::UncertainMutation { + request_id: 41, + reason: crowdb_sdk_native::error::MutationInterruption::Transport + }) + ); + assert_eq!(backend.deletes.load(Ordering::Acquire), 0); + loss.store(false, Ordering::Release); + assert_eq!(read(&client, b"uncertain").await, b"applied"); + assert_eq!(runtime.resources().status().requests, 0); + runtime.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn dropping_a_pending_input_terminates_the_writer_and_releases_request_admission() { + let (runtime, client, backend) = fixture(None).await; + backend.block_write.store(true, Ordering::Release); + let length = 32 * MAX_FRAME_PAYLOAD_BYTES; + let mut put = client + .begin_put(context(81), b"bucket", b"dropped", options(length)) + .await + .unwrap(); + let first = runtime + .allocate_producer(1 << 20) + .unwrap() + .initialize_framed(16 * MAX_FRAME_PAYLOAD_BYTES, 0x61, |_, _| {}) + .unwrap(); + put.write_framed(first).await.unwrap(); + backend.write_started.notified().await; + let second = runtime + .allocate_producer(1 << 20) + .unwrap() + .initialize_framed(16 * MAX_FRAME_PAYLOAD_BYTES, 0x62, |_, _| {}) + .unwrap(); + let mut pending = Box::pin(put.write_framed(second)); + assert!(tokio::time::timeout(Duration::from_millis(20), &mut pending) + .await + .is_err()); + drop(pending); + assert_eq!(runtime.resources().status().requests, 0); + assert_eq!(put.finish().await, Err(Error::Cancelled)); + assert!(matches!( + client.resolve_object(context(82), b"bucket", b"dropped").await, + Err(Error::S3(crowdb_access_s3::S3ErrorCode::NoSuchKey)) + )); + backend.block_write.store(false, Ordering::Release); + backend.write_allowed.notify_waiters(); + runtime.shutdown().await.unwrap(); +} From 66d8084da71b379e2757ec092a58292b8336e5a7 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 15:02:37 +0800 Subject: [PATCH 069/258] Share verified S3 copies with generation-bound native operations --- .../src/s3/operations/copy.rs | 144 +++++--------- doc/working/plan-rdma-fat-client-sdk.md | 9 + lib/crowdb-access-s3/src/execution.rs | 2 + lib/crowdb-access-s3/src/execution/copy.rs | 153 +++++++++++++++ lib/crowdb-access-s3/src/execution/put.rs | 14 +- lib/crowdb-sdk-native/src/s3.rs | 3 + lib/crowdb-sdk-native/src/s3/copy.rs | 80 ++++++++ lib/crowdb-sdk-native/src/s3/write.rs | 2 +- .../tests/common/write_fixture.rs | 61 ++++++ lib/crowdb-sdk-native/tests/s3_copy_test.rs | 182 ++++++++++++++++++ lib/crowdb-sdk-native/tests/s3_write_test.rs | 71 ++----- 11 files changed, 559 insertions(+), 162 deletions(-) create mode 100644 lib/crowdb-access-s3/src/execution/copy.rs create mode 100644 lib/crowdb-sdk-native/src/s3/copy.rs create mode 100644 lib/crowdb-sdk-native/tests/common/write_fixture.rs create mode 100644 lib/crowdb-sdk-native/tests/s3_copy_test.rs diff --git a/app/crowdb-access-server/src/s3/operations/copy.rs b/app/crowdb-access-server/src/s3/operations/copy.rs index a8e177752..9be5a42ba 100644 --- a/app/crowdb-access-server/src/s3/operations/copy.rs +++ b/app/crowdb-access-server/src/s3/operations/copy.rs @@ -5,19 +5,16 @@ use crowdb_access_s3::condition::ObjectConditions; use crowdb_access_s3::error::S3ErrorCode; -use crowdb_access_s3::integrity::SinglePartIntegrity; +use crowdb_access_s3::execution::{CopyFailure, CopyMetadata, S3WritePolicy}; use crowdb_access_s3::metadata::{MultipartPartRecord, ObjectRecord}; -use crowdb_access_s3::publication::PublicationRequest; use crowdb_access_s3::route::{S3Operation, S3Route}; -use crowdb_access_s3::streaming::{publish_completed_locations, PutOutcome}; use crowdb_access_s3::{copy, retrieval, wire}; -use crowdb_chunk_client::{ChunkIoWriter, ProtoLocation}; use hyper::{body::Incoming, Request, Response}; use std::sync::Arc; use super::multipart::{map_multipart_error, parse_md5}; use super::{ - map_put_outcome, map_retrieval, required_bucket, required_key, strict_header, unix_millis, ObjectWriter, + map_put_outcome, map_retrieval, required_bucket, required_key, strict_header, unix_millis, ProductionS3Operations, ResponseBody, }; @@ -78,9 +75,8 @@ impl ProductionS3Operations { Some("REPLACE") => true, _ => return Err(S3ErrorCode::InvalidRequest), }; - let attributes = - crowdb_access_s3::metadata::UserMetadata::from_headers(request.headers())?.encode()?; - let mut record = self.copy_source(request, false).await?; + let attributes = crowdb_access_s3::metadata::UserMetadata::from_headers(request.headers())?; + let record = self.copy_source(request, false).await?; let destination = self.resolve_bucket(required_bucket(&route)?).await?; let key = required_key(&route)?.to_vec(); copy::validate_object(&record, destination, &key, replace)?; @@ -91,45 +87,40 @@ impl ProductionS3Operations { } else { record.content_type.clone() }; - let mut writer_key = self.config.tenant.as_bytes().to_vec(); - writer_key.extend_from_slice(destination.as_bytes()); - writer_key.extend_from_slice(&key); - let source = record.clone(); - record.bucket_id = destination; - record.key = key; - record.content_type = content_type; - if replace { - record.attributes = attributes; - } + let metadata = replace.then_some(CopyMetadata { + content_type, + user_metadata: attributes, + }); let operations = self.clone(); crate::s3::copy_body::response( - async move { operations.finish_object_copy(source, record, writer_key).await }, + async move { + let selected = operations + .execution + .copy_object_record( + &record, + destination, + &key, + metadata, + &S3WritePolicy { + small_object_limit: operations.config.small_object_limit, + large_write: operations.config.large_write.clone(), + }, + ) + .await + .map_err(|error| match error { + CopyFailure::S3(code) => code, + CopyFailure::Publication(outcome) => map_put_outcome(&outcome), + })?; + Ok(wire::copy_result( + &selected.record.etag, + selected.record.modified_at_ms, + false, + )) + }, request.uri().path().to_owned(), ) } - async fn finish_object_copy( - &self, - source: ObjectRecord, - mut record: ObjectRecord, - writer_key: Vec, - ) -> Result { - let (etag, checksum, locations) = self.stream_copy(&source, None, &writer_key).await?; - let now = unix_millis(); - record.etag = etag.clone(); - record.checksum = checksum; - record.created_at_ms = now; - record.modified_at_ms = now; - let mut publication = PublicationRequest { - tenant: self.config.tenant.clone(), - object: record, - }; - match publish_completed_locations(&self.storage.metadata, &mut publication, &locations).await { - PutOutcome::Success => Ok(wire::copy_result(&etag, now, false)), - outcome => Err(map_put_outcome(&outcome)), - } - } - pub(super) async fn upload_part_copy( self: &Arc, route: S3Route, @@ -157,9 +148,20 @@ impl ProductionS3Operations { let operations = self.clone(); crate::s3::copy_body::response( async move { - let (etag, _, locations) = operations - .stream_copy(&record, range.as_deref(), &writer_key) + let copied = operations + .execution + .stream_copy( + &record, + range.as_deref(), + &writer_key, + &S3WritePolicy { + small_object_limit: operations.config.small_object_limit, + large_write: operations.config.large_write.clone(), + }, + ) .await?; + let etag = copied.etag; + let locations = copied.locations; let now = unix_millis(); let part = MultipartPartRecord { bucket_id: session.bucket_id, @@ -182,62 +184,6 @@ impl ProductionS3Operations { request.uri().path().to_owned(), ) } - - async fn stream_copy( - &self, - record: &ObjectRecord, - range: Option<&str>, - key: &[u8], - ) -> Result<(String, Vec, Vec), S3ErrorCode> { - let mut prepared = - retrieval::prepare_get(&self.storage.chunks, record, range, &ObjectConditions::default()) - .map_err(|error| map_retrieval(&error))?; - let length = prepared.headers.content_length; - let mut writer = self.prepare_writer(Some(length), key).await?; - let result = feed_copy(&mut prepared.stream, &mut writer, length).await; - match result { - Ok((etag, checksum)) => { - let Ok(locations) = writer.on_finish().await else { - let _ = writer.on_error().await; - return Err(S3ErrorCode::ServiceUnavailable); - }; - Ok((etag, checksum, locations)) - } - Err(error) => { - let _ = writer.on_error().await; - Err(error) - } - } - } -} - -async fn feed_copy( - stream: &mut Option, - writer: &mut ObjectWriter, - length: u64, -) -> Result<(String, Vec), S3ErrorCode> { - let mut integrity = SinglePartIntegrity::default(); - let mut actual = 0_u64; - if let Some(stream) = stream { - while let Some(bytes) = stream.next_chunk().await { - let bytes = bytes.map_err(|_| S3ErrorCode::ServiceUnavailable)?; - actual = actual - .checked_add(bytes.len() as u64) - .ok_or(S3ErrorCode::InternalError)?; - integrity.update(&bytes); - while !writer.require_data() { - writer.wait_for_capacity().await; - } - writer - .on_data(bytes) - .await - .map_err(|_| S3ErrorCode::ServiceUnavailable)?; - } - } - if actual != length { - return Err(S3ErrorCode::InternalError); - } - Ok(integrity.finish()) } fn source_conditions(request: &Request) -> Result, S3ErrorCode> { diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index 51651561c..1f39282e2 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -339,6 +339,15 @@ transport acceptance alone cannot establish SDK completion. ## Immediate dependent implementation +- HTTP object copy and upload-part copy now use shared verified copy execution. + Native copy accepts ordinary source names or a tenant/authority-bound retained + selection; overwrites cannot replace that selection. Metadata COPY/REPLACE, + self-copy validation, conditions, source integrity, durable completion and + uncertain publication remain shared. Three native cases cover these contracts + and lost applied replies. Native/S3 all-target suites (41 native cases), + affected HTTP tests, focused clippy and formatting pass. Full service parity + and native multipart attachment remain pending. + - Native `begin_put`/`put_vec` now use the shared durable PUT session and return the publication's exact revision. Input calls accept at most 1 MiB and retain actual allocation capacity; foreign-runtime owners are rejected. Registered diff --git a/lib/crowdb-access-s3/src/execution.rs b/lib/crowdb-access-s3/src/execution.rs index 76c5bd49a..273a8e73f 100644 --- a/lib/crowdb-access-s3/src/execution.rs +++ b/lib/crowdb-access-s3/src/execution.rs @@ -3,9 +3,11 @@ //! Trusted S3 execution shared by HTTP parsing and native business handles. +mod copy; mod namespace; mod put; mod write; +pub use copy::{CompletedCopy, CopyFailure, CopyMetadata}; pub use namespace::{DeleteObjectFailure, DeleteObjectResult, DeleteObjectStream, ObjectListRequest}; pub use put::{PreparedPut, PutObjectInfo}; pub use write::{ObjectWriter, S3WritePolicy}; diff --git a/lib/crowdb-access-s3/src/execution/copy.rs b/lib/crowdb-access-s3/src/execution/copy.rs new file mode 100644 index 000000000..0b7dd7b52 --- /dev/null +++ b/lib/crowdb-access-s3/src/execution/copy.rs @@ -0,0 +1,153 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Bounded verified copies over a captured source generation. + +use super::{ObjectWriter, PutObjectInfo, S3Executor, S3WritePolicy}; +use crate::condition::ObjectConditions; +use crate::integrity::SinglePartIntegrity; +use crate::metadata::{BucketId, ObjectRecord, UserMetadata}; +use crate::object::SelectedObject; +use crate::retrieval::{self, RetrievalError, VerifiedGetStream}; +use crate::streaming::PutOutcome; +use crate::S3ErrorCode; +use crowdb_chunk_client::{ChunkIoWriter, ProtoLocation}; + +#[derive(Clone, Debug)] +pub struct CopyMetadata { + pub content_type: String, + pub user_metadata: UserMetadata, +} + +pub enum CopyFailure { + S3(S3ErrorCode), + Publication(PutOutcome), +} + +pub struct CompletedCopy { + pub etag: String, + pub checksum: Vec, + pub locations: Vec, +} + +impl S3Executor { + /// Copies one captured source and publishes only after verified durable data. + /// # Errors + /// Returns source validation, storage failure or uncertain publication. + pub async fn copy_object_record( + &self, + source: &ObjectRecord, + destination: BucketId, + key: &[u8], + replace: Option, + policy: &S3WritePolicy, + ) -> Result { + crate::copy::validate_object(source, destination, key, replace.is_some()).map_err(CopyFailure::S3)?; + let info = match replace { + Some(replace) => PutObjectInfo { + content_length: Some(source.logical_length), + content_type: replace.content_type, + user_metadata: replace.user_metadata, + }, + None => PutObjectInfo { + content_length: Some(source.logical_length), + content_type: source.content_type.clone(), + user_metadata: UserMetadata::decode(&source.attributes).map_err(CopyFailure::S3)?, + }, + }; + let mut prepared = + retrieval::prepare_get(&self.storage.chunks, source, None, &ObjectConditions::default()) + .map_err(|error| CopyFailure::S3(retrieval_error(&error)))?; + let mut session = self + .prepare_put_in_bucket(destination, key, info, policy) + .await + .map_err(CopyFailure::S3)?; + let result = feed_copy(&mut prepared.stream, session.writer_mut(), source.logical_length).await; + let (etag, checksum) = match result { + Ok(result) => result, + Err(error) => { + session.abort().await; + return Err(CopyFailure::S3(error)); + } + }; + session + .finish(etag, checksum) + .await + .map_err(CopyFailure::Publication) + } + + /// Prepares verified copied locations for a multipart part publication. + /// # Errors + /// Returns range, checksum, writer or durable completion failure. + pub async fn stream_copy( + &self, + source: &ObjectRecord, + range: Option<&str>, + key: &[u8], + policy: &S3WritePolicy, + ) -> Result { + let mut prepared = + retrieval::prepare_get(&self.storage.chunks, source, range, &ObjectConditions::default()) + .map_err(|error| retrieval_error(&error))?; + let mut writer = self + .prepare_writer(Some(prepared.headers.content_length), key, policy) + .await?; + let result = feed_copy(&mut prepared.stream, &mut writer, prepared.headers.content_length).await; + let (etag, checksum) = match result { + Ok(result) => result, + Err(error) => { + let _ = writer.on_error().await; + return Err(error); + } + }; + let Ok(locations) = writer.on_finish().await else { + let _ = writer.on_error().await; + return Err(S3ErrorCode::ServiceUnavailable); + }; + Ok(CompletedCopy { + etag, + checksum, + locations, + }) + } +} + +async fn feed_copy( + stream: &mut Option, + writer: &mut ObjectWriter, + length: u64, +) -> Result<(String, Vec), S3ErrorCode> { + let mut integrity = SinglePartIntegrity::default(); + let mut actual = 0u64; + if let Some(stream) = stream { + while let Some(bytes) = stream + .next_verified() + .await + .map_err(|_| S3ErrorCode::ServiceUnavailable)? + { + actual = actual + .checked_add(bytes.len() as u64) + .ok_or(S3ErrorCode::InternalError)?; + integrity.update(&bytes); + while !writer.require_data() { + writer.wait_for_capacity().await; + } + writer + .on_data(bytes) + .await + .map_err(|_| S3ErrorCode::ServiceUnavailable)?; + } + } + if actual != length { + return Err(S3ErrorCode::InternalError); + } + Ok(integrity.finish()) +} + +fn retrieval_error(error: &RetrievalError) -> S3ErrorCode { + match error { + RetrievalError::Range(_) => S3ErrorCode::InvalidRange, + RetrievalError::PreconditionFailed | RetrievalError::NotModified => S3ErrorCode::PreconditionFailed, + _ => S3ErrorCode::ServiceUnavailable, + } +} diff --git a/lib/crowdb-access-s3/src/execution/put.rs b/lib/crowdb-access-s3/src/execution/put.rs index 6f2560e65..b1d6916cf 100644 --- a/lib/crowdb-access-s3/src/execution/put.rs +++ b/lib/crowdb-access-s3/src/execution/put.rs @@ -4,7 +4,7 @@ //! Durable object completion followed by exactly one metadata publication. use super::{ObjectWriter, S3Executor, S3WritePolicy}; -use crate::metadata::{ChunkKvMetadataStore, MetadataKey, ObjectRecord, UserMetadata}; +use crate::metadata::{BucketId, ChunkKvMetadataStore, MetadataKey, ObjectRecord, UserMetadata}; use crate::object::SelectedObject; use crate::publication::{publish_versioned, PublicationRequest}; use crate::streaming::{attach_completed_locations, classify_publication_error, PutErrorCode, PutOutcome}; @@ -46,11 +46,21 @@ impl S3Executor { key: &[u8], info: PutObjectInfo, policy: &S3WritePolicy, + ) -> Result { + let bucket_id = self.resolve_bucket(bucket).await?; + self.prepare_put_in_bucket(bucket_id, key, info, policy).await + } + + pub(super) async fn prepare_put_in_bucket( + &self, + bucket_id: BucketId, + key: &[u8], + info: PutObjectInfo, + policy: &S3WritePolicy, ) -> Result { if info.content_type.is_empty() || info.content_length.is_some_and(|length| length > 5 << 40) { return Err(S3ErrorCode::InvalidRequest); } - let bucket_id = self.resolve_bucket(bucket).await?; MetadataKey::object(&self.tenant, bucket_id, key).map_err(|_| S3ErrorCode::InvalidRequest)?; let mut route_key = Vec::with_capacity(self.tenant.as_bytes().len() + 16 + key.len()); route_key.extend_from_slice(self.tenant.as_bytes()); diff --git a/lib/crowdb-sdk-native/src/s3.rs b/lib/crowdb-sdk-native/src/s3.rs index 77865a8d3..f5079dab0 100644 --- a/lib/crowdb-sdk-native/src/s3.rs +++ b/lib/crowdb-sdk-native/src/s3.rs @@ -13,6 +13,7 @@ use std::sync::Arc; mod body; mod config; +mod copy; mod locality; mod namespace; mod object; @@ -23,7 +24,9 @@ pub use namespace::{DeleteObjects, ListObject, ListObjects, ListOptions}; pub use body::ObjectBody; pub use config::S3ClientConfig; +pub use copy::CopyOptions; pub use crowdb_access_s3::condition::{EtagSelection, ReadPreconditions}; +pub use crowdb_access_s3::execution::CopyMetadata; pub use crowdb_access_s3::execution::{PutObjectInfo, S3WritePolicy}; pub use crowdb_access_s3::metadata::UserMetadata; pub use crowdb_access_s3::range::ByteRange; diff --git a/lib/crowdb-sdk-native/src/s3/copy.rs b/lib/crowdb-sdk-native/src/s3/copy.rs new file mode 100644 index 000000000..efa7278f0 --- /dev/null +++ b/lib/crowdb-sdk-native/src/s3/copy.rs @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{CopyMetadata, ObjectMetadata, ReadPreconditions, ResolvedObject, S3Client}; +use crate::{Error, OperationContext, Result}; +use crowdb_access_s3::condition::{ConditionOutcome, ReadCondition}; +use crowdb_access_s3::execution::CopyFailure; +use crowdb_access_s3::S3ErrorCode; + +#[derive(Default)] +pub struct CopyOptions { + pub source_conditions: ReadPreconditions, + /// None preserves source metadata; Some replaces it and allows self-copy. + pub metadata: Option, +} + +impl S3Client { + /// Selects one source generation, then copies it through the shared engine. + /// # Errors + /// Returns namespace, condition, storage or uncertain publication failure. + pub async fn copy_object( + &self, + context: OperationContext, + source_bucket: &[u8], + source_key: &[u8], + destination_bucket: &[u8], + destination_key: &[u8], + options: CopyOptions, + ) -> Result { + let source = self + .resolve_object(context.clone(), source_bucket, source_key) + .await?; + Box::pin(self.copy_resolved(context, &source, destination_bucket, destination_key, options)).await + } + + /// Copies the retained source generation even after its mutable name changes. + /// # Errors + /// Returns authority/condition, storage or uncertain publication failure. + pub async fn copy_resolved( + &self, + context: OperationContext, + source: &ResolvedObject, + destination_bucket: &[u8], + destination_key: &[u8], + options: CopyOptions, + ) -> Result { + self.validate_selection(source)?; + if options.source_conditions.evaluate(&source.record) != ConditionOutcome::Proceed { + return Err(Error::S3(S3ErrorCode::PreconditionFailed)); + } + let id = context.request_id.get(); + let request = self.runtime.resources.admit(context)?; + let destination = request + .run(async { + self.execution + .resolve_bucket(destination_bucket) + .await + .map_err(Error::S3) + }) + .await?; + let selected = request + .run_mutation(async { + self.execution + .copy_object_record( + &source.record, + destination, + destination_key, + options.metadata, + &self.write_policy, + ) + .await + .map_err(|error| match error { + CopyFailure::S3(code) => Error::S3(code), + CopyFailure::Publication(outcome) => super::write::map_outcome(outcome, id), + }) + }) + .await?; + Ok(self.bind_object(selected)?.metadata().clone()) + } +} diff --git a/lib/crowdb-sdk-native/src/s3/write.rs b/lib/crowdb-sdk-native/src/s3/write.rs index 7c6462d98..c4dd5fa8c 100644 --- a/lib/crowdb-sdk-native/src/s3/write.rs +++ b/lib/crowdb-sdk-native/src/s3/write.rs @@ -249,7 +249,7 @@ impl PutObject { } } -fn map_outcome(outcome: PutOutcome, request_id: u64) -> Error { +pub(super) fn map_outcome(outcome: PutOutcome, request_id: u64) -> Error { match outcome { PutOutcome::Timeout => Error::UncertainMutation { request_id, diff --git a/lib/crowdb-sdk-native/tests/common/write_fixture.rs b/lib/crowdb-sdk-native/tests/common/write_fixture.rs new file mode 100644 index 000000000..15ece5e28 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/common/write_fixture.rs @@ -0,0 +1,61 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{metadata, write_storage}; +use std::num::NonZeroU64; +use std::sync::atomic::AtomicBool; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crowdb_access_s3::metadata::TenantId; +use crowdb_sdk_native::s3::{GetOptions, PutObjectInfo, PutOptions, S3Client, S3ClientConfig, S3WritePolicy}; +use crowdb_sdk_native::{ClientRuntime, OperationContext}; +use write_storage::TestWriteStorage; + +pub fn context(id: u64) -> OperationContext { + OperationContext::new( + NonZeroU64::new(id).unwrap(), + Instant::now() + Duration::from_secs(5), + ) +} + +pub async fn fixture(loss: Option>) -> (ClientRuntime, S3Client, Arc) { + let backend = TestWriteStorage::new(); + let metadata = match loss { + Some(loss) => metadata::client_with_loss(loss).await, + None => metadata::client().await, + }; + let runtime = backend.runtime(metadata); + let mut config = S3ClientConfig::from(TenantId::new(b"tenant".to_vec()).unwrap()); + let mut policy = S3WritePolicy::basic(0); + let large = Arc::make_mut(&mut policy.large_write.client); + large.large_mirror_copies = Some(1); + large.max_chunk_size = 2 << 20; + large.large_parallel_strip_writes = 1; + config.write_policy = Some(policy); + let client = S3Client::new(runtime.clone(), config).unwrap(); + client.create_bucket(context(1), b"bucket").await.unwrap(); + (runtime, client, backend) +} + +pub fn options(length: usize) -> PutOptions { + PutOptions { + object: PutObjectInfo { + content_length: Some(length as u64), + ..PutObjectInfo::default() + }, + ..PutOptions::default() + } +} + +pub async fn read(client: &S3Client, key: &[u8]) -> Vec { + let mut get = client + .get_object(context(900), b"bucket", key, &GetOptions::default()) + .await + .unwrap(); + let mut output = Vec::new(); + while let Some(lease) = get.body.next().await.unwrap() { + output.extend_from_slice(lease.as_slice()); + } + output +} diff --git a/lib/crowdb-sdk-native/tests/s3_copy_test.rs b/lib/crowdb-sdk-native/tests/s3_copy_test.rs new file mode 100644 index 000000000..a1a81a8ed --- /dev/null +++ b/lib/crowdb-sdk-native/tests/s3_copy_test.rs @@ -0,0 +1,182 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/write_fixture.rs"] +mod fixture; +#[path = "common/metadata.rs"] +mod metadata; +#[path = "common/write_storage.rs"] +mod write_storage; + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use crowdb_access_s3::S3ErrorCode; +use crowdb_sdk_native::s3::{CopyMetadata, CopyOptions, EtagSelection, ReadPreconditions, UserMetadata}; +use crowdb_sdk_native::Error; +use fixture::{context, fixture, options, read}; + +#[tokio::test] +async fn copy_keeps_selected_generation_and_replaces_metadata_only_when_requested() { + let (runtime, client, _) = fixture(None).await; + let mut source = options(8); + source.object.content_type = "application/source".into(); + source.object.user_metadata = + UserMetadata::from_values(BTreeMap::from([("owner".into(), "original".into())])).unwrap(); + let original = client + .put_vec(context(10), b"bucket", b"source", b"original".to_vec(), source) + .await + .unwrap(); + let selected = client + .resolve_object(context(11), b"bucket", b"source") + .await + .unwrap(); + client + .put_vec(context(12), b"bucket", b"source", b"new".to_vec(), options(3)) + .await + .unwrap(); + let copied = client + .copy_resolved(context(13), &selected, b"bucket", b"old", CopyOptions::default()) + .await + .unwrap(); + assert_eq!(read(&client, b"old").await, b"original"); + assert_eq!(copied.content_type, original.content_type); + assert_eq!(copied.user_metadata, original.user_metadata); + assert_eq!(copied.checksum, original.checksum); + assert_eq!( + copied.revision, + client + .resolve_object(context(14), b"bucket", b"old") + .await + .unwrap() + .metadata() + .revision + ); + assert_eq!( + client + .copy_resolved( + context(15), + &selected, + b"bucket", + b"source", + CopyOptions::default() + ) + .await, + Err(Error::S3(S3ErrorCode::InvalidRequest)) + ); + let replacement = CopyOptions { + metadata: Some(CopyMetadata { + content_type: "application/replaced".into(), + user_metadata: UserMetadata::default(), + }), + ..CopyOptions::default() + }; + let copied = client + .copy_resolved(context(16), &selected, b"bucket", b"source", replacement) + .await + .unwrap(); + assert_eq!(copied.content_type, "application/replaced"); + assert!(copied.user_metadata.values().is_empty()); + assert_eq!(read(&client, b"source").await, b"original"); + runtime.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn copy_conditions_and_corrupt_source_stop_before_destination_publication() { + let (runtime, client, backend) = fixture(None).await; + client + .put_vec( + context(10), + b"bucket", + b"source", + b"original".to_vec(), + options(8), + ) + .await + .unwrap(); + let before = backend.writes.load(Ordering::Acquire); + let conditions = CopyOptions { + source_conditions: ReadPreconditions { + if_match: Some(EtagSelection::Values(vec!["wrong".into()])), + ..ReadPreconditions::default() + }, + ..CopyOptions::default() + }; + assert_eq!( + client + .copy_object( + context(11), + b"bucket", + b"source", + b"bucket", + b"condition", + conditions + ) + .await, + Err(Error::S3(S3ErrorCode::PreconditionFailed)) + ); + assert_eq!(backend.writes.load(Ordering::Acquire), before); + let entry = backend.data.front().unwrap(); + let key = *entry.key(); + let mut corrupt = entry.value().to_vec(); + corrupt[14] ^= 1; + drop(entry); + backend.data.insert(key, corrupt.into()); + assert_eq!( + client + .copy_object( + context(12), + b"bucket", + b"source", + b"bucket", + b"corrupt", + CopyOptions::default() + ) + .await, + Err(Error::S3(S3ErrorCode::ServiceUnavailable)) + ); + assert!(matches!( + client.resolve_object(context(13), b"bucket", b"corrupt").await, + Err(Error::S3(S3ErrorCode::NoSuchKey)) + )); + assert_eq!(runtime.resources().status().requests, 0); + runtime.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn copied_data_remains_recoverable_after_a_lost_publication_reply() { + let loss = Arc::new(AtomicBool::new(false)); + let (runtime, client, backend) = fixture(Some(loss.clone())).await; + client + .put_vec( + context(10), + b"bucket", + b"source", + b"original".to_vec(), + options(8), + ) + .await + .unwrap(); + loss.store(true, Ordering::Release); + assert_eq!( + client + .copy_object( + context(51), + b"bucket", + b"source", + b"bucket", + b"uncertain", + CopyOptions::default() + ) + .await, + Err(Error::UncertainMutation { + request_id: 51, + reason: crowdb_sdk_native::error::MutationInterruption::Transport + }) + ); + loss.store(false, Ordering::Release); + assert_eq!(read(&client, b"uncertain").await, b"original"); + assert_eq!(backend.deletes.load(Ordering::Acquire), 0); + runtime.shutdown().await.unwrap(); +} diff --git a/lib/crowdb-sdk-native/tests/s3_write_test.rs b/lib/crowdb-sdk-native/tests/s3_write_test.rs index 6864a964e..9e8d9adc3 100644 --- a/lib/crowdb-sdk-native/tests/s3_write_test.rs +++ b/lib/crowdb-sdk-native/tests/s3_write_test.rs @@ -6,64 +6,15 @@ mod metadata; #[path = "common/write_storage.rs"] mod write_storage; -use std::num::NonZeroU64; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use std::time::{Duration, Instant}; +#[path = "common/write_fixture.rs"] +mod fixture; -use crowdb_access_s3::metadata::TenantId; use crowdb_protocol::frame::{MAX_FRAME_BYTES, MAX_FRAME_PAYLOAD_BYTES}; -use crowdb_sdk_native::s3::{GetOptions, PutObjectInfo, PutOptions, S3Client, S3ClientConfig, S3WritePolicy}; -use crowdb_sdk_native::{ClientRuntime, Error, OperationContext}; -use write_storage::TestWriteStorage; - -fn context(id: u64) -> OperationContext { - OperationContext::new( - NonZeroU64::new(id).unwrap(), - Instant::now() + Duration::from_secs(5), - ) -} - -async fn fixture(loss: Option>) -> (ClientRuntime, S3Client, Arc) { - let backend = TestWriteStorage::new(); - let metadata = match loss { - Some(loss) => metadata::client_with_loss(loss).await, - None => metadata::client().await, - }; - let runtime = backend.runtime(metadata); - let mut config = S3ClientConfig::from(TenantId::new(b"tenant".to_vec()).unwrap()); - let mut policy = S3WritePolicy::basic(0); - let large = Arc::make_mut(&mut policy.large_write.client); - large.large_mirror_copies = Some(1); - large.max_chunk_size = 2 << 20; - large.large_parallel_strip_writes = 1; - config.write_policy = Some(policy); - let client = S3Client::new(runtime.clone(), config).unwrap(); - client.create_bucket(context(1), b"bucket").await.unwrap(); - (runtime, client, backend) -} - -fn options(length: usize) -> PutOptions { - PutOptions { - object: PutObjectInfo { - content_length: Some(length as u64), - ..PutObjectInfo::default() - }, - ..PutOptions::default() - } -} - -async fn read(client: &S3Client, key: &[u8]) -> Vec { - let mut get = client - .get_object(context(900), b"bucket", key, &GetOptions::default()) - .await - .unwrap(); - let mut output = Vec::new(); - while let Some(lease) = get.body.next().await.unwrap() { - output.extend_from_slice(lease.as_slice()); - } - output -} +use crowdb_sdk_native::Error; +use fixture::{context, fixture, options, read}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; #[tokio::test] async fn ordinary_put_streams_mixed_sizes_and_returns_the_published_revision() { @@ -178,18 +129,18 @@ async fn invalid_length_checksum_and_cancel_never_publish_an_object() { .await, Err(Error::Integrity) ); - let context = context(12); + let cancellation_context = context(12); let mut cancelled = client - .begin_put(context.clone(), b"bucket", b"cancel", options(7)) + .begin_put(cancellation_context.clone(), b"bucket", b"cancel", options(7)) .await .unwrap(); - context.cancellation.cancel(); + cancellation_context.cancellation.cancel(); let input = runtime.resources().retain_vec(vec![1; 7], 12, 12, 0, 0).unwrap(); assert_eq!(cancelled.write(input).await, Err(Error::Cancelled)); assert_eq!(cancelled.finish().await, Err(Error::Cancelled)); for key in [b"short".as_slice(), b"digest", b"cancel"] { assert!(matches!( - client.resolve_object(self::context(30), b"bucket", key).await, + client.resolve_object(context(30), b"bucket", key).await, Err(Error::S3(crowdb_access_s3::S3ErrorCode::NoSuchKey)) )); } From d5833842decfaf3218f5e4f9ca4f18ee2a59c042 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 15:22:15 +0800 Subject: [PATCH 070/258] Share native multipart execution and preserve uncertain publication --- app/crowdb-access-server/src/s3/operations.rs | 21 +- .../src/s3/operations/copy.rs | 40 +-- .../src/s3/operations/multipart.rs | 181 +++-------- doc/working/plan-rdma-fat-client-sdk.md | 18 +- lib/crowdb-access-s3/src/execution.rs | 2 + .../src/execution/multipart.rs | 201 ++++++++++++ .../src/execution/multipart/part.rs | 157 +++++++++ lib/crowdb-access-s3/src/integrity.rs | 6 + lib/crowdb-sdk-native/src/s3.rs | 5 + lib/crowdb-sdk-native/src/s3/multipart.rs | 168 ++++++++++ .../src/s3/multipart/listing.rs | 119 +++++++ .../src/s3/multipart/part.rs | 130 ++++++++ lib/crowdb-sdk-native/src/s3/write.rs | 54 +++- lib/crowdb-sdk-native/src/s3/write/session.rs | 69 ++++ .../tests/common/metadata.rs | 56 +++- .../tests/common/write_fixture.rs | 8 +- .../tests/s3_multipart_test.rs | 305 ++++++++++++++++++ 17 files changed, 1339 insertions(+), 201 deletions(-) create mode 100644 lib/crowdb-access-s3/src/execution/multipart.rs create mode 100644 lib/crowdb-access-s3/src/execution/multipart/part.rs create mode 100644 lib/crowdb-sdk-native/src/s3/multipart.rs create mode 100644 lib/crowdb-sdk-native/src/s3/multipart/listing.rs create mode 100644 lib/crowdb-sdk-native/src/s3/multipart/part.rs create mode 100644 lib/crowdb-sdk-native/src/s3/write/session.rs create mode 100644 lib/crowdb-sdk-native/tests/s3_multipart_test.rs diff --git a/app/crowdb-access-server/src/s3/operations.rs b/app/crowdb-access-server/src/s3/operations.rs index 137a845d3..284b5004c 100644 --- a/app/crowdb-access-server/src/s3/operations.rs +++ b/app/crowdb-access-server/src/s3/operations.rs @@ -10,9 +10,7 @@ use base64::Engine as _; use crowdb_access_s3::condition::ObjectConditions; use crowdb_access_s3::continuation::ContinuationTokenSigner; use crowdb_access_s3::error::{S3Error, S3ErrorCode}; -use crowdb_access_s3::execution::{ - ObjectListRequest, ObjectWriter, PutObjectInfo, S3Executor, S3WritePolicy, -}; +use crowdb_access_s3::execution::{ObjectListRequest, PutObjectInfo, S3Executor, S3WritePolicy}; use crowdb_access_s3::metadata::{ObjectRecord, TenantId}; use crowdb_access_s3::retrieval::{ObjectHeaders, RetrievalError}; use crowdb_access_s3::route::{S3Operation, S3Route}; @@ -382,23 +380,6 @@ impl ProductionS3Operations { .object(required_bucket(route)?, required_key(route)?) .await } - - async fn prepare_writer( - &self, - content_length: Option, - key: &[u8], - ) -> Result { - self.execution - .prepare_writer( - content_length, - key, - &S3WritePolicy { - small_object_limit: self.config.small_object_limit, - large_write: self.config.large_write.clone(), - }, - ) - .await - } } impl S3Operations for ProductionS3Operations { diff --git a/app/crowdb-access-server/src/s3/operations/copy.rs b/app/crowdb-access-server/src/s3/operations/copy.rs index 9be5a42ba..910c28e15 100644 --- a/app/crowdb-access-server/src/s3/operations/copy.rs +++ b/app/crowdb-access-server/src/s3/operations/copy.rs @@ -6,16 +6,15 @@ use crowdb_access_s3::condition::ObjectConditions; use crowdb_access_s3::error::S3ErrorCode; use crowdb_access_s3::execution::{CopyFailure, CopyMetadata, S3WritePolicy}; -use crowdb_access_s3::metadata::{MultipartPartRecord, ObjectRecord}; +use crowdb_access_s3::metadata::ObjectRecord; use crowdb_access_s3::route::{S3Operation, S3Route}; use crowdb_access_s3::{copy, retrieval, wire}; use hyper::{body::Incoming, Request, Response}; use std::sync::Arc; -use super::multipart::{map_multipart_error, parse_md5}; use super::{ - map_put_outcome, map_retrieval, required_bucket, required_key, strict_header, unix_millis, - ProductionS3Operations, ResponseBody, + map_put_outcome, map_retrieval, required_bucket, required_key, strict_header, ProductionS3Operations, + ResponseBody, }; impl ProductionS3Operations { @@ -140,45 +139,26 @@ impl ProductionS3Operations { if length > session.max_part_bytes { return Err(S3ErrorCode::InvalidRequest); } - let mut writer_key = self.config.tenant.as_bytes().to_vec(); - writer_key.extend_from_slice(session.bucket_id.as_bytes()); - writer_key.extend_from_slice(&session.upload_id); - writer_key.extend_from_slice(&number.to_be_bytes()); let range = range.map(str::to_owned); let operations = self.clone(); crate::s3::copy_body::response( async move { - let copied = operations + let part = operations .execution - .stream_copy( + .copy_part_record( &record, + &session, + number, range.as_deref(), - &writer_key, &S3WritePolicy { small_object_limit: operations.config.small_object_limit, large_write: operations.config.large_write.clone(), }, ) - .await?; - let etag = copied.etag; - let locations = copied.locations; - let now = unix_millis(); - let part = MultipartPartRecord { - bucket_id: session.bucket_id, - upload_id: session.upload_id, - number, - revision: 1, - modified_ms: now, - length, - raw_md5: parse_md5(&etag)?, - locations, - }; - operations - .multipart() - .put_stream_part(&session, &part, now) .await - .map_err(|error| map_multipart_error(&error))? - .ok_or(S3ErrorCode::SlowDown)?; + .map_err(|error| error.code())?; + let etag = crowdb_access_s3::integrity::raw_md5_etag(part.raw_md5); + let now = part.modified_ms; Ok(wire::copy_result(&etag, now, true)) }, request.uri().path().to_owned(), diff --git a/app/crowdb-access-server/src/s3/operations/multipart.rs b/app/crowdb-access-server/src/s3/operations/multipart.rs index 6c69195e0..e2b257db2 100644 --- a/app/crowdb-access-server/src/s3/operations/multipart.rs +++ b/app/crowdb-access-server/src/s3/operations/multipart.rs @@ -5,14 +5,13 @@ use crowdb_access_s3::bucket; use crowdb_access_s3::error::S3ErrorCode; +use crowdb_access_s3::execution::{CopyMetadata, S3WritePolicy}; use crowdb_access_s3::integrity::{IntegrityError, SinglePartIntegrity}; use crowdb_access_s3::metadata::{ - new_upload_id, CompletionPart, MultipartPartRecord, MultipartPhase, MultipartRepository, - MultipartRepositoryError, MultipartSessionRecord, + CompletionPart, MultipartRepository, MultipartRepositoryError, MultipartSessionRecord, }; use crowdb_access_s3::route::{S3Operation, S3Route}; use crowdb_access_s3::wire; -use crowdb_chunk_client::ChunkIoWriter; use http_body_util::BodyExt as _; use hyper::body::Bytes; use hyper::body::Incoming; @@ -26,11 +25,6 @@ use super::{ }; use crate::multipart_complete::{CompleteRequestError, CompleteSelection}; -const MAX_PARTS: u16 = 10_000; -const MAX_PART_BYTES: u64 = 5 * 1024 * 1024 * 1024; -const MAX_OBJECT_BYTES: u64 = 5 * 1024 * 1024 * 1024 * 1024; -const MAX_STAGED_BYTES: u64 = 10_000 * MAX_PART_BYTES; -const UPLOAD_LIFETIME_MS: u64 = 7 * 24 * 60 * 60 * 1000; const MAX_COMPLETE_BODY: usize = 2 * 1024 * 1024; impl ProductionS3Operations { @@ -67,34 +61,22 @@ impl ProductionS3Operations { } pub(super) fn multipart(&self) -> MultipartRepository { - MultipartRepository::new(self.storage.metadata.clone(), self.config.tenant.clone()) + self.execution.multipart_repository() } pub(super) async fn multipart_identity( &self, route: &S3Route, ) -> Result { - let bucket = self.resolve_bucket(required_bucket(route)?).await?; - let key = required_key(route)?; - let upload_id = route.upload_id.ok_or(S3ErrorCode::InvalidRequest)?; - let session = self - .multipart() - .load_identity(bucket, key, &upload_id) + self.execution + .multipart_session( + required_bucket(route)?, + required_key(route)?, + &route.upload_id.ok_or(S3ErrorCode::InvalidRequest)?, + route.operation == S3Operation::AbortMultipartUpload, + ) .await - .map_err(|error| map_multipart_error(&error))? - .ok_or(S3ErrorCode::NoSuchUpload)?; - if session.phase == MultipartPhase::Open && session.expires_ms <= unix_millis() { - let expired = self - .multipart() - .abort(&session) - .await - .map_err(|error| map_multipart_error(&error))?; - if route.operation != S3Operation::AbortMultipartUpload { - return Err(S3ErrorCode::NoSuchUpload); - } - return Ok(expired); - } - Ok(session) + .map_err(|error| error.code()) } pub(super) async fn create_multipart_upload( @@ -104,43 +86,23 @@ impl ProductionS3Operations { ) -> Result, S3ErrorCode> { let bucket_name = required_bucket(&route)?; let key = required_key(&route)?; - let bucket_id = self.resolve_bucket(bucket_name).await?; if content_length(request)?.is_some_and(|length| length != 0) { return Err(S3ErrorCode::InvalidRequest); } - let now = unix_millis(); - let session = MultipartSessionRecord { - bucket_id, - object_key: key.to_vec(), - upload_id: new_upload_id(now), - revision: 1, - phase: MultipartPhase::Open, - created_ms: now, - expires_ms: now - .checked_add(UPLOAD_LIFETIME_MS) - .ok_or(S3ErrorCode::InternalError)?, - content_type: strict_header(request, "content-type", S3ErrorCode::InvalidRequest)? - .unwrap_or("application/octet-stream") - .to_owned(), - attributes: crowdb_access_s3::metadata::UserMetadata::from_headers(request.headers())? - .encode()?, - max_parts: MAX_PARTS, - max_part_bytes: MAX_PART_BYTES, - max_object_bytes: MAX_OBJECT_BYTES, - max_staged_bytes: MAX_STAGED_BYTES, - part_count: 0, - staged_bytes: 0, - pending: None, - selection: None, - completion_request_digest: None, - publication_ms: None, - object_predecessor: None, - etag: None, - }; - self.multipart() - .begin(&session) + let session = self + .execution + .begin_multipart( + bucket_name, + key, + CopyMetadata { + content_type: strict_header(request, "content-type", S3ErrorCode::InvalidRequest)? + .unwrap_or("application/octet-stream") + .to_owned(), + user_metadata: crowdb_access_s3::metadata::UserMetadata::from_headers(request.headers())?, + }, + ) .await - .map_err(|error| map_multipart_error(&error))?; + .map_err(|error| error.code())?; xml_response(wire::create_multipart_upload( bucket_name, key, @@ -165,12 +127,20 @@ impl ProductionS3Operations { let payload_sha256 = strict_header(&request, "x-amz-content-sha256", S3ErrorCode::InvalidRequest)? .filter(|value| *value != "UNSIGNED-PAYLOAD" && streaming.is_none()) .map(str::to_owned); - let mut route_key = self.config.tenant.as_bytes().to_vec(); - route_key.extend_from_slice(session.bucket_id.as_bytes()); - route_key.extend_from_slice(&session.upload_id); - route_key.extend_from_slice(&number.to_be_bytes()); - let mut writer = self.prepare_writer(Some(length), &route_key).await?; - let native_receiver = if writer.is_large() && streaming.is_none() { + let mut part = self + .execution + .prepare_part( + &session, + number, + length, + &S3WritePolicy { + small_object_limit: self.config.small_object_limit, + large_write: self.config.large_write.clone(), + }, + ) + .await + .map_err(|error| error.code())?; + let native_receiver = if part.is_large() && streaming.is_none() { install_body_receive_provider(&mut request) } else { None @@ -189,7 +159,7 @@ impl ProductionS3Operations { body.defer_md5(); let written = write_object_body( &mut body, - &mut writer, + part.writer_mut(), native_receiver.as_deref(), Some(length), content_md5.as_deref(), @@ -198,39 +168,16 @@ impl ProductionS3Operations { self.metrics.as_deref(), ) .await; - let (etag, _) = match written { + let (etag, checksum) = match written { Ok(value) => value, Err(outcome) => { - let _ = writer.on_error().await; + part.abort().await; return Err(body .failure() .map_or_else(|| map_put_outcome(&outcome), super::encoding_error)); } }; - let locations = writer.on_finish().await.map_err(|error| { - tracing::error!(%error, "multipart part finalization failed; returning ServiceUnavailable"); - S3ErrorCode::ServiceUnavailable - })?; - let actual: u64 = locations.iter().map(|location| location.logical_length).sum(); - if actual != length { - return Err(S3ErrorCode::InvalidRequest); - } - let part = MultipartPartRecord { - bucket_id: session.bucket_id, - upload_id: session.upload_id, - number, - revision: 1, - modified_ms: unix_millis(), - length, - raw_md5: parse_md5(&etag)?, - locations, - }; - let _saved = self - .multipart() - .put_stream_part(&session, &part, part.modified_ms) - .await - .map_err(|error| map_multipart_error(&error))? - .ok_or(S3ErrorCode::SlowDown)?; + part.finish(checksum).await.map_err(|error| error.code())?; Response::builder() .status(StatusCode::OK) .header(ETAG, format!("\"{etag}\"")) @@ -306,10 +253,10 @@ impl ProductionS3Operations { route: S3Route, ) -> Result, S3ErrorCode> { let session = self.multipart_identity(&route).await?; - self.multipart() - .abort(&session) + self.execution + .abort_multipart(&session) .await - .map_err(|error| map_multipart_error(&error))?; + .map_err(|error| error.code())?; response(StatusCode::NO_CONTENT, Vec::new()) } @@ -343,17 +290,11 @@ impl ProductionS3Operations { etag: part.etag.clone(), }) .collect(); - let frozen = self - .multipart() - .freeze_completion(&session, &requested, unix_millis()) - .await - .map_err(|error| map_multipart_error(&error))? - .ok_or(S3ErrorCode::ServiceUnavailable)?; let published = self - .multipart() - .publish_completion(&frozen) + .execution + .complete_multipart(&session, &requested) .await - .map_err(|error| map_multipart_error(&error))?; + .map_err(|error| error.code())?; xml_response(wire::complete_multipart_upload( &location, required_bucket(&route)?, @@ -364,16 +305,7 @@ impl ProductionS3Operations { } pub(super) fn map_multipart_error(error: &MultipartRepositoryError) -> S3ErrorCode { - match error { - MultipartRepositoryError::Key(_) | MultipartRepositoryError::Record(_) => S3ErrorCode::InvalidRequest, - MultipartRepositoryError::Store(_) => S3ErrorCode::ServiceUnavailable, - MultipartRepositoryError::Conflict => S3ErrorCode::NoSuchUpload, - MultipartRepositoryError::Busy | MultipartRepositoryError::ScanBudgetExhausted => { - S3ErrorCode::SlowDown - } - MultipartRepositoryError::InvalidPart => S3ErrorCode::InvalidPart, - MultipartRepositoryError::EntityTooSmall => S3ErrorCode::EntityTooSmall, - } + crowdb_access_s3::execution::map_multipart_error(error) } fn map_integrity_error(error: IntegrityError) -> S3ErrorCode { @@ -411,21 +343,6 @@ fn parse_upload_id(value: &str) -> Result<[u8; 16], S3ErrorCode> { (id != [0; 16]).then_some(id).ok_or(S3ErrorCode::InvalidRequest) } -pub(super) fn parse_md5(value: &str) -> Result<[u8; 16], S3ErrorCode> { - if value.len() != 32 { - return Err(S3ErrorCode::InternalError); - } - let mut md5 = [0; 16]; - for (byte, pair) in md5.iter_mut().zip(value.as_bytes().chunks_exact(2)) { - *byte = u8::from_str_radix( - std::str::from_utf8(pair).map_err(|_| S3ErrorCode::InternalError)?, - 16, - ) - .map_err(|_| S3ErrorCode::InternalError)?; - } - Ok(md5) -} - async fn read_completion(mut body: Incoming) -> Result, S3ErrorCode> { let mut bytes = Vec::new(); while let Some(frame) = body.frame().await { diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index 1f39282e2..52fcefe2c 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -115,7 +115,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. `app/crowdb-access-server/src/s3/operations.rs`, `lib/crowdb-sdk-native/src/s3/`, `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`. -- [~] **Extract PUT and copy**: separate HTTP decoding from bounded input, +- [x] **Extract PUT and copy**: separate HTTP decoding from bounded input, checksums, ChunkIO writes, publication, and definite-failure cleanup. Add ordinary-input and frozen native-buffer entry points. Preserve `PutOutcome::Timeout` uncertainty; final success requires durable data and @@ -124,7 +124,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. `app/crowdb-access-server/src/s3/operations/copy.rs`, `app/crowdb-access-server/src/s3/operations/upload.rs`, `lib/crowdb-sdk-native/src/s3/`. -- [ ] **Attach multipart**: reuse the repository and multipart engine for +- [x] **Attach multipart**: reuse the repository and multipart engine for begin, upload/list parts, list uploads, complete, and abort. Preserve upload identity, validation, publication uncertainty, and deferred cleanup. Files: `lib/crowdb-access-s3/src/metadata/multipart_repository/`, @@ -339,6 +339,20 @@ transport acceptance alone cannot establish SDK completion. ## Immediate dependent implementation +- HTTP and native multipart now share session creation/expiry, part writer + preparation, copied-part execution, exact generation publication, completion + freeze and abort. Retained native upload identities are tenant/authority-bound; + admission refreshes durable phase/expiry instead of trusting an old handle. + Native part input shares PUT's bounded ordinary/framed buffers and cancellation. + Five new cases cover 5 MiB plus framed-tail composition, paged parts/uploads, + replacement/captured-source copy, abort/reconnect, confirmed lost replies and + an applied object publication whose confirmation reads also fail. That failure + remains uncertain; the same frozen selection later confirms without data + replay. Native/S3/Access Server all-target suites pass (46 native cases), as + do three-crate clippy and formatting. Access Server required more than the + default 60-second outer compilation limit; continuing its cached build and + complete test run passes without changing any test timeout. + - HTTP object copy and upload-part copy now use shared verified copy execution. Native copy accepts ordinary source names or a tenant/authority-bound retained selection; overwrites cannot replace that selection. Metadata COPY/REPLACE, diff --git a/lib/crowdb-access-s3/src/execution.rs b/lib/crowdb-access-s3/src/execution.rs index 273a8e73f..fe19adbda 100644 --- a/lib/crowdb-access-s3/src/execution.rs +++ b/lib/crowdb-access-s3/src/execution.rs @@ -4,10 +4,12 @@ //! Trusted S3 execution shared by HTTP parsing and native business handles. mod copy; +mod multipart; mod namespace; mod put; mod write; pub use copy::{CompletedCopy, CopyFailure, CopyMetadata}; +pub use multipart::{repository_error as map_multipart_error, MultipartFailure, PreparedPart}; pub use namespace::{DeleteObjectFailure, DeleteObjectResult, DeleteObjectStream, ObjectListRequest}; pub use put::{PreparedPut, PutObjectInfo}; pub use write::{ObjectWriter, S3WritePolicy}; diff --git a/lib/crowdb-access-s3/src/execution/multipart.rs b/lib/crowdb-access-s3/src/execution/multipart.rs new file mode 100644 index 000000000..e7dbb237f --- /dev/null +++ b/lib/crowdb-access-s3/src/execution/multipart.rs @@ -0,0 +1,201 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Shared multipart identity, admission and completion over the durable repository. + +mod part; +pub use part::PreparedPart; + +use super::{CopyMetadata, S3Executor}; +use crate::metadata::{ + new_upload_id, CompletionPart, MetadataStoreError, MultipartPhase, MultipartRepository, + MultipartRepositoryError, MultipartSessionRecord, +}; +use crate::S3ErrorCode; +use crowdb_chunk_kv_client::ClientError; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[derive(Clone, Copy, Debug)] +pub enum MultipartFailure { + S3(S3ErrorCode), + Uncertain, +} + +impl MultipartFailure { + #[must_use] + pub fn code(&self) -> S3ErrorCode { + match self { + Self::S3(code) => *code, + Self::Uncertain => S3ErrorCode::ServiceUnavailable, + } + } +} + +impl S3Executor { + #[must_use] + pub fn multipart_repository(&self) -> MultipartRepository { + MultipartRepository::new(self.storage.metadata.clone(), self.tenant.clone()) + } + + /// # Errors + /// Returns namespace/admission failures or an unconfirmed session mutation. + pub async fn begin_multipart( + &self, + bucket: &[u8], + key: &[u8], + metadata: CopyMetadata, + ) -> Result { + let bucket_id = self.resolve_bucket(bucket).await.map_err(MultipartFailure::S3)?; + if metadata.content_type.is_empty() { + return Err(MultipartFailure::S3(S3ErrorCode::InvalidRequest)); + } + let now = unix_millis(); + let session = MultipartSessionRecord { + bucket_id, + object_key: key.to_vec(), + upload_id: new_upload_id(now), + revision: 1, + phase: MultipartPhase::Open, + created_ms: now, + expires_ms: now.saturating_add(7 * 24 * 60 * 60 * 1000), + content_type: metadata.content_type, + attributes: metadata.user_metadata.encode().map_err(MultipartFailure::S3)?, + max_parts: 10_000, + max_part_bytes: 5 << 30, + max_object_bytes: 5 << 40, + max_staged_bytes: 10_000 * (5 << 30), + part_count: 0, + staged_bytes: 0, + pending: None, + selection: None, + completion_request_digest: None, + publication_ms: None, + object_predecessor: None, + etag: None, + }; + self.multipart_repository() + .begin(&session) + .await + .map_err(mutation_error) + } + + /// # Errors + /// Returns a missing/expired identity or unavailable metadata. Expiry follows + /// the same durable abort path for HTTP and native callers. + pub async fn multipart_session( + &self, + bucket: &[u8], + key: &[u8], + upload_id: &[u8; 16], + abort: bool, + ) -> Result { + let bucket_id = self.resolve_bucket(bucket).await.map_err(MultipartFailure::S3)?; + let repository = self.multipart_repository(); + let session = repository + .load_identity(bucket_id, key, upload_id) + .await + .map_err(|error| MultipartFailure::S3(repository_error(&error)))? + .ok_or(MultipartFailure::S3(S3ErrorCode::NoSuchUpload))?; + self.check_multipart_expiry(session, abort).await + } + + /// Refreshes retained identity before admitting another part or completion. + /// # Errors + /// Returns missing/expired identity or metadata/expiry-abort failure. + pub async fn refresh_multipart( + &self, + session: &MultipartSessionRecord, + abort: bool, + ) -> Result { + let current = self + .multipart_repository() + .load(session) + .await + .map_err(|error| MultipartFailure::S3(repository_error(&error)))? + .ok_or(MultipartFailure::S3(S3ErrorCode::NoSuchUpload))?; + self.check_multipart_expiry(current, abort).await + } + + async fn check_multipart_expiry( + &self, + session: MultipartSessionRecord, + abort: bool, + ) -> Result { + let repository = self.multipart_repository(); + if session.phase == MultipartPhase::Open && session.expires_ms <= unix_millis() { + let expired = repository.abort(&session).await.map_err(mutation_error)?; + if !abort { + return Err(MultipartFailure::S3(S3ErrorCode::NoSuchUpload)); + } + return Ok(expired); + } + Ok(session) + } + + /// # Errors + /// Returns a frozen-selection conflict or an unconfirmed object publication. + pub async fn complete_multipart( + &self, + session: &MultipartSessionRecord, + requested: &[CompletionPart], + ) -> Result { + if requested.is_empty() || requested.len() > 10_000 { + return Err(MultipartFailure::S3(S3ErrorCode::InvalidRequest)); + } + if requested.windows(2).any(|pair| pair[0].number >= pair[1].number) { + return Err(MultipartFailure::S3(S3ErrorCode::InvalidPartOrder)); + } + let repository = self.multipart_repository(); + let current = self.refresh_multipart(session, false).await?; + let frozen = repository + .freeze_completion(¤t, requested, unix_millis()) + .await + .map_err(mutation_error)? + .ok_or(MultipartFailure::S3(S3ErrorCode::ServiceUnavailable))?; + repository + .publish_completion(&frozen) + .await + .map_err(mutation_error) + } + + /// # Errors + /// Returns a conflicting identity or unconfirmed abort mutation. + pub async fn abort_multipart(&self, session: &MultipartSessionRecord) -> Result<(), MultipartFailure> { + self.multipart_repository() + .abort(session) + .await + .map_err(mutation_error)?; + Ok(()) + } +} + +pub(super) fn mutation_error(error: MultipartRepositoryError) -> MultipartFailure { + match error { + MultipartRepositoryError::Store(MetadataStoreError::Client( + ClientError::Deadline | ClientError::Transport(_), + )) => MultipartFailure::Uncertain, + other => MultipartFailure::S3(repository_error(&other)), + } +} + +#[must_use] +pub fn repository_error(error: &MultipartRepositoryError) -> S3ErrorCode { + match error { + MultipartRepositoryError::Key(_) | MultipartRepositoryError::Record(_) => S3ErrorCode::InvalidRequest, + MultipartRepositoryError::Store(_) => S3ErrorCode::ServiceUnavailable, + MultipartRepositoryError::Conflict => S3ErrorCode::NoSuchUpload, + MultipartRepositoryError::Busy | MultipartRepositoryError::ScanBudgetExhausted => { + S3ErrorCode::SlowDown + } + MultipartRepositoryError::InvalidPart => S3ErrorCode::InvalidPart, + MultipartRepositoryError::EntityTooSmall => S3ErrorCode::EntityTooSmall, + } +} + +pub(super) fn unix_millis() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) + }) +} diff --git a/lib/crowdb-access-s3/src/execution/multipart/part.rs b/lib/crowdb-access-s3/src/execution/multipart/part.rs new file mode 100644 index 000000000..177ead708 --- /dev/null +++ b/lib/crowdb-access-s3/src/execution/multipart/part.rs @@ -0,0 +1,157 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{mutation_error, unix_millis, MultipartFailure}; +use crate::condition::ObjectConditions; +use crate::execution::{ObjectWriter, S3Executor, S3WritePolicy}; +use crate::metadata::ObjectRecord; +use crate::metadata::{MultipartPartRecord, MultipartPhase, MultipartSessionRecord}; +use crate::S3ErrorCode; +use crowdb_chunk_client::{ChunkIoWriter, ProtoLocation}; + +pub struct PreparedPart { + writer: ObjectWriter, + executor: S3Executor, + session: Box, + number: u16, + length: u64, +} + +impl S3Executor { + /// Verifies a captured source and publishes one copied part generation. + /// # Errors + /// Returns invalid ranges/parts, source corruption or uncertain publication. + pub async fn copy_part_record( + &self, + source: &ObjectRecord, + session: &MultipartSessionRecord, + number: u16, + range: Option<&str>, + policy: &S3WritePolicy, + ) -> Result { + let session = self.refresh_multipart(session, false).await?; + if let Some(range) = range { + crate::copy::validate_range(range, source.logical_length).map_err(MultipartFailure::S3)?; + } + let prepared = + crate::retrieval::prepare_get(self.chunks(), source, range, &ObjectConditions::default()) + .map_err(|_| MultipartFailure::S3(S3ErrorCode::InvalidRange))?; + let length = prepared.headers.content_length; + validate_part(&session, number, length)?; + let key = self.part_writer_key(&session, number); + let copied = self + .stream_copy(source, range, &key, policy) + .await + .map_err(MultipartFailure::S3)?; + self.save_completed_part(&session, number, length, &copied.checksum, copied.locations) + .await + } + + /// # Errors + /// Rejects closed uploads, invalid part selection or writer preparation failure. + pub async fn prepare_part( + &self, + session: &MultipartSessionRecord, + number: u16, + length: u64, + policy: &S3WritePolicy, + ) -> Result { + let session = self.refresh_multipart(session, false).await?; + validate_part(&session, number, length)?; + let key = self.part_writer_key(&session, number); + let writer = self + .prepare_writer(Some(length), &key, policy) + .await + .map_err(MultipartFailure::S3)?; + Ok(PreparedPart { + writer, + executor: self.clone(), + session: Box::new(session.clone()), + number, + length, + }) + } + + fn part_writer_key(&self, session: &MultipartSessionRecord, number: u16) -> Vec { + let mut key = self.tenant().as_bytes().to_vec(); + key.extend_from_slice(session.bucket_id.as_bytes()); + key.extend_from_slice(&session.upload_id); + key.extend_from_slice(&number.to_be_bytes()); + key + } + + /// Saves completed locations through the same generation fence as uploads. + /// # Errors + /// Returns invalid digest/length, a closed upload or an unconfirmed mutation. + pub async fn save_completed_part( + &self, + session: &MultipartSessionRecord, + number: u16, + length: u64, + checksum: &[u8], + locations: Vec, + ) -> Result { + let raw_md5 = checksum + .try_into() + .map_err(|_| MultipartFailure::S3(S3ErrorCode::InvalidDigest))?; + let actual = locations + .iter() + .try_fold(0u64, |total, location| total.checked_add(location.logical_length)); + if actual != Some(length) { + return Err(MultipartFailure::S3(S3ErrorCode::InvalidRequest)); + } + let now = unix_millis(); + let part = MultipartPartRecord { + bucket_id: session.bucket_id, + upload_id: session.upload_id, + number, + revision: 1, + modified_ms: now, + length, + raw_md5, + locations, + }; + self.multipart_repository() + .put_stream_part(session, &part, now) + .await + .map_err(mutation_error)? + .ok_or(MultipartFailure::S3(S3ErrorCode::SlowDown)) + } +} + +fn validate_part(session: &MultipartSessionRecord, number: u16, length: u64) -> Result<(), MultipartFailure> { + if number == 0 || number > session.max_parts || length > session.max_part_bytes { + return Err(MultipartFailure::S3(S3ErrorCode::InvalidRequest)); + } + if session.phase != MultipartPhase::Open || session.expires_ms <= unix_millis() { + return Err(MultipartFailure::S3(S3ErrorCode::NoSuchUpload)); + } + Ok(()) +} + +impl PreparedPart { + pub fn writer_mut(&mut self) -> &mut ObjectWriter { + &mut self.writer + } + + #[must_use] + pub fn is_large(&self) -> bool { + self.writer.is_large() + } + + pub async fn abort(&mut self) { + let _ = self.writer.on_error().await; + } + + /// # Errors + /// Returns durability, length/digest or uncertain part publication failure. + pub async fn finish(mut self, checksum: Vec) -> Result { + let Ok(locations) = self.writer.on_finish().await else { + self.abort().await; + return Err(MultipartFailure::S3(S3ErrorCode::ServiceUnavailable)); + }; + self.executor + .save_completed_part(&self.session, self.number, self.length, &checksum, locations) + .await + } +} diff --git a/lib/crowdb-access-s3/src/integrity.rs b/lib/crowdb-access-s3/src/integrity.rs index 3525fac21..924fe5dfa 100644 --- a/lib/crowdb-access-s3/src/integrity.rs +++ b/lib/crowdb-access-s3/src/integrity.rs @@ -139,6 +139,12 @@ fn hex_digest(bytes: &[u8]) -> String { value } +/// Formats a retained part digest using the canonical single-part `ETag` form. +#[must_use] +pub fn raw_md5_etag(bytes: [u8; 16]) -> String { + hex_digest(&bytes) +} + /// Encodes a composite multipart `ETag` as a distinct 18-byte metadata /// checksum marker: 16 digest bytes followed by the part count. #[must_use] diff --git a/lib/crowdb-sdk-native/src/s3.rs b/lib/crowdb-sdk-native/src/s3.rs index f5079dab0..abc783083 100644 --- a/lib/crowdb-sdk-native/src/s3.rs +++ b/lib/crowdb-sdk-native/src/s3.rs @@ -15,6 +15,7 @@ mod body; mod config; mod copy; mod locality; +mod multipart; mod namespace; mod object; mod read; @@ -28,12 +29,16 @@ pub use copy::CopyOptions; pub use crowdb_access_s3::condition::{EtagSelection, ReadPreconditions}; pub use crowdb_access_s3::execution::CopyMetadata; pub use crowdb_access_s3::execution::{PutObjectInfo, S3WritePolicy}; +pub use crowdb_access_s3::metadata::CompletionPart; pub use crowdb_access_s3::metadata::UserMetadata; pub use crowdb_access_s3::range::ByteRange; pub use locality::{ ChunkLayoutObservation, FragmentPlacement, FragmentRole, LocalityOptions, ObjectExtent, ObjectLocality, StripPlacement, }; +pub use multipart::{ + CompletedMultipart, ListUploads, MultipartUpload, PartMetadata, PartPage, UploadPart, UploadsPage, +}; pub use object::{ObjectMetadata, ResolvedObject}; pub use read::{GetObject, GetOptions}; pub use write::{PutObject, PutOptions}; diff --git a/lib/crowdb-sdk-native/src/s3/multipart.rs b/lib/crowdb-sdk-native/src/s3/multipart.rs new file mode 100644 index 000000000..f65b3fdd8 --- /dev/null +++ b/lib/crowdb-sdk-native/src/s3/multipart.rs @@ -0,0 +1,168 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Native multipart handles remain bound to their tenant and metadata authority. + +mod listing; +mod part; +pub use listing::{ListUploads, PartPage, UploadsPage}; +pub use part::{PartMetadata, UploadPart}; + +use super::{CompletionPart, CopyMetadata, S3Client}; +use crate::{Error, OperationContext, Result}; +use crowdb_access_s3::execution::MultipartFailure; +use crowdb_access_s3::metadata::{MultipartSessionRecord, TenantId}; +use crowdb_chunk_kv_client::ChunkKvClient; +use std::sync::Arc; + +#[derive(Clone)] +pub struct MultipartUpload { + session: MultipartSessionRecord, + tenant: TenantId, + authority: Arc, +} + +impl MultipartUpload { + #[must_use] + pub fn upload_id(&self) -> [u8; 16] { + self.session.upload_id + } + #[must_use] + pub fn key(&self) -> &[u8] { + &self.session.object_key + } + #[must_use] + pub fn created_at_ms(&self) -> u64 { + self.session.created_ms + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CompletedMultipart { + pub upload_id: [u8; 16], + pub key: Vec, + pub etag: String, + pub logical_length: u64, +} + +impl S3Client { + /// # Errors + /// Returns namespace, metadata validation or uncertain session creation. + pub async fn begin_multipart( + &self, + context: OperationContext, + bucket: &[u8], + key: &[u8], + metadata: CopyMetadata, + ) -> Result { + let id = context.request_id.get(); + let request = self.runtime.resources.admit(context)?; + let session = request + .run_mutation(async { + self.execution + .begin_multipart(bucket, key, metadata) + .await + .map_err(|error| map_error(error, id)) + }) + .await?; + Ok(self.bind_upload(session)) + } + + /// Reattaches an upload after reconnecting without trusting caller state. + /// # Errors + /// Returns missing/expired upload or unavailable metadata. + pub async fn resolve_multipart( + &self, + context: OperationContext, + bucket: &[u8], + key: &[u8], + upload_id: &[u8; 16], + ) -> Result { + let id = context.request_id.get(); + let request = self.runtime.resources.admit(context)?; + let session = request + .run(async { + self.execution + .multipart_session(bucket, key, upload_id, false) + .await + .map_err(|error| map_error(error, id)) + }) + .await?; + Ok(self.bind_upload(session)) + } + + /// Freezes exact part generations and publishes through the shared CAS engine. + /// # Errors + /// Returns part validation/conflict or uncertain completion; a retry with the + /// same selection confirms that publication instead of replaying payloads. + pub async fn complete_multipart( + &self, + context: OperationContext, + upload: &MultipartUpload, + parts: &[CompletionPart], + ) -> Result { + self.validate_upload(upload)?; + let id = context.request_id.get(); + let request = self.runtime.resources.admit(context)?; + let session = request + .run_mutation(async { + Box::pin(self.execution.complete_multipart(&upload.session, parts)) + .await + .map_err(|error| map_error(error, id)) + }) + .await?; + Ok(CompletedMultipart { + upload_id: session.upload_id, + key: session.object_key, + logical_length: session.staged_bytes, + etag: session + .etag + .ok_or_else(|| Error::Storage("published multipart object has no ETag".into()))?, + }) + } + + /// # Errors + /// Returns foreign identity, closed/conflicting upload or uncertain abort. + pub async fn abort_multipart(&self, context: OperationContext, upload: &MultipartUpload) -> Result<()> { + self.validate_upload(upload)?; + let id = context.request_id.get(); + let request = self.runtime.resources.admit(context)?; + request + .run_mutation(async { + self.execution + .abort_multipart(&upload.session) + .await + .map_err(|error| map_error(error, id)) + }) + .await + } + + fn bind_upload(&self, session: MultipartSessionRecord) -> MultipartUpload { + MultipartUpload { + session, + tenant: self.execution.tenant().clone(), + authority: self.runtime.storage.metadata.clone(), + } + } + + fn validate_upload(&self, upload: &MultipartUpload) -> Result<()> { + if upload.tenant != *self.execution.tenant() + || !Arc::ptr_eq(&upload.authority, &self.runtime.storage.metadata) + { + return Err(Error::Configuration( + "upload belongs to another tenant or storage authority".into(), + )); + } + Ok(()) + } +} + +pub(super) fn map_error(error: MultipartFailure, request_id: u64) -> Error { + match error { + MultipartFailure::S3(code) => Error::S3(code), + MultipartFailure::Uncertain => Error::UncertainMutation { + request_id, + reason: crate::error::MutationInterruption::Transport, + }, + } +} diff --git a/lib/crowdb-sdk-native/src/s3/multipart/listing.rs b/lib/crowdb-sdk-native/src/s3/multipart/listing.rs new file mode 100644 index 000000000..94de1ce9c --- /dev/null +++ b/lib/crowdb-sdk-native/src/s3/multipart/listing.rs @@ -0,0 +1,119 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{MultipartUpload, PartMetadata}; +use crate::s3::S3Client; +use crate::{Error, OperationContext, Result}; +use crowdb_access_s3::execution::map_multipart_error; +use crowdb_access_s3::S3ErrorCode; +use std::time::{SystemTime, UNIX_EPOCH}; + +pub struct PartPage { + pub parts: Vec, + pub next_part_number_marker: Option, +} + +pub struct ListUploads { + pub prefix: Vec, + pub key_marker: Option>, + pub upload_marker: Option<[u8; 16]>, + pub limit: usize, +} + +impl Default for ListUploads { + fn default() -> Self { + Self { + prefix: Vec::new(), + key_marker: None, + upload_marker: None, + limit: 1000, + } + } +} + +pub struct UploadsPage { + pub uploads: Vec, + pub next: Option<(Vec, [u8; 16])>, +} + +impl S3Client { + /// # Errors + /// Returns invalid paging, foreign/closed session or bounded metadata failure. + pub async fn list_parts( + &self, + context: OperationContext, + upload: &MultipartUpload, + after_number: u16, + limit: usize, + ) -> Result { + self.validate_upload(upload)?; + if !(1..=1000).contains(&limit) || after_number > 10_000 { + return Err(Error::S3(S3ErrorCode::InvalidRequest)); + } + let request = self.runtime.resources.admit(context)?; + let page = request + .run(async { + let current = self + .execution + .refresh_multipart(&upload.session, false) + .await + .map_err(|error| super::map_error(error, request.context().request_id.get()))?; + self.execution + .multipart_repository() + .list_parts(¤t, after_number, limit) + .await + .map_err(|error| Error::S3(map_multipart_error(&error))) + }) + .await?; + Ok(PartPage { + parts: page.parts.into_iter().map(Into::into).collect(), + next_part_number_marker: page.next_part_number_marker, + }) + } + + /// # Errors + /// Returns invalid markers, namespace failure or exhausted bounded scan budget. + pub async fn list_multipart_uploads( + &self, + context: OperationContext, + bucket: &[u8], + options: &ListUploads, + ) -> Result { + if !(1..=1000).contains(&options.limit) + || options.upload_marker.is_some() && options.key_marker.is_none() + { + return Err(Error::S3(S3ErrorCode::InvalidRequest)); + } + let request = self.runtime.resources.admit(context)?; + let page = request + .run(async { + let bucket = self.execution.resolve_bucket(bucket).await.map_err(Error::S3)?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) + }); + self.execution + .multipart_repository() + .list_uploads( + bucket, + &options.prefix, + options.key_marker.as_deref(), + options.upload_marker.as_ref(), + options.limit, + now, + ) + .await + .map_err(|error| Error::S3(map_multipart_error(&error))) + }) + .await?; + Ok(UploadsPage { + uploads: page + .uploads + .into_iter() + .map(|session| self.bind_upload(session)) + .collect(), + next: page.next, + }) + } +} diff --git a/lib/crowdb-sdk-native/src/s3/multipart/part.rs b/lib/crowdb-sdk-native/src/s3/multipart/part.rs new file mode 100644 index 000000000..56c4e2417 --- /dev/null +++ b/lib/crowdb-sdk-native/src/s3/multipart/part.rs @@ -0,0 +1,130 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{map_error, MultipartUpload}; +use crate::buffer::{BufferLease, FrozenObjectBuffer}; +use crate::s3::{ByteRange, PutObject, ReadPreconditions, ResolvedObject, S3Client}; +use crate::{Error, OperationContext, Result}; +use crowdb_access_s3::condition::{ConditionOutcome, ReadCondition}; +use crowdb_access_s3::metadata::MultipartPartRecord; +use crowdb_access_s3::S3ErrorCode; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PartMetadata { + pub number: u16, + pub generation: u64, + pub length: u64, + pub etag: String, + pub checksum: [u8; 16], + pub modified_at_ms: u64, +} + +impl From for PartMetadata { + fn from(part: MultipartPartRecord) -> Self { + Self { + number: part.number, + generation: part.revision, + length: part.length, + etag: crowdb_access_s3::integrity::raw_md5_etag(part.raw_md5), + checksum: part.raw_md5, + modified_at_ms: part.modified_ms, + } + } +} + +pub struct UploadPart { + input: PutObject, +} + +impl UploadPart { + #[must_use] + pub fn accepted_bytes(&self) -> u64 { + self.input.accepted_bytes() + } + + /// # Errors + /// Returns the same bounded input/cancellation rules as a single-part PUT. + pub async fn write(&mut self, buffer: BufferLease) -> Result<()> { + self.input.write(buffer).await + } + + /// # Errors + /// Rejects invalid frame boundaries or returns storage/cancellation failure. + pub async fn write_framed(&mut self, buffer: FrozenObjectBuffer) -> Result<()> { + self.input.write_framed(buffer).await + } + + /// # Errors + /// Returns durability, digest/length, session fence or uncertain part failure. + pub async fn finish(self) -> Result { + self.input.finish_part().await.map(Into::into) + } +} + +impl S3Client { + /// Copies a retained source generation through the shared multipart engine. + /// # Errors + /// Returns foreign identity, invalid condition/range/part or publication uncertainty. + pub async fn copy_part_resolved( + &self, + context: OperationContext, + upload: &MultipartUpload, + number: u16, + source: &ResolvedObject, + range: Option, + conditions: &ReadPreconditions, + ) -> Result { + self.validate_upload(upload)?; + self.validate_selection(source)?; + if conditions.evaluate(&source.record) != ConditionOutcome::Proceed { + return Err(Error::S3(S3ErrorCode::PreconditionFailed)); + } + if range.is_some_and(|range| range.start >= range.end || range.end > source.metadata().logical_length) + { + return Err(Error::S3(S3ErrorCode::InvalidRange)); + } + let range = range.map(|range| format!("bytes={}-{}", range.start, range.end - 1)); + let id = context.request_id.get(); + let request = self.runtime.resources.admit(context)?; + request + .run_mutation(async { + Box::pin(self.execution.copy_part_record( + &source.record, + &upload.session, + number, + range.as_deref(), + &self.write_policy, + )) + .await + .map(Into::into) + .map_err(|error| map_error(error, id)) + }) + .await + } + + /// # Errors + /// Returns invalid upload/part selection, admission or writer preparation failure. + pub async fn begin_upload_part( + &self, + context: OperationContext, + upload: &MultipartUpload, + number: u16, + length: u64, + expected_md5: Option<[u8; 16]>, + ) -> Result { + self.validate_upload(upload)?; + let id = context.request_id.get(); + let request = self.runtime.resources.admit(context)?; + let part = request + .run(async { + self.execution + .prepare_part(&upload.session, number, length, &self.write_policy) + .await + .map_err(|error| map_error(error, id)) + }) + .await?; + Ok(UploadPart { + input: PutObject::from_part(self.clone(), request, part, length, expected_md5), + }) + } +} diff --git a/lib/crowdb-sdk-native/src/s3/write.rs b/lib/crowdb-sdk-native/src/s3/write.rs index c4dd5fa8c..b7a606a88 100644 --- a/lib/crowdb-sdk-native/src/s3/write.rs +++ b/lib/crowdb-sdk-native/src/s3/write.rs @@ -7,12 +7,14 @@ use super::{ObjectMetadata, PutObjectInfo, S3Client}; use crate::buffer::{BufferLease, FrozenObjectBuffer}; use crate::runtime::RequestLease; use crate::{Error, OperationContext, Result}; -use crowdb_access_s3::execution::PreparedPut; +use crowdb_access_s3::execution::PreparedPart; +mod session; use crowdb_access_s3::integrity::SinglePartIntegrity; use crowdb_access_s3::streaming::{PutErrorCode, PutOutcome}; use crowdb_access_s3::S3ErrorCode; use crowdb_chunk_client::ChunkIoWriter; use crowdb_protocol::frame::MAX_FRAME_PAYLOAD_BYTES; +use session::{UploadResult, UploadSession}; #[derive(Default)] pub struct PutOptions { @@ -23,7 +25,7 @@ pub struct PutOptions { pub struct PutObject { client: S3Client, request: Option, - session: Option, + session: Option, integrity: SinglePartIntegrity, expected_md5: Option<[u8; 16]>, expected_length: Option, @@ -56,7 +58,7 @@ impl S3Client { Ok(PutObject { client: self.clone(), request: Some(request), - session: Some(session), + session: Some(UploadSession::Object(session)), integrity: SinglePartIntegrity::default(), expected_md5: options.expected_md5, expected_length, @@ -91,6 +93,26 @@ impl S3Client { } impl PutObject { + pub(super) fn from_part( + client: S3Client, + request: RequestLease, + session: PreparedPart, + length: u64, + expected_md5: Option<[u8; 16]>, + ) -> Self { + Self { + client, + request: Some(request), + session: Some(UploadSession::Part(session)), + integrity: SinglePartIntegrity::default(), + expected_md5, + expected_length: Some(length), + length: 0, + final_frame: false, + failure: None, + } + } + #[must_use] pub fn accepted_bytes(&self) -> u64 { self.length @@ -181,7 +203,17 @@ impl PutObject { /// Returns success only after data durability and object publication. /// # Errors /// Returns incomplete input, checksum failure or an uncertain publication. - pub async fn finish(mut self) -> Result { + pub async fn finish(self) -> Result { + let client = self.client.clone(); + let selected = self.finish_upload().await?.object()?; + Ok(client.bind_object(selected)?.metadata().clone()) + } + + pub(super) async fn finish_part(self) -> Result { + self.finish_upload().await?.part() + } + + async fn finish_upload(mut self) -> Result { let (request, mut session) = self.take_active()?; if let Some(expected) = self.expected_length.filter(|expected| *expected != self.length) { session.abort().await; @@ -199,15 +231,7 @@ impl PutObject { return Err(Error::Integrity); } let id = request.context().request_id.get(); - let selected = request - .run_mutation(async { - session - .finish(etag, checksum) - .await - .map_err(|outcome| map_outcome(outcome, id)) - }) - .await?; - Ok(self.client.bind_object(selected)?.metadata().clone()) + request.run_mutation(session.finish(etag, checksum, id)).await } fn check_input(&self, count: u64) -> Result { @@ -222,7 +246,7 @@ impl PutObject { Ok(next) } - fn take_active(&mut self) -> Result<(RequestLease, PreparedPut)> { + fn take_active(&mut self) -> Result<(RequestLease, UploadSession)> { if let Some(error) = &self.failure { return Err(error.clone()); } @@ -233,7 +257,7 @@ impl PutObject { )) } - fn restore(&mut self, result: Result<()>, request: RequestLease, session: PreparedPut) -> Result<()> { + fn restore(&mut self, result: Result<()>, request: RequestLease, session: UploadSession) -> Result<()> { match result { Ok(()) => { self.request = Some(request); diff --git a/lib/crowdb-sdk-native/src/s3/write/session.rs b/lib/crowdb-sdk-native/src/s3/write/session.rs new file mode 100644 index 000000000..95726196c --- /dev/null +++ b/lib/crowdb-sdk-native/src/s3/write/session.rs @@ -0,0 +1,69 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::{Error, Result}; +use crowdb_access_s3::execution::{ObjectWriter, PreparedPart, PreparedPut}; +use crowdb_access_s3::metadata::MultipartPartRecord; +use crowdb_access_s3::object::SelectedObject; + +pub(super) enum UploadSession { + Object(PreparedPut), + Part(PreparedPart), +} + +pub(super) enum UploadResult { + Object(SelectedObject), + Part(MultipartPartRecord), +} + +impl UploadSession { + pub fn writer_mut(&mut self) -> &mut ObjectWriter { + match self { + Self::Object(session) => session.writer_mut(), + Self::Part(session) => session.writer_mut(), + } + } + + pub fn is_large(&self) -> bool { + match self { + Self::Object(session) => session.is_large(), + Self::Part(session) => session.is_large(), + } + } + + pub async fn abort(&mut self) { + match self { + Self::Object(session) => session.abort().await, + Self::Part(session) => session.abort().await, + } + } + + pub async fn finish(self, etag: String, checksum: Vec, request_id: u64) -> Result { + match self { + Self::Object(session) => Box::pin(session.finish(etag, checksum)) + .await + .map(UploadResult::Object) + .map_err(|outcome| super::map_outcome(outcome, request_id)), + Self::Part(session) => Box::pin(session.finish(checksum)) + .await + .map(UploadResult::Part) + .map_err(|error| crate::s3::multipart::map_error(error, request_id)), + } + } +} + +impl UploadResult { + pub fn object(self) -> Result { + match self { + Self::Object(object) => Ok(object), + Self::Part(_) => Err(Error::Configuration("part writer used as object writer".into())), + } + } + + pub fn part(self) -> Result { + match self { + Self::Part(part) => Ok(part), + Self::Object(_) => Err(Error::Configuration("object writer used as part writer".into())), + } + } +} diff --git a/lib/crowdb-sdk-native/tests/common/metadata.rs b/lib/crowdb-sdk-native/tests/common/metadata.rs index f3b3e0e92..84e9ae528 100644 --- a/lib/crowdb-sdk-native/tests/common/metadata.rs +++ b/lib/crowdb-sdk-native/tests/common/metadata.rs @@ -30,17 +30,47 @@ enum TestCommand { pub struct TestMetadataTransport { sender: mpsc::Sender, lose_mutation_reply: Arc, + blackout: Option>, +} + +#[derive(Default)] +pub struct TestMutationBlackout { + pub key: arc_swap::ArcSwapOption>, + pub blocked: AtomicBool, } #[async_trait] impl ChunkKvTransport for TestMetadataTransport { async fn point(&self, _: &str, request: &PointRequest) -> Result { + if self + .blackout + .as_ref() + .is_some_and(|fault| fault.blocked.load(Ordering::Acquire)) + { + return Err(crowdb_chunk_kv_client::ClientError::Transport( + "test metadata blackout".into(), + )); + } let (sender, receiver) = oneshot::channel(); self.sender .send(TestCommand::Point(request.operation.clone(), sender)) .await .unwrap(); let response = receiver.await.unwrap(); + if !matches!(request.operation, PointOperation::Get { .. }) { + if let Some(fault) = self.blackout.as_ref().filter(|fault| { + fault + .key + .load() + .as_ref() + .is_some_and(|key| key.as_slice() == request.operation.key()) + }) { + fault.blocked.store(true, Ordering::Release); + return Err(crowdb_chunk_kv_client::ClientError::Transport( + "test lost applied publication and confirmation reads".into(), + )); + } + } if !matches!(request.operation, PointOperation::Get { .. }) && self.lose_mutation_reply.load(Ordering::Acquire) { @@ -193,7 +223,13 @@ pub fn catalog( } pub async fn client_with_source(source: Arc) -> Arc { - client_with_options(source, Arc::new(AtomicBool::new(false)), ClientConfig::default()).await + client_with_options( + source, + Arc::new(AtomicBool::new(false)), + ClientConfig::default(), + None, + ) + .await } #[allow(dead_code)] // Used by namespace uncertainty tests only. @@ -206,6 +242,22 @@ pub async fn client_with_loss(loss: Arc) -> Arc { max_attempts: 1, ..ClientConfig::default() }, + None, + ) + .await +} + +#[allow(dead_code)] // Used by multipart publication fault acceptance. +pub async fn client_with_blackout(blackout: Arc) -> Arc { + let (head, pages) = catalog(1, 1); + client_with_options( + Arc::new(TestCatalog(head, pages)), + Arc::new(AtomicBool::new(false)), + ClientConfig { + max_attempts: 1, + ..ClientConfig::default() + }, + Some(blackout), ) .await } @@ -214,6 +266,7 @@ async fn client_with_options( source: Arc, loss: Arc, config: ClientConfig, + blackout: Option>, ) -> Arc { let (sender, mut receiver) = mpsc::channel(64); tokio::spawn(async move { @@ -240,6 +293,7 @@ async fn client_with_options( Arc::new(TestMetadataTransport { sender, lose_mutation_reply: loss, + blackout, }), ) .unwrap(), diff --git a/lib/crowdb-sdk-native/tests/common/write_fixture.rs b/lib/crowdb-sdk-native/tests/common/write_fixture.rs index 15ece5e28..76c1041b0 100644 --- a/lib/crowdb-sdk-native/tests/common/write_fixture.rs +++ b/lib/crowdb-sdk-native/tests/common/write_fixture.rs @@ -20,11 +20,17 @@ pub fn context(id: u64) -> OperationContext { } pub async fn fixture(loss: Option>) -> (ClientRuntime, S3Client, Arc) { - let backend = TestWriteStorage::new(); let metadata = match loss { Some(loss) => metadata::client_with_loss(loss).await, None => metadata::client().await, }; + fixture_with_metadata(metadata).await +} + +pub async fn fixture_with_metadata( + metadata: Arc, +) -> (ClientRuntime, S3Client, Arc) { + let backend = TestWriteStorage::new(); let runtime = backend.runtime(metadata); let mut config = S3ClientConfig::from(TenantId::new(b"tenant".to_vec()).unwrap()); let mut policy = S3WritePolicy::basic(0); diff --git a/lib/crowdb-sdk-native/tests/s3_multipart_test.rs b/lib/crowdb-sdk-native/tests/s3_multipart_test.rs new file mode 100644 index 000000000..b1532a24c --- /dev/null +++ b/lib/crowdb-sdk-native/tests/s3_multipart_test.rs @@ -0,0 +1,305 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/write_fixture.rs"] +mod fixture; +#[path = "common/metadata.rs"] +mod metadata; +#[path = "common/write_storage.rs"] +mod write_storage; + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use crowdb_access_s3::S3ErrorCode; +use crowdb_sdk_native::s3::{ + CompletionPart, CopyMetadata, ListUploads, MultipartUpload, PartMetadata, ReadPreconditions, S3Client, + UserMetadata, +}; +use crowdb_sdk_native::{ClientRuntime, Error}; +use fixture::{context, fixture, options, read}; + +fn attributes() -> CopyMetadata { + CopyMetadata { + content_type: "application/native".into(), + user_metadata: UserMetadata::default(), + } +} + +async fn part( + runtime: &ClientRuntime, + client: &S3Client, + upload: &MultipartUpload, + number: u16, + data: Vec, + id: u64, +) -> PartMetadata { + let mut input = client + .begin_upload_part(context(id), upload, number, data.len() as u64, None) + .await + .unwrap(); + let lease = runtime.resources().retain_vec(data, id, id, 0, 0).unwrap(); + for start in (0..lease.as_slice().len()).step_by(1 << 20) { + input + .write( + lease + .slice(start..(start + (1 << 20)).min(lease.as_slice().len())) + .unwrap(), + ) + .await + .unwrap(); + } + input.finish().await.unwrap() +} + +fn selection(part: &PartMetadata) -> CompletionPart { + CompletionPart { + number: part.number, + etag: part.etag.clone(), + } +} + +#[tokio::test] +async fn multipart_streams_ordinary_and_framed_parts_and_replays_frozen_completion() { + let (runtime, client, _) = fixture(None).await; + let upload = client + .begin_multipart(context(10), b"bucket", b"joined", attributes()) + .await + .unwrap(); + let first = part(&runtime, &client, &upload, 1, vec![0x52; 5 << 20], 11).await; + let mut input = client + .begin_upload_part(context(12), &upload, 2, 513, None) + .await + .unwrap(); + let frozen = runtime + .allocate_producer(65_536) + .unwrap() + .initialize_framed(513, 0x63, |_, _| {}) + .unwrap(); + input.write_framed(frozen).await.unwrap(); + let second = input.finish().await.unwrap(); + let first_page = client.list_parts(context(13), &upload, 0, 1).await.unwrap(); + assert_eq!(first_page.parts, vec![first.clone()]); + assert_eq!(first_page.next_part_number_marker, Some(1)); + let second_page = client.list_parts(context(14), &upload, 1, 1).await.unwrap(); + assert_eq!(second_page.parts, vec![second.clone()]); + assert_eq!(second_page.next_part_number_marker, None); + assert_eq!( + client + .complete_multipart(context(15), &upload, &[selection(&second), selection(&first)]) + .await, + Err(Error::S3(S3ErrorCode::InvalidPartOrder)) + ); + let selected = [selection(&first), selection(&second)]; + let completed = client + .complete_multipart(context(16), &upload, &selected) + .await + .unwrap(); + assert_eq!(completed.logical_length, (5 << 20) + 513); + assert!(completed.etag.ends_with("-2")); + assert_eq!( + client + .complete_multipart(context(17), &upload, &selected) + .await + .unwrap(), + completed + ); + let data = read(&client, b"joined").await; + assert!(data[..5 << 20].iter().all(|byte| *byte == 0x52)); + assert_eq!(&data[5 << 20..], &[0x63; 513]); + let metadata = client + .resolve_object(context(18), b"bucket", b"joined") + .await + .unwrap(); + assert_eq!(metadata.metadata().content_type, "application/native"); + assert!(matches!( + client.begin_upload_part(context(19), &upload, 3, 7, None).await, + Err(Error::S3(S3ErrorCode::NoSuchUpload)) + )); + assert_eq!(runtime.resources().status().requests, 0); + runtime.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn copied_parts_bind_the_source_generation_and_replacement_is_visible_in_listing() { + let (runtime, client, _) = fixture(None).await; + client + .put_vec( + context(10), + b"bucket", + b"source", + b"original".to_vec(), + options(8), + ) + .await + .unwrap(); + let source = client + .resolve_object(context(11), b"bucket", b"source") + .await + .unwrap(); + let upload = client + .begin_multipart(context(12), b"bucket", b"copied", attributes()) + .await + .unwrap(); + let old = part(&runtime, &client, &upload, 1, b"old".to_vec(), 13).await; + client + .put_vec( + context(14), + b"bucket", + b"source", + b"replacement".to_vec(), + options(11), + ) + .await + .unwrap(); + let copied = client + .copy_part_resolved( + context(15), + &upload, + 1, + &source, + None, + &ReadPreconditions::default(), + ) + .await + .unwrap(); + assert!(copied.generation > old.generation); + assert_eq!(copied.length, 8); + assert_eq!( + client + .list_parts(context(16), &upload, 0, 1000) + .await + .unwrap() + .parts, + vec![copied.clone()] + ); + client + .complete_multipart(context(17), &upload, &[selection(&copied)]) + .await + .unwrap(); + assert_eq!(read(&client, b"copied").await, b"original"); + runtime.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn upload_paging_abort_and_reattachment_preserve_exact_identity() { + let (runtime, client, _) = fixture(None).await; + let first = client + .begin_multipart(context(10), b"bucket", b"a", attributes()) + .await + .unwrap(); + let second = client + .begin_multipart(context(11), b"bucket", b"b", attributes()) + .await + .unwrap(); + let first_page = client + .list_multipart_uploads( + context(12), + b"bucket", + &ListUploads { + limit: 1, + ..ListUploads::default() + }, + ) + .await + .unwrap(); + assert_eq!(first_page.uploads[0].upload_id(), first.upload_id()); + let (key, id) = first_page.next.unwrap(); + let second_page = client + .list_multipart_uploads( + context(13), + b"bucket", + &ListUploads { + key_marker: Some(key), + upload_marker: Some(id), + limit: 1, + ..ListUploads::default() + }, + ) + .await + .unwrap(); + assert_eq!(second_page.uploads[0].upload_id(), second.upload_id()); + let attached = client + .resolve_multipart(context(14), b"bucket", b"a", &first.upload_id()) + .await + .unwrap(); + assert_eq!(attached.key(), b"a"); + client.abort_multipart(context(15), &attached).await.unwrap(); + assert!(matches!( + client.begin_upload_part(context(16), &first, 1, 7, None).await, + Err(Error::S3(S3ErrorCode::NoSuchUpload)) + )); + let page = client + .list_multipart_uploads(context(17), b"bucket", &ListUploads::default()) + .await + .unwrap(); + assert_eq!(page.uploads.len(), 1); + assert_eq!(page.uploads[0].upload_id(), second.upload_id()); + runtime.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn confirmed_lost_multipart_replies_do_not_repeat_data_writes() { + let loss = Arc::new(AtomicBool::new(false)); + let (runtime, client, backend) = fixture(Some(loss.clone())).await; + loss.store(true, Ordering::Release); + let upload = client + .begin_multipart(context(10), b"bucket", b"confirmed", attributes()) + .await + .unwrap(); + let saved = part(&runtime, &client, &upload, 1, b"retained".to_vec(), 11).await; + let writes = backend.writes.load(Ordering::Acquire); + let completed = client + .complete_multipart(context(12), &upload, &[selection(&saved)]) + .await + .unwrap(); + assert_eq!( + client + .complete_multipart(context(13), &upload, &[selection(&saved)]) + .await + .unwrap(), + completed + ); + assert_eq!(backend.writes.load(Ordering::Acquire), writes); + loss.store(false, Ordering::Release); + assert_eq!(read(&client, b"confirmed").await, b"retained"); + assert_eq!(runtime.resources().status().requests, 0); + runtime.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn unconfirmed_publication_is_uncertain_and_retry_confirms_the_frozen_generation() { + let fault = Arc::new(metadata::TestMutationBlackout::default()); + let (runtime, client, backend) = + fixture::fixture_with_metadata(metadata::client_with_blackout(fault.clone()).await).await; + let upload = client + .begin_multipart(context(60), b"bucket", b"uncertain", attributes()) + .await + .unwrap(); + let saved = part(&runtime, &client, &upload, 1, b"retained".to_vec(), 61).await; + let bucket = client.head_bucket(context(62), b"bucket").await.unwrap(); + let tenant = crowdb_access_s3::metadata::TenantId::new(b"tenant".to_vec()).unwrap(); + let key = crowdb_access_s3::metadata::MetadataKey::object(&tenant, bucket, b"uncertain").unwrap(); + fault.key.store(Some(Arc::new(key))); + let writes = backend.writes.load(Ordering::Acquire); + assert_eq!( + client + .complete_multipart(context(63), &upload, &[selection(&saved)]) + .await, + Err(Error::UncertainMutation { + request_id: 63, + reason: crowdb_sdk_native::error::MutationInterruption::Transport + }) + ); + assert_eq!(backend.deletes.load(Ordering::Acquire), 0); + assert_eq!(runtime.resources().status().requests, 0); + fault.key.store(None); + fault.blocked.store(false, Ordering::Release); + client + .complete_multipart(context(64), &upload, &[selection(&saved)]) + .await + .unwrap(); + assert_eq!(backend.writes.load(Ordering::Acquire), writes); + assert_eq!(read(&client, b"uncertain").await, b"retained"); + runtime.shutdown().await.unwrap(); +} From 11c8fd999bf5221535f8188fe76ae0a9ac0a9b4e Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 15:36:55 +0800 Subject: [PATCH 071/258] Coordinate owned runtime shutdown while preserving borrowed clients and leases --- doc/working/plan-rdma-fat-client-sdk.md | 28 +++-- lib/crowdb-chunk-client/src/client.rs | 10 ++ .../src/client/connection.rs | 4 +- .../src/client/disk_metrics.rs | 3 + .../src/disk_io/disk_writer.rs | 3 + .../src/disk_io/routing.rs | 4 + .../src/transport/rpc.rs | 21 +++- .../src/rpc_transport.rs | 97 +-------------- .../src/rpc_transport/lifecycle.rs | 117 ++++++++++++++++++ lib/crowdb-diskio-client/src/client.rs | 4 + lib/crowdb-diskio-client/src/semantic.rs | 22 +++- .../src/semantic/connection.rs | 2 + .../src/semantic/native_routes.rs | 12 +- .../src/semantic/shutdown.rs | 29 +++++ .../src/semantic/write.rs | 1 + .../src/semantic/write_views.rs | 1 + lib/crowdb-sdk-native/src/runtime.rs | 19 +-- .../src/runtime/attachment.rs | 16 +++ lib/crowdb-sdk-native/src/runtime/shutdown.rs | 74 +++++++++++ lib/crowdb-sdk-native/src/runtime/storage.rs | 4 + .../src/runtime/writer_policy.rs | 4 + .../tests/common/read_storage.rs | 2 +- .../tests/common/write_storage.rs | 31 ++++- .../tests/runtime_shutdown_test.rs | 80 ++++++++++++ .../tests/s3_rpc_read_test.rs | 7 ++ 25 files changed, 466 insertions(+), 129 deletions(-) create mode 100644 lib/crowdb-chunkdb-client/src/rpc_transport/lifecycle.rs create mode 100644 lib/crowdb-diskio-client/src/semantic/shutdown.rs create mode 100644 lib/crowdb-sdk-native/src/runtime/shutdown.rs create mode 100644 lib/crowdb-sdk-native/tests/runtime_shutdown_test.rs diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index 52fcefe2c..703b9e55f 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -45,7 +45,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. Run focused existing S3/RPC/access and standalone SDK baselines before moving operation code. Files: `Cargo.toml`, `lib/crowdb-sdk-native/`, `lib/crowdb-access-s3/tests/`, `app/crowdb-access-server/tests/`. -- [~] **Define runtime ownership**: implement `ClientRuntime::connect(config)` +- [x] **Define runtime ownership**: implement `ClientRuntime::connect(config)` and shutdown, sharing discovery, compatible transport groups, producer allocation, and hard request/byte limits across handles. Preserve distinct S3/Dataset chunk types and writer policies; do not create a generic Repo @@ -171,7 +171,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. blocking RPC workers. Release backing storage before waking waiters. Files: `lib/crowdb-sdk-native/src/runtime.rs`, `lib/crowdb-sdk-native/src/buffer.rs`, `lib/crowdb-rpc/ffi/tests/`, `lib/crowdb-sdk-native/tests/`. -- [ ] **Implement cancel and shutdown**: close admission, stop producers, +- [x] **Implement cancel and shutdown**: close admission, stop producers, resolve operations once, and preserve storage through disk/CQ/caller ownership. Release after shutdown retires slots without accessing destroyed transport. Files: `lib/crowdb-sdk-native/src/runtime.rs`, @@ -339,6 +339,19 @@ transport acceptance alone cannot establish SDK completion. ## Immediate dependent implementation +- Owned shutdown now has one lock-free coordinator and retained terminal result. + Closing admission starts cleanup independently of any caller's wait future; + owned small pools drain before owned ChunkIO/ChunkDB/Chunk-KV RPC workers and + reapers join on a blocking worker. Attached `from_clients` groups remain + borrowed; `adopt_clients` explicitly transfers shutdown responsibility, and + independently created writer policies own only their new small pools. + DiskIO closes admission and wakes native-route refresh workers. Held native + receive storage remains readable after actual worker shutdown; new DiskIO + operations are rejected. Two new cases cover borrowed ownership and cancelled + close waiters with a paused owned close. Explicit TCP/HCA receive tests pass + with the stronger worker-stop assertion. Native and four affected storage + client all-target suites (48 native cases), clippy and formatting pass. + - HTTP and native multipart now share session creation/expiry, part writer preparation, copied-part execution, exact generation publication, completion freeze and abort. Retained native upload identities are tenant/authority-bound; @@ -386,17 +399,6 @@ transport acceptance alone cannot establish SDK completion. pass. Explicit HCA native producer framing still sends 1 MiB directly with zero staging-copy bytes. Native PUT attachment is the next task. -- Extract HTTP `ObjectWriter` selection/finalization and publication into - shared S3 write sessions before native PUT/copy/multipart. Preserve the - existing digest pipeline, native framed owners, small-writer pools, large - EC policy, and `PutOutcome::Timeout`; frontend parsing/encoding stays HTTP. - Ordinary native input and frozen registered input must use these sessions, - with reserved frame space for in-place preparation when eligible. -- Complete owned runtime shutdown without prematurely destroying held leases. - Distinguish borrowed attached clients from runtime-owned transports and - newly created writer groups. Shared shutdown must run once, drain owned - producers, close RPC workers/reapers and let all shutdown callers observe - the same terminal result. Do not stop externally borrowed clients implicitly. - Audit accepted RDMA read failures above semantic DiskIO: the routed adapter and strip recovery currently have read retry paths. Ensure a native accepted failure cannot turn into a TCP suffix across these layers, while retaining diff --git a/lib/crowdb-chunk-client/src/client.rs b/lib/crowdb-chunk-client/src/client.rs index 75d9fa331..3e496c980 100644 --- a/lib/crowdb-chunk-client/src/client.rs +++ b/lib/crowdb-chunk-client/src/client.rs @@ -100,10 +100,20 @@ struct ClientTopology { service: ServiceRegistryClient, hardware: HardwareClient, chunkdb: Arc, + chunkdb_transport: Arc, disk_writer: Arc, } impl ChunkIoClient { + /// Stops owned data and chunk-metadata RPC workers after writer drain. + /// Embedded borrowers must leave transport shutdown to their original owner. + pub fn shutdown_transports(&self) { + if let Some(topology) = &self.topology { + topology.chunkdb_transport.shutdown(); + } + self.disk_writer.shutdown(); + } + /// Reuse the DiskIO transport's bounded native producer allocation source. #[must_use] pub fn producer_allocator(&self) -> Option { diff --git a/lib/crowdb-chunk-client/src/client/connection.rs b/lib/crowdb-chunk-client/src/client/connection.rs index 94bf01829..a9dea1eef 100644 --- a/lib/crowdb-chunk-client/src/client/connection.rs +++ b/lib/crowdb-chunk-client/src/client/connection.rs @@ -46,7 +46,8 @@ impl ChunkIoClient { let service = ServiceRegistryClient::from_shared(kv.clone()); let hardware = HardwareClient::from_shared(kv.clone()); let range_binding = discover_current_range_bindings(&service, kv.clone()).await?; - let mut chunkdb = ChunkdbClient::new(service.clone(), Arc::new(ChunkdbRpcTransport::new())); + let chunkdb_transport = Arc::new(ChunkdbRpcTransport::new()); + let mut chunkdb = ChunkdbClient::new(service.clone(), Arc::clone(&chunkdb_transport)); if let Some(range_binding) = range_binding { chunkdb = chunkdb.with_range_binding(range_binding); } @@ -84,6 +85,7 @@ impl ChunkIoClient { service, hardware, chunkdb, + chunkdb_transport, disk_writer, })), metrics: None, diff --git a/lib/crowdb-chunk-client/src/client/disk_metrics.rs b/lib/crowdb-chunk-client/src/client/disk_metrics.rs index e210e9dad..8ff819d13 100644 --- a/lib/crowdb-chunk-client/src/client/disk_metrics.rs +++ b/lib/crowdb-chunk-client/src/client/disk_metrics.rs @@ -12,6 +12,9 @@ pub(super) struct MetricsDiskWriter { #[async_trait] impl DiskWriter for MetricsDiskWriter { + fn shutdown(&self) { + self.inner.shutdown(); + } fn producer_allocator(&self) -> Option { self.inner.producer_allocator() } diff --git a/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs b/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs index 6b16f6720..174d93b10 100644 --- a/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs +++ b/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs @@ -13,6 +13,9 @@ use crate::{IoError, Result}; /// Block-IO seam. Production routing is owned by `crowdb-diskio-client`. #[async_trait] pub trait DiskWriter: Send + Sync { + /// Explicit owned shutdown; borrowed client groups do not invoke this hook. + fn shutdown(&self) {} + /// Retained route inspection never opens a connection or publishes placement. fn disk_topology(&self) -> Option { None diff --git a/lib/crowdb-chunk-client/src/disk_io/routing.rs b/lib/crowdb-chunk-client/src/disk_io/routing.rs index 8e88fc0f5..27ade059a 100644 --- a/lib/crowdb-chunk-client/src/disk_io/routing.rs +++ b/lib/crowdb-chunk-client/src/disk_io/routing.rs @@ -128,6 +128,10 @@ impl RoutedDiskWriter { #[async_trait] impl DiskWriter for RoutedDiskWriter { + fn shutdown(&self) { + self.client.shutdown(); + } + fn disk_topology(&self) -> Option { Some(self.topology()) } diff --git a/lib/crowdb-chunk-kv-client/src/transport/rpc.rs b/lib/crowdb-chunk-kv-client/src/transport/rpc.rs index 4e9df2880..07c1b317c 100644 --- a/lib/crowdb-chunk-kv-client/src/transport/rpc.rs +++ b/lib/crowdb-chunk-kv-client/src/transport/rpc.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use crowdb_protocol::chunk_kv::ChunkKvResponse; @@ -17,6 +17,7 @@ use super::endpoint::{normalize_endpoint, parse_endpoint}; use crate::{ClientError, Result}; pub(super) struct RpcGroup { + closed: AtomicBool, pub(super) server: Arc, pub(super) connections: ConnectionPoolIndex, pub(super) data_capacity: Option, @@ -42,6 +43,7 @@ impl RpcGroup { }; server.start(); Ok(Self { + closed: AtomicBool::new(false), server: Arc::new(server), connections: ConnectionPoolIndex::new(pool_size, Some(max_owners)), data_capacity: config.map_or(Some(capacity), RdmaConfig::control_data_capacity), @@ -49,6 +51,9 @@ impl RpcGroup { } fn connection(&self, endpoint: &str, rpc: &RpcClient) -> Result { + if self.closed.load(Ordering::Acquire) { + return Err(ClientError::Transport("Chunk-KV transport is closed".into())); + } let (host, port) = parse_endpoint(endpoint)?; self.connections .get_or_try_install(endpoint, || { @@ -79,6 +84,20 @@ pub struct ChunkKvRpcTransport { } impl ChunkKvRpcTransport { + /// Closes this owned transport and joins its RPC workers and timeout reaper. + /// Existing response buffers retain their storage independently. + pub fn shutdown(&self) { + if self.metadata.closed.swap(true, Ordering::AcqRel) { + return; + } + self.aggregate.closed.store(true, Ordering::Release); + self.metadata.server.stop(); + self.aggregate.server.stop(); + self.metadata.connections.clear(); + self.aggregate.connections.clear(); + self.rpc.stop_reaper(); + } + /// Create the fixed service layouts with environment transport policy. /// /// # Panics diff --git a/lib/crowdb-chunkdb-client/src/rpc_transport.rs b/lib/crowdb-chunkdb-client/src/rpc_transport.rs index abf1109d0..999850a00 100644 --- a/lib/crowdb-chunkdb-client/src/rpc_transport.rs +++ b/lib/crowdb-chunkdb-client/src/rpc_transport.rs @@ -6,7 +6,7 @@ //! Builds typed `FlatBuffer` requests, awaits correlated replies and validates //! response records before returning them to the routing/retry client. -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use flatbuffers::FlatBufferBuilder; @@ -50,17 +50,17 @@ use crowdb_protocol::fb_wrappers::chunkdb::{ FBAdvanceChunkWriteResponseRef, FBAllocateChunkResponseRef, FBAppendChunkResponseRef, FBDeleteChunkRangeResponseRef, FBListChunksResponseRef, FBQueryChunkResponseRef, }; -use crowdb_rpc_ffi::{ - Buffer, ConnectionPoolError, ConnectionPoolIndex, OwnedClientRoute, RpcClient, RpcError, RpcServer, - SelectedConnection, -}; +use crowdb_rpc_ffi::{Buffer, ConnectionPoolIndex, RpcClient, RpcError, RpcServer, SelectedConnection}; use crate::{ChunkdbClientError, Result}; +mod lifecycle; + /// crowdb-rpc transport for `ChunkdbService`. Holds the client-side /// `RpcServer` (manages connections), `RpcClient` (request/response /// correlation), and a `Connection` cache per endpoint. pub struct ChunkdbRpcTransport { + closed: AtomicBool, server: Arc, rpc: Arc, connections: ConnectionPoolIndex, @@ -143,93 +143,6 @@ impl ChunkdbRpcTransport { .map_or_else(Vec::new, |data| data.bytes().to_vec()), }) } - /// Create a new crowdb-rpc transport with 2 I/O workers (default). - /// The `RpcServer` is the client-side transport — it does not listen - /// but is used to establish connections to remote endpoints. - #[must_use] - pub fn new() -> Self { - Self::with_pool_size(1, 2) - } - - /// Create a new crowdb-rpc transport with `workers` I/O worker threads. - #[must_use] - pub fn with_workers(workers: u32) -> Self { - Self::with_pool_size(1, workers) - } - - /// Create a transport with `pool_size` connections per endpoint and - /// `workers` crowdb-rpc I/O workers. - #[must_use] - pub fn with_pool_size(pool_size: usize, workers: u32) -> Self { - let server = Arc::new(RpcServer::with_engines(None, 1, workers)); - server.start(); - server.register_conn_count_gauge("rpc.client.connections"); - let rpc = Arc::new(RpcClient::new()); - rpc.set_completion_pool_size(1024); - // A chunk allocation spans DiskDB allocate + commit and two KV state - // writes. Keep the client deadline aligned with DiskDB's 10-second - // mutation window so a temporary KV tail does not abandon the whole - // chunk after its physical blocks have already been allocated. - rpc.start_reaper(10_000_000_000, 500_000_000); - Self { - server, - rpc, - connections: ConnectionPoolIndex::new(pool_size, None), - next_req_id: AtomicU64::new(1), - } - } - - fn next_id(&self) -> u64 { - REQUEST_IDENTITY - .try_with(|(transport, id)| (*transport == self as *const Self as usize).then_some(*id)) - .ok() - .flatten() - .unwrap_or_else(|| self.next_req_id.fetch_add(1, Ordering::Relaxed)) - } - - pub(crate) async fn with_request_identity(&self, future: F) -> F::Output { - let id = self.next_req_id.fetch_add(1, Ordering::Relaxed); - REQUEST_IDENTITY - .scope((self as *const Self as usize, id), future) - .await - } - - /// Get or create a `Connection` for the given rpc endpoint. - fn conn_for(&self, rpc_endpoint: &str) -> Result { - let normalized = normalize_endpoint(rpc_endpoint); - let (host, port) = parse_endpoint(&normalized).map_err(|reason| { - ChunkdbClientError::Unreachable(format!("invalid endpoint {rpc_endpoint}: {reason}")) - })?; - self.connections - .get_or_try_install(&normalized, || { - let conn = self.server.connect(&host, port).map_err(|error| { - ChunkdbClientError::ConnectFailed(format!("rpc connect to {host}:{port}: {error:?}")) - })?; - self.rpc.attach(&conn); - Ok(conn) - }) - .map_err(|error| match error { - ConnectionPoolError::Closed => { - ChunkdbClientError::ConnectFailed("connection closed during acquisition".into()) - } - ConnectionPoolError::Connect(error) => error, - ConnectionPoolError::Capacity { max_endpoints } => ChunkdbClientError::Unreachable(format!( - "endpoint connection limit {max_endpoints} reached" - )), - }) - } - - /// Resolve an endpoint and retain every crowdb-rpc owner needed by a - /// higher-level FFI transport. - pub fn owned_route(&self, rpc_endpoint: &str) -> Result { - let connection = self.conn_for(rpc_endpoint)?; - Ok(OwnedClientRoute::new( - Arc::clone(&self.rpc), - Arc::clone(&self.server), - connection.into_connection(), - )) - } - // ── AllocateChunk ───────────────────────────────────────────── /// Send an allocation request and decode its typed response. diff --git a/lib/crowdb-chunkdb-client/src/rpc_transport/lifecycle.rs b/lib/crowdb-chunkdb-client/src/rpc_transport/lifecycle.rs new file mode 100644 index 000000000..b33bb017b --- /dev/null +++ b/lib/crowdb-chunkdb-client/src/rpc_transport/lifecycle.rs @@ -0,0 +1,117 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Connection acquisition and owned transport lifecycle. + +use super::{normalize_endpoint, parse_endpoint, ChunkdbRpcTransport, REQUEST_IDENTITY}; +use crate::{ChunkdbClientError, Result}; +use crowdb_rpc_ffi::{ + ConnectionPoolError, ConnectionPoolIndex, OwnedClientRoute, RpcClient, RpcServer, SelectedConnection, +}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; + +impl ChunkdbRpcTransport { + /// Closes this owned transport and joins its workers and timeout reaper. + pub fn shutdown(&self) { + if self.closed.swap(true, Ordering::AcqRel) { + return; + } + self.server.stop(); + self.connections.clear(); + self.rpc.stop_reaper(); + } + + /// Create a new crowdb-rpc transport with 2 I/O workers (default). + /// The `RpcServer` is the client-side transport — it does not listen + /// but is used to establish connections to remote endpoints. + #[must_use] + pub fn new() -> Self { + Self::with_pool_size(1, 2) + } + + /// Create a new crowdb-rpc transport with `workers` I/O worker threads. + #[must_use] + pub fn with_workers(workers: u32) -> Self { + Self::with_pool_size(1, workers) + } + + /// Create a transport with `pool_size` connections per endpoint and + /// `workers` crowdb-rpc I/O workers. + #[must_use] + pub fn with_pool_size(pool_size: usize, workers: u32) -> Self { + let server = Arc::new(RpcServer::with_engines(None, 1, workers)); + server.start(); + server.register_conn_count_gauge("rpc.client.connections"); + let rpc = Arc::new(RpcClient::new()); + rpc.set_completion_pool_size(1024); + // A chunk allocation spans DiskDB allocate + commit and two KV state + // writes. Keep the client deadline aligned with DiskDB's 10-second + // mutation window so a temporary KV tail does not abandon the whole + // chunk after its physical blocks have already been allocated. + rpc.start_reaper(10_000_000_000, 500_000_000); + Self { + closed: AtomicBool::new(false), + server, + rpc, + connections: ConnectionPoolIndex::new(pool_size, None), + next_req_id: AtomicU64::new(1), + } + } + + pub(super) fn next_id(&self) -> u64 { + REQUEST_IDENTITY + .try_with(|(transport, id)| (*transport == self as *const Self as usize).then_some(*id)) + .ok() + .flatten() + .unwrap_or_else(|| self.next_req_id.fetch_add(1, Ordering::Relaxed)) + } + + pub(crate) async fn with_request_identity(&self, future: F) -> F::Output { + let id = self.next_req_id.fetch_add(1, Ordering::Relaxed); + REQUEST_IDENTITY + .scope((self as *const Self as usize, id), future) + .await + } + + /// Get or create a `Connection` for the given rpc endpoint. + pub(super) fn conn_for(&self, rpc_endpoint: &str) -> Result { + if self.closed.load(Ordering::Acquire) { + return Err(ChunkdbClientError::Unreachable( + "ChunkDB RPC transport is closed".into(), + )); + } + let normalized = normalize_endpoint(rpc_endpoint); + let (host, port) = parse_endpoint(&normalized).map_err(|reason| { + ChunkdbClientError::Unreachable(format!("invalid endpoint {rpc_endpoint}: {reason}")) + })?; + self.connections + .get_or_try_install(&normalized, || { + let conn = self.server.connect(&host, port).map_err(|error| { + ChunkdbClientError::ConnectFailed(format!("rpc connect to {host}:{port}: {error:?}")) + })?; + self.rpc.attach(&conn); + Ok(conn) + }) + .map_err(|error| match error { + ConnectionPoolError::Closed => { + ChunkdbClientError::ConnectFailed("connection closed during acquisition".into()) + } + ConnectionPoolError::Connect(error) => error, + ConnectionPoolError::Capacity { max_endpoints } => ChunkdbClientError::Unreachable(format!( + "endpoint connection limit {max_endpoints} reached" + )), + }) + } + + /// Resolve an endpoint and retain every crowdb-rpc owner needed by a + /// higher-level FFI transport. + pub fn owned_route(&self, rpc_endpoint: &str) -> Result { + let connection = self.conn_for(rpc_endpoint)?; + Ok(OwnedClientRoute::new( + Arc::clone(&self.rpc), + Arc::clone(&self.server), + connection.into_connection(), + )) + } +} diff --git a/lib/crowdb-diskio-client/src/client.rs b/lib/crowdb-diskio-client/src/client.rs index fc8ea43be..8ce2d0042 100644 --- a/lib/crowdb-diskio-client/src/client.rs +++ b/lib/crowdb-diskio-client/src/client.rs @@ -118,6 +118,10 @@ impl std::fmt::Debug for WireClient { } impl WireClient { + pub(crate) fn stop_reaper(&self) { + self.rpc.stop_reaper(); + } + /// Create a wire client. Call `attach()` before issuing requests. #[must_use] pub fn new() -> Self { diff --git a/lib/crowdb-diskio-client/src/semantic.rs b/lib/crowdb-diskio-client/src/semantic.rs index 5c6206502..70242d467 100644 --- a/lib/crowdb-diskio-client/src/semantic.rs +++ b/lib/crowdb-diskio-client/src/semantic.rs @@ -6,7 +6,7 @@ use std::collections::HashMap; #[cfg(feature = "test-util")] use std::collections::HashSet; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -25,6 +25,7 @@ use crate::{DiskId, DiskioError, DiskioResult, DiskioStatus, SegmentTarget}; mod connection; mod native_routes; mod observation; +mod shutdown; pub use native_routes::NativeDiskRouteResolver; mod drain; pub use observation::{DiskPlacement, DiskTopologyView}; @@ -296,6 +297,8 @@ impl Drop for InflightGuard<'_> { /// Complete routed semantic `DiskIO` client. pub struct DiskioClient { + closed: AtomicBool, + closing: tokio::sync::Notify, config: DiskioClientConfig, service: Option, hardware: Option, @@ -361,6 +364,8 @@ impl DiskioClient { .map_err(|error| DiskioError::TransportUnavailable(format!("start RPC client: {error}")))?; server.start(); let client = Self { + closed: AtomicBool::new(false), + closing: tokio::sync::Notify::new(), wire: Arc::new(WireClient::with_completion_capacity(config.max_pending_calls)), server, normal: ConnectionPoolIndex::new( @@ -423,6 +428,7 @@ impl DiskioClient { /// Returns a control-plane, topology, or connection error without /// publishing a partial generation. pub async fn refresh(&self) -> DiskioResult { + self.check_open()?; let observed = self.routes.load_full(); let service = self.service.as_ref().ok_or_else(|| { DiskioError::TopologyUnavailable("static test topology cannot be refreshed".into()) @@ -442,6 +448,9 @@ impl DiskioClient { } async fn refresh_topology_if_due(&self) { + if self.closed.load(Ordering::Acquire) { + return; + } if self.service.is_none() { return; } @@ -476,6 +485,7 @@ impl DiskioClient { observed: &Arc, draft: topology::TopologyDraft, ) -> DiskioResult { + self.check_open()?; let mut endpoints = HashMap::, Arc>::new(); for route in draft.routes.values() { endpoints @@ -540,6 +550,7 @@ impl DiskioClient { length: u32, options: OperationOptions, ) -> DiskioResult { + self.check_open()?; self.refresh_topology_if_due().await; let (zone_offset, _) = target.checked_range(offset, length as usize)?; if length == 0 { @@ -595,6 +606,7 @@ impl DiskioClient { /// Returns a typed topology, transport, durability, protocol, or disk /// error. pub async fn fsync(&self, disk_id: DiskId, options: OperationOptions) -> DiskioResult<()> { + self.check_open()?; self.refresh_topology_if_due().await; self.fsync_inner(disk_id, options).await } @@ -635,6 +647,7 @@ impl DiskioClient { } fn begin_operation(&self, kind: OperationKind, lane: TrafficLane) -> DiskioResult> { + self.check_open()?; let pending_limit = u64::try_from(self.config.max_pending_calls).unwrap_or(u64::MAX); let lane_inflight = match lane { TrafficLane::Normal => &self.counters.normal_inflight, @@ -651,16 +664,19 @@ impl DiskioClient { ))); } self.counters.inflight.fetch_add(1, Ordering::Relaxed); - Ok(InflightGuard { + let guard = InflightGuard { counters: &self.counters, kind, started: Instant::now(), counted_inflight: true, lane_inflight: Some(lane_inflight), - }) + }; + self.check_open()?; + Ok(guard) } fn select(&self, disk_id: DiskId, lane: TrafficLane) -> DiskioResult<(DiskRoute, SelectedConnection)> { + self.check_open()?; let route = self.routes.load().routes.get(&disk_id).cloned().ok_or_else(|| { DiskioError::TopologyUnavailable(format!( "disk {}:{} has no published route", diff --git a/lib/crowdb-diskio-client/src/semantic/connection.rs b/lib/crowdb-diskio-client/src/semantic/connection.rs index 70209da20..b02109384 100644 --- a/lib/crowdb-diskio-client/src/semantic/connection.rs +++ b/lib/crowdb-diskio-client/src/semantic/connection.rs @@ -103,6 +103,8 @@ impl DiskioClient { .map_err(|error| DiskioError::TransportUnavailable(format!("start RPC client: {error}")))?; server.start(); let client = Self { + closed: std::sync::atomic::AtomicBool::new(false), + closing: tokio::sync::Notify::new(), wire: Arc::new(WireClient::with_completion_capacity(config.max_pending_calls)), server, normal: ConnectionPoolIndex::new( diff --git a/lib/crowdb-diskio-client/src/semantic/native_routes.rs b/lib/crowdb-diskio-client/src/semantic/native_routes.rs index 6e94f98e1..e6f3ae763 100644 --- a/lib/crowdb-diskio-client/src/semantic/native_routes.rs +++ b/lib/crowdb-diskio-client/src/semantic/native_routes.rs @@ -59,11 +59,21 @@ impl DiskioClient { let mut tick = tokio::time::interval(Duration::from_secs(5)); tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { + let closing = client.closing.notified(); + tokio::pin!(closing); + closing.as_mut().enable(); + if client.closed.load(std::sync::atomic::Ordering::Acquire) { + return; + } tokio::select! { + () = &mut closing => return, _ = tick.tick() => {}, () = worker_refresh.notified() => {}, } - client.refresh_topology_if_due().await; + tokio::select! { + () = &mut closing => return, + () = client.refresh_topology_if_due() => {}, + } let current = client.routes.load_full(); if observed.routes == current.routes && worker_routes diff --git a/lib/crowdb-diskio-client/src/semantic/shutdown.rs b/lib/crowdb-diskio-client/src/semantic/shutdown.rs new file mode 100644 index 000000000..5287db969 --- /dev/null +++ b/lib/crowdb-diskio-client/src/semantic/shutdown.rs @@ -0,0 +1,29 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{DiskioClient, DiskioError, DiskioResult, Ordering}; + +impl DiskioClient { + /// Closes owned request admission, joins RPC workers and stops timeout reaping. + /// Caller-owned payloads retain their native recyclers after worker shutdown. + pub fn shutdown(&self) { + if self.closed.swap(true, Ordering::AcqRel) { + return; + } + self.closing.notify_waiters(); + self.server.stop(); + self.normal.clear(); + self.priority.clear(); + self.wire.stop_reaper(); + } + + pub(super) fn check_open(&self) -> DiskioResult<()> { + if self.closed.load(Ordering::Acquire) { + Err(DiskioError::TransportUnavailable( + "DiskIO client is closed".into(), + )) + } else { + Ok(()) + } + } +} diff --git a/lib/crowdb-diskio-client/src/semantic/write.rs b/lib/crowdb-diskio-client/src/semantic/write.rs index e2027dc91..ae4e939dc 100644 --- a/lib/crowdb-diskio-client/src/semantic/write.rs +++ b/lib/crowdb-diskio-client/src/semantic/write.rs @@ -46,6 +46,7 @@ impl DiskioClient { return Err(DiskioError::DeadlineExceeded); } let write_create_time_ms = unix_time_ms(); + self.check_open()?; self.refresh_topology_if_due().await; if data.is_empty() && mode != WriteMode::Drain { return Err(DiskioError::InvalidInput("write data must be nonempty".into())); diff --git a/lib/crowdb-diskio-client/src/semantic/write_views.rs b/lib/crowdb-diskio-client/src/semantic/write_views.rs index 73aeeb00d..733d3644a 100644 --- a/lib/crowdb-diskio-client/src/semantic/write_views.rs +++ b/lib/crowdb-diskio-client/src/semantic/write_views.rs @@ -23,6 +23,7 @@ impl DiskioClient { durability: Durability, options: OperationOptions, ) -> DiskioResult<()> { + self.check_open()?; let total = data.iter().try_fold(0usize, |total, view| { if view.is_empty() { return Err(DiskioError::InvalidInput("write views must be nonempty".into())); diff --git a/lib/crowdb-sdk-native/src/runtime.rs b/lib/crowdb-sdk-native/src/runtime.rs index adb0208b8..3dde0961c 100644 --- a/lib/crowdb-sdk-native/src/runtime.rs +++ b/lib/crowdb-sdk-native/src/runtime.rs @@ -5,6 +5,7 @@ mod attachment; mod resources; +mod shutdown; mod storage; mod writer_policy; @@ -58,6 +59,7 @@ pub struct ClientRuntime { pub(crate) storage: Arc, pub(crate) resources: Arc, pub(crate) groups: Arc>>>, + shutdown: Arc, } impl ClientRuntime { @@ -80,6 +82,7 @@ impl ClientRuntime { .map_err(|_| Error::Deadline)??, ); Ok(Self { + shutdown: Arc::new(shutdown::ShutdownState::default()), groups: Arc::new(ArcSwap::from_pointee(vec![Arc::clone(&storage)])), storage, resources, @@ -107,20 +110,4 @@ impl ClientRuntime { pub fn writer_group_count(&self) -> usize { self.groups.load().len() } - - /// Closes shared admission and wakes outstanding cancellable operations. - /// Caller-owned leases retain their backing storage after this call. - /// # Errors - /// Returns a writer group's durability/drain failure after closing admission. - pub async fn shutdown(&self) -> Result<()> { - self.resources.shutdown(); - let groups = self.groups.load_full(); - let mut failure = None; - for group in groups.iter() { - if let Err(error) = group.chunks.shutdown_small_writes().await { - failure.get_or_insert_with(|| Error::Storage(error.to_string())); - } - } - failure.map_or(Ok(()), Err) - } } diff --git a/lib/crowdb-sdk-native/src/runtime/attachment.rs b/lib/crowdb-sdk-native/src/runtime/attachment.rs index 8993ab01d..e5baefdbc 100644 --- a/lib/crowdb-sdk-native/src/runtime/attachment.rs +++ b/lib/crowdb-sdk-native/src/runtime/attachment.rs @@ -22,15 +22,30 @@ pub struct RuntimeClients { } impl ClientRuntime { + /// Borrows client groups; SDK shutdown never stops their original owners. /// # Errors /// Rejects invalid resource limits or a missing initial catalog observation. pub fn from_clients(clients: RuntimeClients, limits: RuntimeLimits) -> Result { + Self::attach_clients(clients, limits, false) + } + + /// Transfers shutdown responsibility for these client groups to the runtime. + /// All external Arc clones of the adopted transports close with this runtime. + /// # Errors + /// Rejects invalid limits or a missing initial catalog observation. + pub fn adopt_clients(clients: RuntimeClients, limits: RuntimeLimits) -> Result { + Self::attach_clients(clients, limits, true) + } + + fn attach_clients(clients: RuntimeClients, limits: RuntimeLimits, owned: bool) -> Result { let catalog = clients .metadata .cached_catalog() .ok_or_else(|| Error::Configuration("attached metadata catalog is not initialized".into()))?; let policy = clients.chunks.small_write_policy().as_ref().clone(); let storage = Arc::new(Storage { + owns_small_pool: owned, + owns_transports: owned, control: clients.control, metadata: clients.metadata, chunks: clients.chunks, @@ -44,6 +59,7 @@ impl ClientRuntime { services: Arc::new(ArcSwap::from_pointee(Vec::new())), }); Ok(Self { + shutdown: Arc::new(super::shutdown::ShutdownState::default()), groups: Arc::new(ArcSwap::from_pointee(vec![Arc::clone(&storage)])), storage, resources: RuntimeResources::new(limits)?, diff --git a/lib/crowdb-sdk-native/src/runtime/shutdown.rs b/lib/crowdb-sdk-native/src/runtime/shutdown.rs new file mode 100644 index 000000000..c9f58ec99 --- /dev/null +++ b/lib/crowdb-sdk-native/src/runtime/shutdown.rs @@ -0,0 +1,74 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! One shared close/drain outcome, independent of each shutdown caller's lifetime. + +use super::ClientRuntime; +use crate::{Error, Result}; +use arc_swap::ArcSwapOption; +use futures::FutureExt; +use std::panic::AssertUnwindSafe; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use tokio::sync::Notify; + +#[derive(Default)] +pub(super) struct ShutdownState { + started: AtomicBool, + result: ArcSwapOption>, + changed: Notify, +} + +impl ClientRuntime { + /// Closes admission, drains owned writer groups, and joins owned RPC workers. + /// Borrowed clients remain usable by their original owner. Held buffers retain + /// storage until final release. Cancelling this call does not cancel cleanup; + /// every subsequent caller observes the same terminal result. + /// # Errors + /// Returns the first drain/worker failure after attempting every owned close. + pub async fn shutdown(&self) -> Result<()> { + self.resources.shutdown(); + if !self.shutdown.started.swap(true, Ordering::AcqRel) { + let runtime = self.clone(); + tokio::spawn(async move { + let result = AssertUnwindSafe(runtime.finish_shutdown()) + .catch_unwind() + .await + .unwrap_or_else(|_| Err(Error::Storage("native runtime shutdown panicked".into()))); + runtime.shutdown.result.store(Some(Arc::new(result))); + runtime.shutdown.changed.notify_waiters(); + }); + } + loop { + let changed = self.shutdown.changed.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + if let Some(result) = self.shutdown.result.load_full() { + return result.as_ref().clone(); + } + changed.await; + } + } + + async fn finish_shutdown(&self) -> Result<()> { + let groups = self.groups.load_full(); + let mut failure = None; + for group in groups.iter().filter(|group| group.owns_small_pool) { + if let Err(error) = group.chunks.shutdown_small_writes().await { + failure.get_or_insert_with(|| Error::Storage(error.to_string())); + } + } + let workers = groups.clone(); + let stopped = tokio::task::spawn_blocking(move || { + for group in workers.iter().filter(|group| group.owns_transports) { + group.chunks.shutdown_transports(); + group.metadata_transport.shutdown(); + } + }) + .await; + if let Err(error) = stopped { + failure.get_or_insert_with(|| Error::Storage(error.to_string())); + } + failure.map_or(Ok(()), Err) + } +} diff --git a/lib/crowdb-sdk-native/src/runtime/storage.rs b/lib/crowdb-sdk-native/src/runtime/storage.rs index 9e6012b26..e6398f227 100644 --- a/lib/crowdb-sdk-native/src/runtime/storage.rs +++ b/lib/crowdb-sdk-native/src/runtime/storage.rs @@ -18,6 +18,8 @@ use crate::topology::{CatalogObservation, ServiceObservation}; use crate::{Error, Result}; pub(crate) struct Storage { + pub owns_small_pool: bool, + pub owns_transports: bool, pub control: Arc, pub metadata: Arc, pub chunks: Arc, @@ -69,6 +71,8 @@ impl Storage { .await .map_err(|error| Error::Storage(error.to_string()))?; Ok(Self { + owns_small_pool: true, + owns_transports: true, control, metadata, chunks: Arc::new(chunks), diff --git a/lib/crowdb-sdk-native/src/runtime/writer_policy.rs b/lib/crowdb-sdk-native/src/runtime/writer_policy.rs index e339727f4..770734814 100644 --- a/lib/crowdb-sdk-native/src/runtime/writer_policy.rs +++ b/lib/crowdb-sdk-native/src/runtime/writer_policy.rs @@ -22,6 +22,7 @@ impl ClientRuntime { let observed = self.groups.load_full(); if let Some(storage) = observed.iter().find(|group| &group.policy == policy) { return Ok(Self { + shutdown: Arc::clone(&self.shutdown), storage: Arc::clone(storage), resources: Arc::clone(&self.resources), groups: Arc::clone(&self.groups), @@ -31,6 +32,8 @@ impl ClientRuntime { return Err(Error::Configuration("writer group limit reached".into())); } let storage = Arc::new(Storage { + owns_small_pool: true, + owns_transports: false, control: Arc::clone(&self.storage.control), metadata: Arc::clone(&self.storage.metadata), chunks: Arc::new( @@ -53,6 +56,7 @@ impl ClientRuntime { // A racing shutdown cannot leave an untracked active producer. self.resources.check_open()?; return Ok(Self { + shutdown: Arc::clone(&self.shutdown), storage, resources: Arc::clone(&self.resources), groups: Arc::clone(&self.groups), diff --git a/lib/crowdb-sdk-native/tests/common/read_storage.rs b/lib/crowdb-sdk-native/tests/common/read_storage.rs index deb996b7c..dc407eb07 100644 --- a/lib/crowdb-sdk-native/tests/common/read_storage.rs +++ b/lib/crowdb-sdk-native/tests/common/read_storage.rs @@ -167,7 +167,7 @@ impl TestFixture { disk: Arc, ) -> Self { let metadata = super::metadata::client().await; - let runtime = ClientRuntime::from_clients( + let runtime = ClientRuntime::adopt_clients( RuntimeClients { control: Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))), metadata: metadata.clone(), diff --git a/lib/crowdb-sdk-native/tests/common/write_storage.rs b/lib/crowdb-sdk-native/tests/common/write_storage.rs index c2975bd70..f33d46c5b 100644 --- a/lib/crowdb-sdk-native/tests/common/write_storage.rs +++ b/lib/crowdb-sdk-native/tests/common/write_storage.rs @@ -25,6 +25,10 @@ pub struct TestWriteStorage { pub data: SkipMap, pub writes: AtomicUsize, pub deletes: AtomicUsize, + pub shutdowns: AtomicUsize, + pub pause_shutdown: AtomicBool, + pub shutdown_entered: tokio::sync::Notify, + pub shutdown_resume: tokio::sync::Notify, pub fail_write: AtomicBool, pub block_write: AtomicBool, pub write_started: tokio::sync::Notify, @@ -39,6 +43,10 @@ impl TestWriteStorage { data: SkipMap::new(), writes: AtomicUsize::new(0), deletes: AtomicUsize::new(0), + shutdowns: AtomicUsize::new(0), + pause_shutdown: AtomicBool::new(false), + shutdown_entered: tokio::sync::Notify::new(), + shutdown_resume: tokio::sync::Notify::new(), fail_write: AtomicBool::new(false), block_write: AtomicBool::new(false), write_started: tokio::sync::Notify::new(), @@ -69,7 +77,21 @@ impl TestWriteStorage { } pub fn runtime(self: &Arc, metadata: Arc) -> ClientRuntime { - ClientRuntime::from_clients( + self.attach_runtime(metadata, true) + } + + #[allow(dead_code)] // Used by shutdown ownership tests. + pub fn runtime_borrowed(self: &Arc, metadata: Arc) -> ClientRuntime { + self.attach_runtime(metadata, false) + } + + fn attach_runtime(self: &Arc, metadata: Arc, owned: bool) -> ClientRuntime { + let attach = if owned { + ClientRuntime::adopt_clients + } else { + ClientRuntime::from_clients + }; + attach( RuntimeClients { control: Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))), metadata, @@ -188,6 +210,13 @@ impl ChunkAllocator for TestWriteStorage { #[async_trait] impl DiskWriter for TestWriteStorage { + fn shutdown(&self) { + self.shutdowns.fetch_add(1, Ordering::AcqRel); + self.shutdown_entered.notify_one(); + if self.pause_shutdown.load(Ordering::Acquire) { + futures::executor::block_on(self.shutdown_resume.notified()); + } + } fn producer_allocator(&self) -> Option { Some(self.producer.clone()) } diff --git a/lib/crowdb-sdk-native/tests/runtime_shutdown_test.rs b/lib/crowdb-sdk-native/tests/runtime_shutdown_test.rs new file mode 100644 index 000000000..b9d6aef13 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/runtime_shutdown_test.rs @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/metadata.rs"] +mod metadata; +#[path = "common/write_storage.rs"] +mod write_storage; + +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::Duration; + +use crowdb_chunk_client::SmallWritePolicy; +use crowdb_protocol::chunkdb::rpc::ChunkType; +use crowdb_sdk_native::Error; +use write_storage::TestWriteStorage; + +struct TestPausedShutdown(Arc); + +impl Drop for TestPausedShutdown { + fn drop(&mut self) { + self.0.pause_shutdown.store(false, Ordering::Release); + self.0.shutdown_resume.notify_one(); + } +} + +#[tokio::test] +async fn borrowed_clients_are_not_stopped_when_the_sdk_closes_its_own_admission() { + let backend = TestWriteStorage::new(); + let runtime = backend.runtime_borrowed(metadata::client().await); + let dataset = runtime + .with_writer_policy(&SmallWritePolicy::new(ChunkType::Dataset)) + .unwrap(); + runtime.shutdown().await.unwrap(); + dataset.shutdown().await.unwrap(); + assert!(runtime.resources().status().closed); + assert_eq!(backend.shutdowns.load(Ordering::Acquire), 0); + assert!(matches!( + runtime.with_writer_policy(&SmallWritePolicy::new(ChunkType::S3)), + Err(Error::Shutdown) + )); +} + +#[tokio::test] +async fn cancelled_shutdown_waiters_do_not_cancel_shared_owned_cleanup_or_release_held_data() { + let backend = TestWriteStorage::new(); + backend.pause_shutdown.store(true, Ordering::Release); + let pause = TestPausedShutdown(backend.clone()); + let runtime = backend.runtime(metadata::client().await); + let dataset = runtime + .with_writer_policy(&SmallWritePolicy::new(ChunkType::Dataset)) + .unwrap(); + let held = runtime + .resources() + .retain_vec(b"retained".to_vec(), 1, 1, 0, 0) + .unwrap(); + let first_runtime = runtime.clone(); + let first = tokio::spawn(async move { first_runtime.shutdown().await }); + tokio::time::timeout(Duration::from_secs(2), backend.shutdown_entered.notified()) + .await + .unwrap(); + first.abort(); + assert!(first.await.unwrap_err().is_cancelled()); + let mut second = Box::pin(dataset.shutdown()); + assert!(tokio::time::timeout(Duration::from_millis(20), &mut second) + .await + .is_err()); + assert_eq!(held.as_slice(), b"retained"); + assert_eq!( + runtime.resources().status().held_bytes, + held.metadata().allocation_capacity + ); + drop(pause); + second.await.unwrap(); + runtime.shutdown().await.unwrap(); + assert_eq!(backend.shutdowns.load(Ordering::Acquire), 1); + assert_eq!(held.as_slice(), b"retained"); + drop(held); + assert_eq!(runtime.resources().status().held_bytes, 0); +} diff --git a/lib/crowdb-sdk-native/tests/s3_rpc_read_test.rs b/lib/crowdb-sdk-native/tests/s3_rpc_read_test.rs index a72b63a10..e0996617a 100644 --- a/lib/crowdb-sdk-native/tests/s3_rpc_read_test.rs +++ b/lib/crowdb-sdk-native/tests/s3_rpc_read_test.rs @@ -90,6 +90,13 @@ async fn assert_rpc_delivery(rdma: bool) { assert_eq!(disk.status().read_operations, 2); assert_eq!(disk.status().retries, 0); fixture.runtime.shutdown().await.unwrap(); + let target = + crowdb_diskio_client::SegmentTarget::new(crowdb_diskio_client::DiskId::new(1, 1), 0, 0, 1, 1024) + .unwrap(); + assert!(matches!( + disk.read(target, 0, 8, disk.normal_options()).await, + Err(crowdb_diskio_client::DiskioError::TransportUnavailable(_)) + )); drop(first); drop(second); drop(fixture); From 14bf37f5a5ac99f1a2d2b791ea62c2aa07223607 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 15:50:54 +0800 Subject: [PATCH 072/258] Preserve terminal accepted RDMA read failures through object delivery --- doc/design/rpc/design-crowdb-rpc-rdma.md | 7 ++ doc/working/plan-rdma-fat-client-sdk.md | 20 ++- .../src/chunk/chunk_reader/location.rs | 1 + .../src/chunk/strip_reader.rs | 89 ++++++++----- .../src/disk_io/routing.rs | 1 + lib/crowdb-chunk-client/src/error.rs | 4 + .../tests/common/read_interruption.rs | 35 ++++++ .../tests/read_interruption_test.rs | 117 ++++++++++++++++++ lib/crowdb-diskio-client/src/error.rs | 4 + lib/crowdb-diskio-client/src/semantic.rs | 12 ++ .../tests/rdma_failure_test.rs | 17 ++- lib/crowdb-sdk-native/src/error.rs | 1 + lib/crowdb-sdk-native/src/s3/body.rs | 1 + .../tests/common/rpc_read.rs | 21 +++- .../tests/s3_rpc_read_test.rs | 70 +++++++++++ 15 files changed, 357 insertions(+), 43 deletions(-) create mode 100644 lib/crowdb-chunk-client/tests/common/read_interruption.rs create mode 100644 lib/crowdb-chunk-client/tests/read_interruption_test.rs diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index 41953fe8a..19b71b638 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -455,6 +455,13 @@ uncertain operation through TCP. Writes report their ambiguous outcome and fsync preserves durability failure mapping. Known pre-submit rejection and the existing TCP retry policy retain their separate handling. +An accepted read interruption retains its terminal type through the routed +DiskIO adapter, mirror/EC readers, frame verification and native object stream. +It cannot become a route refresh, replica switch, EC recovery or layout retry. +Ordinary disk failures and verified corruption retain their qualified recovery +rules. A native object body keeps the interruption visible on later pulls; +caller-held buffers remain valid independently of that failure. + `Connection::close()` also wakes its assigned CQ through a private transport notifier. The CQ owner requests `rdma_disconnect` once, guarded by the same per-connection operation admission that protects posting and CM-ID lifetime. diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index 703b9e55f..b068f43a0 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -399,11 +399,21 @@ transport acceptance alone cannot establish SDK completion. pass. Explicit HCA native producer framing still sends 1 MiB directly with zero staging-copy bytes. Native PUT attachment is the next task. -- Audit accepted RDMA read failures above semantic DiskIO: the routed adapter - and strip recovery currently have read retry paths. Ensure a native accepted - failure cannot turn into a TCP suffix across these layers, while retaining - qualified ordinary mirror/EC recovery. Add native GET fault coverage before - claiming the final fallback boundary. +- Accepted RDMA read transport, deadline and malformed-response failures now + retain `AcceptedReadInterrupted` through routed DiskIO, mirror/EC readers, + physical/frame windows and native `ReadFailureKind::Interrupted`. No route, + replica, recovery-candidate or layout retry can replay that accepted read. + Ordinary disk/CRC recovery controls still pass. Four new CPU cases cover + direct mirror/EC, interrupted recovery and framed-stream propagation. + Explicit shared-HCA GET accepts a read, stops the peer with TCP fallback + enabled and three retries configured, and reports the same sticky terminal + error without any additional semantic read, connect, reconnect or retry. + A prior caller-held receive remains readable through owned runtime shutdown. + The three DiskIO HCA failure cases also pass. Native (49 cases), DiskIO and + ChunkIO all-target suites, focused three-crate clippy and formatting pass. + +- Next: native RPC write/registered producer acceptance and the complete + real-service native/OCI S3 corpus, including frontend parity and restart. ## Subsequent Dataset integration under R220 diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader/location.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader/location.rs index ff261b2e2..14592c2bb 100644 --- a/lib/crowdb-chunk-client/src/chunk/chunk_reader/location.rs +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader/location.rs @@ -169,6 +169,7 @@ impl ChunkReader { end: range_end, data, }), + Err(error @ ReadError::Interrupted(_)) => return Err(error), Err(error) => partial.failures.push(FailedReadRange { start: range_start, end: range_end, diff --git a/lib/crowdb-chunk-client/src/chunk/strip_reader.rs b/lib/crowdb-chunk-client/src/chunk/strip_reader.rs index ed1e85ac0..91de3cc04 100644 --- a/lib/crowdb-chunk-client/src/chunk/strip_reader.rs +++ b/lib/crowdb-chunk-client/src/chunk/strip_reader.rs @@ -159,6 +159,9 @@ impl StripReader { } match self.read_target(segment, unit_bytes, offset, length).await { Ok(data) => return Ok((data, failed_segments, vec![*segment])), + Err(crate::IoError::ReadInterrupted(detail)) => { + return Err((ReadError::Interrupted(detail), failed_segments)); + } Err(error) => { failures.push(error.to_string()); let _ = error; @@ -193,36 +196,17 @@ impl StripReader { end, } = geometry; - let mut pieces = Vec::new(); let mut failed_segments = strip.unavailable_segments.clone(); - let first = usize::try_from(offset / shard_bytes).unwrap_or(usize::MAX); - let last = usize::try_from((end - 1) / shard_bytes).unwrap_or(usize::MAX); - for (order, shard_index) in (first..=last).enumerate() { - let shard_start = shard_index as u64 * shard_bytes; - let local_start = offset.max(shard_start) - shard_start; - let local_end = end.min(shard_start + shard_bytes) - shard_start; - let read_len = u32::try_from(local_end - local_start).map_err(|_| { - ( - ReadError::InvalidLocations("EC read exceeds RPC size".into()), - failed_segments.clone(), - ) - })?; - let unavailable = strip.unavailable_segments.contains(&ec.segments[shard_index]); - let segment = ec.segments[shard_index]; - let result = if unavailable { - Err(crate::IoError::ReadFailed("segment is unavailable".into())) - } else { - self.read_target(&segment, unit_bytes, local_start, read_len) - .await - }; - pieces.push((order, shard_index, local_start, read_len, result)); - } - if let [(_, shard_index, _, _, Ok(data))] = pieces.as_slice() { + let pieces = self + .read_ec_pieces(strip, ec, offset..end, shard_bytes, unit_bytes) + .await + .map_err(|error| (error, failed_segments.clone()))?; + if let [(shard_index, _, _, Ok(data))] = pieces.as_slice() { return Ok((data.clone(), failed_segments, vec![ec.segments[*shard_index]])); } let mut output = Vec::with_capacity(usize::try_from(length).unwrap_or(usize::MAX)); let mut served_segments = Vec::new(); - for (_, shard_index, local_start, read_len, result) in pieces { + for (shard_index, local_start, read_len, result) in pieces { if let Ok(data) = result { output.extend_from_slice(&data); push_unique(&mut served_segments, ec.segments[shard_index]); @@ -285,6 +269,38 @@ impl StripReader { Ok((OwnedPayload::from(output), failed_segments, served_segments)) } + async fn read_ec_pieces( + &self, + strip: &ChunkStrip, + ec: &crowdb_protocol::chunkdb::rpc::EcStrip, + range: std::ops::Range, + shard_bytes: u64, + unit_bytes: u64, + ) -> ReadResult> { + let mut pieces = Vec::new(); + let first = usize::try_from(range.start / shard_bytes).unwrap_or(usize::MAX); + let last = usize::try_from((range.end - 1) / shard_bytes).unwrap_or(usize::MAX); + for shard_index in first..=last { + let shard_start = shard_index as u64 * shard_bytes; + let local_start = range.start.max(shard_start) - shard_start; + let local_end = range.end.min(shard_start + shard_bytes) - shard_start; + let read_len = u32::try_from(local_end - local_start) + .map_err(|_| ReadError::InvalidLocations("EC read exceeds RPC size".into()))?; + let segment = ec.segments[shard_index]; + let result = if strip.unavailable_segments.contains(&segment) { + Err(crate::IoError::ReadFailed("segment is unavailable".into())) + } else { + self.read_target(&segment, unit_bytes, local_start, read_len) + .await + }; + if let Err(crate::IoError::ReadInterrupted(detail)) = result { + return Err(ReadError::Interrupted(detail)); + } + pieces.push((shard_index, local_start, read_len, result)); + } + Ok(pieces) + } + #[allow(clippy::too_many_arguments)] async fn recover_slices( &self, @@ -425,11 +441,17 @@ impl StripReader { }; let (index, result) = result.map_err(|error| (ReadError::DiskIo(error.to_string()), failed_segments.clone()))?; - if let Ok(data) = result { - let mut shard = vec![0; length as usize]; - shard[..data.len()].copy_from_slice(&data); - shards[index] = Some(shard); - available = available.saturating_add(1); + match result { + Ok(data) => { + let mut shard = vec![0; length as usize]; + shard[..data.len()].copy_from_slice(&data); + shards[index] = Some(shard); + available = available.saturating_add(1); + } + Err(crate::IoError::ReadInterrupted(detail)) => { + return Err((ReadError::Interrupted(detail), failed_segments)); + } + Err(_) => {} } } let decoded = decode_recoverable(strip, scheme, shards, &failed_segments)?; @@ -450,6 +472,8 @@ impl StripReader { } } +type EcReadPiece = (usize, u64, u32, crate::Result); + async fn read_target( disk_io: &dyn DiskWriter, segment: &Segment, @@ -462,7 +486,10 @@ async fn read_target( match disk_io.read_payload(segment, unit_bytes, offset, length).await { Ok(data) => return Ok(data), Err(error) => { - let connection_failure = matches!(error, crate::IoError::Topology(_)); + let connection_failure = matches!( + error, + crate::IoError::Topology(_) | crate::IoError::ReadInterrupted(_) + ); last_error = Some(error); if connection_failure || attempt == 2 { break; diff --git a/lib/crowdb-chunk-client/src/disk_io/routing.rs b/lib/crowdb-chunk-client/src/disk_io/routing.rs index 27ade059a..a3aa87255 100644 --- a/lib/crowdb-chunk-client/src/disk_io/routing.rs +++ b/lib/crowdb-chunk-client/src/disk_io/routing.rs @@ -411,6 +411,7 @@ fn map_write_error(error: DiskioError) -> IoError { #[allow(clippy::needless_pass_by_value, reason = "used directly as a map_err adapter")] fn map_read_error(error: DiskioError) -> IoError { match error { + DiskioError::AcceptedReadInterrupted(detail) => IoError::ReadInterrupted(detail), DiskioError::TopologyUnavailable(_) | DiskioError::TransportUnavailable(_) | DiskioError::Backpressure(_) diff --git a/lib/crowdb-chunk-client/src/error.rs b/lib/crowdb-chunk-client/src/error.rs index c3b44dbd7..c28303297 100644 --- a/lib/crowdb-chunk-client/src/error.rs +++ b/lib/crowdb-chunk-client/src/error.rs @@ -18,6 +18,8 @@ pub enum IoError { ReadFailed(String), #[error("transient disk read failed: {0}")] TransientRead(String), + #[error("accepted disk read interrupted: {0}")] + ReadInterrupted(String), #[error("chunk not found: {0}")] ChunkNotFound(String), #[error("chunk metadata conflict: {0}")] @@ -65,6 +67,8 @@ pub enum ReadError { Metadata(String), #[error("disk read failed: {0}")] DiskIo(String), + #[error("accepted read interrupted: {0}")] + Interrupted(String), #[error("EC reconstruction failed: {0}")] EcDecode(String), #[error("object bytes [{start}, {end}) could not be read: {message}")] diff --git a/lib/crowdb-chunk-client/tests/common/read_interruption.rs b/lib/crowdb-chunk-client/tests/common/read_interruption.rs new file mode 100644 index 000000000..2d36e4043 --- /dev/null +++ b/lib/crowdb-chunk-client/tests/common/read_interruption.rs @@ -0,0 +1,35 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::atomic::{AtomicUsize, Ordering}; + +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_chunk_client::{DiskWriter, IoError, Result}; +use crowdb_protocol::diskdb::rpc::Segment; + +pub struct TestInterruptedDisk { + pub calls: AtomicUsize, + /// A normal disk failure triggers EC recovery before another read interrupts. + pub first_disk_unavailable: bool, +} + +#[async_trait] +impl DiskWriter for TestInterruptedDisk { + async fn write(&self, _: &Segment, _: u64, _: Bytes) -> Result<()> { + panic!("read wrote storage") + } + + async fn write_at_byte_offset(&self, _: &Segment, _: u64, _: u64, _: Bytes) -> Result<()> { + panic!("read wrote storage") + } + + async fn read(&self, segment: &Segment, _: u64, _: u64, _: u32) -> Result { + self.calls.fetch_add(1, Ordering::Relaxed); + if self.first_disk_unavailable && segment.disk_id.unwrap().low == 1 { + Err(IoError::ReadFailed("ordinary disk failure".into())) + } else { + Err(IoError::ReadInterrupted("accepted RDMA connection failed".into())) + } + } +} diff --git a/lib/crowdb-chunk-client/tests/read_interruption_test.rs b/lib/crowdb-chunk-client/tests/read_interruption_test.rs new file mode 100644 index 000000000..e7a9e4e4b --- /dev/null +++ b/lib/crowdb-chunk-client/tests/read_interruption_test.rs @@ -0,0 +1,117 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/read_interruption.rs"] +mod fault; +#[path = "common/read_payload.rs"] +mod storage; + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use crowdb_chunk_client::{ChunkReadPolicy, ChunkReader, ReadError, StripReader}; +use crowdb_protocol::chunkdb::rpc::{ + Chunk, ChunkState, ChunkStrip, EcState, EcStrip, Location, MirrorStrip, Strip, +}; +use crowdb_protocol::common::{ChunkId, DiskId}; +use crowdb_protocol::diskdb::rpc::Segment; +use crowdb_rpc_ffi::OwnedPayload; +use tokio::sync::Semaphore; + +fn segments(count: u64) -> Vec { + (1..=count) + .map(|index| Segment { + disk_id: Some(DiskId { high: 1, low: index }), + unit_count: 1, + ..Segment::default() + }) + .collect() +} + +fn strip(ec: bool) -> ChunkStrip { + ChunkStrip { + unit_kb: 1, + capacity: if ec { 2 } else { 1 }, + sealed_length: if ec { 2 } else { 1 }, + sealed_ts_ms: 1, + strip: Some(if ec { + Strip::EcStrip(EcStrip { + data_num: 2, + code_num: 2, + ec_state: EcState::Parity as i32, + segments: segments(4), + }) + } else { + Strip::MirrorStrip(MirrorStrip { + segments: segments(2), + }) + }), + ..ChunkStrip::default() + } +} + +async fn assert_terminal_strip(ec: bool, recovery: bool) { + let disk = Arc::new(fault::TestInterruptedDisk { + calls: AtomicUsize::new(0), + first_disk_unavailable: recovery, + }); + let reader = StripReader::new(disk.clone(), Arc::new(Semaphore::new(4096)), 4096); + let error = reader.read(&strip(ec), 0, 0, 8).await.unwrap_err(); + assert!(matches!(error, ReadError::Interrupted(_)), "{error:?}"); + let calls = disk.calls.load(Ordering::Relaxed); + if recovery { + // Three ordinary target retries and at most the two necessary shards. + assert!((4..=5).contains(&calls), "unexpected recovery read count {calls}"); + } else { + assert_eq!(calls, 1, "interrupted reads must not retry or switch replicas"); + } +} + +#[tokio::test] +async fn interrupted_mirror_read_does_not_try_another_replica() { + assert_terminal_strip(false, false).await; +} + +#[tokio::test] +async fn interrupted_ec_target_does_not_fetch_recovery_shards() { + assert_terminal_strip(true, false).await; +} + +#[tokio::test] +async fn interrupted_ec_recovery_does_not_fetch_additional_candidates() { + assert_terminal_strip(true, true).await; +} + +#[tokio::test] +async fn interrupted_framed_stream_preserves_type_without_layout_retry() { + let id = ChunkId { high: 1, low: 1 }; + let allocator = Arc::new(storage::TestPayloadStorage { + chunk: Chunk { + id: Some(id), + state: ChunkState::Sealed as i32, + capacity: 1, + sealed_length: 1, + strips: vec![strip(false)], + ..Chunk::default() + }, + data: OwnedPayload::new(), + }); + let disk = Arc::new(fault::TestInterruptedDisk { + calls: AtomicUsize::new(0), + first_disk_unavailable: false, + }); + let reader = ChunkReader::new(allocator, disk.clone(), ChunkReadPolicy::default()).unwrap(); + let mut stream = reader + .read_stream(&[Location { + chunk_id: Some(id), + length: 42, + logical_length: 8, + ..Location::default() + }]) + .unwrap(); + assert!(matches!( + stream.next_payload().await, + Some(Err(ReadError::Interrupted(_))) + )); + assert_eq!(disk.calls.load(Ordering::Relaxed), 1); +} diff --git a/lib/crowdb-diskio-client/src/error.rs b/lib/crowdb-diskio-client/src/error.rs index 323fd0370..bdf2b6e0b 100644 --- a/lib/crowdb-diskio-client/src/error.rs +++ b/lib/crowdb-diskio-client/src/error.rs @@ -16,6 +16,10 @@ pub enum DiskioError { TopologyInconsistent(String), #[error("DiskIO transport unavailable: {0}")] TransportUnavailable(String), + /// An RDMA read was accepted before its transport or response failed. + /// Higher layers must terminate this read rather than reconnect or replay it. + #[error("accepted RDMA read interrupted: {0}")] + AcceptedReadInterrupted(String), #[error("DiskIO queue rejected the operation: {0}")] Backpressure(String), #[error("DiskIO has not opened the requested disk")] diff --git a/lib/crowdb-diskio-client/src/semantic.rs b/lib/crowdb-diskio-client/src/semantic.rs index 70242d467..62502a3aa 100644 --- a/lib/crowdb-diskio-client/src/semantic.rs +++ b/lib/crowdb-diskio-client/src/semantic.rs @@ -584,6 +584,18 @@ impl DiskioClient { let result = self.await_read(future, length, options.deadline).await; match result { Ok(data) => return Ok(data), + Err(error) + if selected.connection().is_rdma() + && matches!( + error, + DiskioError::TransportUnavailable(_) + | DiskioError::Backpressure(_) + | DiskioError::DeadlineExceeded + | DiskioError::Protocol(_) + ) => + { + return Err(DiskioError::AcceptedReadInterrupted(error.to_string())); + } Err(error) if !selected.connection().is_rdma() && error.is_retryable_read() diff --git a/lib/crowdb-diskio-client/tests/rdma_failure_test.rs b/lib/crowdb-diskio-client/tests/rdma_failure_test.rs index d7d4e02b0..acfe136ce 100644 --- a/lib/crowdb-diskio-client/tests/rdma_failure_test.rs +++ b/lib/crowdb-diskio-client/tests/rdma_failure_test.rs @@ -19,6 +19,17 @@ enum TestOperation { Fsync, } +fn assert_terminal_error(operation: TestOperation, error: &DiskioError) { + match operation { + TestOperation::Write => assert!(matches!(error, DiskioError::AmbiguousWrite(_)), "{error:?}"), + TestOperation::Read => assert!( + matches!(error, DiskioError::AcceptedReadInterrupted(_)), + "{error:?}" + ), + TestOperation::Fsync => assert!(matches!(error, DiskioError::TransportUnavailable(_)), "{error:?}"), + } +} + fn test_rdma_config(address: &str) -> RdmaConfig { RdmaConfig { enabled: true, @@ -124,11 +135,7 @@ async fn assert_accepted_rdma_failure(operation: TestOperation) { .unwrap() .unwrap() .unwrap_err(); - if matches!(operation, TestOperation::Write) { - assert!(matches!(error, DiskioError::AmbiguousWrite(_)), "{error:?}"); - } else { - assert!(matches!(error, DiskioError::TransportUnavailable(_)), "{error:?}"); - } + assert_terminal_error(operation, &error); let status = client.status(); assert_eq!(received.load(Ordering::Acquire), 1); assert_eq!(status.retries, baseline.retries); diff --git a/lib/crowdb-sdk-native/src/error.rs b/lib/crowdb-sdk-native/src/error.rs index d85cf41c7..5fe758abc 100644 --- a/lib/crowdb-sdk-native/src/error.rs +++ b/lib/crowdb-sdk-native/src/error.rs @@ -28,6 +28,7 @@ pub enum ReadFailureKind { DataLoss, Metadata, DiskIo, + Interrupted, EcDecode, FailedRange, } diff --git a/lib/crowdb-sdk-native/src/s3/body.rs b/lib/crowdb-sdk-native/src/s3/body.rs index 9206dde15..6fdbbd891 100644 --- a/lib/crowdb-sdk-native/src/s3/body.rs +++ b/lib/crowdb-sdk-native/src/s3/body.rs @@ -143,6 +143,7 @@ fn read_error(error: GetStreamError) -> Error { ReadError::DataLoss(_) => ReadFailureKind::DataLoss, ReadError::Metadata(_) => ReadFailureKind::Metadata, ReadError::DiskIo(_) => ReadFailureKind::DiskIo, + ReadError::Interrupted(_) => ReadFailureKind::Interrupted, ReadError::EcDecode(_) => ReadFailureKind::EcDecode, ReadError::FailedRange { .. } => ReadFailureKind::FailedRange, }; diff --git a/lib/crowdb-sdk-native/tests/common/rpc_read.rs b/lib/crowdb-sdk-native/tests/common/rpc_read.rs index 2ffdb3cd5..91473f6e4 100644 --- a/lib/crowdb-sdk-native/tests/common/rpc_read.rs +++ b/lib/crowdb-sdk-native/tests/common/rpc_read.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; use crowdb_diskio_client::{DiskId, DiskioClient, DiskioClientConfig, TestDiskRoute}; @@ -17,6 +17,8 @@ use super::storage::TestReadStorage; pub struct TestRpcReadServer { pub server: Arc, pub replies: Arc, + pub requests: Arc, + pub reply_enabled: Arc, address: String, config: RdmaConfig, } @@ -51,6 +53,10 @@ impl TestRpcReadServer { server.listen(&address, 0).unwrap(); let replies = Arc::new(AtomicUsize::new(0)); let count = replies.clone(); + let requests = Arc::new(AtomicUsize::new(0)); + let received = requests.clone(); + let reply_enabled = Arc::new(AtomicBool::new(true)); + let enabled = reply_enabled.clone(); let data = backend.data[0].clone(); server.register_owned_handler( u16::try_from(FBMsgType::EDiskReadRequest.0).unwrap(), @@ -58,6 +64,10 @@ impl TestRpcReadServer { let read = flatbuffers::root::>(request.control()).unwrap(); assert_eq!(read.zone_offset(), 0); assert_eq!(read.size() as usize, data.len()); + received.fetch_add(1, Ordering::Release); + if !enabled.load(Ordering::Acquire) { + return; + } let mut builder = FlatBufferBuilder::new(); let response = FBDiskReadResponse::create( &mut builder, @@ -82,12 +92,18 @@ impl TestRpcReadServer { Self { server, replies, + requests, + reply_enabled, address, config, } } pub fn client(&self) -> Arc { + self.client_with_retries(1, !self.config.enabled) + } + + pub fn client_with_retries(&self, retry_attempts: usize, allow_tcp_fallback: bool) -> Arc { Arc::new( DiskioClient::connect_for_tests( vec![TestDiskRoute { @@ -101,12 +117,13 @@ impl TestRpcReadServer { DiskioClientConfig { rdma: Some(RdmaConfig { listen_enabled: false, + allow_tcp_fallback, ..self.config.clone() }), normal_connections_per_endpoint: 1, priority_connections_per_endpoint: 1, rpc_workers: 1, - retry_attempts: 1, + retry_attempts, ..DiskioClientConfig::default() }, ) diff --git a/lib/crowdb-sdk-native/tests/s3_rpc_read_test.rs b/lib/crowdb-sdk-native/tests/s3_rpc_read_test.rs index e0996617a..8cd1b7aa6 100644 --- a/lib/crowdb-sdk-native/tests/s3_rpc_read_test.rs +++ b/lib/crowdb-sdk-native/tests/s3_rpc_read_test.rs @@ -14,6 +14,7 @@ use std::time::Duration; use crowdb_chunk_client::RoutedDiskWriter; use crowdb_sdk_native::s3::{GetOptions, LocalityOptions}; +use crowdb_sdk_native::{error::ReadFailureKind, Error}; use storage::{context, TestFixture, TestReadStorage}; async fn assert_rpc_delivery(rdma: bool) { @@ -119,3 +120,72 @@ async fn hardware_rdma_get_preserves_registered_receive_capacity_and_shutdown_li assert_rpc_delivery(true).await; } } + +#[tokio::test] +async fn hardware_accepted_rdma_get_failure_is_sticky_without_routed_or_replica_replay() { + if std::env::var_os("CROWDB_RDMA_E2E").is_none() { + return; + } + let backend = Arc::new(TestReadStorage::new()); + let server = rpc::TestRpcReadServer::start(&backend, true); + let disk = server.client_with_retries(3, true); + let fixture = TestFixture::with_disk( + 65_536, + backend, + Arc::new(RoutedDiskWriter::from_client(disk.clone())), + ) + .await; + fixture.publish(&fixture.record(0)).await; + let mut successful = fixture + .client + .get_object(context(201), b"bucket", b"key", &GetOptions::default()) + .await + .unwrap(); + let held = successful.body.next().await.unwrap().unwrap(); + assert_ne!(held.metadata().registration_id, 0); + assert!(successful.body.next().await.unwrap().is_none()); + server.reply_enabled.store(false, Ordering::Release); + let baseline = disk.status(); + let mut interrupted = fixture + .client + .get_object(context(202), b"bucket", b"key", &GetOptions::default()) + .await + .unwrap(); + let pending = tokio::spawn(async move { + let error = interrupted.body.next().await.err().expect("read must fail"); + (interrupted, error) + }); + tokio::time::timeout(Duration::from_secs(1), async { + while server.requests.load(Ordering::Acquire) < 2 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + server.server.stop(); + let (mut interrupted, error) = pending.await.unwrap(); + assert!( + matches!( + error, + Error::Read { + kind: ReadFailureKind::Interrupted, + .. + } + ), + "{error:?}" + ); + assert_eq!( + interrupted.body.next().await.err().expect("failure must persist"), + error + ); + assert!(!interrupted.body.is_complete()); + let status = disk.status(); + assert_eq!(status.read_operations, baseline.read_operations + 1); + assert_eq!(status.retries, baseline.retries); + assert_eq!(status.connect_attempts, baseline.connect_attempts); + assert_eq!(status.reconnect_attempts, baseline.reconnect_attempts); + assert_eq!(server.requests.load(Ordering::Acquire), 2); + assert_eq!(fixture.runtime.resources().status().requests, 0); + fixture.runtime.shutdown().await.unwrap(); + assert_eq!(held.as_slice(), b"original"); +} From 7889dd6acfacae038c8291a7a4ff0c075d163fdb Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 16:38:49 +0800 Subject: [PATCH 073/258] Preserve payload ownership and transport observations with metrics --- doc/working/plan-rdma-fat-client-sdk.md | 43 ++++++--- .../src/client/disk_metrics.rs | 30 +++++++ .../tests/common/read_payload.rs | 4 + .../tests/common/read_payload_fixture.rs | 50 +++++++++++ .../tests/read_payload_test.rs | 88 ++++++++++--------- 5 files changed, 163 insertions(+), 52 deletions(-) create mode 100644 lib/crowdb-chunk-client/tests/common/read_payload_fixture.rs diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index b068f43a0..823b16308 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -53,13 +53,13 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. shared clients instead of reconnecting for each facade. Files: `lib/crowdb-sdk-native/src/runtime.rs`, `lib/crowdb-access-s3/src/storage.rs`, `lib/crowdb-chunk-client/src/client.rs`. -- [~] **Define business context**: expose `S3Client::new(runtime, config)` with +- [x] **Define business context**: expose `S3Client::new(runtime, config)` with tenant/write policy and typed per-operation request identity, deadline, cancellation, conditions, and range. Preserve S3 errors and uncertain outcomes without HTTP headers, XML, or `CredentialProvider`. Files: `lib/crowdb-sdk-native/src/s3/`, `lib/crowdb-access-s3/src/error.rs`, `lib/crowdb-access-s3/src/condition.rs`, `lib/crowdb-access-s3/src/range.rs`. -- [~] **Define common delivery**: reads return metadata plus a pull-based +- [x] **Define common delivery**: reads return metadata plus a pull-based `next() -> Result, _>` body; `Ok(None)` means applicable length/integrity checks passed. Preserve terminal failure across subsequent polling; early drop is cancellation, not successful EOF. Define unique @@ -89,7 +89,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. `lib/crowdb-sdk-native/src/topology.rs`, `lib/crowdb-access-s3/src/metadata/record.rs`, `lib/crowdb-chunk-client/src/client.rs`. -- [~] **Handle stale observations**: retain catalog invalidation, endpoint +- [ ] **Handle stale observations**: retain catalog invalidation, endpoint refresh, ownership epochs, and generation fencing. Old views stay safe to inspect but cannot force execution past validation. Return typed stale/fenced/deadline outcomes when refresh fails; never replay ambiguous @@ -107,7 +107,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. `app/crowdb-access-server/src/s3/operations.rs`, `app/crowdb-access-server/src/s3/operations/delete.rs`, `lib/crowdb-sdk-native/src/s3/`. -- [~] **Extract streaming GET**: adapt `PreparedGet`/`VerifiedGetStream` to +- [x] **Extract streaming GET**: adapt `PreparedGet`/`VerifiedGetStream` to typed metadata and a lazy native body with logical offsets. Preserve selected generation, range checks, storage checksums, and final checksum errors. Do not expose HTTP strings or RPC fragments as native object data. @@ -131,12 +131,16 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. `lib/crowdb-access-multipart/`, `app/crowdb-access-server/src/s3/operations/multipart.rs`, `lib/crowdb-sdk-native/src/s3/`. -- [ ] **Verify native TCP parity first**: compare supported operations through +- [x] **Verify native TCP parity first**: compare supported operations through HTTP and native entry points, then run native access without an HTTP listener or S3 credential store. Compare bytes, metadata, conditions, errors, publication, and cancellation before enabling RDMA. Files: `lib/crowdb-sdk-native/tests/`, `lib/crowdb-access-s3/tests/`, - `app/crowdb-access-server/tests/`. + `app/crowdb-access-server/tests/`, `e2e-test/tests/access_s3/native/`. + The `s3.native` owned fixture starts only KV, Diskdb, DiskIO, Chunkdb and + Chunk-KV. Its client corpus covers mixed sizes, framed input, multipart, + generation-bound copy, paged listing, range/topology and restart. A separate + `s3.native.parity` fixture compares the native and trusted HTTP entry points. ## Phase 4: retained leases, RDMA, and bounded lifecycle @@ -158,7 +162,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. `lib/crowdb-diskio-client/src/semantic.rs`, `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`, `lib/crowdb-access-s3/src/retrieval.rs`. -- [~] **Wire producer buffers**: reuse `ProducerBufferAllocator` and +- [x] **Wire producer buffers**: reuse `ProducerBufferAllocator` and `WritableBuffer`, expose only initialized frozen prefixes, and retain submitted owners through transport/disk completion. Ordinary input adapters stay bounded and account for copies. Files: @@ -186,7 +190,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. ## Phase 5: S3 acceptance and Dataset handoff -- [ ] **Run native and OCI profiles**: extend accepted launcher/runtime +- [~] **Run native and OCI profiles**: extend accepted launcher/runtime conventions with the S3 SDK corpus, no-device TCP and shared-HCA profiles. Record topology, actual path/classes, independent QPs, held-capacity high water, errors, and exact HTTP/native parity. Native-only execution must not @@ -412,8 +416,27 @@ transport acceptance alone cannot establish SDK completion. The three DiskIO HCA failure cases also pass. Native (49 cases), DiskIO and ChunkIO all-target suites, focused three-crate clippy and formatting pass. -- Next: native RPC write/registered producer acceptance and the complete - real-service native/OCI S3 corpus, including frontend parity and restart. +- The real-service `s3.native` and `s3.native.parity` TCP profiles pass in + `run-0ae636d32f1c4b228918a4bf1099708c`. Twelve storage services run without + Access Server or credential lookup; host and unprivileged no-device OCI + clients verify the corpus before and after all storage services restart. + The separate thirteen-service fixture verifies HTTP/native bytes, ETags, + user metadata, conditions/range errors, signed copy and multipart delivery. + Corpus sizes span empty through 4 MiB plus 31 bytes, crossing frame and + small/large boundaries. Reader compatibility copies are zero; held capacity + reaches exactly the configured 8 MiB pressure limit and resumes on release. + A simulated asynchronous copy retains the last lease through owned shutdown. + Small writer input remains retained for open-strip mirror repair until pool + drain; after owned shutdown both producer and held charges return to zero. + Fresh reader runtime artifacts intentionally have zero producer high water. + The fixture suite (157 cases), focused E2E clippy and workspace Rust lint pass. +- [x] **Preserve observed reads with metrics**: forward owned payloads and + transport/topology observations through `MetricsDiskWriter`; test original + allocation identity and read accounting through the public client facade. + Both read-owner cases, all-target ChunkIO clippy, Rust formatting and + tree-lint pass. +- Next: native RPC write/registered producer and shared-HCA native/OCI corpus, + stale endpoint acceptance, distribution documentation and final gates. ## Subsequent Dataset integration under R220 diff --git a/lib/crowdb-chunk-client/src/client/disk_metrics.rs b/lib/crowdb-chunk-client/src/client/disk_metrics.rs index 8ff819d13..7304e3468 100644 --- a/lib/crowdb-chunk-client/src/client/disk_metrics.rs +++ b/lib/crowdb-chunk-client/src/client/disk_metrics.rs @@ -12,6 +12,36 @@ pub(super) struct MetricsDiskWriter { #[async_trait] impl DiskWriter for MetricsDiskWriter { + fn disk_topology(&self) -> Option { + self.inner.disk_topology() + } + + fn transport_observation(&self) -> Result>> { + self.inner.transport_observation() + } + + async fn read_payload( + &self, + seg: &Segment, + unit_bytes: u64, + segment_offset: u64, + length: u32, + ) -> Result { + let mut operation = self.metrics.diskio_read.start(); + let result = self + .inner + .read_payload(seg, unit_bytes, segment_offset, length) + .await; + if let Ok(data) = &result { + self.metrics + .diskio_read_bytes + .observe(u64::try_from(data.len()).unwrap_or(u64::MAX)); + operation.mark_success(); + } + result + } + + fn shutdown(&self) { self.inner.shutdown(); } diff --git a/lib/crowdb-chunk-client/tests/common/read_payload.rs b/lib/crowdb-chunk-client/tests/common/read_payload.rs index afba83c58..c9563a15b 100644 --- a/lib/crowdb-chunk-client/tests/common/read_payload.rs +++ b/lib/crowdb-chunk-client/tests/common/read_payload.rs @@ -44,6 +44,10 @@ impl ChunkAllocator for TestPayloadStorage { #[async_trait] impl DiskWriter for TestPayloadStorage { + fn transport_observation(&self) -> Result>> { + Ok(Some(br#"{"source":"original"}"#.to_vec())) + } + async fn write(&self, _: &Segment, _: u64, _: Bytes) -> Result<()> { panic!("read wrote storage") } diff --git a/lib/crowdb-chunk-client/tests/common/read_payload_fixture.rs b/lib/crowdb-chunk-client/tests/common/read_payload_fixture.rs new file mode 100644 index 000000000..bbefb62b1 --- /dev/null +++ b/lib/crowdb-chunk-client/tests/common/read_payload_fixture.rs @@ -0,0 +1,50 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::storage::TestPayloadStorage; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, ChunkStrip, Location, MirrorStrip, Strip}; +use crowdb_protocol::common::{ChunkId, DiskId}; +use crowdb_protocol::diskdb::rpc::Segment; +use crowdb_protocol::frame::{encode_frame, FrameMagic}; +use crowdb_rpc_ffi::OwnedPayload; +use std::sync::Arc; + +pub fn storage() -> (Arc, Location) { + let id = ChunkId { high: 13, low: 37 }; + let mut data = Vec::with_capacity(4096); + data.extend_from_slice(&encode_frame(FrameMagic::RepoSmallV1, id, b"payload", 1).unwrap()); + let frame_length = data.len(); + let original = OwnedPayload::from(data); + let segment = Segment { + disk_id: Some(DiskId { high: 1, low: 1 }), + unit_count: 1, + ..Segment::default() + }; + let storage = Arc::new(TestPayloadStorage { + chunk: Chunk { + id: Some(id), + state: ChunkState::Sealed as i32, + capacity: 1, + sealed_length: 1, + strips: vec![ChunkStrip { + unit_kb: 1, + capacity: 1, + sealed_length: 1, + sealed_ts_ms: 1, + strip: Some(Strip::MirrorStrip(MirrorStrip { + segments: vec![segment], + })), + ..ChunkStrip::default() + }], + ..Chunk::default() + }, + data: original, + }); + let location = Location { + chunk_id: Some(id), + length: frame_length as u64, + logical_length: 7, + ..Location::default() + }; + (storage, location) +} diff --git a/lib/crowdb-chunk-client/tests/read_payload_test.rs b/lib/crowdb-chunk-client/tests/read_payload_test.rs index 1fa646a6b..3af7c40de 100644 --- a/lib/crowdb-chunk-client/tests/read_payload_test.rs +++ b/lib/crowdb-chunk-client/tests/read_payload_test.rs @@ -1,57 +1,20 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#[path = "common/read_payload_fixture.rs"] +mod fixture; #[path = "common/read_payload.rs"] mod storage; use crowdb_chunk_client::{ChunkReadPolicy, ChunkReader}; -use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, ChunkStrip, Location, MirrorStrip, Strip}; -use crowdb_protocol::common::{ChunkId, DiskId}; -use crowdb_protocol::diskdb::rpc::Segment; -use crowdb_protocol::frame::{encode_frame, FrameMagic}; -use crowdb_rpc_ffi::OwnedPayload; use std::sync::Arc; #[tokio::test] async fn framing_and_stream_credits_preserve_original_allocation_metadata() { - let id = ChunkId { high: 13, low: 37 }; - let mut data = Vec::with_capacity(4096); - data.extend_from_slice(&encode_frame(FrameMagic::RepoSmallV1, id, b"payload", 1).unwrap()); - let frame_length = data.len(); - let original = OwnedPayload::from(data); - let backing = Arc::clone(original.allocation().unwrap()); - let segment = Segment { - disk_id: Some(DiskId { high: 1, low: 1 }), - unit_count: 1, - ..Segment::default() - }; - let storage = Arc::new(storage::TestPayloadStorage { - chunk: Chunk { - id: Some(id), - state: ChunkState::Sealed as i32, - capacity: 1, - sealed_length: 1, - strips: vec![ChunkStrip { - unit_kb: 1, - capacity: 1, - sealed_length: 1, - sealed_ts_ms: 1, - strip: Some(Strip::MirrorStrip(MirrorStrip { - segments: vec![segment], - })), - ..ChunkStrip::default() - }], - ..Chunk::default() - }, - data: original, - }); + let (storage, location) = fixture::storage(); + let frame_length = storage.data.len(); + let backing = Arc::clone(storage.data.allocation().unwrap()); let reader = ChunkReader::new(storage.clone(), storage, ChunkReadPolicy::default()).unwrap(); - let location = Location { - chunk_id: Some(id), - length: frame_length as u64, - logical_length: 7, - ..Location::default() - }; let mut stream = reader.read_stream(&[location]).unwrap(); let payload = stream.next_payload().await.unwrap().unwrap(); assert_eq!(payload.as_ref(), b"payload"); @@ -72,3 +35,44 @@ async fn framing_and_stream_credits_preserve_original_allocation_metadata() { frame_length as u64 ); } + +#[tokio::test] +async fn metrics_preserve_payload_ownership_and_record_one_read() { + use crowdb_chunk_client::{ChunkClientMetrics, ChunkIoClient}; + use crowdb_common::metrics::{MetricPoint, MetricsRegistry}; + use crowdb_protocol::chunkdb::rpc::ChunkType; + + let (storage, location) = fixture::storage(); + let backing = Arc::clone(storage.data.allocation().unwrap()); + let frame_length = storage.data.len(); + let mut registry = MetricsRegistry::new(); + let metrics = Arc::new(ChunkClientMetrics::register(&mut registry)); + let client = ChunkIoClient::from_parts(storage.clone(), storage, ChunkType::S3).with_metrics(&metrics); + assert_eq!( + client.transport_status_json().unwrap(), + br#"{"source":"original"}"# + ); + let mut stream = client.read_stream(&[location]).unwrap(); + let payload = stream.next_payload().await.unwrap().unwrap(); + assert_eq!(payload.as_ref(), b"payload"); + assert!(Arc::ptr_eq(payload.allocation().unwrap(), &backing)); + assert_eq!(payload.allocation().unwrap().capacity(), 4096); + assert_eq!( + payload.as_ptr(), + backing.bytes()[payload.allocation_offset()..].as_ptr() + ); + assert!(stream.next_payload().await.is_none()); + drop(stream); + assert_eq!(metrics.read_flow.snapshot().stream_bytes_released, 0); + drop(payload); + assert_eq!( + metrics.read_flow.snapshot().stream_bytes_released, + frame_length as u64 + ); + assert!(matches!( + registry.snapshot_named("chunkio.diskio.read.e2e.lh", 1.0), + Some(MetricPoint::Histogram { total: 1, .. }) + )); + assert!(matches!(registry.snapshot_named("chunkio.diskio.read.bw", 1.0), + Some(MetricPoint::Bandwidth { total_bytes, .. }) if total_bytes == frame_length as u64)); +} From 5b9778cba00e33f9b2ae8b377b576a855e15523b Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 17:21:28 +0800 Subject: [PATCH 074/258] Verify native S3 ownership across TCP and shared HCA --- Cargo.lock | 1 + doc/backlog/R220-dataset-fat-client-plan.md | 30 ++- .../design-crowdb-access-server.md | 43 ++-- .../access-server/design-crowdb-native-sdk.md | 146 +++++++++++ doc/doc_index.md | 4 +- doc/working/plan-rdma-fat-client-sdk.md | 58 ++++- doc/working/plan-rdma.md | 164 ++++++------ e2e-test/Cargo.toml | 1 + e2e-test/allocation.py | 17 ++ e2e-test/chunkdb_fixture.py | 7 +- e2e-test/clients/s3/http_exchange.py | 5 + e2e-test/diskdb_fixture.py | 4 +- e2e-test/fixture.py | 4 +- e2e-test/native_s3_fixture.py | 59 +++++ e2e-test/native_s3_hca.py | 88 +++++++ e2e-test/run.py | 2 +- e2e-test/scenarios.py | 9 + e2e-test/tests/access_s3.rs | 1 + e2e-test/tests/access_s3/native.rs | 235 ++++++++++++++++++ e2e-test/tests/access_s3/native/parity.rs | 113 +++++++++ e2e-test/tests/common.rs | 1 + e2e-test/tests/common/native_s3.rs | 219 ++++++++++++++++ e2e-test/tests/common/native_s3/multipart.rs | 128 ++++++++++ .../tests/common/native_s3/signed_copy.rs | 79 ++++++ e2e-test/tests/common/native_s3/transport.rs | 59 +++++ e2e-test/tests/native_s3_hca_test.py | 55 ++++ e2e-test/tests/s3_http_exchange_test.py | 58 +++++ lib/crowdb-sdk-native/README.md | 142 +++++++++++ .../tests/common/metadata.rs | 48 +++- lib/crowdb-sdk-native/tests/topology_test.rs | 38 +++ sdk/README.md | 4 + tools/ci-checks/check-ci-test-tasks.py | 2 +- tools/pixi-tasks/test.py | 2 +- tools/rdma/run-native-s3.py | 55 ++++ 34 files changed, 1744 insertions(+), 137 deletions(-) create mode 100644 doc/design/access-server/design-crowdb-native-sdk.md create mode 100644 e2e-test/native_s3_fixture.py create mode 100644 e2e-test/native_s3_hca.py create mode 100644 e2e-test/tests/access_s3/native.rs create mode 100644 e2e-test/tests/access_s3/native/parity.rs create mode 100644 e2e-test/tests/common/native_s3.rs create mode 100644 e2e-test/tests/common/native_s3/multipart.rs create mode 100644 e2e-test/tests/common/native_s3/signed_copy.rs create mode 100644 e2e-test/tests/common/native_s3/transport.rs create mode 100644 e2e-test/tests/native_s3_hca_test.py create mode 100644 lib/crowdb-sdk-native/README.md create mode 100644 tools/rdma/run-native-s3.py diff --git a/Cargo.lock b/Cargo.lock index 492f3a2c4..cf58da0ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1558,6 +1558,7 @@ dependencies = [ "crowdb-monitor", "crowdb-protocol", "crowdb-rpc-ffi", + "crowdb-sdk-native", "flatbuffers", "futures", "hmac", diff --git a/doc/backlog/R220-dataset-fat-client-plan.md b/doc/backlog/R220-dataset-fat-client-plan.md index 87e1b2533..afc9efe7a 100644 --- a/doc/backlog/R220-dataset-fat-client-plan.md +++ b/doc/backlog/R220-dataset-fat-client-plan.md @@ -3,12 +3,12 @@ ### R220: Dataset — Fat-client direct access plan -Status: Deferred until the shared runtime and first S3 frontend in -[R242](R242-rdma-fat-client-sdk.md) are accepted and dedicated CPU HTTP -measurements establish latency, throughput, memory, and digest costs. The -public Rust HTTP SDK and Python wrapper are independently packaged under -`sdk/`; functional container acceptance is complete. These Dataset follow-up -gates do not defer R242's shared foundation or S3 delivery. +Status: Deferred until dedicated CPU HTTP measurements establish latency, +throughput, memory, and digest costs. The shared native runtime, topology, +buffer leases and S3 frontend are available in `crowdb-sdk-native`; native and +unprivileged OCI consumers pass TCP and shared-HCA functional acceptance. +The public Rust HTTP SDK and Python wrapper remain independently packaged +under `sdk/`; functional container acceptance is complete. ## Problem @@ -19,11 +19,14 @@ worker partitioning must remain correct without duplicating authority logic. Performance claims need an HTTP workload baseline. The root contract is [Dataset design](../design/access-server/dataset/design-crowdb-access-dataset.md). +Reuse the [native runtime contract](../design/access-server/design-crowdb-native-sdk.md) +and [SDK API guide](../../lib/crowdb-sdk-native/README.md). ## Solution -1. Attach Dataset after S3 to R242's native runtime in the main workspace. - Reuse topology, transports, admission, and buffer leases; do not introduce +1. Attach Dataset to `ClientRuntime` in the main workspace. Reuse + `TopologyView`, `OperationContext`, `BufferLease`, frozen producer storage + and resource admission; do not introduce another runtime or make the independent HTTP SDK depend on storage FFI. The fat client shares Access Server's internal trust domain, takes explicit Dataset identity/context, and needs neither SigV4 verification, the S3 @@ -46,12 +49,17 @@ The root contract is [Dataset design](../design/access-server/dataset/design-cro existing lease ABI rather than making a new business ABI a prerequisite. 6. Use the runtime's TCP/RDMA policy, with eligible pre-payload fallback only. Native access does not automatically select HTTP on transport failure or - replay ambiguous mutations on another path. GPU delivery reuses R242's + replay ambiguous mutations on another path. GPU delivery reuses the runtime's host-copy lifetime contract; direct GPU memory remains later work. ## Dependencies -- Accepted R242 runtime/topology/lease APIs and first native S3 frontend. +- Existing `ClientRuntime`, topology/locality observations, owner-backed + delivery and first native S3 frontend. Dataset uses the common transport + and budget contracts while retaining its own selection and acknowledgement. + Successful stream EOF, durable publication, caller release and owned runtime + shutdown remain distinct events. The HCA acceptance is same-host functional + evidence; it does not replace Dataset workload or cross-host measurements. - Existing public REST API, packaged HTTP SDK, Dataset CPU container acceptance, and measured Chunk-KV/DiskIO flows. Keep HTTP available as an explicitly selected client while native work is unlanded. @@ -89,7 +97,7 @@ The root contract is [Dataset design](../design/access-server/dataset/design-cro cleanup, and explicit acknowledgement agree. Invariant: binding parity. Integration test. -Run after the R242 native crate is available: +Run: ```bash pixi run -- cargo fmt --all -- --check diff --git a/doc/design/access-server/design-crowdb-access-server.md b/doc/design/access-server/design-crowdb-access-server.md index b57b3b0fd..1226f8f88 100644 --- a/doc/design/access-server/design-crowdb-access-server.md +++ b/doc/design/access-server/design-crowdb-access-server.md @@ -5,9 +5,10 @@ CROWDB presents three first-class upper access models: S3, Iceberg, and Dataset. Each model owns its semantics while sharing the same durable CROWDB -storage substrate. HTTP is the portable server interface. Dataset additionally -provides a topology-aware native client for the shortest path from an -application to CROWDB data and, where supported, GPU memory. +storage substrate. HTTP is the portable server interface. A trusted internal +native runtime exposes topology-aware storage and host-buffer delivery, with +S3 as its first typed frontend. Dataset retains its own semantic authority +when attaching to this shared runtime. ## Table of contents @@ -39,13 +40,14 @@ is an implementation choice and is not part of the architecture contract. | Model | Semantic authority | Portable surface | Performance surface | | ------- | ------------------------------------------ | ---------------- | ---------------------------- | -| S3 | Buckets, objects, multipart, S3 lifecycle | HTTP | Optional direct data plane | -| Iceberg | Catalogs, namespaces, tables, snapshots | HTTP | Delegated immutable FileIO | -| Dataset | Namespaces, snapshots, samples, fields | HTTP | Topology-aware native client | +| S3 | Buckets, objects, multipart, S3 lifecycle | HTTP | Trusted native storage SDK | +| Iceberg | Catalogs, namespaces, tables, snapshots | HTTP | Delegated immutable FileIO | +| Dataset | Namespaces, snapshots, samples, fields | HTTP | Topology-aware native client | Related designs: - [S3](s3/design-crowdb-access-s3.md) +- [Trusted native SDK](design-crowdb-native-sdk.md) - [Iceberg](iceberge/design-crowdb-iceberg.md) - [Dataset](dataset/design-crowdb-access-dataset.md) - [Accelerated transfer](s3/design-crowdb-access-s3-rdma.md) @@ -75,7 +77,7 @@ Related designs: | routed RPC / optional RDMA data plane | - Dataset native client + Shared native runtime ^ | AI runtime / CPU or GPU memory @@ -83,11 +85,11 @@ Related designs: topology-aware direct access ``` -S3 and Iceberg are HTTP protocols. Dataset offers HTTP for interoperability -and a linked native client for maximum performance. The native client is not an -HTTP wrapper: it consumes a validated CROWDB routing view and contacts the -responsible metadata and data services directly, removing the Access Server -routing and payload hop. +S3 and Iceberg expose HTTP protocols. The trusted native S3 facade reuses S3 +execution over the shared runtime and contacts metadata/data services directly. +Dataset's native adapter uses this same runtime while retaining Dataset +selection and acknowledgement semantics. The native runtime removes the +Access Server routing and payload hop. ## 4. Semantic ownership @@ -179,8 +181,8 @@ Each access model stores only its own logical authority and opaque references to data below this boundary. Physical segments, disks, replicas, and erasure-coding shards remain storage -concerns. The native Dataset client may observe the minimum versioned topology -needed for routing, but cannot publish or alter physical placement. +concerns. Native clients observe versioned topology for scheduling and routing, +but cannot publish or alter physical placement. ## 7. Scale-out and security @@ -188,11 +190,12 @@ Access Server instances keep no authoritative namespace or operation outcome. Durable state lives in CROWDB, so any healthy instance can serve a later request or retry. Protocol-local caches are discardable hints. -Direct Dataset access also preserves the storage authority boundary. A client -authenticates before receiving topology, metadata, or payload authority. Direct -operations use short-lived capabilities bound to the principal, logical object, -immutable snapshot, operation, ranges, limits, expiry, and topology epoch. -Topology and memory descriptors alone grant no access. +Trusted internal native clients share Access Server's trust domain and receive +explicit tenant and operation context from their embedding system. They do not +obtain the S3 secret database or verify SigV4. External HTTP authentication +remains in the HTTP frontend. Dataset HTTP capability contracts remain in the +Dataset design; native embedding does not depend on external-user capability +issuance. Topology and memory descriptors alone grant no access. All paths bound connections, metadata work, stream windows, retained buffers, registered memory, retries, and background work independently of total object, @@ -238,7 +241,7 @@ Frontend query failures do not mutate these backend health observations. namespace, publication, authorization, compatibility, and reclamation rules. - **AS-I2 — HTTP boundary:** S3 and Iceberg are HTTP server protocols; Dataset HTTP is the portable Dataset server surface. -- **AS-I3 — Native directness:** Dataset native access reaches responsible +- **AS-I3 — Native directness:** trusted native access reaches responsible CROWDB services without an Access Server request or payload hop. - **AS-I4 — Semantic equivalence:** Dataset HTTP, native CPU, and accelerated GPU paths select the same immutable snapshot and logical data. diff --git a/doc/design/access-server/design-crowdb-native-sdk.md b/doc/design/access-server/design-crowdb-native-sdk.md new file mode 100644 index 000000000..d58aef150 --- /dev/null +++ b/doc/design/access-server/design-crowdb-native-sdk.md @@ -0,0 +1,146 @@ + + + +# CROWDB - Design: Trusted native storage SDK + +The native SDK embeds internal storage access in the application. Its shared +runtime, topology and host-buffer delivery support a typed S3 frontend. + +Depends on: [Access Server](design-crowdb-access-server.md), +[RDMA RPC](../rpc/design-crowdb-rpc-rdma.md), +[Chunk I/O](../chunkio/design-crowdb-chunkio.md). + +Satisfies: Access Server invariants AS-I3, AS-I5, AS-I7 and AS-I9. + +## Table of contents + +1. [Trust and semantic ownership](#1-trust-and-semantic-ownership) +2. [Shared runtime](#2-shared-runtime) +3. [Topology and selection](#3-topology-and-selection) +4. [Data and completion](#4-data-and-completion) +5. [Budgets and shutdown](#5-budgets-and-shutdown) +6. [Transport and integration](#6-transport-and-integration) +7. [Correctness invariants](#7-correctness-invariants) + +## 1. Trust and semantic ownership + +The embedding process is a trusted internal cluster component in the Access +Server trust domain. It supplies tenant and operation context and reaches +management, Chunk-KV, Chunkdb and DiskIO directly. Native operations do not +verify SigV4, receive a user-secret database or call Access Server to authorize +an operation. Network access to internal services belongs to the embedding +system's deployment boundary. + +`S3Executor` owns shared S3 namespace, metadata selection, conditions, +publication and multipart orchestration. HTTP supplies parsing, authentication +and HTTP/XML encoding; `S3Client` supplies typed context and results. Both +preserve S3 generation, checksum, visibility and uncertain-publication rules. +Copy captures a source generation. Multipart completion binds an exact selected +part set; a repeated completed request returns its recorded outcome. + +## 2. Shared runtime + +`ClientRuntime` owns or borrows control, metadata and data clients. Business +handles share transport groups, discovery, topology observations and resource +budgets. Compatible writer policies share one bounded small-writer pool; +incompatible policies use distinct pools while sharing the transport owners. +S3 chunks retain type `S3`; native embedding introduces no generic user-data +chunk type. + +`connect` creates owned groups. `adopt_clients` transfers shutdown +responsibility for existing groups. `from_clients` borrows them, so shutting +down that facade cannot stop its original owner's workers. These ownership +choices are explicit rather than inferred from an Arc reference count. + +## 3. Topology and selection + +`TopologyView` retains catalog generation, independently versioned disk routes, +service observations and actual transport configuration/counters. Sources have +separate observation ages and are not a globally atomic topology epoch. An +independent refresh with unknown observation time reports unknown freshness. +Catalog regression or invalid pages cannot replace a valid cached generation. + +`ResolvedObject` binds one published object record to a tenant and metadata +authority. `object_locality` returns a bounded page of logical extents with +layout revisions, fragment roles and rack/node/disk observations. Inspection +does not dial storage, publish placement or replace routing checks. Execution +uses current routes and layout fencing while preserving the selected object's +identity. A retained old view remains inspectable but cannot pin execution to +an old owner or force a deleted/expired layout into a read. + +## 4. Data and completion + +`ObjectBody` is a lazy pull stream of `BufferLease` values. Each lease exposes +request/stream identity, sequence, logical offset, length, allocation capacity +and registration identity. Storage frame/control bytes are excluded from the +logical object stream. `OwnedPayload` retains the allocation through RPC, +DiskIO, ChunkIO, metrics wrappers and S3 adaptation. + +Successful EOF requires all applicable length and integrity checks. Delivered +metadata or a valid prefix is not full-read success. Terminal errors remain +sticky across polling; early body drop is cancellation. Cancelling one pending +pull preserves its position when the body itself remains live. + +Ordinary input retains its allocation or uses a bounded adapter. A unique +producer allocation can be filled directly in final storage frame slots, +frozen and consumed by `write_framed`. Submitted input is immutable and retained +through physical completion. Acceptance of an input buffer is distinct from +durable object publication; only successful finalization establishes success. +A lost publication reply retains uncertainty and cannot authorize a blind +replay with a new identity. + +## 5. Budgets and shutdown + +Admission is lock-free and bounded across requests, producers and caller-held +allocation capacity. Shared views charge physical capacity once until the final +owner releases it. An allocation's registration budget is separate from SDK +held bytes, posted receives, SQ work requests and writer windows. + +Held-capacity exhaustion pauses production without blocking RPC completion +workers. Releasing the final owner returns capacity and wakes waiters; deadlines, +cancellation and shutdown remain observable. Unknown external owners reserve +bounded capacity before a compatibility copy. Open small strips can retain +input for replica repair after operation completion until seal or pool drain. + +Shutdown closes admission, drains owned writer groups and joins owned workers. +All waiters observe one shutdown result; cancelling a waiter does not cancel +the shared cleanup. Caller-held leases retain storage after shutdown, and final +release never reposts to a destroyed transport. A host-to-GPU copy holds its +host lease until the copy completes, independently of stream completion. + +## 6. Transport and integration + +Metadata and data groups retain their own TCP/RDMA policies. Disabled RDMA uses +TCP; eligible setup failures follow the configured fallback policy. Accepted +or partial RDMA reads fail terminally through routing, mirror/EC recovery and +object delivery; they cannot append a TCP suffix or switch replicas to conceal +an interrupted transfer. Ambiguous writes preserve uncertainty. + +Eligible registered producers avoid staging; eligible receives transfer their +allocation without a second SDK payload copy. Framing/control decoding, +alignment and EC construction/reconstruction retain their required copies. +The base lease API exposes host memory and opaque ownership, not verbs objects, +GPU pointers or RDMA READ/WRITE authority. + +The independent Dataset HTTP distribution has no dependency on this storage/FFI +runtime. A Dataset facade attaches to the common runtime and delivery contracts +while retaining Dataset authority, Snapshot/Split binding and explicit +acknowledgement. Buffer release, download completion and processing +acknowledgement remain separate events. + +## 7. Correctness invariants + +- **NS-I1 — Trusted embedding:** native tenant/context execute inside the + internal-service trust domain without a per-operation HTTP authentication hop. +- **NS-I2 — Shared semantics:** HTTP and native S3 use the same selection, + durability, publication, multipart and cleanup authorities. +- **NS-I3 — Non-authoritative observation:** topology inspection never changes + placement or bypasses owner, generation or layout validation. +- **NS-I4 — Verified completion:** only successful EOF establishes a full read; + a terminal failure cannot later become successful EOF. +- **NS-I5 — Final-owner accounting:** completion does not release caller-held + capacity; the allocation and its charge survive every dependent owner. +- **NS-I6 — Safe shutdown:** closed admission and drained workers do not + invalidate bytes still held by a caller or asynchronous copy. +- **NS-I7 — Bounded transport failure:** accepted RDMA failures never append + TCP data or replay an ambiguous mutation. diff --git a/doc/doc_index.md b/doc/doc_index.md index 37753703a..7b8f689c2 100644 --- a/doc/doc_index.md +++ b/doc/doc_index.md @@ -21,7 +21,7 @@ listed document or section needed by the task. | `doc/design/console/design-crowdb-console.md` | Console architecture and service lifecycle. | | `doc/design/config/design-crowdb-config.md` | Configuration ownership, precedence, validation, reload. | | `doc/design/deploy/design-crowdb-deploy.md` | Node discovery, Group-0 bootstrap, multi-node deployment. | -| `doc/design/access-server/design-crowdb-access-server.md` | S3, Iceberg, native Dataset access, and GPU delivery. | +| `doc/design/access-server/design-crowdb-access-server.md` | S3, Iceberg, Dataset and shared native storage access. | ## Backlog (`doc/backlog/`) @@ -127,6 +127,8 @@ Temporary plans live under `doc/working/`; flow analyses live under | `doc/design/access-server/dataset/spec-crowdb-dataset.md` | Normative Dataset behavior and compatibility. | | `doc/design/access-server/dataset/openapi-crowdb-dataset-v1.yaml` | Dataset REST v1 schemas, framing, bounds and retries. | | `doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml` | Dataset REST v2 capability-gated control operations. | +| `doc/design/access-server/design-crowdb-native-sdk.md` | Trusted runtime, topology, S3 and host-lease ownership. | +| `lib/crowdb-sdk-native/README.md` | Native S3 configuration, streaming and HCA acceptance. | | `sdk/README.md` | Rust/Python SDK installation and resource ownership. | Prefer the most specific match. Open a root design only for cross-topic work; diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md index 823b16308..82cc74de6 100644 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ b/doc/working/plan-rdma-fat-client-sdk.md @@ -15,7 +15,10 @@ native runtime, implement S3 first, and let Dataset attach to the accepted API. Status: mixed-size transport acceptance passes with repeatable CPU/request benefit and unchanged per-size latency bounds. Continue this requirement under `/implement-requirement`; the accepted prerequisite is recorded in the root -RDMA plan. Native SDK implementation and acceptance remain outstanding. +RDMA plan. Native S3 implementation, CPU/TCP/HCA acceptance, workspace gates, +all 45 S3 fixtures and independent HTTP SDK checks pass. The additional serial +protocol suite passes all 23 fixtures. Implementation is complete; requirement +cleanup remains. ## Scope and current boundary @@ -89,7 +92,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. `lib/crowdb-sdk-native/src/topology.rs`, `lib/crowdb-access-s3/src/metadata/record.rs`, `lib/crowdb-chunk-client/src/client.rs`. -- [ ] **Handle stale observations**: retain catalog invalidation, endpoint +- [x] **Handle stale observations**: retain catalog invalidation, endpoint refresh, ownership epochs, and generation fencing. Old views stay safe to inspect but cannot force execution past validation. Return typed stale/fenced/deadline outcomes when refresh fails; never replay ambiguous @@ -181,7 +184,7 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. transport. Files: `lib/crowdb-sdk-native/src/runtime.rs`, `lib/crowdb-sdk-native/src/buffer.rs`, `lib/crowdb-rpc/ffi/src/client.rs`, `lib/crowdb-sdk-native/tests/`. -- [ ] **Verify fallback and GPU-copy lifetime**: retain existing TCP/RDMA +- [x] **Verify fallback and GPU-copy lifetime**: retain existing TCP/RDMA policy, request identity, and eligible pre-payload fallback; fail accepted or partial RDMA operations without TCP suffixes or ambiguous replay. Add an asynchronous host-to-GPU copy simulation holding a lease beyond stream @@ -190,13 +193,13 @@ RDMA plan. Native SDK implementation and acceptance remain outstanding. ## Phase 5: S3 acceptance and Dataset handoff -- [~] **Run native and OCI profiles**: extend accepted launcher/runtime +- [x] **Run native and OCI profiles**: extend accepted launcher/runtime conventions with the S3 SDK corpus, no-device TCP and shared-HCA profiles. Record topology, actual path/classes, independent QPs, held-capacity high water, errors, and exact HTTP/native parity. Native-only execution must not require Access Server. Files: `tools/rdma/`, `e2e-test/tests/access_s3/`, `lib/crowdb-sdk-native/tests/`. -- [ ] **Verify distribution and document API**: run standalone SDK boundary +- [x] **Verify distribution and document API**: run standalone SDK boundary and functional controls; document shared runtime, topology scheduling, stream termination, lease release, publication, and shutdown. Record R242 evidence and the shared API consumed by R220. Files: @@ -435,8 +438,49 @@ transport acceptance alone cannot establish SDK completion. allocation identity and read accounting through the public client facade. Both read-owner cases, all-target ChunkIO clippy, Rust formatting and tree-lint pass. -- Next: native RPC write/registered producer and shared-HCA native/OCI corpus, - stale endpoint acceptance, distribution documentation and final gates. +- Full shared-HCA native and OCI S3 corpus passes in + `native-s3-hca/ee6b87cbef394f3880d877e0178d7263`: twelve real service containers, + explicit HCA `rocep35s0f0`, `10.200.1.1`, separate data directories and CM ports. + Both consumers write their own corpus and verify after full storage restart. + Data clients disable TCP fallback and use twelve independent data connections; + all four reader runs report zero errors/fallback/CQ errors, zero SDK compatibility + copies, 8 MiB held high water and 436404224 registered bytes within 512 MiB. + Each producer run has 1114112 producer bytes high water, 5374424 held high water + and zero SDK compatibility copies. Registered framed input sends 2099385 bytes + directly; ordinary input and EC construction still stage 21247554 bytes. + The small framed-input delta is explicitly zero transport payload copies. + Server control clients retain TCP connectivity; the data client independently + enforces RDMA. OCI uses matched system verbs/provider libraries and no root user. + This is shared-HCA local-loop functional evidence, not cross-host throughput. +- Native topology acceptance now moves a catalog owner/endpoint while an old + view is retained. S3 receives one typed refresh response, refreshes to the + new owner and completes; the retained old view stays unchanged. Existing + object-generation, expired-layout and failed-refresh tests cover the remaining + authority boundary. +- Added native SDK README and shared runtime design, updated Access Server + trust/semantic boundary and the independent HTTP SDK pointer. +- Final HCA run `native-s3-hca/3522f5c97f7c46e880e09ff8ed158605` also verifies + a nonzero registration identity on the retained pressure-test lease. All + native/OCI write and restart/read phases pass with the same counters above. + Image identity is `sha256:685c3632b0b27fdb8a684fda65914f509a34972bc2a9b1b8c6959ea5f92fe28b`. +- Final affected native/S3/RPC/access Rust tests, workspace all-feature clippy, + `rs-lint`, formatting and independent SDK gates pass. The HTTP SDK's first + sandboxed run could not bind loopback sockets; rerunning with local network + permission passes all twenty tests without source changes. +- Full S3 run `run-677eebfc1cbb43859f96a2492f57f2a0` passes 44 of 45 fixtures. + The trailer rejection test races its 110022-byte GET payload against an early + 403 response, raising `BrokenPipeError` before reading the HTTP response; + the exact case passes alone. The raw HTTP test client now reads a complete + error response after an interrupted upload without retrying. A missing or + malformed response still fails, and a 2xx response cannot conceal incomplete + transmission. Four deterministic socket-seam cases preserve these boundaries; + all 163 fixture cases pass. The full default-concurrency S3 suite passes + all 45 fixtures in `run-c3ded34f0ac84d70aae9f0ff913d63d1`; the additional + serial protocol recheck passes all 23 fixtures in + `run-6dd0488a66be4fd5bd480fdf157f06cc`. No server code or deadline was changed. +- R220 now points to the implemented runtime/lease/topology contract and API + guide. Its Dataset measurements and Python binding remain separate work. +- Next: requirement cleanup. ## Subsequent Dataset integration under R220 diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index 8822864c6..9fe349b26 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -4,16 +4,17 @@ # CROWDB RDMA implementation Plan Upstream: [R5](../backlog/R5-rdma-alloc.md), -[RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md), and the remaining -child backlog [R242](../backlog/R242-rdma-fat-client-sdk.md). +[RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md), and the +[native SDK contract](../design/access-server/design-crowdb-native-sdk.md). Goal: implement the host-memory RDMA SEND/RECV path end to end, with RDMA as the preferred connection path, TCP as the one-way fallback, shared aggregation semantics, bounded buffer leases, and automated OCI verification using a single-node HCA local-loop with multiple containers. -This root plan persists until the umbrella goal is complete. Completed child -plans are deleted; remaining child plans retain their implementation detail. +Persistent-plan exception: retain this root record for accepted local-HCA +measurements and the physical multi-node handoff. Completed child plans and +implementation checklists are removed; the evidence remains reproducible. ## Current implementation status @@ -59,10 +60,9 @@ and RPC flow analysis. The foundation, transport, datapath and Chunk-KV continuation is complete. The subsequent mixed-size send optimization also passes its local-HCA -acceptance, as recorded below. The user authorized continuation with R242 -after a repeatable benefit; its implementation now follows the existing -[SDK plan](plan-rdma-fat-client-sdk.md). Physical multi-node validation remains -in its separate plan. +acceptance, as recorded below. The resulting shared native runtime and S3 +frontend pass their TCP and shared-HCA ownership profiles. Physical multi-node +validation remains in its separate plan. ## Accepted mixed-size send optimization @@ -181,88 +181,78 @@ under `plan-rdma-multi-node.md`. - Treat `run-local-loop.sh local` as the smoke test for every local hardware setup. The current acceptance scope is one physical node with multiple containers sharing the configured HCA and opening independent QPs. -- Use host networking only for the multi-container profile. Assign each - container a `CROWDB_PORT_OFFSET`; apply it uniformly to all default - TCP/RPC and RDMA CM service ports, while keeping the HCA physical port - unchanged. +- Use host networking only for the shared-HCA multi-container profile. + Benchmark servers use `CROWDB_PORT_OFFSET`. The real-service SDK fixture + assigns separate loopback endpoints and nonoverlapping RPC/CM ports, while + keeping the HCA physical port unchanged. - Physical multi-node testing is deferred to [the separate follow-up plan](plan-rdma-multi-node.md) and is not part of the current backlog scope. -## Remaining dependency order - -```text -Accepted registered host buffers and common RPC - | -Accepted HTTP/ChunkIO/DiskIO and Chunk-KV/access - | - v - R242 shared runtime/topology, S3 SDK and leases - | - v - R220 Dataset adapter and measurements -``` - -The remaining child uses the existing [fat-client SDK plan](plan-rdma-fat-client-sdk.md) -when the user includes that scope. Do not create a duplicate plan or repeat the -completed service integration. The shared benchmark/functional `Runtime` -already launches native, no-device TCP OCI and shared-HCA OCI profiles. - -Before changing the SDK frontend, run its focused baseline corpus, retain the -accepted transport/service settings and compare exact result bytes and errors. -Transport echo acceptance alone does not prove SDK lease semantics. - -## Phase 4: R242 fat client SDK - -- [ ] **Execute R242 child plan**: build the shared trusted internal runtime, - expose storage topology and FFI-safe host leases, and deliver native S3 - using existing S3 components and Chunk-KV/ChunkIO transports. Native calls - take explicit tenant/context without SigV4 or a user-secret database; - Dataset attaches afterward under R220 using the same runtime. -- [ ] **Host-to-GPU simulation**: hold each lease until the explicit host-to-GPU - copy completes, then release it; no GPU pointer is exposed by the base API. -- [ ] **SDK pressure tests**: hold all returned leases, verify bounded admission, - cancellation, shutdown, and recovery after release. - -Exit condition: native S3 and HTTP return equivalent semantic results, topology -is exposed for caller scheduling with existing freshness/fencing rules, and -caller-controlled lease lifetime is testable through FFI. R220's later Dataset -adapter and measurements do not gate R242 or this RDMA umbrella acceptance. - -## Phase 5: SDK OCI E2E matrix - -- [ ] **SDK TCP-only profile**: extend the shared launcher with the SDK corpus, - omit RDMA devices and require zero RDMA attempts with exact frontend parity. -- [ ] **SDK shared-HCA profile**: run the SDK corpus in two non-root host-network - containers, requiring independent QPs, retained caller leases, bounded - pressure and setup-only TCP fallback. -- [ ] **SDK topology selection**: reuse existing runtime/readiness/report - contracts for the SDK fixture when each profile's prerequisites are present. - -Exit condition: SDK profiles report topology, actual path/classes/QPs, fallback, -held lease pressure, typed errors and exact frontend result equality. Existing -HTTP/DiskIO and Chunk-KV native/OCI corpora remain regression controls. - -## Invariants to retain in SDK work - -The accepted transport/service tests already cover these properties. Reapply -them to the new SDK frontend and its caller-controlled leases: - -- [ ] **No mixed stream**: never append TCP bytes to a partially completed RDMA - stream. -- [ ] **Bounded aggregation**: application message boundaries remain intact; - transport coalescing preserves complete frames, order and per-RPC identity - and stays within both peers' slot limits. -- [ ] **No premature reuse**: completion, disk completion, and caller release - all precede buffer reuse for their respective ownership path. -- [ ] **Bounded memory**: registered bytes, posted receives, send depth, disk - queue bytes, and SDK-held bytes remain within configuration. -- [ ] **Capability optionality**: a host or OCI container without RDMA remains - a supported TCP deployment. -- [ ] **NIC ownership**: each single-node container uses the explicitly - configured shared HCA; no test depends on moving a link or entering a network - namespace. -- [ ] **Shared identity**: request ID, generation, stream sequence, and logical - offset retain their meaning across TCP and RDMA. +## Accepted native runtime and S3 delivery + +- `crowdb-sdk-native` embeds trusted internal S3 over existing metadata and + ChunkIO/DiskIO clients. `ClientRuntime` shares transport, topology and + lock-free request/producer/held-capacity admission. The portable Dataset + HTTP SDK remains independently packaged without storage FFI dependencies. +- HTTP and native use the same S3 execution for namespace, immutable selection, + conditional reads, copy and multipart publication. Native execution needs + explicit tenant/context, with no Access listener or credential lookup. +- `OwnedPayload` preserves physical allocation ownership through RPC, DiskIO, + ChunkIO, metrics and S3. Ordered leases expose request, sequence, offset, + length, capacity and registration identity. Registered framed producers fill + final storage slots before freezing; ordinary input and EC processing retain + their required copies. +- Successful EOF includes final length/checksum verification. Interrupted + accepted RDMA reads fail terminally through mirror/EC recovery; they cannot + append a TCP suffix or change replicas to conceal an interrupted transfer. + Publication uncertainty never authorizes automatic mutation replay. +- Shared views charge an allocation once until final release. Pressure blocks + further delivery at the configured limit. Cancellation and owned shutdown + close admission, drain writers and join RPC workers without invalidating + caller-held leases. Borrowed groups retain their original shutdown owner. +- Topology views retain separate source generations/freshness. A moved owner + test uses the real routed metadata client with an injected catalog/transport: + one stale-owner response refreshes execution while the old view remains + inspectable. Selected-object, expired-layout and refresh-failure regressions + preserve the logical generation and authority boundary. +- Native and no-device OCI TCP acceptance passes before and after full storage + restart in `container-e2e/run-0ae636d32f1c4b228918a4bf1099708c`. The native-only + fixture has twelve real services; the separate thirteen-service parity + fixture compares HTTP/native bytes, ETags, metadata, errors, copy and multipart. +- Final shared-HCA acceptance passes in + `native-s3-hca/3522f5c97f7c46e880e09ff8ed158605`, using image + `sha256:685c3632b0b27fdb8a684fda65914f509a34972bc2a9b1b8c6959ea5f92fe28b`. + Native and non-root OCI consumers each write a corpus, then verify after all + storage services restart. Sizes include 0, 7, 513, 65502, 65503, 1048589, + 2097152 and 4194335 bytes, plus framed producer and multipart cases. +- Each HCA reader reports twelve data connections, 38 receives, zero + fallback/CQ/protocol/post errors, zero SDK compatibility-copy bytes, + 8 MiB held high water and 436404224 registered bytes within a 512 MiB budget. + A registered pressure-test lease survives owned shutdown while a simulated + asynchronous host-to-GPU copy holds it; release returns its final charge. +- Each producer records 1114112 producer bytes high water and 5374424 held + bytes high water. Registered framed input directly sends 2099385 bytes; + ordinary input and EC construction still stage 21247554 bytes. The small + framed-input transport copy delta is zero. Open-strip repair retains small + writer inputs until drain; producer and held charges then return to zero. +- This is shared-HCA local-loop functional evidence. Container io_uring can + select its existing blocking fallback; these checks make no cross-host or + storage-throughput claim. +- Final full S3 acceptance passes all 45 fixtures in + `container-e2e/run-c3ded34f0ac84d70aae9f0ff913d63d1`, including native/HTTP + parity, recovery and Java/JavaScript/Go SDK clients. All 163 fixture tests, + affected Rust suites, workspace all-feature clippy, `rs-lint`, formatting and + independent HTTP SDK boundary/twenty functional tests pass. The additional + serial protocol recheck passes all 23 fixtures in + `container-e2e/run-6dd0488a66be4fd5bd480fdf157f06cc`. The final C++ source + previously passed tree-lint and the native transport gates above. + +The [native SDK guide](../../lib/crowdb-sdk-native/README.md) and +[architecture](../design/access-server/design-crowdb-native-sdk.md) define the +supported API. [R220](../backlog/R220-dataset-fat-client-plan.md) attaches +Dataset authority, Snapshot/Split binding and explicit acknowledgement to the +same runtime after its HTTP workload measurements. Dataset binding work, +GPUDirect and physical multi-node measurements remain separate follow-ups. ## Final verification diff --git a/e2e-test/Cargo.toml b/e2e-test/Cargo.toml index 42257b80e..9ac7a4eeb 100644 --- a/e2e-test/Cargo.toml +++ b/e2e-test/Cargo.toml @@ -24,6 +24,7 @@ crowdb-access-server = { path = "../app/crowdb-access-server" } crowdb-access-iceberg = { path = "../lib/crowdb-access-iceberg" } crowdb-access-dataset = { path = "../lib/crowdb-access-dataset" } crowdb-access-s3 = { path = "../lib/crowdb-access-s3" } +crowdb-sdk-native = { path = "../lib/crowdb-sdk-native" } async-trait = "0.1" axum = "0.7" uuid = { version = "1", features = ["v4"] } diff --git a/e2e-test/allocation.py b/e2e-test/allocation.py index 6f67a7f48..269ea7a1a 100644 --- a/e2e-test/allocation.py +++ b/e2e-test/allocation.py @@ -10,6 +10,9 @@ def create_network(cluster): + if getattr(cluster, "host_network", False): + from native_s3_hca import loopback_subnet + return loopback_subnet(cluster) # Record before allocation because a timed-out request may have succeeded. cluster.network_created = True network_args, subnet = (), None @@ -43,6 +46,12 @@ def start_node(cluster, index, data, arguments, address=None, environment=(), mo raise ValueError("node PID limit must be a bounded positive count") if not running and address is None: raise ValueError("a deferred node needs an explicit advertised address") + if getattr(cluster, "host_network", False): + if not running or preserve_network or network_container is not None: + raise ValueError("HCA host profile requires directly running owned nodes") + from native_s3_hca import start_host_node + return start_host_node(cluster, index, data, arguments, address, environment, mounts, + memory, pids_limit) name = f"{cluster.run_id}-node{index}" network_owner = None if network_container is not None: @@ -93,3 +102,11 @@ def start_node(cluster, index, data, arguments, address=None, environment=(), mo raise AssertionError("deferred node lost its explicit network address") return address return network["IPAddress"] + + +def observed_address(cluster, info, expected): + if getattr(cluster, "host_network", False): + if info["HostConfig"]["NetworkMode"] != "host" or info["Config"]["Labels"].get(LABEL) != cluster.run_id: + raise ValueError("HCA node lost its owned host network") + return expected + return info["NetworkSettings"]["Networks"][cluster.run_id]["IPAddress"] diff --git a/e2e-test/chunkdb_fixture.py b/e2e-test/chunkdb_fixture.py index 76a6fb318..8003bb6bb 100644 --- a/e2e-test/chunkdb_fixture.py +++ b/e2e-test/chunkdb_fixture.py @@ -137,6 +137,7 @@ def start(self): for offset in range(6): index = self.chunkdb_index + len(self.service_instances) + offset address = str(base + index) + rpc_port = self.diskio_port(index) if hasattr(self, "diskio_port") else 7200 data = node_data(self, index) disks = [] if self.storage_backend == "file": @@ -148,7 +149,7 @@ def start(self): path.chmod(0o666) disks.extend(("--disk", f"{disk_id:x}:/opt/crowdb/data/{path.name}:134217728")) self.addresses.append(start_node(self, index, data, ( - "server", "diskio", "--port", "7200", "--bind", address, + "server", "diskio", "--port", str(rpc_port), "--bind", address, *(("--dummy-disk", self.storage_backend) if not disks else ("--no-o-direct", *disks)), "--fault-error-rate", str(self.fault_rate), "--kv-seeds", self.origins[0], "--instance-id", str(8000 + offset), "--rack-id", str(100 + offset), @@ -156,7 +157,7 @@ def start(self): "--sync-interval-ms", "200", "--auto-discover-disks", "--log-dir", "/opt/crowdb/data/log/diskio"), address, preserve_network=self.diskio_process_exit)) - endpoints.append({"node": index, "ip": address, "rpc_port": 7200, + endpoints.append({"node": index, "ip": address, "rpc_port": rpc_port, "instance_id": 8000 + offset, "disk_group_id": 1000 + offset, "data": str(data), "backend": self.storage_backend, "fault_rate": self.fault_rate}) for endpoint in endpoints: @@ -164,7 +165,7 @@ def reachable(endpoint=endpoint): state = json.loads(self.command("inspect", self.servers[endpoint["node"]]))[0]["State"] if not state["Running"]: raise AssertionError("Diskio exited: " + self.command("logs", self.servers[endpoint["node"]])) - with socket.create_connection((endpoint["ip"], 7200), timeout=1): + with socket.create_connection((endpoint["ip"], endpoint["rpc_port"]), timeout=1): return True wait(reachable, bool, "packaged storage Diskio RPC listener") (self.artifacts / "storage-endpoints.json").write_text(json.dumps(endpoints, indent=2)) diff --git a/e2e-test/clients/s3/http_exchange.py b/e2e-test/clients/s3/http_exchange.py index b68aca194..915fac3b6 100644 --- a/e2e-test/clients/s3/http_exchange.py +++ b/e2e-test/clients/s3/http_exchange.py @@ -13,3 +13,8 @@ def request_response(connection, method, path, body, headers): raise return response return connection.getresponse() + + +def exchange(connection, method, path, body, headers): + response = request_response(connection, method, path, body, headers) + return response.status, dict(response.getheaders()), response.read() diff --git a/e2e-test/diskdb_fixture.py b/e2e-test/diskdb_fixture.py index 083a33819..c39a6ea49 100644 --- a/e2e-test/diskdb_fixture.py +++ b/e2e-test/diskdb_fixture.py @@ -5,7 +5,7 @@ import ipaddress import json import time -from allocation import node_data, start_node +from allocation import observed_address, node_data, start_node from fixture import Cluster, request, wait @@ -78,7 +78,7 @@ def crash_restart(self): self.command("start", self.servers[self.diskdb_index]) for index, name in enumerate(self.servers[:self.diskdb_index + 1]): inspected = json.loads(self.command("inspect", name))[0] - if inspected["NetworkSettings"]["Networks"][self.run_id]["IPAddress"] != self.addresses[index]: + if observed_address(self, inspected, self.addresses[index]) != self.addresses[index]: raise AssertionError("Diskdb dependency IP changed during persisted restart") wait(lambda: request(f"http://{self.addresses[self.diskdb_index]}:7101", "/ready"), lambda _: True, "recovered Diskdb readiness") diff --git a/e2e-test/fixture.py b/e2e-test/fixture.py index bdaae20b9..a6107a440 100644 --- a/e2e-test/fixture.py +++ b/e2e-test/fixture.py @@ -12,7 +12,7 @@ import uuid from pathlib import Path -from allocation import LABEL, create_network, node_data, start_node +from allocation import observed_address, LABEL, create_network, node_data, start_node from client import run_client from kv_node import arguments as kv_arguments from control import Control @@ -152,7 +152,7 @@ def refresh_endpoints(self): origins = [] for index, name in enumerate(self.servers): info = json.loads(self.command("inspect", name))[0] - address = info["NetworkSettings"]["Networks"][self.run_id]["IPAddress"] + address = observed_address(self, info, self.addresses[index]) if address != self.addresses[index]: raise AssertionError("persisted peer RPC address changed during restart") origins.append(f"http://{address}:7000") diff --git a/e2e-test/native_s3_fixture.py b/e2e-test/native_s3_fixture.py new file mode 100644 index 000000000..789ea4622 --- /dev/null +++ b/e2e-test/native_s3_fixture.py @@ -0,0 +1,59 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Native and no-device OCI SDK consumers of storage with no Access process.""" + +import json +import os +from pathlib import Path +import time + +from chunk_kv_fixture import ChunkKvCluster +from fixture import LABEL + + +class NativeS3Cluster(ChunkKvCluster): + def client(self, bundle, phase, test_name): + super().client(bundle, phase, test_name) + if test_name.startswith("access_s3::native::"): + self.oci_client(bundle, phase, test_name) + + def oci_client(self, bundle, phase, test_name): + bundle = Path(bundle).resolve() + name = self.run_id + "-sdk-client" + self.containers.append(name) + environment = { + "CROWDB_E2E_RPC": json.dumps(self.addresses), + "CROWDB_E2E_PHASE": phase if getattr(self, "host_network", False) else "verify", + "CROWDB_E2E_ARTIFACTS": str(self.artifacts), + "CROWDB_NATIVE_S3_EXECUTION": "oci", + "CROWDB_RDMA_ENABLED": "0", + "LD_LIBRARY_PATH": str(bundle / "lib"), + } + environment.update(getattr(self, "sdk_environment", {})) + if getattr(self, "host_network", False): + environment.update( + LD_LIBRARY_PATH=f"/usr/lib/x86_64-linux-gnu:{bundle / 'lib'}", + LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libibverbs.so.1:/usr/lib/x86_64-linux-gnu/librdmacm.so.1", + RDMAV_DRIVERS_DIR="/usr/lib/x86_64-linux-gnu/libibverbs", + ) + options = tuple(item for key, value in environment.items() for item in ("--env", f"{key}={value}")) + started = time.monotonic() + output = self.command("run", "--rm", "--name", name, "--label", f"{LABEL}={self.run_id}", + "--network", "host" if getattr(self, "host_network", False) else self.run_id, + *getattr(self, "hca_devices", ()), "--user", str(os.getuid()), + "--cpu-shares", "1024", "--memory", "1024m", "--pids-limit", "128", + "--ulimit", "memlock=-1:-1", + "--mount", f"type=bind,source={bundle},target={bundle},readonly", + "--mount", f"type=bind,source={self.artifacts},target={self.artifacts}", + *options, "--entrypoint", str(bundle / "domain-test"), self.image, + "--ignored", "--exact", test_name, "--nocapture", timeout=getattr(self, "client_timeout", 120)) + (self.artifacts / f"client-{phase}-oci.log").write_text(output) + print(output, end="", flush=True) + if "test result: ok. 1 passed; 0 failed;" not in output: + raise RuntimeError(f"OCI native S3 client failed: {output}") + (self.artifacts / f"client-{phase}-oci.json").write_text(json.dumps({ + "case": test_name, "phase": phase, "execution": "oci", "status": "pass", + "exit_code": 0, + "seconds": time.monotonic() - started, "rdma_devices": list(getattr(self, "hca_devices", ())), "access_process": False, + }, indent=2)) + self.containers.remove(name) diff --git a/e2e-test/native_s3_hca.py b/e2e-test/native_s3_hca.py new file mode 100644 index 000000000..bf7848308 --- /dev/null +++ b/e2e-test/native_s3_hca.py @@ -0,0 +1,88 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Explicit shared-HCA profile; isolated storage directories and loopback endpoints.""" + +import ipaddress +import json +import socket +import uuid +from pathlib import Path + +from allocation import LABEL, observed_address +from native_s3_fixture import NativeS3Cluster + + +def loopback_subnet(cluster): + identity = uuid.UUID(cluster.run_id.removeprefix("crowdb-e2e-")) + subnet = ipaddress.ip_network(f"127.{identity.bytes[0]}.{identity.bytes[1]}.0/24") + (cluster.artifacts / "host-network.json").write_text(json.dumps({ + "mode": "host", "loopback_subnet": str(subnet), "shared_hca": cluster.sdk_environment, + "note": "Same-host HCA local loop, not cross-host or isolated-NIC performance", + }, indent=2)) + return subnet + + +def start_host_node(cluster, index, data, arguments, address, environment, mounts, memory, pids_limit): + if not ipaddress.ip_address(address).is_loopback: + raise ValueError("HCA fixture service endpoints must use its owned loopback range") + arguments = list(arguments) + role = arguments[1] + if role == "kv": + with socket.socket() as reservation: + reservation.bind(("0.0.0.0", 0)) + arguments[arguments.index("--ports") + 1] = str(reservation.getsockname()[1]) + for mount in mounts: + fields = dict(part.split("=", 1) for part in mount.split(",") if "=" in part) + source = Path(fields["source"]).resolve() + if source.parent != cluster.artifacts.resolve() or source.suffix != ".toml": + raise ValueError("HCA profile only rewrites its own generated service configuration") + source.write_text(source.read_text().replace('"0.0.0.0:', f'"{address}:')) + env = {"CROWDB_RDMA_ENABLED": "0"} + devices = () + if role == "diskio": + env.update(cluster.server_rdma_environment) + devices = cluster.hca_devices + name = f"{cluster.run_id}-node{index}" + cluster.containers.append(name) + cluster.servers.append(name) + env_args = tuple(item for value in (*environment, *(f"{k}={v}" for k, v in env.items())) + for item in ("--env", value)) + mount_args = tuple(item for value in mounts for item in ("--mount", value)) + cluster.command("run", "-d", "--name", name, "--label", f"{LABEL}={cluster.run_id}", + "--network", "host", "--cpu-shares", "1024", "--memory", memory, + "--pids-limit", str(pids_limit), "--env", "TOKIO_WORKER_THREADS=1", + "--ulimit", "memlock=-1:-1", *devices, *env_args, *mount_args, + "--mount", f"type=bind,source={data},target=/opt/crowdb/data", + cluster.image, *arguments) + info = json.loads(cluster.command("inspect", name))[0] + return observed_address(cluster, info, address) + + +class NativeS3HcaCluster(NativeS3Cluster): + host_network = True + + def __init__(self, image, artifacts, bundle, *, address, device, **kwargs): + ipaddress.ip_address(address) + verbs = sorted((Path("/sys/class/infiniband") / device / "device/infiniband_verbs").glob("uverbs*")) + if len(verbs) != 1 or not Path("/dev/infiniband/rdma_cm").exists(): + raise ValueError("explicit HCA profile requires one visible verbs device and rdma_cm") + paths = ("/dev/infiniband/rdma_cm", f"/dev/infiniband/{verbs[0].name}") + self.hca_devices = tuple(item for path in paths for item in ("--device", path)) + self.sdk_environment = {"CROWDB_NATIVE_S3_RDMA": "1", "CROWDB_RDMA_SOURCE_ADDR": address, + "CROWDB_RDMA_DEVICE": device} + self.server_rdma_environment = { + "CROWDB_RDMA_ENABLED": "1", "CROWDB_RDMA_ALLOW_TCP_FALLBACK": "1", + "CROWDB_RDMA_SOURCE_ADDR": address, "CROWDB_RDMA_DEVICE": device, + "CROWDB_RDMA_WIRE_LAYOUT": "1", "CROWDB_RDMA_CQ_COUNT": "1", + "CROWDB_RDMA_CQ_DEPTH": "512", "CROWDB_RDMA_SQ_DEPTH": "16", + "CROWDB_RDMA_RQ_DEPTH": "16", "CROWDB_RDMA_POSTED_RECEIVE_TARGET": "16", + "CROWDB_RDMA_SEND_ARENA_BYTES": str(16 << 20), + "CROWDB_RDMA_RECEIVE_ARENA_BYTES": str(16 * ((1 << 20) + 1024)), + "CROWDB_RDMA_AGGREGATION_ENABLED": "0", + } + super().__init__(image, artifacts, bundle, **kwargs) + + def diskio_port(self, _index): + with socket.socket() as reservation: + reservation.bind((self.sdk_environment["CROWDB_RDMA_SOURCE_ADDR"], 0)) + return reservation.getsockname()[1] diff --git a/e2e-test/run.py b/e2e-test/run.py index 4d361be3b..9702d6600 100644 --- a/e2e-test/run.py +++ b/e2e-test/run.py @@ -88,7 +88,7 @@ def scenario(index): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--layer", choices=("kv", "diskdb", "diskio", "chunkdb", "chunk-stream", "chunk-client", "chunk-kv", "s3", "iceberg", "dataset", "console", "deployment", "all"), default="all") - parser.add_argument("--scope", choices=("all", "dataset.http", "dataset.sdk", "console.managed", "console.native", "console.native.binding", "console.native.s3", "console.native.recovery", "console.native.load", "console.native.inspection", "console.native.browser.s3", "console.native.browser.ownership", "console.native.browser.journal", "console.native.browser.journal-windows", "console.native.browser.scanner", "console.native.browser.storage-windows", "console.native.browser.lifecycle", "console.native.browser.defaults", "console.native.browser.plan", "console.native.browser.prerequisites", "console.native.browser.transition", "console.native.geometry", "console.native.browser.hardware", "console.native.browser.maintenance", "console.native.browser.disk", "console.native.browser.binding", "console.native.browser.diskdb", "console.native.browser.crud", "console.native.browser.shell", "console.native.browser.retry", "console.native.browser.topology", "console.native.browser.six-node", "console.native.browser.dataset", "console.browser", "console.browser.lifecycle", "console.browser.discovery", "console.browser.shell", "console.browser.registry", "console.browser.kv", "console.browser.canvas", "console.browser.inspector", "console.browser.logical", "console.browser.node", "console.discovery", "console.hardware", "console.logical", "console.bootstrap", "console.cleanup", "console.deployment", "console.cli", "console.cli.container", "console.operations", "console.web", "console.web.disposable", "console.web.recovery", "console.web.access", "console.web.authority", "console.web.slots", "console.web.catalog", "console.web.bootstrap", "console.web.standalone", "console.web.registry", "console.web.lifecycle", "console.web.wal", "console.clients", "iceberg.manual.tpc", "iceberg.benchmark", "iceberg.metadata", "iceberg.namespace", "iceberg.files", "iceberg.files.listing", "iceberg.catalog", "iceberg.gc", "iceberg.sdk", "iceberg.sdk.java", "iceberg.sdk.rust", "iceberg.sdk.rck", "s3.protocol", "s3.recovery", "s3.responses", "s3.benchmark", "s3.sdk", "s3.credentials", "s3.policy", "s3.nodes", "kv.management", "kv.membership", "kv.routing", "kv.recovery", "kv.isolation", "kv.group0", "kv.client", "kv.metadata", "diskdb.allocation", "diskdb.ownership", "diskdb.reuse", "diskdb.recovery", "chunkdb.allocation", "chunkdb.lifecycle", "chunkdb.recovery", "chunkdb.placement", "chunkdb.io", "chunkdb.conversion", "chunkdb.repair", "chunkdb.slots", "chunkdb.handoff", "chunk-kv.protocol", "chunk-kv.recovery", "chunk-client.reader", "chunk-client.large_repair", "chunk-client.small_flow", "chunk-client.benchmark", "chunk-client.large", "chunk-client.small", "chunk-stream.objects", "chunk-stream.recovery", "chunk-stream.errors", "deployment.reference", "deployment.selection", "deployment.supervision", "deployment.hardware", "deployment.bootstrap", "deployment.balance", "deployment.listeners", "deployment.browser", "deployment.anonymous", "deployment.failures", "deployment.clients", "deployment.storage", "deployment.validation", "deployment.kv", "diskio.durability", "diskio.direct", "diskio.io.mem", "diskio.io.null", "diskio.discovery", "diskio.errors", "diskio.faults", "diskio.semantic"), + parser.add_argument("--scope", choices=("all", "dataset.http", "dataset.sdk", "console.managed", "console.native", "console.native.binding", "console.native.s3", "console.native.recovery", "console.native.load", "console.native.inspection", "console.native.browser.s3", "console.native.browser.ownership", "console.native.browser.journal", "console.native.browser.journal-windows", "console.native.browser.scanner", "console.native.browser.storage-windows", "console.native.browser.lifecycle", "console.native.browser.defaults", "console.native.browser.plan", "console.native.browser.prerequisites", "console.native.browser.transition", "console.native.geometry", "console.native.browser.hardware", "console.native.browser.maintenance", "console.native.browser.disk", "console.native.browser.binding", "console.native.browser.diskdb", "console.native.browser.crud", "console.native.browser.shell", "console.native.browser.retry", "console.native.browser.topology", "console.native.browser.six-node", "console.native.browser.dataset", "console.browser", "console.browser.lifecycle", "console.browser.discovery", "console.browser.shell", "console.browser.registry", "console.browser.kv", "console.browser.canvas", "console.browser.inspector", "console.browser.logical", "console.browser.node", "console.discovery", "console.hardware", "console.logical", "console.bootstrap", "console.cleanup", "console.deployment", "console.cli", "console.cli.container", "console.operations", "console.web", "console.web.disposable", "console.web.recovery", "console.web.access", "console.web.authority", "console.web.slots", "console.web.catalog", "console.web.bootstrap", "console.web.standalone", "console.web.registry", "console.web.lifecycle", "console.web.wal", "console.clients", "iceberg.manual.tpc", "iceberg.benchmark", "iceberg.metadata", "iceberg.namespace", "iceberg.files", "iceberg.files.listing", "iceberg.catalog", "iceberg.gc", "iceberg.sdk", "iceberg.sdk.java", "iceberg.sdk.rust", "iceberg.sdk.rck", "s3.native", "s3.native.parity", "s3.protocol", "s3.recovery", "s3.responses", "s3.benchmark", "s3.sdk", "s3.credentials", "s3.policy", "s3.nodes", "kv.management", "kv.membership", "kv.routing", "kv.recovery", "kv.isolation", "kv.group0", "kv.client", "kv.metadata", "diskdb.allocation", "diskdb.ownership", "diskdb.reuse", "diskdb.recovery", "chunkdb.allocation", "chunkdb.lifecycle", "chunkdb.recovery", "chunkdb.placement", "chunkdb.io", "chunkdb.conversion", "chunkdb.repair", "chunkdb.slots", "chunkdb.handoff", "chunk-kv.protocol", "chunk-kv.recovery", "chunk-client.reader", "chunk-client.large_repair", "chunk-client.small_flow", "chunk-client.benchmark", "chunk-client.large", "chunk-client.small", "chunk-stream.objects", "chunk-stream.recovery", "chunk-stream.errors", "deployment.reference", "deployment.selection", "deployment.supervision", "deployment.hardware", "deployment.bootstrap", "deployment.balance", "deployment.listeners", "deployment.browser", "deployment.anonymous", "deployment.failures", "deployment.clients", "deployment.storage", "deployment.validation", "deployment.kv", "diskio.durability", "diskio.direct", "diskio.io.mem", "diskio.io.null", "diskio.discovery", "diskio.errors", "diskio.faults", "diskio.semantic"), default="all") parser.add_argument("--profile", choices=("bridge", "bridge-admission", "host", "containerd-host", "containerd-cluster", "ecosystem", "ecosystem-python"), default="bridge") parser.add_argument("--concurrency", type=int, default=2) diff --git a/e2e-test/scenarios.py b/e2e-test/scenarios.py index 490a67989..b5933fbc0 100644 --- a/e2e-test/scenarios.py +++ b/e2e-test/scenarios.py @@ -15,6 +15,7 @@ from diskdb_fixture import DiskdbCluster from chunkdb_fixture import ChunkdbCluster, ChunkdbStorageCluster from chunk_kv_fixture import ChunkKvCluster +from native_s3_fixture import NativeS3Cluster from access_s3_fixture import AccessS3Cluster from s3_node_fixture import S3NodeCluster from access_protocol_fixture import AccessProtocolCluster @@ -339,6 +340,10 @@ class Case: Case("s3.benchmark", "s3.benchmark::s3_request_path", sdk_client=True, s3_script="benchmark")) S3_LANGUAGE_CASES = tuple(Case("s3.sdk", "s3.sdk::" + language, sdk_client=True, s3_language=language) for language in ("java", "js", "go")) +S3_SPECIAL_CASES += ( + Case("s3.native", "access_s3::native::complete_corpus_survives_storage_restart_without_access_or_credentials", restart=True), + Case("s3.native.parity", "access_s3::native::parity::http_and_native_share_bytes_metadata_conditions_copy_and_multipart", access_protocol=True), +) ICEBERG_CASES = ( Case("iceberg.files.listing", "access_iceberg::file_http::listing::native_list_objects_v2_serves_selected_files_and_rejects_foreign_scope", iceberg_deferred_access=True), Case("iceberg.files.listing", "access_iceberg::file_http::listing::official_pyarrow_native_fileio_creates_exact_files_and_lists_table_prefixes", iceberg_deferred_access=True), @@ -600,6 +605,8 @@ def case_nodes(case): return 13 if case.scope.startswith("iceberg."): return 14 + if case.scope == "s3.native": + return 12 if case.scope.startswith("s3."): return 14 if case.scope.startswith("chunk-kv."): @@ -718,6 +725,8 @@ def allocate_cluster(case, image, artifacts, bundle, data_root): return ChunkKvCluster(image, artifacts, bundle, service_instances=(11, 12, 13), storage_groups=(1, 2, 3), diskio_process_exit=case.diskio_process_exit, data_root=data_root) + if case.scope == "s3.native": + return NativeS3Cluster(image, artifacts, bundle, data_root=data_root) if case.scope.startswith("s3."): return AccessS3Cluster(image, artifacts, bundle, data_root=data_root) if case.scope.startswith("chunk-kv."): diff --git a/e2e-test/tests/access_s3.rs b/e2e-test/tests/access_s3.rs index eaf7cf2c0..612ef7b92 100644 --- a/e2e-test/tests/access_s3.rs +++ b/e2e-test/tests/access_s3.rs @@ -4,6 +4,7 @@ //! Host S3 clients, credential authority and protected storage recovery. mod benchmark; +mod native; mod node_failure; mod policy; mod policy_failure; diff --git a/e2e-test/tests/access_s3/native.rs b/e2e-test/tests/access_s3/native.rs new file mode 100644 index 000000000..4643f9320 --- /dev/null +++ b/e2e-test/tests/access_s3/native.rs @@ -0,0 +1,235 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +mod parity; + +use crate::common::native_s3::{context, payload, TestNativeS3}; +use crate::common::native_s3::{multipart, transport}; +use crowdb_access_s3::S3ErrorCode; +use crowdb_sdk_native::s3::{ + ByteRange, CopyOptions, EtagSelection, GetOptions, ListOptions, LocalityOptions, ReadPreconditions, +}; +use crowdb_sdk_native::Error; + +const SIZES: &[usize] = &[0, 7, 513, 65_502, 65_503, (1 << 20) + 13, 2 << 20, (4 << 20) + 31]; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires owned protected storage services without Access Server"] +async fn complete_corpus_survives_storage_restart_without_access_or_credentials() { + let addresses = crate::common::management::addresses(12); + let verify = std::env::var("CROWDB_E2E_PHASE").as_deref() == Ok("verify"); + if !verify { + write_corpus(&addresses).await; + } + let fixture = TestNativeS3::connect(&addresses, b"native-internal").await; + for &length in SIZES { + let key = format!("size-{length:08}"); + assert_eq!( + fixture.read(key.as_bytes(), GetOptions::default()).await.1, + payload(length) + ); + } + assert_eq!( + fixture.read(b"framed-small", GetOptions::default()).await.1, + payload(513) + ); + assert_eq!( + fixture.read(b"framed-large", GetOptions::default()).await.1, + payload((2 << 20) + 17) + ); + multipart::verify(&fixture).await; + assert_topology_and_range(&fixture).await; + let topology = fixture.runtime.topology().unwrap(); + transport::assert_selected(&fixture.runtime); + let resources = fixture.runtime.resources().clone(); + assert_eq!(resources.status().compatibility_copy_bytes, 0); + assert_eq!(resources.status().held_bytes, 0); + assert_eq!(resources.status().requests, 0); + fixture.assert_pressure_and_shutdown().await; + let status = resources.status(); + assert!(status.held_bytes_high_water <= 8 << 20); + assert!(status.producer_bytes_high_water <= 8 << 20); + let execution = std::env::var("CROWDB_NATIVE_S3_EXECUTION").unwrap_or_else(|_| "native".into()); + let phase = if verify { "verify" } else { "write" }; + let evidence = serde_json::json!({ + "execution": execution, "phase": phase, "transport": if transport::enabled() { "rdma" } else { "tcp" }, "access_server": false, + "credential_lookup": false, "sizes": SIZES, "errors": 0, + "held_bytes_high_water": status.held_bytes_high_water, + "producer_bytes_high_water": status.producer_bytes_high_water, + "compatibility_copy_bytes": status.compatibility_copy_bytes, + "catalog_generation": topology.catalog().generation(), + "metadata_transport": serde_json::from_slice::(&topology.metadata_transport_json).unwrap(), + "data_transport": topology.data_transport_json.as_ref().map(|json| serde_json::from_slice::(json).unwrap()), + }); + let artifacts = std::path::PathBuf::from(std::env::var("CROWDB_E2E_ARTIFACTS").unwrap()); + std::fs::write( + artifacts.join(format!("native-s3-{execution}-{phase}.json")), + serde_json::to_vec_pretty(&evidence).unwrap(), + ) + .unwrap(); +} + +async fn write_corpus(addresses: &[String]) { + let fixture = TestNativeS3::connect(addresses, b"native-internal").await; + fixture + .client + .create_bucket(context(), fixture.bucket) + .await + .unwrap(); + for &length in SIZES { + let key = format!("size-{length:08}"); + let metadata = fixture.put(key.as_bytes(), payload(length)).await; + assert_eq!(metadata.logical_length, length as u64); + } + fixture.put_framed(b"framed-small", 513).await; + fixture.put_framed(b"framed-large", (2 << 20) + 17).await; + Box::pin(assert_conditions_copy_and_listing(&fixture)).await; + Box::pin(multipart::exercise(&fixture)).await; + let resources = fixture.runtime.resources().clone(); + assert!(resources.status().held_bytes_high_water <= 8 << 20); + assert!(resources.status().producer_bytes_high_water <= 8 << 20); + assert_eq!(resources.status().compatibility_copy_bytes, 0); + transport::assert_selected(&fixture.runtime); + let evidence = serde_json::json!({ + "held_bytes_high_water": resources.status().held_bytes_high_water, + "producer_bytes_high_water": resources.status().producer_bytes_high_water, + "compatibility_copy_bytes": resources.status().compatibility_copy_bytes, + "data_transport": transport::data_status(&fixture.runtime), + }); + let artifacts = std::path::PathBuf::from(std::env::var("CROWDB_E2E_ARTIFACTS").unwrap()); + let execution = std::env::var("CROWDB_NATIVE_S3_EXECUTION").unwrap_or_else(|_| "native".into()); + std::fs::write( + artifacts.join(format!("producer-{execution}.json")), + serde_json::to_vec_pretty(&evidence).unwrap(), + ) + .unwrap(); + // Open small strips retain their input views for mirror repair. Closing + // the owned pool seals those strips and must release every such owner. + fixture.runtime.shutdown().await.unwrap(); + assert_eq!(resources.status().held_bytes, 0); + assert_eq!(resources.status().producer_bytes, 0); +} + +async fn assert_conditions_copy_and_listing(fixture: &TestNativeS3) { + let key = b"size-00000513"; + let selected = fixture + .client + .resolve_object(context(), fixture.bucket, key) + .await + .unwrap(); + let conditions = ReadPreconditions { + if_match: Some(EtagSelection::Values(vec!["wrong".into()])), + ..ReadPreconditions::default() + }; + assert!(matches!( + fixture + .client + .get_object( + context(), + fixture.bucket, + key, + &GetOptions { + conditions, + ..GetOptions::default() + } + ) + .await, + Err(Error::S3(S3ErrorCode::PreconditionFailed)) + )); + fixture.put(b"temporary", b"original".to_vec()).await; + let source = fixture + .client + .resolve_object(context(), fixture.bucket, b"temporary") + .await + .unwrap(); + fixture.put(b"temporary", b"replacement".to_vec()).await; + fixture + .client + .copy_resolved( + context(), + &source, + fixture.bucket, + b"captured-copy", + CopyOptions::default(), + ) + .await + .unwrap(); + assert_eq!( + fixture.read(b"captured-copy", GetOptions::default()).await.1, + b"original" + ); + assert_eq!(selected.metadata().logical_length, 513); + let mut options = ListOptions { + prefix: b"size-".to_vec(), + limit: 2, + ..ListOptions::default() + }; + let mut listed = Vec::new(); + loop { + let page = fixture + .client + .list_objects(context(), fixture.bucket, &options) + .await + .unwrap(); + listed.extend(page.objects.into_iter().map(|object| object.key)); + options.continuation = page.next_continuation; + if options.continuation.is_none() { + break; + } + } + assert_eq!(listed.len(), SIZES.len()); + assert!(listed.windows(2).all(|pair| pair[0] < pair[1])); + let mut deletion = fixture + .client + .delete_objects( + context(), + fixture.bucket, + vec![b"temporary".to_vec(), b"missing".to_vec()], + ) + .await + .unwrap(); + while let Some(result) = deletion.next().await.unwrap() { + assert!(result.1.is_ok()); + } +} + +async fn assert_topology_and_range(fixture: &TestNativeS3) { + let key = b"framed-large"; + let selected = fixture + .client + .resolve_object(context(), fixture.bucket, key) + .await + .unwrap(); + let range = ByteRange { + start: 65_499, + end: 65_521, + }; + let view = fixture + .client + .object_locality( + context(), + &selected, + &LocalityOptions { + range: Some(range), + limit: 1, + ..LocalityOptions::default() + }, + ) + .await + .unwrap(); + assert_eq!(view.object, *selected.metadata()); + assert!(view.extents.iter().all(|extent| !extent.strips.is_empty())); + let fresh = fixture.runtime.refresh_topology(context()).await.unwrap(); + assert!(fresh.services.iter().all(|service| !service.instances.is_empty())); + let result = fixture + .read( + key, + GetOptions { + range: Some(range), + ..GetOptions::default() + }, + ) + .await + .1; + assert_eq!(result, payload((2 << 20) + 17)[65_499..65_521]); +} diff --git a/e2e-test/tests/access_s3/native/parity.rs b/e2e-test/tests/access_s3/native/parity.rs new file mode 100644 index 000000000..1abf065d9 --- /dev/null +++ b/e2e-test/tests/access_s3/native/parity.rs @@ -0,0 +1,113 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::common::native_s3::{context, payload, TestNativeS3}; +use crowdb_access_s3::S3ErrorCode; +use crowdb_sdk_native::s3::{GetOptions, ReadPreconditions}; +use crowdb_sdk_native::Error; +use reqwest::{Client, StatusCode}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires owned protected storage and trusted HTTP Access frontend"] +async fn http_and_native_share_bytes_metadata_conditions_copy_and_multipart() { + let addresses = crate::common::management::addresses(13); + let fixture = TestNativeS3::connect(&addresses, b"local").await; + fixture + .client + .create_bucket(context(), fixture.bucket) + .await + .unwrap(); + let http = Client::builder().no_proxy().build().unwrap(); + let base = format!("http://{}:7500/native-corpus", addresses[12]); + for length in [0, 513, 65_503, (2 << 20) + 17] { + let key = format!("native-{length}"); + let expected = payload(length); + let metadata = fixture.put(key.as_bytes(), expected.clone()).await; + let response = http.get(format!("{base}/{key}")).send().await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()["etag"].to_str().unwrap().trim_matches('"'), + metadata.etag + ); + assert_eq!(response.bytes().await.unwrap().as_ref(), expected); + let key = format!("http-{length}"); + let response = http + .put(format!("{base}/{key}")) + .header("content-type", "application/parity") + .header("x-amz-meta-source", "http") + .body(expected.clone()) + .send() + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::OK, + "{}", + response.text().await.unwrap() + ); + let (metadata, received) = fixture.read(key.as_bytes(), GetOptions::default()).await; + assert_eq!(received, expected); + assert_eq!(metadata.content_type, "application/parity"); + assert_eq!( + metadata.user_metadata.values().get("source").map(String::as_str), + Some("http") + ); + } + assert_error_parity(&fixture, &http, &base, &addresses).await; + // The same durable multipart object is subsequently read through HTTP. + fixture.put_framed(b"framed-small", 513).await; + Box::pin(crate::common::native_s3::multipart::exercise(&fixture)).await; + let response = http.get(format!("{base}/multipart")).send().await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let mut expected = payload(5 << 20); + expected.extend_from_slice(&payload(513)); + assert_eq!(response.bytes().await.unwrap().as_ref(), expected); + fixture.runtime.shutdown().await.unwrap(); +} + +async fn assert_error_parity(fixture: &TestNativeS3, http: &Client, base: &str, addresses: &[String]) { + let response = http + .get(format!("{base}/native-513")) + .header("if-match", "\"wrong\"") + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::PRECONDITION_FAILED); + let response = http + .get(format!("{base}/native-513")) + .header("range", "bytes=999-1000") + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); + assert!(matches!( + fixture + .client + .head_object( + context(), + fixture.bucket, + b"missing", + &ReadPreconditions::default() + ) + .await, + Err(Error::S3(S3ErrorCode::NoSuchKey)) + )); + let response = http.get(format!("{base}/missing")).send().await.unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + let response = crate::common::native_s3::signed_copy::copy(addresses, http, base).await; + assert_eq!( + response.status(), + StatusCode::OK, + "{}", + response.text().await.unwrap() + ); + let result = response.text().await.unwrap(); + assert!( + result.contains(""), + "COPY did not publish: {result}" + ); + assert_eq!( + fixture.read(b"http-copy", GetOptions::default()).await.1, + payload(513) + ); +} diff --git a/e2e-test/tests/common.rs b/e2e-test/tests/common.rs index d3a5e92f7..72902549b 100644 --- a/e2e-test/tests/common.rs +++ b/e2e-test/tests/common.rs @@ -4,6 +4,7 @@ pub(crate) mod control; pub(crate) mod hardware; pub(crate) mod management; +pub(crate) mod native_s3; mod test_client; use bytes::Bytes; diff --git a/e2e-test/tests/common/native_s3.rs b/e2e-test/tests/common/native_s3.rs new file mode 100644 index 000000000..3632de526 --- /dev/null +++ b/e2e-test/tests/common/native_s3.rs @@ -0,0 +1,219 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Native S3 consumer using only live management, metadata and storage services. + +pub mod multipart; +pub mod signed_copy; +pub mod transport; + +use std::num::NonZeroU64; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use crowdb_access_s3::metadata::TenantId; +use crowdb_access_s3::storage::{S3LargeWriteSettings, S3WriteSettings}; +use crowdb_chunk_client::SmallWritePolicy; +use crowdb_protocol::chunkdb::rpc::ChunkType; +use crowdb_protocol::frame::{MAX_FRAME_BYTES, MAX_FRAME_PAYLOAD_BYTES}; +use crowdb_sdk_native::s3::{ + GetOptions, ObjectMetadata, PutObjectInfo, PutOptions, S3Client, S3ClientConfig, S3WritePolicy, +}; +use crowdb_sdk_native::{ClientRuntime, OperationContext, RuntimeConfig}; + +pub fn context() -> OperationContext { + static NEXT: AtomicU64 = AtomicU64::new(1); + OperationContext::new( + NonZeroU64::new(NEXT.fetch_add(1, Ordering::Relaxed)).unwrap(), + Instant::now() + Duration::from_secs(30), + ) +} + +pub fn payload(length: usize) -> Vec { + (0..length) + .map(|offset| u8::try_from((offset * 13 + length) % 251).unwrap()) + .collect() +} + +pub struct TestNativeS3 { + pub runtime: ClientRuntime, + pub client: S3Client, + pub bucket: &'static [u8], +} + +impl TestNativeS3 { + pub async fn connect(addresses: &[String], tenant: &[u8]) -> Self { + let artifacts = std::env::var("CROWDB_E2E_ARTIFACTS").unwrap(); + crowdb_rpc_ffi::init_logging(&artifacts, "debug", 30, 5, "native-client"); + crowdb_rpc_ffi::add_log_stderr("error"); + let policies = S3WriteSettings { + small: SmallWritePolicy { + mirror_copies: 2, + conversion_enabled: false, + memory_budget: 64 << 20, + max_pipelines: 4, + ..SmallWritePolicy::new(ChunkType::S3) + }, + threshold_ratio: 0.5, + disk_block_bytes: 1 << 20, + ec_data: 2, + ec_code: 1, + large: S3LargeWriteSettings { + max_chunk_size: Some(4 << 20), + memory_budget_bytes: Some(64 << 20), + ..S3LargeWriteSettings::default() + }, + } + .policies() + .unwrap(); + let mut config = RuntimeConfig::new( + addresses[..3] + .iter() + .map(|ip| super::management::origin(ip)) + .collect(), + ); + config.small_write = policies.small; + config.limits.held_bytes = 8 << 20; + config.limits.producer_bytes = 8 << 20; + config.data_rdma = Some(transport::configuration()); + let runtime = ClientRuntime::connect(config).await.unwrap(); + let mut config = S3ClientConfig::from(TenantId::new(tenant.to_vec()).unwrap()); + config.write_policy = Some(S3WritePolicy { + small_object_limit: policies.small_threshold, + large_write: policies.large, + }); + let client = S3Client::new(runtime.clone(), config).unwrap(); + Self { + runtime, + client, + bucket: if transport::enabled() + && std::env::var("CROWDB_NATIVE_S3_EXECUTION").as_deref() == Ok("oci") + { + b"native-corpus-oci" + } else { + b"native-corpus" + }, + } + } + + pub async fn put(&self, key: &[u8], data: Vec) -> ObjectMetadata { + self.client + .put_vec(context(), self.bucket, key, data, PutOptions::default()) + .await + .unwrap() + } + + pub async fn put_framed(&self, key: &[u8], length: usize) -> ObjectMetadata { + let before = transport::data_status(&self.runtime); + let mut writer = self + .client + .begin_put( + context(), + self.bucket, + key, + PutOptions { + object: PutObjectInfo { + content_length: Some(length as u64), + ..PutObjectInfo::default() + }, + ..PutOptions::default() + }, + ) + .await + .unwrap(); + let full = 16 * MAX_FRAME_PAYLOAD_BYTES; + for start in (0..length).step_by(full) { + let count = (length - start).min(full); + let frozen = self + .runtime + .allocate_producer( + u32::try_from(count.div_ceil(MAX_FRAME_PAYLOAD_BYTES) * MAX_FRAME_BYTES).unwrap(), + ) + .unwrap() + .initialize_framed(count, 0, |offset, bytes| { + for (index, byte) in bytes.iter_mut().enumerate() { + *byte = u8::try_from(((start + offset + index) * 13 + length) % 251).unwrap(); + } + }) + .unwrap(); + assert_eq!(frozen.registration_id() != 0, transport::enabled()); + writer.write_framed(frozen).await.unwrap(); + } + let result = writer.finish().await.unwrap(); + if transport::enabled() { + let after = transport::data_status(&self.runtime); + let direct = after["rdma"]["direct_payload_bytes"].as_u64().unwrap() + - before["rdma"]["direct_payload_bytes"].as_u64().unwrap(); + assert!( + direct >= length as u64, + "registered producer did not send directly: {after}" + ); + if length == 513 { + assert_eq!( + after["rdma"]["copied_payload_bytes"], + before["rdma"]["copied_payload_bytes"] + ); + } + } + result + } + + pub async fn read(&self, key: &[u8], options: GetOptions) -> (ObjectMetadata, Vec) { + let mut get = self + .client + .get_object(context(), self.bucket, key, &options) + .await + .unwrap_or_else(|error| panic!("native GET {} failed: {error}", String::from_utf8_lossy(key))); + let mut data = Vec::new(); + let mut sequence = 0; + let start = options.range.map_or(0, |range| range.start); + while let Some(lease) = get.body.next().await.unwrap() { + let metadata = lease.metadata(); + assert_eq!(metadata.sequence, sequence); + assert_eq!(metadata.logical_offset, start + data.len() as u64); + assert_eq!(metadata.length, lease.as_slice().len()); + assert!(metadata.allocation_capacity >= metadata.length); + if transport::enabled() && metadata.registration_id != 0 { + assert!(matches!( + metadata.allocation_capacity, + 65_536 | 262_144 | 1_048_576 + )); + } + data.extend_from_slice(lease.as_slice()); + sequence += 1; + } + assert!(get.body.is_complete()); + assert!(get.body.next().await.unwrap().is_none()); + (get.metadata, data) + } + + pub async fn assert_pressure_and_shutdown(&self) { + let resources = self.runtime.resources().clone(); + let held = resources.retain_vec(vec![0; 8 << 20], 1, 1, 0, 0).unwrap(); + let mut get = self + .client + .get_object(context(), self.bucket, b"framed-small", &GetOptions::default()) + .await + .unwrap(); + assert!(tokio::time::timeout(Duration::from_millis(20), get.body.next()) + .await + .is_err()); + assert_eq!(resources.status().held_bytes, 8 << 20); + drop(held); + let lease = get.body.next().await.unwrap().unwrap(); + assert_eq!(lease.as_slice(), payload(513)); + assert_eq!(lease.metadata().registration_id != 0, transport::enabled()); + assert!(get.body.next().await.unwrap().is_none()); + let (release, wait) = tokio::sync::oneshot::channel(); + let copy = tokio::spawn(async move { + wait.await.unwrap(); + lease.as_slice().to_vec() + }); + self.runtime.shutdown().await.unwrap(); + assert!(resources.status().held_bytes > 0); + release.send(()).unwrap(); + assert_eq!(copy.await.unwrap(), payload(513)); + assert_eq!(resources.status().held_bytes, 0); + assert_eq!(resources.status().requests, 0); + } +} diff --git a/e2e-test/tests/common/native_s3/multipart.rs b/e2e-test/tests/common/native_s3/multipart.rs new file mode 100644 index 000000000..6c52c47e7 --- /dev/null +++ b/e2e-test/tests/common/native_s3/multipart.rs @@ -0,0 +1,128 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{context, payload, TestNativeS3}; +use crowdb_sdk_native::s3::{ + CompletionPart, CopyMetadata, GetOptions, ListUploads, MultipartUpload, PartMetadata, ReadPreconditions, + UserMetadata, +}; + +pub async fn exercise(fixture: &TestNativeS3) { + let upload = fixture + .client + .begin_multipart( + context(), + fixture.bucket, + b"multipart", + CopyMetadata { + content_type: "application/native".into(), + user_metadata: UserMetadata::default(), + }, + ) + .await + .unwrap(); + let first = write_part(fixture, &upload, 1, 5 << 20).await; + let replaced = write_part(fixture, &upload, 2, 7).await; + let source = fixture + .client + .resolve_object(context(), fixture.bucket, b"framed-small") + .await + .unwrap(); + let tail = fixture + .client + .copy_part_resolved( + context(), + &upload, + 2, + &source, + None, + &ReadPreconditions::default(), + ) + .await + .unwrap(); + assert!(tail.generation > replaced.generation); + let listed = fixture.client.list_parts(context(), &upload, 0, 1).await.unwrap(); + assert_eq!(listed.parts, vec![first.clone()]); + assert_eq!(listed.next_part_number_marker, Some(1)); + assert_eq!( + fixture + .client + .list_parts(context(), &upload, 1, 1) + .await + .unwrap() + .parts, + vec![tail.clone()] + ); + let selection = [first, tail].map(|part| CompletionPart { + number: part.number, + etag: part.etag, + }); + let completed = fixture + .client + .complete_multipart(context(), &upload, &selection) + .await + .unwrap(); + assert_eq!(completed.logical_length, (5 << 20) + 513); + assert_eq!( + fixture + .client + .complete_multipart(context(), &upload, &selection) + .await + .unwrap(), + completed + ); + let abort = fixture + .client + .begin_multipart( + context(), + fixture.bucket, + b"aborted", + CopyMetadata { + content_type: "application/native".into(), + user_metadata: UserMetadata::default(), + }, + ) + .await + .unwrap(); + fixture.client.abort_multipart(context(), &abort).await.unwrap(); + assert!(fixture + .client + .list_multipart_uploads(context(), fixture.bucket, &ListUploads::default()) + .await + .unwrap() + .uploads + .is_empty()); +} + +async fn write_part( + fixture: &TestNativeS3, + upload: &MultipartUpload, + number: u16, + length: usize, +) -> PartMetadata { + let mut writer = fixture + .client + .begin_upload_part(context(), upload, number, length as u64, None) + .await + .unwrap(); + let lease = fixture + .runtime + .resources() + .retain_vec(payload(length), 1, 1, 0, 0) + .unwrap(); + for start in (0..length).step_by(1 << 20) { + writer + .write(lease.slice(start..length.min(start + (1 << 20))).unwrap()) + .await + .unwrap(); + } + writer.finish().await.unwrap() +} + +pub async fn verify(fixture: &TestNativeS3) { + let (metadata, data) = fixture.read(b"multipart", GetOptions::default()).await; + assert_eq!(metadata.content_type, "application/native"); + let mut expected = payload(5 << 20); + expected.extend_from_slice(&payload(513)); + assert_eq!(data, expected); +} diff --git a/e2e-test/tests/common/native_s3/signed_copy.rs b/e2e-test/tests/common/native_s3/signed_copy.rs new file mode 100644 index 000000000..45c0018f9 --- /dev/null +++ b/e2e-test/tests/common/native_s3/signed_copy.rs @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! The HTTP parity fixture uses its ordinary credential authority for COPY. + +use std::fmt::Write; +use std::sync::Arc; + +use crowdb_access_s3::auth::{CredentialCipher, MasterKey}; +use crowdb_access_server::credentials::CredentialAuthority; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use hmac::{Hmac, Mac}; +use reqwest::{Client, Response}; +use sha2::{Digest, Sha256}; + +pub async fn copy(addresses: &[String], http: &Client, base: &str) -> Response { + let control = Arc::new(CrowdbKvClient::new(ClientConfig::new( + addresses[..3] + .iter() + .map(|ip| super::super::management::origin(ip)) + .collect(), + ))); + let master = MasterKey::from_hex(&"11".repeat(32)).unwrap(); + let authority = CredentialAuthority::new(control, Arc::new(CredentialCipher::new(&master))); + let credential = authority.ensure_user(b"local").await.unwrap(); + let seconds = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let now = chrono::DateTime::::from_timestamp(i64::try_from(seconds).unwrap(), 0).unwrap(); + let date = now.format("%Y%m%dT%H%M%SZ").to_string(); + let short = now.format("%Y%m%d").to_string(); + let host = base.strip_prefix("http://").unwrap().split('/').next().unwrap(); + let path = "/native-corpus/http-copy"; + let source = "/native-corpus/native-513"; + let hash = hex(&Sha256::digest([])); + let names = "host;x-amz-content-sha256;x-amz-copy-source;x-amz-date"; + let canonical = format!( + "PUT\n{path}\n\nhost:{host}\nx-amz-content-sha256:{hash}\nx-amz-copy-source:{source}\nx-amz-date:{date}\n\n{names}\n{hash}" + ); + let scope = format!("{short}/us-east-1/s3/aws4_request"); + let date_key = mac(format!("AWS4{}", credential.secret_key).as_bytes(), &short); + let region_key = mac(&date_key, "us-east-1"); + let service_key = mac(®ion_key, "s3"); + let signing_key = mac(&service_key, "aws4_request"); + let to_sign = format!( + "AWS4-HMAC-SHA256\n{date}\n{scope}\n{}", + hex(&Sha256::digest(canonical)) + ); + let authorization = format!( + "AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders={names}, Signature={}", + credential.access_key_id, + hex(&mac(&signing_key, &to_sign)) + ); + http.put(format!("http://{host}{path}")) + .header("host", host) + .header("x-amz-date", date) + .header("x-amz-content-sha256", hash) + .header("x-amz-copy-source", source) + .header("authorization", authorization) + .header("content-length", 0) + .send() + .await + .unwrap() +} + +fn mac(key: &[u8], input: &str) -> Vec { + let mut signer = Hmac::::new_from_slice(key).unwrap(); + signer.update(input.as_bytes()); + signer.finalize().into_bytes().to_vec() +} + +fn hex(bytes: &[u8]) -> String { + let mut output = String::new(); + for byte in bytes { + write!(output, "{byte:02x}").unwrap(); + } + output +} diff --git a/e2e-test/tests/common/native_s3/transport.rs b/e2e-test/tests/common/native_s3/transport.rs new file mode 100644 index 000000000..5a5aff7fc --- /dev/null +++ b/e2e-test/tests/common/native_s3/transport.rs @@ -0,0 +1,59 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_rpc_ffi::RdmaConfig; +use crowdb_sdk_native::ClientRuntime; +use serde_json::Value; + +pub fn enabled() -> bool { + std::env::var("CROWDB_NATIVE_S3_RDMA").as_deref() == Ok("1") +} + +pub fn configuration() -> RdmaConfig { + if !enabled() { + return RdmaConfig { + enabled: false, + ..RdmaConfig::default() + }; + } + RdmaConfig { + enabled: true, + allow_tcp_fallback: false, + listen_enabled: false, + device: std::env::var("CROWDB_RDMA_DEVICE").unwrap(), + source_address: std::env::var("CROWDB_RDMA_SOURCE_ADDR").unwrap(), + cq_count: 1, + cq_depth: 512, + sq_depth: 16, + rq_depth: 16, + posted_receive_target: 16, + registered_memory_budget_bytes: 512 << 20, + aggregation_enabled: false, + ..RdmaConfig::default() + } + .with_control_data_capacity(1 << 20) + .unwrap() +} + +pub fn data_status(runtime: &ClientRuntime) -> Value { + serde_json::from_slice(&runtime.topology().unwrap().data_transport_json.unwrap()).unwrap() +} + +pub fn assert_selected(runtime: &ClientRuntime) { + let status = data_status(runtime); + if !enabled() { + assert!(status["rdma"].is_null()); + return; + } + assert_eq!(status["rdma"]["ready"], true, "{status}"); + assert_eq!(status["allow_tcp_fallback"], false, "{status}"); + for field in [ + "fallback_count", + "protocol_errors", + "cq_errors", + "send_errors", + "receive_post_errors", + ] { + assert_eq!(status["rdma"][field], 0, "{field}: {status}"); + } +} diff --git a/e2e-test/tests/native_s3_hca_test.py b/e2e-test/tests/native_s3_hca_test.py new file mode 100644 index 000000000..d17fbfaff --- /dev/null +++ b/e2e-test/tests/native_s3_hca_test.py @@ -0,0 +1,55 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Host-network HCA fixture retains ownership without inventing Docker IPs.""" + +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock + +from allocation import create_network, observed_address, start_node +from fixture import Cluster, LABEL + + +class NativeS3HcaTest(unittest.TestCase): + def test_explicit_profile_binds_owned_loopback_and_passes_only_selected_devices(self): + with tempfile.TemporaryDirectory() as directory: + cluster = Cluster("image", directory) + cluster.host_network = True + cluster.sdk_environment = {"CROWDB_NATIVE_S3_RDMA": "1"} + cluster.server_rdma_environment = {"CROWDB_RDMA_ENABLED": "1"} + cluster.hca_devices = ("--device", "/dev/infiniband/rdma_cm") + cluster.command = Mock() + subnet = create_network(cluster) + self.assertTrue(subnet.is_loopback) + self.assertFalse(cluster.network_created) + cluster.command.assert_not_called() + address = str(subnet.network_address + 2) + info = {"HostConfig": {"NetworkMode": "host"}, + "Config": {"Labels": {LABEL: cluster.run_id}}} + cluster.command.return_value = json.dumps([info]) + config = cluster.artifacts / "disk.toml" + config.write_text('http_listen_addr = "0.0.0.0:7101"') + self.assertEqual(start_node(cluster, 0, Path(directory), ("server", "diskio"), address, + mounts=(f"type=bind,source={config},target=/config,readonly",)), address) + self.assertIn(address, config.read_text()) + self.assertIn("CROWDB_RDMA_ENABLED=1", cluster.command.call_args_list[0].args) + self.assertIn("/dev/infiniband/rdma_cm", cluster.command.call_args_list[0].args) + self.assertEqual(cluster.servers, cluster.containers) + info["Config"]["Labels"][LABEL] = "foreign" + with self.assertRaisesRegex(ValueError, "owned host network"): + observed_address(cluster, info, address) + + def test_profile_rejects_foreign_config_before_starting_process(self): + with tempfile.TemporaryDirectory() as directory: + cluster = Cluster("image", directory) + cluster.host_network = True + cluster.command = Mock() + foreign = Path(directory) / "foreign.toml" + foreign.write_text('address = "0.0.0.0:80"') + with self.assertRaisesRegex(ValueError, "own generated"): + start_node(cluster, 0, Path(directory), ("server", "diskio"), "127.1.2.3", + mounts=(f"type=bind,source={foreign},target=/config,readonly",)) + cluster.command.assert_not_called() + self.assertEqual(foreign.read_text(), 'address = "0.0.0.0:80"') diff --git a/e2e-test/tests/s3_http_exchange_test.py b/e2e-test/tests/s3_http_exchange_test.py index d89784a13..924794cae 100644 --- a/e2e-test/tests/s3_http_exchange_test.py +++ b/e2e-test/tests/s3_http_exchange_test.py @@ -35,3 +35,61 @@ def test_disconnect_without_an_http_response_still_fails(self): connection.getresponse.side_effect = ConnectionResetError('no response') with self.assertRaisesRegex(ConnectionResetError, 'no response'): exchange.request_response(connection, 'PUT', '/key', b'body', {}) + + +from http.client import BadStatusLine, HTTPConnection, RemoteDisconnected +from io import BytesIO +from pathlib import Path +import unittest + +SPEC = importlib.util.spec_from_file_location( + "s3_http_exchange", Path(__file__).parents[1] / "clients/s3/http_exchange.py") +HTTP = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(HTTP) + + +class TestEarlyResponseSocket: + def __init__(self, response): + self.response = response + self.sent = [] + + def sendall(self, data): + self.sent.append(data) + if len(self.sent) > 1: + raise BrokenPipeError("peer rejected the body") + + def makefile(self, _mode): + return BytesIO(self.response) + + def close(self): + pass + + +class S3HttpSocketExchangeTest(unittest.TestCase): + def exchange(self, response): + connection = HTTPConnection("test.invalid") + sock = TestEarlyResponseSocket(response) + connection.sock = sock + try: + return HTTP.exchange(connection, "GET", "/bucket/key", b"body", {}) + finally: + connection.close() + self.assertEqual(len(sock.sent), 2) + self.assertTrue(sock.sent[0].startswith(b"GET /bucket/key HTTP/1.1\r\n")) + + def test_rejection_status_and_complete_body_survive_failed_upload(self): + response = b"HTTP/1.1 403 Forbidden\r\nContent-Length: 6\r\nConnection: close\r\n\r\ndenied" + status, _, body = self.exchange(response) + self.assertEqual((status, body), (403, b"denied")) + + def test_missing_response_still_fails(self): + with self.assertRaises(RemoteDisconnected): + self.exchange(b"") + + def test_malformed_response_still_fails(self): + with self.assertRaises(BadStatusLine): + self.exchange(b"invalid response\r\n") + + def test_success_cannot_hide_an_incomplete_upload(self): + with self.assertRaises(BrokenPipeError): + self.exchange(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") diff --git a/lib/crowdb-sdk-native/README.md b/lib/crowdb-sdk-native/README.md new file mode 100644 index 000000000..683b9b55b --- /dev/null +++ b/lib/crowdb-sdk-native/README.md @@ -0,0 +1,142 @@ + + + +# CrowDB native SDK + +`crowdb-sdk-native` embeds trusted internal S3 access through Chunk-KV and +ChunkIO/DiskIO. It belongs to the main Rust workspace and links the native RPC +stack. Supply management seeds, a tenant and per-operation context. The +embedding process has the same internal-service trust as Access Server; this +API performs no SigV4 verification or user-secret lookup. + +The [native runtime design](../../doc/design/access-server/design-crowdb-native-sdk.md) +defines ownership and routing. The separately packaged +[Dataset HTTP SDK](../../sdk/README.md) is the portable Rust/Python distribution. + +## Connect and read + +The embedding application supplies Tokio. Clone one `ClientRuntime` across +handles to share connection groups, topology caches and budgets: + +```rust +use std::num::NonZeroU64; +use std::time::{Duration, Instant}; +use crowdb_access_s3::metadata::TenantId; +use crowdb_sdk_native::{ClientRuntime, OperationContext, RuntimeConfig}; +use crowdb_sdk_native::s3::{GetOptions, S3Client}; + +async fn read_object(seeds: Vec) -> Result<(), Box> { + let mut config = RuntimeConfig::new(seeds); + config.data_rdma = Some(crowdb_rpc_ffi::RdmaConfig { + enabled: false, + ..Default::default() + }); + let runtime = ClientRuntime::connect(config).await?; + let client = S3Client::new(runtime.clone(), TenantId::new(b"tenant".to_vec())?)?; + let context = OperationContext::new( + NonZeroU64::new(1).unwrap(), + Instant::now() + Duration::from_secs(10), + ); + let mut object = client.get_object(context, b"bucket", b"key", &GetOptions::default()).await?; + while let Some(lease) = object.body.next().await? { + let bytes = lease.as_slice(); + // Consume bytes here, or retain the lease until an asynchronous copy completes. + assert_eq!(bytes.len(), lease.metadata().length); + } + assert!(object.body.is_complete()); + runtime.shutdown().await?; + Ok(()) +} +``` + +- Give each operation a nonzero request ID and deadline; its cancellation + token belongs to that operation. GET remains lazy until the body is polled. +- Only `next() == Ok(None)` establishes verified completion. A valid prefix + does not establish full-read success. Terminal errors remain sticky on + subsequent polls; dropping a body early cancels it. +- `GetOptions` supports conditions and a logical half-open `ByteRange`. + `resolve_object` captures one published generation; `get_resolved` reads + that selection after an overwrite without looking the mutable key up again. +- The S3 facade supports bucket operations, HEAD/GET, PUT, paged listing, + ordered batch deletion, copy and multipart begin/parts/list/complete/abort. + Typed S3 failures and `NotModified` do not require HTTP/XML parsing. + +## Produce directly into native buffers + +- `put_vec` accepts bounded ordinary input and charges the Vec's actual + allocation capacity. Large input can instead be fed through `begin_put` + with `write`, using leases from `runtime.resources()` and views of at most + 1 MiB per call. A small view can retain a larger allocation and its charge. +- `allocate_producer(capacity)` returns unique writable storage from the + runtime's native allocator. Under RDMA this can be registered memory; + inspect the resulting registration ID rather than inferring it from config. +- `initialize_framed(length, initial, fill)` exposes logical payload regions + within their final storage frame slots. `write_framed` consumes the frozen + owner; callers cannot mutate submitted bytes. Capacity must include storage + framing. Each 64 KiB frame carries at most `MAX_FRAME_PAYLOAD_BYTES` bytes; + each submitted physical buffer is bounded by 1 MiB. Finish the logical + payload directly in these regions to avoid a later payload staging copy. +- `write`/`write_framed` input acceptance is separate from object visibility. + `finish` waits for durability, checksums and publication. Multipart complete + binds the selected part generations and preserves repeat-completion results. +- `UncertainMutation` means a final outcome is unknown. Preserve request and + operation identity; do not blindly replay the mutation with a new identity. +- Small writer pools can retain input owners for open-strip repair after a + PUT completes. Pool drain/seal releases them; budget this lifetime. + +## Budgets, topology and shutdown + +- Defaults: 1024 requests, 64 MiB of producers, 256 MiB of caller-held buffers. + These limits are independent of the transport's registered-memory budget, + posted receive pools, SQ depth, in-flight byte window and writer policy. +- Shared allocation views charge capacity once until the final owner releases + it. Finishing an RPC or stream does not return caller-held credits. A read + waiting for held capacity resumes after release or returns its deadline/ + cancellation outcome. Unknown external owners use an accounted bounded copy. +- `resources().status()` reports live charges, high water and compatibility + copy bytes. `topology()` exposes current transport counters and separately + versioned catalog/disk observations. `refresh_topology(context)` explicitly + refreshes authorities; it does not create a globally atomic snapshot. +- `object_locality` resolves bounded extents, layout revisions and fragment + locations for a selected object. Observations aid scheduling and prefetch; + current routing/fencing still decides execution. Old views grant no placement + authority and cannot force an expired layout into a read. +- `connect` and `adopt_clients` own worker shutdown. `from_clients` borrows + existing groups and leaves their shutdown to the original owner. + `shutdown()` closes admission, drains owned writers and joins owned RPC + workers. Cancellation of one shutdown waiter does not cancel cleanup. +- Leases remain readable after shutdown until their final release. For a GPU + host-copy operation, retain the host lease until copy completion. The base + API exposes host memory; it does not provide GPU pointers or GPUDirect. + +## TCP and RDMA + +`RuntimeConfig::data_rdma` selects the DiskIO group; `chunk_kv_transport` owns +metadata transport policy. `None` retains the existing service defaults. +Explicitly disabled RDMA uses TCP. An enabled `RdmaConfig` supplies device, +source address, queue depths, post batches, receive class and registration +budget; use `with_control_data_capacity` to select a supported control/data +class. Setup fallback follows `allow_tcp_fallback`. An accepted or partial +RDMA read fails terminally instead of appending a TCP suffix or retrying another +replica; ambiguous mutations are never replayed by the SDK. + +Eligible receive allocations reach `BufferLease` without a second SDK payload +copy. Ordinary unregistered input, EC construction/reconstruction, alignment +and control decoding can still require copies. The opaque RPC C ABI uses +single final release; releasing a freed handle again is invalid. + +## Verification + +```sh +pixi run -- cargo test -p crowdb-sdk-native --all-targets +pixi run clean-env && pixi run -- python e2e-test/run.py --layer s3 --scope s3.native --concurrency 1 --cpus 4 --memory-mib 8192 --image crowdb-node:dev +pixi run clean-env && pixi run -- python tools/rdma/run-native-s3.py --address 10.200.1.1 --device rocep35s0f0 +``` + +Build `crowdb-node:dev` from current server sources first. The explicit HCA +profile uses one host HCA, twelve independent service containers and native/ +unprivileged OCI consumers. Services have separate persistent directories, +loopback endpoints and CM ports. It validates mixed sizes, registered producers, +held-capacity pressure, lease lifetime and restart; it is a local-loop functional +profile, not a cross-host performance measurement. The TCP profile includes a +separate HTTP/native semantic comparison and OCI consumers with no RDMA device. diff --git a/lib/crowdb-sdk-native/tests/common/metadata.rs b/lib/crowdb-sdk-native/tests/common/metadata.rs index 84e9ae528..07f7e3f93 100644 --- a/lib/crowdb-sdk-native/tests/common/metadata.rs +++ b/lib/crowdb-sdk-native/tests/common/metadata.rs @@ -31,6 +31,14 @@ pub struct TestMetadataTransport { sender: mpsc::Sender, lose_mutation_reply: Arc, blackout: Option>, + route: Option>, +} + +#[derive(Default)] +pub struct TestRouteChange { + pub generation: std::sync::atomic::AtomicU64, + pub rejected: std::sync::atomic::AtomicU64, + pub accepted: std::sync::atomic::AtomicU64, } #[derive(Default)] @@ -41,7 +49,25 @@ pub struct TestMutationBlackout { #[async_trait] impl ChunkKvTransport for TestMetadataTransport { - async fn point(&self, _: &str, request: &PointRequest) -> Result { + async fn point(&self, endpoint: &str, request: &PointRequest) -> Result { + if let Some(route) = &self.route { + let generation = route.generation.load(Ordering::Acquire); + if endpoint != format!("test-owner-{generation}") || request.routing.owner_epoch != generation { + route.rejected.fetch_add(1, Ordering::Relaxed); + return Ok(ChunkKvResponse { + map_revision: generation, + journal_position: None, + result: Err(RpcFailure { + code: ChunkKvRpcErrorCode::RefreshRequired, + message: "owner moved before operation".into(), + retry_after_ms: None, + latest_map_revision: Some(generation), + owner_hint: None, + }), + }); + } + route.accepted.store(generation, Ordering::Release); + } if self .blackout .as_ref() @@ -228,6 +254,7 @@ pub async fn client_with_source(source: Arc) -> A Arc::new(AtomicBool::new(false)), ClientConfig::default(), None, + None, ) .await } @@ -243,6 +270,7 @@ pub async fn client_with_loss(loss: Arc) -> Arc { ..ClientConfig::default() }, None, + None, ) .await } @@ -258,6 +286,22 @@ pub async fn client_with_blackout(blackout: Arc) -> Arc, + route: Arc, +) -> Arc { + client_with_options( + source, + Arc::new(AtomicBool::new(false)), + ClientConfig::default(), + None, + Some(route), ) .await } @@ -267,6 +311,7 @@ async fn client_with_options( loss: Arc, config: ClientConfig, blackout: Option>, + route: Option>, ) -> Arc { let (sender, mut receiver) = mpsc::channel(64); tokio::spawn(async move { @@ -294,6 +339,7 @@ async fn client_with_options( sender, lose_mutation_reply: loss, blackout, + route, }), ) .unwrap(), diff --git a/lib/crowdb-sdk-native/tests/topology_test.rs b/lib/crowdb-sdk-native/tests/topology_test.rs index e813db20c..1c9c36af0 100644 --- a/lib/crowdb-sdk-native/tests/topology_test.rs +++ b/lib/crowdb-sdk-native/tests/topology_test.rs @@ -67,3 +67,41 @@ async fn retained_catalog_views_do_not_regress_and_independent_refresh_age_is_un assert_eq!(old.catalog().route(b"key").unwrap().owner.instance_id, 1); assert_eq!(refreshed.catalog().generation(), 2); } + +#[tokio::test] +async fn retained_view_cannot_pin_s3_operations_to_a_moved_owner() { + use crowdb_access_s3::metadata::TenantId; + use crowdb_sdk_native::s3::S3Client; + + let source = Arc::new(topology::TestCatalogSource { + generation: 1.into(), + unavailable: false.into(), + }); + let route = Arc::new(metadata::TestRouteChange { + generation: 1.into(), + ..Default::default() + }); + let metadata = metadata::client_with_route(source.clone(), route.clone()).await; + let runtime = storage::runtime_with_metadata(metadata); + let client = S3Client::new(runtime.clone(), TenantId::new(b"tenant".to_vec()).unwrap()).unwrap(); + client.create_bucket(context(10), b"bucket").await.unwrap(); + client.head_bucket(context(11), b"bucket").await.unwrap(); + let old = runtime.topology().unwrap(); + source.generation.store(2, Ordering::Release); + route.generation.store(2, Ordering::Release); + client.head_bucket(context(12), b"bucket").await.unwrap(); + assert_eq!(route.rejected.load(Ordering::Acquire), 1); + assert_eq!(route.accepted.load(Ordering::Acquire), 2); + let current = runtime.topology().unwrap(); + assert_eq!(current.catalog().generation(), 2); + assert_eq!( + current.catalog().route(b"key").unwrap().owner.rpc_endpoint, + "test-owner-2" + ); + assert_eq!(old.catalog().generation(), 1); + assert_eq!( + old.catalog().route(b"key").unwrap().owner.rpc_endpoint, + "test-owner-1" + ); + runtime.shutdown().await.unwrap(); +} diff --git a/sdk/README.md b/sdk/README.md index e2572999a..c6bf5dfd7 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -21,6 +21,10 @@ integrity, worker partition and explicit progress replay. defines JSON and binary framing. Legacy bincode routes remain internal compatibility interfaces. S3/Iceberg standard access uses ecosystem SDKs. +Trusted internal S3 embedding uses the main-workspace +[native SDK](../lib/crowdb-sdk-native/README.md), with shared topology and host +buffer leases. This HTTP distribution has no dependency on that native stack. + ## Python Build a wheel with the interpreter it will be installed into: diff --git a/tools/ci-checks/check-ci-test-tasks.py b/tools/ci-checks/check-ci-test-tasks.py index d883547ea..746390fbd 100644 --- a/tools/ci-checks/check-ci-test-tasks.py +++ b/tools/ci-checks/check-ci-test-tasks.py @@ -21,7 +21,7 @@ }, "test-access": { "crowdb-access-multipart", "crowdb-access-s3", "crowdb-access-iceberg", - "crowdb-access-server", "crowdb-access-dataset", "crowdb-monitor", + "crowdb-access-server", "crowdb-access-dataset", "crowdb-monitor", "crowdb-sdk-native", }, "test-console": {"crowdb-console-shared", "crowdb-cli", "crowdb-web"}, } diff --git a/tools/pixi-tasks/test.py b/tools/pixi-tasks/test.py index d9659cccc..79de1cb39 100644 --- a/tools/pixi-tasks/test.py +++ b/tools/pixi-tasks/test.py @@ -18,7 +18,7 @@ "storage": ("chunkdb-client", "chunk-kv", "chunk-kv-client", "chunk-kv-server", "chunk-stream", "kv-server", "diskdb", "diskdb-client", "chunkdb", "chunk-client", "diskio-client"), - "access": ("access-multipart", "access-s3", "access-iceberg", "access-server", + "access": ("access-multipart", "access-s3", "sdk-native", "access-iceberg", "access-server", "access-dataset", "monitor"), "console": ("console-shared", "cli", "web"), "cpp": ("tree-ffi", "rpc-ffi", "rpc-test"), diff --git a/tools/rdma/run-native-s3.py b/tools/rdma/run-native-s3.py new file mode 100644 index 000000000..b5829cb33 --- /dev/null +++ b/tools/rdma/run-native-s3.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Native and OCI S3 consumers against twelve real services using a shared HCA.""" + +import argparse +import json +import os +from pathlib import Path +import signal +import sys +import uuid + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "e2e-test")) +from client import bundle_client +from fixture import docker +from image_identity import verify_server_sources +from native_s3_hca import NativeS3HcaCluster + + +def interrupted(_signal, _frame): + raise KeyboardInterrupt("shared HCA S3 acceptance interrupted") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--address", default=os.environ.get("CROWDB_RDMA_SOURCE_ADDR", "10.200.1.1")) + parser.add_argument("--device", default=os.environ.get("CROWDB_RDMA_DEVICE", "rocep35s0f0")) + parser.add_argument("--image", default="crowdb-node:dev") + args = parser.parse_args() + signal.signal(signal.SIGTERM, interrupted) + signal.signal(signal.SIGINT, interrupted) + output = ROOT / ".crowdb-runtime/artifacts/native-s3-hca" / uuid.uuid4().hex + output.mkdir(parents=True) + print(f"Artifacts: {output}", flush=True) + image = json.loads(docker("image", "inspect", args.image))[0] + labels = image["Config"].get("Labels") or {} + verify_server_sources(ROOT, labels.get("org.opencontainers.image.revision"), + labels.get("org.crowdb.source.tree") or None) + os.environ.update(CROWDB_NATIVE_S3_RDMA="1", CROWDB_RDMA_SOURCE_ADDR=args.address, + CROWDB_RDMA_DEVICE=args.device, CROWDB_RDMA_ENABLED="0") + bundle = output / "client" + bundle_client(bundle) + cluster = NativeS3HcaCluster(image["Id"], output, bundle, address=args.address, device=args.device) + case = "access_s3::native::complete_corpus_survives_storage_restart_without_access_or_credentials" + with cluster: + cluster.client(bundle, "write", case) + cluster.crash_restart() + cluster.client(bundle, "verify", case) + print("Native and OCI shared-HCA S3 acceptance passed", flush=True) + + +if __name__ == "__main__": + main() From 4051adcca13be587d9bc3f78cfabd9065d47ed51 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 17:22:33 +0800 Subject: [PATCH 075/258] Retire completed native SDK work and update follow-up dependencies --- doc/backlog/R242-rdma-fat-client-sdk.md | 176 -------- doc/backlog/R5-rdma-alloc.md | 12 +- doc/backlog/backlog.md | 16 +- doc/working/plan-rdma-fat-client-sdk.md | 567 ------------------------ 4 files changed, 13 insertions(+), 758 deletions(-) delete mode 100644 doc/backlog/R242-rdma-fat-client-sdk.md delete mode 100644 doc/working/plan-rdma-fat-client-sdk.md diff --git a/doc/backlog/R242-rdma-fat-client-sdk.md b/doc/backlog/R242-rdma-fat-client-sdk.md deleted file mode 100644 index 8246ea637..000000000 --- a/doc/backlog/R242-rdma-fat-client-sdk.md +++ /dev/null @@ -1,176 +0,0 @@ - - - -### R242: Access SDK — Trusted internal S3 fat client and shared RDMA runtime - -## Problem - -Data-processing and AI systems inside the trusted cluster need to embed -storage, inspect storage topology, schedule work near data, and control -concurrency and buffer lifetime. Routing every operation through Access Server -adds an application-to-server socket and async hop and hides this topology. - -A fat client is explicitly a trusted internal cluster component in the same -trust domain as Access Server, not an ordinary external-user SDK. It connects -directly to metadata and data services. It does not authenticate its own -operations by verifying SigV4 or receive the server's user-secret database. - -`crowdb-access-s3` already owns metadata, conditions, retrieval, publication, -and multipart components, but complete operations remain partly coupled to -HTTP handlers. The independent `sdk/` workspace is a portable Dataset HTTP -client. Existing RPC buffers carry lease metadata, but completion and access -adapters still include copying paths. Complete these boundaries rather than -replacing accepted storage clients or transport. - -The root contracts are [Access Server](../design/access-server/design-crowdb-access-server.md) -and [RDMA service interactions](../design/rpc/design-crowdb-rpc-rdma.md#7-service-interactions). - -## Solution - -1. Provide a shared native cluster runtime in the main Rust workspace. Reuse - `crowdb-kv-client`, `crowdb-chunk-kv-client`, `crowdb-chunk-client`, - `crowdb-diskio-client`, and `crowdb-rpc-ffi` for discovery, routing, - TCP/RDMA connections, producer allocation, admission, and shutdown. Business - handles share these resources. Keep the existing `sdk/` distribution - independent of server/storage FFI dependencies. -2. Expose storage topology as a supported native API: service endpoints, - node/disk relationships, Chunk-KV owners, and resolved chunk placement for - caller scheduling and prefetch. Views retain existing source generations - and freshness information, without promising a globally atomic snapshot. - Stale views refresh or fail through existing routing/fencing rules. - Observing topology cannot publish placement or bypass generation checks, - durability, or publication rules. -3. Implement S3 first with typed bucket/object, HEAD/GET/range, PUT, listing, - delete, copy, and multipart operations backed by `crowdb-access-s3`. - Extract shared execution from `app/crowdb-access-server/src/s3/`. - HTTP retains parsing, SigV4 verification, and HTTP/XML encoding; native - callers supply explicit tenant and operation context. Both preserve the - same object visibility, conditions, integrity, generation, mutation - outcomes, and cleanup. Native execution needs neither a user-secret lookup - nor an Access Server authentication round trip. -4. Separate business APIs from common data delivery. Reads return typed - metadata and an ordered, pull-based host-buffer stream. Metadata or a - delivered prefix does not establish full-read success; successful - termination includes applicable checksum and length checks. A terminal - error remains observable; later polling cannot turn failure into success. - Writes accept bounded ordinary input or uniquely writable buffers frozen before - submit. Input acceptance is distinct from durable object publication; - only successful finalization establishes PUT or multipart completion. - Ambiguous publication retains existing uncertain-outcome semantics. -5. Complete the existing opaque RPC buffer lease contract through C++/Rust - completion and the native API. A lease exposes request identity, sequence, - logical offset, actual length, capacity class, and release without verbs - objects. Caller-owned storage stays valid until final release, including - after cancellation or runtime shutdown. Eligible payload paths transfer - ownership without a second SDK payload copy; decoding, alignment staging, - and EC reconstruction retain necessary copies. Wire-control fragments are - not object data. -6. Keep hot-path admission lock-free and bounded across requests, producers, - and caller-held buffer capacity. Shared views charge an allocation once - until final release; operation completion does not return held-buffer - credits. Exhaustion pauses production or returns a typed capacity/deadline - outcome without an unbounded waiting queue. Completion callbacks never - block RPC workers. Cancellation stops new work and resolves once; - uncertain in-flight storage stays retained or quarantined until safe. -7. Preserve TCP/RDMA path selection. Disabled RDMA uses TCP; eligible setup - or known pre-payload failures may fall back with the same request identity - under existing retry rules. Accepted/partial RDMA operations never append - a TCP suffix or replay ambiguous mutations. This fallback does not select - HTTP. GPU delivery initially means explicit host-to-GPU copy completion - before host-lease release, with no GPU pointer or RDMA READ/WRITE in the - base ABI. -8. Keep runtime, topology, and buffer APIs independent of S3 so Dataset can - attach after S3 acceptance under [R220](R220-dataset-fat-client-plan.md). - Dataset keeps Snapshot/Read Plan/Split and explicit acknowledgement in - `crowdb-access-dataset`; it reuses the runtime and data delivery without - encoding samples as S3 objects. Buffer release, operation completion, and - Dataset acknowledgement are separate events. R242 delivers the shared - foundation and S3; R220 owns the later Dataset adapter, measurements, and - Python integration. - -## Dependencies - -- Accepted registered buffers/RDMA RPC and - [Chunk-KV/access integration](../design/rpc/design-crowdb-rpc-rdma.md#7-service-interactions). -- Existing S3 library components, HTTP operation orchestration, ChunkIO - read/write policies, and RPC FFI ownership conventions. -- Existing Dataset HTTP SDK packaging and boundary checks are regression - controls, not a place to introduce native storage dependencies. -- R220 consumes the accepted runtime and S3-first API pattern. Its Dataset - measurements and bindings do not gate R242 or require a second transport, - topology cache, or buffer pool. -- Missing RDMA hardware uses the accepted bounded TCP path. Host-memory tests - and simulated asynchronous GPU copies need no GPU; real-HCA acceptance is - an explicit separate profile. - -## Acceptance - -- Given cluster endpoints and a tenant but no S3 secret database or HTTP - listener, when native S3 executes an object operation, it reaches storage - directly. Invariant: trusted internal execution. Integration test. -- Given two handles sharing a runtime, when they execute and close - independently, they share connection/admission owners without invalidating - the other's data. Invariant: shared runtime lifetime. Integration test. -- Given service, disk, catalog, and object-placement observations, when the - caller inspects topology and schedules range reads, the API exposes their - identities/freshness and returns the requested logical bytes. Invariant: - supported topology surface. Integration test. -- Given a view whose owner or endpoint changes, when a request uses it, - routing refreshes or returns a typed stale/fenced outcome without changing - logical object identity or publishing placement. Invariant: - non-authoritative topology. Integration test. -- Given identical supported bucket/object, copy, and multipart operations, - when HTTP and native entry points encounter success, condition/range errors, - conflicts, or publication failures, logical bytes, metadata, and semantic - outcomes agree. Invariant: S3 frontend parity. Integration test. -- Given truncated data or a final checksum mismatch, when a caller drains a - stream after receiving a valid prefix and polls after the error, termination - remains failed rather than becoming successful EOF. - Invariant: prefix delivery is not full-read success. Integration test. -- Given writable producer storage, when frozen input is submitted, callers - cannot mutate it; final success waits for durability/publication and lost - publication replies retain uncertainty. Invariant: input ownership and - publication completion. Integration test. -- Given an eligible direct RDMA payload path, when a multi-buffer read - completes, ordered owner-backed views preserve request, sequence, offset, - and length without a second SDK payload copy. Invariant: caller-owned - payload lifetime. Integration test. -- Given opaque leases and shared views, when access and final release cross - FFI, ABI/access rules hold and each allocation is released once. Invariant: - lease ABI ownership. Unit test. -- Given all admitted capacity is caller-held, when another buffer is needed, - bounded backpressure resumes after final release or ends at its deadline; - stream completion alone does not return held credits. Invariant: bounded - SDK memory. Integration test. -- Given in-flight operations and held leases, when cancellation or shutdown - closes new admission, completion resolves once and held bytes stay valid - until release without reposting to destroyed transport. Invariant: - cancellation and shutdown ownership. Integration test. -- Given an asynchronous host-to-GPU copy simulation, when the stream closes - before copy completion, the host lease stays valid until copy completion and - release. Invariant: copy dependency lifetime. Integration test. -- Given disabled RDMA or an eligible pre-payload failure, when native access - runs, TCP preserves result/request identity; accepted or partial RDMA failure - instead fails without a TCP suffix or ambiguous mutation replay. Invariant: - transport parity and failure boundary. Integration test. -- Given the standalone Dataset SDK, when boundary and functional gates run - alongside the native client, HTTP behavior and independent packaging stay - intact. Invariant: independent HTTP SDK. Integration test. -- Given native and OCI S3 clients, when callers retain leases under pressure, - no-device TCP and shared-HCA profiles record actual topology/transport, - exact frontend results, bounded memory, and safe shutdown. Invariant: - embedded storage integration. E2E test. - -Run after registering the native crate: - -```bash -pixi run -- cargo fmt --all -- --check -pixi run -- cargo test -p crowdb-sdk-native --all-targets -pixi run -- cargo test -p crowdb-access-s3 --all-targets -pixi run -- cargo test -p crowdb-rpc-ffi --all-targets -pixi run -- cargo test -p crowdb-access-server --all-targets -pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings -pixi run check-dataset-sdk-boundary -pixi run test-dataset-sdk -pixi run test-e2e-s3 -``` diff --git a/doc/backlog/R5-rdma-alloc.md b/doc/backlog/R5-rdma-alloc.md index a2f4e04e8..7b2b7f0f6 100644 --- a/doc/backlog/R5-rdma-alloc.md +++ b/doc/backlog/R5-rdma-alloc.md @@ -6,8 +6,9 @@ ## Status The registered host-buffer foundation, common RDMA RPC transport, -HTTP/ChunkIO/DiskIO datapath and Chunk-KV/access integration are accepted. -Remaining child requirement R242 retains the fat-client goal. +HTTP/ChunkIO/DiskIO datapath, Chunk-KV/access integration and native S3 runtime +are accepted. The deployment-policy questions below remain separate from the +completed native SDK functional acceptance. ## Problem @@ -30,9 +31,10 @@ defined in [`design-crowdb-rpc-rdma.md`](../design/rpc/design-crowdb-rpc-rdma.md 4. Reuse the accepted [Chunk-KV/access stream](../design/rpc/design-crowdb-rpc-rdma.md#7-service-interactions), using 64 KiB metadata and 256 KiB aggregate classes, bounded logical control assembly, shared request identity and final-caller byte admission ownership. -5. Implement the native fat client and FFI lease API in - [R242](R242-rdma-fat-client-sdk.md), with host memory first and explicit - host-to-GPU copy for the initial GPU simulation. +5. Reuse the accepted [native SDK and lease contract](../design/access-server/design-crowdb-native-sdk.md), + with host memory first and explicit host-to-GPU copy completion before final + lease release. Native and unprivileged OCI TCP/shared-HCA acceptance evidence + is retained in the [root plan](../working/plan-rdma.md). The base transfer mechanism is SEND/RECV. Large client buffers are split into ordered messages with request ID, sequence, and logical offset. RDMA READ and diff --git a/doc/backlog/backlog.md b/doc/backlog/backlog.md index 3249521fb..3175f7958 100644 --- a/doc/backlog/backlog.md +++ b/doc/backlog/backlog.md @@ -17,9 +17,9 @@ live in the linked document; implementation follows and recoverable checkpoint staging with explicit remote durability. **Area:** SDK. **Complexity:** High. **Status:** Deferred until dedicated HTTP workload measurements. - **[R220](R220-dataset-fat-client-plan.md)** — trusted internal Dataset adapter - on the R242 runtime, measurements, and Python integration. - **Area:** Dataset SDK. **Complexity:** High. **Status:** Deferred until R242 - S3 acceptance and dedicated Dataset HTTP workload measurements. + on the existing native runtime, measurements, and Python integration. + **Area:** Dataset SDK. **Complexity:** High. **Status:** Deferred until + dedicated Dataset HTTP workload measurements. ## Access server and external data @@ -33,11 +33,6 @@ live in the linked document; implementation follows - **[R170](R170-s3-cuobject-rdma.md)** — optional cuObject RDMA data plane. **Area:** access server / S3 / DiskIO / RDMA. **Complexity:** High. **Status:** Deferred until the basic TCP S3 path is stable and measured. -- **[R242](R242-rdma-fat-client-sdk.md)** — trusted internal cluster runtime, - exposed storage topology, native S3 first, and caller-owned buffer leases; - Dataset follows under R220. **Area:** access server / SDK / RDMA. - **Complexity:** High. **Dependencies:** accepted RDMA RPC and Chunk-KV/access - integration. - **[R185](R185-access-iceberg-cache-invalidation.md)** — bounded Iceberg cache and invalidation. **Area:** access server / Iceberg / Group 0 / Chunk-KV. **Complexity:** Medium. **Status:** Deferred pending focused cache measurements. @@ -104,8 +99,9 @@ live in the linked document; implementation follows **Area:** crowdb-tree engine. **Complexity:** Medium. - **[R5](R5-rdma-alloc.md)** — RDMA host-buffer RPC and data transport umbrella. **Area:** RPC / access server / Chunk-KV / DiskIO / SDK / RDMA. - **Complexity:** High. **Status:** Foundation/transport/datapath/Chunk-KV - accepted; R242 remains. + **Complexity:** High. **Status:** Foundation, transport, datapath, Chunk-KV + and native SDK accepted; coarse discovery/default-enablement policy + questions remain in the requirement. - **[R60](R60-tree-scan-sibling-leaf-readahead.md)** — sibling-leaf readahead on cold scans. **Area:** crowdb-tree / scan / DiskIO. **Complexity:** Medium. **Status:** Deferred pending cold file/block-backed measurements. diff --git a/doc/working/plan-rdma-fat-client-sdk.md b/doc/working/plan-rdma-fat-client-sdk.md deleted file mode 100644 index 82cc74de6..000000000 --- a/doc/working/plan-rdma-fat-client-sdk.md +++ /dev/null @@ -1,567 +0,0 @@ - - - -# Trusted internal S3 fat client and shared RDMA runtime Plan - -Upstream: [R242](../backlog/R242-rdma-fat-client-sdk.md), -[root RDMA plan](plan-rdma.md), -[registered buffers](../design/rpc/design-crowdb-rpc-rdma.md#4-registered-arenas-and-slot-ownership), -[service integration](../design/rpc/design-crowdb-rpc-rdma.md#7-service-interactions), -and subsequent [Dataset integration R220](../backlog/R220-dataset-fat-client-plan.md). - -Goal: embed trusted internal storage access through a shared topology-aware -native runtime, implement S3 first, and let Dataset attach to the accepted API. - -Status: mixed-size transport acceptance passes with repeatable CPU/request -benefit and unchanged per-size latency bounds. Continue this requirement under -`/implement-requirement`; the accepted prerequisite is recorded in the root -RDMA plan. Native S3 implementation, CPU/TCP/HCA acceptance, workspace gates, -all 45 S3 fixtures and independent HTTP SDK checks pass. The additional serial -protocol suite passes all 23 fixtures. Implementation is complete; requirement -cleanup remains. - -## Scope and current boundary - -- The embedding data-processing or AI system is in the Access Server trust - domain and can reach internal services. Native execution takes an explicit - tenant/context; it does not construct or verify SigV4, obtain a user-secret - database, or call HTTP for per-operation authorization. -- Reuse `crowdb-access-s3` metadata, conditions, `PreparedGet`, - `VerifiedGetStream`, publication, and multipart components. Complete S3 - operations still live partly in `app/crowdb-access-server/src/s3/`. -- `BufferLeaseMetadata`, `BufferOwnership`, `WritableBuffer`, and completion - callbacks already exist in `crowdb-rpc-ffi`. `standalone_buffer` and - `receive_data_buffer` have copy/detach branches; audit rather than assume - end-to-end ownership is already preserved. -- Add `lib/crowdb-sdk-native/` to the main workspace. Keep the independently - published `sdk/` HTTP crates and Python wheel free of native storage/FFI - dependencies. New Python/C++ business bindings are not R242 gates; the - existing C ABI buffer lease boundary remains in scope. -- R242 accepts shared runtime/topology/data APIs and native S3. Dataset - attaches afterward under R220; its measurements and Python work must not - block S3 or create a parallel native runtime. - -## Phase 1: shared runtime and native API - -- [x] **Create native crate and baseline**: register `crowdb-sdk-native` with - `runtime`, `topology`, `buffer`, and `s3` modules and tests under `tests/`. - Run focused existing S3/RPC/access and standalone SDK baselines before - moving operation code. Files: `Cargo.toml`, `lib/crowdb-sdk-native/`, - `lib/crowdb-access-s3/tests/`, `app/crowdb-access-server/tests/`. -- [x] **Define runtime ownership**: implement `ClientRuntime::connect(config)` - and shutdown, sharing discovery, compatible transport groups, producer - allocation, and hard request/byte limits across handles. Preserve distinct - S3/Dataset chunk types and writer policies; do not create a generic Repo - chunk type or mix incompatible writer pools. Let `S3StorageClients` accept - shared clients instead of reconnecting for each facade. Files: - `lib/crowdb-sdk-native/src/runtime.rs`, `lib/crowdb-access-s3/src/storage.rs`, - `lib/crowdb-chunk-client/src/client.rs`. -- [x] **Define business context**: expose `S3Client::new(runtime, config)` with - tenant/write policy and typed per-operation request identity, deadline, - cancellation, conditions, and range. Preserve S3 errors and uncertain - outcomes without HTTP headers, XML, or `CredentialProvider`. Files: - `lib/crowdb-sdk-native/src/s3/`, `lib/crowdb-access-s3/src/error.rs`, - `lib/crowdb-access-s3/src/condition.rs`, `lib/crowdb-access-s3/src/range.rs`. -- [x] **Define common delivery**: reads return metadata plus a pull-based - `next() -> Result, _>` body; `Ok(None)` means applicable - length/integrity checks passed. Preserve terminal failure across subsequent - polling; early drop is cancellation, not successful EOF. Define unique - writable storage, freeze/submit ownership, and write finalization. Keep business - results separate from common buffer types so later Dataset batches can own - sample/field streams. Files: `lib/crowdb-sdk-native/src/buffer.rs`, - `lib/crowdb-sdk-native/src/s3/`, `lib/crowdb-access-s3/src/retrieval.rs`. - -## Phase 2: supported topology and scheduling surface - -- [x] **Expose retained topology views**: provide inspection and explicit - refresh of service endpoints, rack/node/disk membership, Chunk-KV range - owners, transport capabilities, and source generation/age. Reuse existing - catalog and route owners; independently observed generations must not - become an invented global topology epoch. Files: - `lib/crowdb-sdk-native/src/topology.rs`, - `lib/crowdb-chunk-kv-client/src/catalog.rs`, - `lib/crowdb-diskio-client/src/topology.rs`, - `lib/crowdb-diskio-client/src/status.rs`, - `lib/crowdb-diskio-client/src/semantic/native_routes.rs`. -- [x] **Expose object locality**: resolve a logical object generation/range to - chunk placement and route observations for caller scheduling/prefetch. - Preserve the metadata-selected identity during execution instead of - re-resolving a mutable key to a different generation. Observation cannot - publish placement. Reuse `ChunkIoClient::native_storage_routes` where - suitable. Files: `lib/crowdb-sdk-native/src/s3/`, - `lib/crowdb-sdk-native/src/topology.rs`, - `lib/crowdb-access-s3/src/metadata/record.rs`, - `lib/crowdb-chunk-client/src/client.rs`. -- [x] **Handle stale observations**: retain catalog invalidation, endpoint - refresh, ownership epochs, and generation fencing. Old views stay safe to - inspect but cannot force execution past validation. Return typed - stale/fenced/deadline outcomes when refresh fails; never replay ambiguous - writes. Files: `lib/crowdb-sdk-native/tests/`, - `lib/crowdb-chunk-kv-client/src/client.rs`, - `lib/crowdb-diskio-client/src/semantic/`. - -## Phase 3: shared S3 execution and first native frontend - -- [x] **Extract bucket and metadata operations**: move typed bucket, HEAD, - list, and delete orchestration into the S3 library. Preserve bounded - listing, continuation scope, conditions, and shared-range cleanup. HTTP - parses/authenticates, calls shared execution, and encodes. Files: - `lib/crowdb-access-s3/src/`, - `app/crowdb-access-server/src/s3/operations.rs`, - `app/crowdb-access-server/src/s3/operations/delete.rs`, - `lib/crowdb-sdk-native/src/s3/`. -- [x] **Extract streaming GET**: adapt `PreparedGet`/`VerifiedGetStream` to - typed metadata and a lazy native body with logical offsets. Preserve - selected generation, range checks, storage checksums, and final checksum - errors. Do not expose HTTP strings or RPC fragments as native object data. - Files: `lib/crowdb-access-s3/src/retrieval.rs`, - `app/crowdb-access-server/src/s3/operations.rs`, - `lib/crowdb-sdk-native/src/s3/`, - `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`. -- [x] **Extract PUT and copy**: separate HTTP decoding from bounded input, - checksums, ChunkIO writes, publication, and definite-failure cleanup. Add - ordinary-input and frozen native-buffer entry points. Preserve - `PutOutcome::Timeout` uncertainty; final success requires durable data and - object publication. Files: `lib/crowdb-access-s3/src/streaming.rs`, - `lib/crowdb-access-s3/src/publication.rs`, - `app/crowdb-access-server/src/s3/operations/copy.rs`, - `app/crowdb-access-server/src/s3/operations/upload.rs`, - `lib/crowdb-sdk-native/src/s3/`. -- [x] **Attach multipart**: reuse the repository and multipart engine for - begin, upload/list parts, list uploads, complete, and abort. Preserve upload - identity, validation, publication uncertainty, and deferred cleanup. Files: - `lib/crowdb-access-s3/src/metadata/multipart_repository/`, - `lib/crowdb-access-multipart/`, - `app/crowdb-access-server/src/s3/operations/multipart.rs`, - `lib/crowdb-sdk-native/src/s3/`. -- [x] **Verify native TCP parity first**: compare supported operations through - HTTP and native entry points, then run native access without an HTTP - listener or S3 credential store. Compare bytes, metadata, conditions, - errors, publication, and cancellation before enabling RDMA. Files: - `lib/crowdb-sdk-native/tests/`, `lib/crowdb-access-s3/tests/`, - `app/crowdb-access-server/tests/`, `e2e-test/tests/access_s3/native/`. - The `s3.native` owned fixture starts only KV, Diskdb, DiskIO, Chunkdb and - Chunk-KV. Its client corpus covers mixed sizes, framed input, multipart, - generation-bound copy, paged listing, range/topology and restart. A separate - `s3.native.parity` fixture compares the native and trusted HTTP entry points. - -## Phase 4: retained leases, RDMA, and bounded lifecycle - -- [x] **Complete lease ABI**: reuse opaque `crowdb_rpc_buffer_t`, - `BufferLeaseMetadata`, and actual buffer length for native `BufferLease`. - Preserve request/stream identity, sequence, offset, capacity class, and - single final release. Rust borrowing prevents access after release; - document C ownership/misuse rather than promising repeated release of freed - handles is safe. Files: `lib/crowdb-rpc/ffi/src/buffer.rs`, - `lib/crowdb-rpc/ffi/src/sys.rs`, `lib/crowdb-rpc/include/crowdb-rpc/c_api.h`, - `lib/crowdb-sdk-native/src/buffer.rs`. -- [x] **Preserve completion ownership**: audit C++ completion, - `on_complete_cb`, `receive_data_buffer`, DiskIO conversion, ChunkIO views, - and S3 adaptation. Retain eligible receive storage through caller release - rather than a copy/detach shortcut that loses credits. Keep control - decoding/assembly, alignment, and EC copies explicit; cover existing RPC - callers before changing ownership. Files: `lib/crowdb-rpc/ffi/src/client.rs`, - `lib/crowdb-diskio-client/src/client.rs`, - `lib/crowdb-diskio-client/src/semantic.rs`, - `lib/crowdb-chunk-client/src/chunk/chunk_reader.rs`, - `lib/crowdb-access-s3/src/retrieval.rs`. -- [x] **Wire producer buffers**: reuse `ProducerBufferAllocator` and - `WritableBuffer`, expose only initialized frozen prefixes, and retain - submitted owners through transport/disk completion. Ordinary input - adapters stay bounded and account for copies. Files: - `lib/crowdb-rpc/ffi/src/producer_buffer.rs`, - `lib/crowdb-chunk-client/src/io.rs`, `lib/crowdb-sdk-native/src/buffer.rs`, - `lib/crowdb-access-s3/src/streaming.rs`. -- [x] **Enforce held-capacity budgets**: charge actual allocation capacity - once across shared views until final release. Keep request/producer/held - limits explicit and lock-free; waits honor deadlines/cancellation without - blocking RPC workers. Release backing storage before waking waiters. Files: - `lib/crowdb-sdk-native/src/runtime.rs`, `lib/crowdb-sdk-native/src/buffer.rs`, - `lib/crowdb-rpc/ffi/tests/`, `lib/crowdb-sdk-native/tests/`. -- [x] **Implement cancel and shutdown**: close admission, stop producers, - resolve operations once, and preserve storage through disk/CQ/caller - ownership. Release after shutdown retires slots without accessing destroyed - transport. Files: `lib/crowdb-sdk-native/src/runtime.rs`, - `lib/crowdb-sdk-native/src/buffer.rs`, `lib/crowdb-rpc/ffi/src/client.rs`, - `lib/crowdb-sdk-native/tests/`. -- [x] **Verify fallback and GPU-copy lifetime**: retain existing TCP/RDMA - policy, request identity, and eligible pre-payload fallback; fail accepted - or partial RDMA operations without TCP suffixes or ambiguous replay. Add an - asynchronous host-to-GPU copy simulation holding a lease beyond stream - close until copy completion. Files: `lib/crowdb-sdk-native/tests/`, - `lib/crowdb-rpc/ffi/tests/`, `lib/crowdb-diskio-client/tests/rdma_failure_test.rs`. - -## Phase 5: S3 acceptance and Dataset handoff - -- [x] **Run native and OCI profiles**: extend accepted launcher/runtime - conventions with the S3 SDK corpus, no-device TCP and shared-HCA profiles. - Record topology, actual path/classes, independent QPs, held-capacity high - water, errors, and exact HTTP/native parity. Native-only execution must not - require Access Server. Files: `tools/rdma/`, `e2e-test/tests/access_s3/`, - `lib/crowdb-sdk-native/tests/`. -- [x] **Verify distribution and document API**: run standalone SDK boundary - and functional controls; document shared runtime, topology scheduling, - stream termination, lease release, publication, and shutdown. Record R242 - evidence and the shared API consumed by R220. Files: - `lib/crowdb-sdk-native/README.md`, `sdk/README.md`, - `tools/ci-checks/check-dataset-sdk-boundary.py`, - `doc/backlog/R220-dataset-fat-client-plan.md`. - -R242 exit: accepted runtime/topology/lease APIs and native S3 semantics, -TCP/RDMA parity, bounded pressure, and shutdown-safe caller ownership. Existing -transport acceptance alone cannot establish SDK completion. - -## Execution evidence - -- Mixed-size optimization prerequisite is committed as `f3de1109`; repeated - final HCA measurements and service parity pass under the root RDMA plan. -- Existing S3 library all-target tests, Access Server all-target tests, and - standalone SDK boundary/functional controls pass before extraction. -- `crowdb-sdk-native` is registered in the main workspace. Initial tests cover - actual Vec capacity, shared-slice accounting, frozen native producer owners, - request cancellation/deadline, shutdown-safe held data, two tenant handles - without an HTTP listener/credential store, and separate typed writer groups. - Nine native cases pass, including asynchronous-copy and interrupted-mutation - outcomes. The S3, ChunkIO and DiskIO all-target suites, affected HTTP tests - and focused four-crate all-target clippy also pass. -- Runtime connection and attachment reuse control/catalog/chunk clients. - Compatible writer policies share a bounded group; incompatible policies get - separate small-writer pools sharing transports, reader and SDK budgets. - Shutdown currently closes SDK admission and drains small writers. Completing - native transport/completion teardown remains in Phase 4. -- Disk routes now have a retained inspection view with local generation and - publication age. Native catalog/service/transport observations do not invent - a global epoch. Attached catalog age is explicitly unknown until refreshed. - Full stale-view/locality acceptance remains outstanding. -- Basic bucket and point object lookup/delete orchestration is extracted into - `S3Executor`, and HTTP delegates to it. Streaming GET/PUT, copy, multipart, - typed native results and real HTTP/native parity are not complete. -- FFI completions now retain their owned native handles, including logical - control admission and receive recycling. Additive ABI queries expose the - canonical allocation identity, actual capacity, view offset and initialized - span; unknown external owners never substitute their view length for capacity. - `OwnedPayload` preserves this metadata across slices and logical credit owners. - DiskIO/ChunkIO propagation and SDK accounting remain outstanding. -- Explicit shared-HCA FFI acceptance with two posted receives retains both - response payloads, observes the third response remain pending, releases one - and completes the third. Held data stays readable after connection/server - destruction. The complete FFI suite/clippy and native RPC suite pass (211 - native cases, three explicitly unavailable or separately gated cases). -- Shared GET now offers typed half-open ranges and `next_verified`, with sticky - storage/length/integrity failures. Empty full reads also verify their checksum. - The legacy HTTP `next_chunk` remains fused after its first terminal result. - Three focused cases and the complete S3 library suite/clippy pass. Native - object metadata/body adaptation is still pending. -- Before completing native GET delivery, carry retained allocation identity, - actual capacity and source ownership from RPC through DiskIO/ChunkIO. Use an - owner-backed payload type with slices preserving one allocation owner; do - not rediscover receive ranges by scanning connection tables or adding a hot - lock, and do not substitute a guessed capacity for the actual native class. -- DiskIO now exposes `read_payload`, and production ChunkIO mirror/direct - reads, frame slices, logical stream-credit wrappers and S3 `next_payload` - preserve `OwnedPayload`. Existing public `Bytes` adapters erase metadata only - at their compatibility boundary. Required EC/physical coalescing uses fresh - Vec owners with known capacity; metadata-less injected seams remain explicit. - The 1,500-line reader is split by layout, location, frame views, verified - frame reads and stream delivery. A focused framing test proves original - address/capacity retention and logical credit lifetime through final Bytes - release; all existing reader cases pass. Native SDK capacity admission and - S3 body adaptation are the next dependent tasks. -- Runtime-wide held allocation accounting now uses the existing lock-free - skip-list dependency and weak entries removed at final backing release. - Per-view upstream logical read owners remain retained independently of the - shared physical charge. Admission waits register before checking capacity - and publication, honor request cancellation/deadline and wake on shutdown. - Unknown Bytes owners reserve capacity before a bounded copy, counted in - `ResourceStatus::compatibility_copy_bytes`. Five focused tests cover shared - capacity, 64 concurrent publication/release rounds, bounded copy pressure, - cancellation/shutdown and undersized views hiding oversized allocations. -- Native S3 now has typed configuration, entity-tag/date conditions, HEAD, - lazy GET and half-open ranges. Shared condition evaluation serves HTTP and - typed inputs. `ResolvedObject` captures the metadata revision in the same - point read, retains its immutable record, and rejects another tenant or - metadata authority; identical overwrites have distinct revisions. Native - metadata exposes validated user values without HTTP headers. -- `ObjectBody::next` returns retained `BufferLease` views and verifies final - length/integrity before successful EOF. Terminal errors remain sticky; - cancellation/early drop return request admission independently of delivered - buffers. A cancelled pending pull retains its not-yet-delivered payload, so - resuming after capacity release cannot skip it. Seven native read cases - cover original pointer/capacity, range offsets, overwrite/authority binding, - conditions, empty/corrupt checksum, capacity deadline, cancellation/shutdown - and pending-pull cancellation. The native suite now has 22 passing cases; - six allocation cases separately verify retention of each upstream logical - owner while sharing one physical charge. Shared S3 tests and affected HTTP - service/inspection tests, all-target clippy and Rust formatting pass. -- Still outstanding: native S3 PUT/copy/multipart/listing completion and HTTP - extraction, object locality/stale-view - acceptance, owned transport shutdown, and the complete native/OCI S3 corpus. - Current read fixtures use real storage reader/framing with injected metadata - and disk seams; they are not the required real-service TCP/RDMA acceptance. -- Runtime data transport now accepts an explicit `RdmaConfig`, forwarded - through shared ChunkIO discovery before DiskIO connections open. Legacy - constructors keep environment defaults. Connection construction is split - out of the nearly 1,000-line ChunkIO client. -- Native GET now also traverses real TCP and shared-HCA RC RPC through the - production semantic DiskIO adapter, ChunkIO framing and S3 verification. - The RDMA case charges an 8-byte object to its actual 64 KiB receive class, - blocks the next delivery until final release and keeps the delivered lease - readable after all client/server owners are destroyed. Both profiles report - zero SDK compatibility-copy bytes and no read retries. `ServerRequest::reply` - retains the original connection/server via a safe owned-handler registration; - new native test code adds no unsafe exception. FFI/ChunkIO/native all-target - tests, clippy and formatting pass. These focused RPC tests still inject - metadata/layout and disk contents; the final corpus must use real services. -- Retained topology now follows newer catalog generations installed by normal - operations, with unknown original observation age rather than a fabricated - timestamp. Explicit catalog-only refresh reports typed invalid/unavailable - failures and refuses generation regression. Endpoint, disk and catalog - sources remain independent. Typed transport observations expose readiness, - wire classes, TCP fallback policy and configured queue/registration bounds; - configured values do not pretend to be provider connection limits. -- `object_locality` inspects one tenant/authority-bound `ResolvedObject`, - range and bounded 1..128 extent page without fetching payload. Each chunk - observation retains its own modification revision and layout validity; - mirror/EC roles, unavailable fragments and observed disk/rack/node owners - remain explicit. Shared chunk references query once per page. Pagination - stays on the selected generation after name overwrite. Old views remain - inspectable, report typed stale outcomes at expiry and cannot force a deleted - layout into execution. General injected disk writers can expose actual - native route/transport observations; real TCP/RDMA cases verify these owners. - Three new catalog/locality cases pass; the native suite now has 27 cases. - Native/S3/ChunkIO all-target suites and clippy pass. Full service restart, - stale-catalog/endpoint fault acceptance remains part of the final corpus. -- HTTP and native listing/batch deletion now use shared `S3Executor` execution. - Native listing returns bounded typed entries and scoped continuation; handles - share a runtime-local random signer key unless configured for stable cursors. - This key is not a user credential and requires no credential database. - Lazy native batch deletion validates the entire selection, returns each - independent ordered result, keeps duplicate semantics and stops its tail on - drop/cancellation. Shared typed deletion preserves transport/deadline reply - loss as uncertainty; HTTP renders the same stable ServiceUnavailable code. - Four focused native cases cover paging/scope, lazy ordered partial progress, - cancellation and an applied mutation with a deliberately lost reply. The - native suite now has 31 cases. Native/S3 all-target tests, affected HTTP - service/inspection/multipart tests, three-crate clippy and formatting pass. - -## Immediate dependent implementation - -- Owned shutdown now has one lock-free coordinator and retained terminal result. - Closing admission starts cleanup independently of any caller's wait future; - owned small pools drain before owned ChunkIO/ChunkDB/Chunk-KV RPC workers and - reapers join on a blocking worker. Attached `from_clients` groups remain - borrowed; `adopt_clients` explicitly transfers shutdown responsibility, and - independently created writer policies own only their new small pools. - DiskIO closes admission and wakes native-route refresh workers. Held native - receive storage remains readable after actual worker shutdown; new DiskIO - operations are rejected. Two new cases cover borrowed ownership and cancelled - close waiters with a paused owned close. Explicit TCP/HCA receive tests pass - with the stronger worker-stop assertion. Native and four affected storage - client all-target suites (48 native cases), clippy and formatting pass. - -- HTTP and native multipart now share session creation/expiry, part writer - preparation, copied-part execution, exact generation publication, completion - freeze and abort. Retained native upload identities are tenant/authority-bound; - admission refreshes durable phase/expiry instead of trusting an old handle. - Native part input shares PUT's bounded ordinary/framed buffers and cancellation. - Five new cases cover 5 MiB plus framed-tail composition, paged parts/uploads, - replacement/captured-source copy, abort/reconnect, confirmed lost replies and - an applied object publication whose confirmation reads also fail. That failure - remains uncertain; the same frozen selection later confirms without data - replay. Native/S3/Access Server all-target suites pass (46 native cases), as - do three-crate clippy and formatting. Access Server required more than the - default 60-second outer compilation limit; continuing its cached build and - complete test run passes without changing any test timeout. - -- HTTP object copy and upload-part copy now use shared verified copy execution. - Native copy accepts ordinary source names or a tenant/authority-bound retained - selection; overwrites cannot replace that selection. Metadata COPY/REPLACE, - self-copy validation, conditions, source integrity, durable completion and - uncertain publication remain shared. Three native cases cover these contracts - and lost applied replies. Native/S3 all-target suites (41 native cases), - affected HTTP tests, focused clippy and formatting pass. Full service parity - and native multipart attachment remain pending. - -- Native `begin_put`/`put_vec` now use the shared durable PUT session and return - the publication's exact revision. Input calls accept at most 1 MiB and retain - actual allocation capacity; foreign-runtime owners are rejected. Registered - producers initialize payload slots directly with reserved frame space and - freeze into `FrozenObjectBuffer`; all physical/payload views retain their - producer credit, including initializer failure/panic and shutdown. - Large framed streams require aligned interior frames; one small framed owner - must hold the complete declared object. Native writer waits are cancellable; - dropping a pending input terminates that writer and returns request admission. - Seven new cases cover mixed sizes through multiple chunks, original payload - addresses, length/checksum/cancellation, pending-input drop and an applied - publication whose reply is lost. Native/S3 all-target suites pass (38 native - cases), with focused three-crate clippy and formatting. Native RPC write/HCA - acceptance, copy and multipart remain outstanding. - -- Shared writer selection and `PreparedPut` now serve HTTP PUT. Durable - completion precedes one versioned publication, retaining the same metadata - revision and uncertain timeout classification without a subsequent HEAD. - Native physical views require finalization and reject later header/footer - rewrites, including a changed chunk destination. The new ownership test, - full S3 library suite, affected HTTP tests, three-crate clippy and formatting - pass. Explicit HCA native producer framing still sends 1 MiB directly with - zero staging-copy bytes. Native PUT attachment is the next task. - -- Accepted RDMA read transport, deadline and malformed-response failures now - retain `AcceptedReadInterrupted` through routed DiskIO, mirror/EC readers, - physical/frame windows and native `ReadFailureKind::Interrupted`. No route, - replica, recovery-candidate or layout retry can replay that accepted read. - Ordinary disk/CRC recovery controls still pass. Four new CPU cases cover - direct mirror/EC, interrupted recovery and framed-stream propagation. - Explicit shared-HCA GET accepts a read, stops the peer with TCP fallback - enabled and three retries configured, and reports the same sticky terminal - error without any additional semantic read, connect, reconnect or retry. - A prior caller-held receive remains readable through owned runtime shutdown. - The three DiskIO HCA failure cases also pass. Native (49 cases), DiskIO and - ChunkIO all-target suites, focused three-crate clippy and formatting pass. - -- The real-service `s3.native` and `s3.native.parity` TCP profiles pass in - `run-0ae636d32f1c4b228918a4bf1099708c`. Twelve storage services run without - Access Server or credential lookup; host and unprivileged no-device OCI - clients verify the corpus before and after all storage services restart. - The separate thirteen-service fixture verifies HTTP/native bytes, ETags, - user metadata, conditions/range errors, signed copy and multipart delivery. - Corpus sizes span empty through 4 MiB plus 31 bytes, crossing frame and - small/large boundaries. Reader compatibility copies are zero; held capacity - reaches exactly the configured 8 MiB pressure limit and resumes on release. - A simulated asynchronous copy retains the last lease through owned shutdown. - Small writer input remains retained for open-strip mirror repair until pool - drain; after owned shutdown both producer and held charges return to zero. - Fresh reader runtime artifacts intentionally have zero producer high water. - The fixture suite (157 cases), focused E2E clippy and workspace Rust lint pass. -- [x] **Preserve observed reads with metrics**: forward owned payloads and - transport/topology observations through `MetricsDiskWriter`; test original - allocation identity and read accounting through the public client facade. - Both read-owner cases, all-target ChunkIO clippy, Rust formatting and - tree-lint pass. -- Full shared-HCA native and OCI S3 corpus passes in - `native-s3-hca/ee6b87cbef394f3880d877e0178d7263`: twelve real service containers, - explicit HCA `rocep35s0f0`, `10.200.1.1`, separate data directories and CM ports. - Both consumers write their own corpus and verify after full storage restart. - Data clients disable TCP fallback and use twelve independent data connections; - all four reader runs report zero errors/fallback/CQ errors, zero SDK compatibility - copies, 8 MiB held high water and 436404224 registered bytes within 512 MiB. - Each producer run has 1114112 producer bytes high water, 5374424 held high water - and zero SDK compatibility copies. Registered framed input sends 2099385 bytes - directly; ordinary input and EC construction still stage 21247554 bytes. - The small framed-input delta is explicitly zero transport payload copies. - Server control clients retain TCP connectivity; the data client independently - enforces RDMA. OCI uses matched system verbs/provider libraries and no root user. - This is shared-HCA local-loop functional evidence, not cross-host throughput. -- Native topology acceptance now moves a catalog owner/endpoint while an old - view is retained. S3 receives one typed refresh response, refreshes to the - new owner and completes; the retained old view stays unchanged. Existing - object-generation, expired-layout and failed-refresh tests cover the remaining - authority boundary. -- Added native SDK README and shared runtime design, updated Access Server - trust/semantic boundary and the independent HTTP SDK pointer. -- Final HCA run `native-s3-hca/3522f5c97f7c46e880e09ff8ed158605` also verifies - a nonzero registration identity on the retained pressure-test lease. All - native/OCI write and restart/read phases pass with the same counters above. - Image identity is `sha256:685c3632b0b27fdb8a684fda65914f509a34972bc2a9b1b8c6959ea5f92fe28b`. -- Final affected native/S3/RPC/access Rust tests, workspace all-feature clippy, - `rs-lint`, formatting and independent SDK gates pass. The HTTP SDK's first - sandboxed run could not bind loopback sockets; rerunning with local network - permission passes all twenty tests without source changes. -- Full S3 run `run-677eebfc1cbb43859f96a2492f57f2a0` passes 44 of 45 fixtures. - The trailer rejection test races its 110022-byte GET payload against an early - 403 response, raising `BrokenPipeError` before reading the HTTP response; - the exact case passes alone. The raw HTTP test client now reads a complete - error response after an interrupted upload without retrying. A missing or - malformed response still fails, and a 2xx response cannot conceal incomplete - transmission. Four deterministic socket-seam cases preserve these boundaries; - all 163 fixture cases pass. The full default-concurrency S3 suite passes - all 45 fixtures in `run-c3ded34f0ac84d70aae9f0ff913d63d1`; the additional - serial protocol recheck passes all 23 fixtures in - `run-6dd0488a66be4fd5bd480fdf157f06cc`. No server code or deadline was changed. -- R220 now points to the implemented runtime/lease/topology contract and API - guide. Its Dataset measurements and Python binding remain separate work. -- Next: requirement cleanup. - -## Subsequent Dataset integration under R220 - -- [ ] **Attach Dataset after S3**: add a facade using the same runtime, - topology, and buffers. Reuse `DatasetAuthority`, `DatasetSessions`, upload, - composition, and Split logic; extract remaining common HTTP v1/v2 delivery - where needed. Files: `lib/crowdb-sdk-native/src/dataset/`, - `lib/crowdb-access-dataset/`, `app/crowdb-access-server/src/dataset/api/`. -- [ ] **Preserve Dataset completion**: retain Snapshot/Split binding, - selection/partition, active-read protection, explicit acknowledgement, - resume, and cleanup. Legacy `DatasetDirectClient::scan` progress behavior - is not the v2 contract. Buffer release and download completion never - acknowledge batches. Files: `lib/crowdb-access-dataset/src/read_session/`, - `lib/crowdb-sdk-native/tests/`, `app/crowdb-access-server/tests/`. -- [ ] **Complete Dataset follow-up**: perform R220's HTTP/direct workload - comparison and native Python integration using this runtime, preserving - the independent HTTP SDK distribution. Files: - `doc/backlog/R220-dataset-fat-client-plan.md`, `sdk/`, - `e2e-test/tests/access_dataset/`. - -These tasks belong to R220, follow S3 delivery, and are not prerequisites for -R242 acceptance or a second implementation of its runtime. - -## Tests - -Unit tests: - -- typed context/range/conditions and retained topology ownership; -- opaque ABI layout, fixed-width fields, initialized-length and null/error rules; -- unique write/freeze ownership, shared accounting, and single final release; -- nonblocking completion, single terminal result, and asynchronous copy ownership. - -Integration tests: - -- shared runtime handles with isolated operation cancellation; -- topology/locality inspection, stale owner/endpoint refresh, and fencing; -- S3 bucket/object/copy/multipart parity, conditions, and uncertain publication; -- native execution without HTTP listener, SigV4 verifier, or secret database; -- final integrity failure, lease order/offsets, and eligible no-copy payloads; -- held-capacity pressure, deadlines, cancel, shutdown, and release afterward; -- TCP-only results, pre-payload fallback, terminal accepted/partial RDMA failures; -- standalone Dataset HTTP SDK packaging and functional regression controls. - -E2E tests: - -- native S3 against metadata/ChunkIO/DiskIO with explicit topology inspection; -- no-device TCP OCI and shared-HCA local-loop multi-container profiles; -- held-lease pressure, multiple streams, and simulated asynchronous GPU copy; -- HTTP/native parity separate from the native-only execution fixture. - -## Consolidated files - -- `Cargo.toml`, new `lib/crowdb-sdk-native/`, its `tests/` and README. -- `lib/crowdb-access-s3/`, `lib/crowdb-access-multipart/`, S3 HTTP operations. -- `lib/crowdb-chunk-kv-client/`, `lib/crowdb-chunk-client/`, `lib/crowdb-diskio-client/`. -- `lib/crowdb-rpc/ffi/`, `lib/crowdb-rpc/include/crowdb-rpc/c_api.h`, affected C++ ownership paths. -- Access Server tests, `e2e-test/tests/access_s3/`, `tools/rdma/`. -- `sdk/README.md` and standalone boundary controls; no new native dependency in `sdk/`. -- R220 follow-up: Dataset library/HTTP adapters, native Dataset facade and bindings. - -## Verification gates - -Run after registering the native crate; use existing-package gates for the -baseline. Root Cargo gates do not cover the independent SDK workspace. - -```bash -pixi run -- cargo fmt --all -- --check -pixi run -- cargo test -p crowdb-sdk-native --all-targets -pixi run -- cargo test -p crowdb-access-s3 --all-targets -pixi run -- cargo test -p crowdb-rpc-ffi --all-targets -pixi run -- cargo test -p crowdb-access-server --all-targets -pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings -pixi run rs-lint -pixi run tree-lint -pixi run check-dataset-sdk-boundary -pixi run test-dataset-sdk -pixi run sdk-fmt-check -pixi run sdk-lint -pixi run test-e2e-s3 -``` - -Enable `CROWDB_E2E_PRINT_SERVICE_ERRORS=1` in CI or while diagnosing failures. -Capture actual native/OCI profile and lease evidence; passing TCP gates do -not replace the real-HCA profile. From b551599271fcb21e4da730e842298ec260b10ce1 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 17:40:36 +0800 Subject: [PATCH 076/258] Settle RDMA endpoint policy and separate cross-host validation --- doc/backlog/R5-rdma-alloc.md | 27 ++-- doc/design/rpc/design-crowdb-rpc-rdma.md | 27 ++-- doc/working/plan-rdma-multi-node.md | 161 ++++++++++++++++------- doc/working/plan-rdma.md | 33 +++-- 4 files changed, 167 insertions(+), 81 deletions(-) diff --git a/doc/backlog/R5-rdma-alloc.md b/doc/backlog/R5-rdma-alloc.md index 7b2b7f0f6..43e3b0615 100644 --- a/doc/backlog/R5-rdma-alloc.md +++ b/doc/backlog/R5-rdma-alloc.md @@ -7,8 +7,8 @@ The registered host-buffer foundation, common RDMA RPC transport, HTTP/ChunkIO/DiskIO datapath, Chunk-KV/access integration and native S3 runtime -are accepted. The deployment-policy questions below remain separate from the -completed native SDK functional acceptance. +are accepted. Final verification settles endpoint capability and opt-in +deployment policy using the existing TCP preflight and cold configuration. ## Problem @@ -35,6 +35,14 @@ defined in [`design-crowdb-rpc-rdma.md`](../design/rpc/design-crowdb-rpc-rdma.md with host memory first and explicit host-to-GPU copy completion before final lease release. Native and unprivileged OCI TCP/shared-HCA acceptance evidence is retained in the [root plan](../working/plan-rdma.md). +6. Discovery supplies the RPC endpoint; the live TCP preflight supplies + `RdmaEndpointCapability` and CM binds its generation/identity. Do not add + another authoritative Group-0 copy of transient transport capability. + Disabled or incompatible peers follow configured TCP fallback before payload. +7. Keep RDMA opt-in through explicit cold configuration or environment enablement. + Hardware discovery alone cannot enable it. Typed settings override environment; + explicit TCP remains usable even with fallback disabled. Strict enabled RDMA + fails if local setup or peer capability cannot satisfy the connection. The base transfer mechanism is SEND/RECV. Large client buffers are split into ordered messages with request ID, sequence, and logical offset. RDMA READ and @@ -66,13 +74,14 @@ WRITE are outside this requirement. work applies bounded backpressure and resumes after release. Invariant: caller-controlled lifetime. Integration test. -## Open Questions - -- Which service-discovery record supplies the coarse RDMA capability and - capability generation? The TCP handshake remains the authoritative dynamic - endpoint exchange. -- Which deployments enable the RDMA transport by default after hardware - capability is detected? Unsupported deployments remain TCP-only. +- Given a discovered endpoint whose RDMA identity changes after preflight, + when CM validates the offer, the stale identity fails before payload. A + TCP-only peer causes no CM attempt. Invariant: live capability authority. + Integration test. +- Given default, explicitly disabled and strict enabled configuration, when + connections start, defaults remain TCP, explicit TCP overrides environment, + and strict enabled failures cannot fall back. Invariant: explicit deployment + policy. Integration test. Run: diff --git a/doc/design/rpc/design-crowdb-rpc-rdma.md b/doc/design/rpc/design-crowdb-rpc-rdma.md index 19b71b638..8be905824 100644 --- a/doc/design/rpc/design-crowdb-rpc-rdma.md +++ b/doc/design/rpc/design-crowdb-rpc-rdma.md @@ -38,15 +38,24 @@ logical result is an ordered stream of messages with logical offsets. ## 2. Endpoint discovery and path selection -Service discovery first publishes coarse capability: whether the endpoint is -RDMA-capable and whether an RDMA address is currently available. If either -side lacks capability, or local configuration or device probing disables RDMA, -the client uses TCP without attempting RDMA CM. - -When coarse capability permits RDMA, the TCP handshake exchanges the current -RDMA CM address and port, GID/device identity, capability generation, protocol -version, and supported data classes. The client verifies that the generation -and peer identity are current before creating the RDMA connection. +Service discovery resolves the RPC endpoint. The current endpoint's TCP +preflight supplies `RdmaEndpointCapability`, including availability, RDMA CM +address and port, GID/device identity, generation, protocol version and data +classes. Group 0 does not store a second authoritative copy of this transient +transport state. A discovered endpoint alone cannot authorize RDMA CM. + +RDMA is opt-in through explicit cold configuration or `CROWDB_RDMA_ENABLED=1`. +Hardware presence alone does not enable it. Defaults permit TCP fallback; +strict workloads explicitly disable fallback. Typed configuration overrides +environment settings and rejects an unavailable requested RDMA device before +startup. Environment-based probing can remain on TCP when fallback is allowed. +A deliberately disabled typed configuration selects TCP directly, independently +of the fallback flag. + +An enabled, locally ready client checks the live TCP capability before CM. +An unavailable or incompatible peer skips CM and follows the fallback policy. +The CM offer binds that exact generation and identity, so a changed endpoint +between preflight and connection cannot silently replace its negotiated peer. `SocketTransport` performs this control exchange on a temporary nonblocking TCP socket with one deadline. The 160-byte version 4 capability advertises diff --git a/doc/working/plan-rdma-multi-node.md b/doc/working/plan-rdma-multi-node.md index 0a2aa6981..c189da8e1 100644 --- a/doc/working/plan-rdma-multi-node.md +++ b/doc/working/plan-rdma-multi-node.md @@ -3,52 +3,115 @@ # RDMA physical multi-node validation Plan -Upstream: [R5](../backlog/R5-rdma-alloc.md), with transport behavior supplied -by [RDMA RPC transport](../design/rpc/design-crowdb-rpc-rdma.md) and the current -[RDMA implementation plan](plan-rdma.md). - -Goal: validate the already implemented RDMA RPC and data paths across physical -nodes after the single-node, multi-container acceptance is complete. This plan -does not add transport behavior or change the R239 backlog scope. - -## Scope - -- one or more containers per physical node; -- one explicitly selected RDMA NIC/device and address per container; -- shared RPC framing, request correlation, fallback, and bounded buffers; -- host-provided fabric, HCA, device access, and memlock setup; -- no network-namespace manipulation in the application or test path. - -The host-network port rule remains in force when this plan is executed: each -container receives a distinct port offset applied to all service ports. The -HCA physical port is not part of the offset. - -## Tasks - -- [ ] **Record node inventory**: capture each node's HCA, port, GID/IP - mapping, provider libraries, and reachable peer address. -- [ ] **Prepare multi-node OCI runtime**: provide the required devices, - userspace libraries, memlock allowance, and application-user access on each - node. -- [ ] **Bind endpoint devices**: configure one local device/address mapping per - container and verify that the negotiated peer identity matches the intended - node. -- [ ] **Run transport corpus**: exercise CM/QP setup, bounded wire-slot frames, - receive repost, request correlation, disconnect/error handling, and - pre-payload TCP fallback across the fabric. -- [ ] **Run service corpus**: compare TCP and RDMA results for Access Server, - DiskIO, Chunk-KV, and SDK flows byte-for-byte and by error/sequence behavior. -- [ ] **Publish topology evidence**: report node count, device mapping, QP - count, fallback count, buffer pressure, and final result. - -## Exit condition - -The same RPC and service corpus passes across the available physical nodes, -with explicit device ownership and no namespace operations. Failures caused by -missing fabric or unavailable HCA remain environment blockers and do not alter -the R239 transport contract. - -## Verification - -Run the current R239 and service gates through `pixi run`, then execute the OCI -profile with the node inventory and device mappings recorded above. +Upstream: [RDMA transport](../design/rpc/design-crowdb-rpc-rdma.md), +[native SDK](../design/access-server/design-crowdb-native-sdk.md). + +Goal: validate the existing RPC, storage and native SDK across distinct physical +hosts, with reproducible fabric, resource and failure evidence. + +Status: independently scheduled follow-up. This request refines the plan only; +no remote host, fabric or cross-host benchmark is changed or executed. Completion +of the local RDMA implementation and Dataset client work does not depend on this +plan. Hardware inventory must be supplied when execution is scheduled. + +## Scope and prerequisites + +- At least two distinct physical hosts with reachable peer HCA addresses. + Record host identity, NIC model/firmware, provider/kernel versions, link rate, + MTU, port/GID/IP mapping, NUMA placement and switch/fabric settings. +- Use the same source revision, native library hashes and immutable OCI image + digest on every host. Keep data directories, service IDs, endpoints and + cleanup ownership separate for this run. +- Host configuration supplies devices, memlock and non-root access. Application + and test code do not move links or enter other network namespaces. Changes + to shared fabric settings require separately authorized operator work. +- Each container receives an explicit device/address and nonoverlapping service + and CM ports. Benchmark port offsets and real-service explicit endpoints must + produce a recorded complete endpoint map; HCA physical port remains unchanged. +- Missing hosts, HCA access or peer reachability leaves this plan pending. It + cannot be replaced by localhost, two containers on one host or a TCP-only run. + +## Phase 1: inventory and remote fixture ownership + +- [ ] **Capture inventory**: write one machine-readable manifest per host and + a run manifest tying physical hosts, NICs and peer endpoints together. Probe + device access under the application UID, available registration capacity and + bidirectional reachability. Files: `tools/rdma/`, `e2e-test/`. +- [ ] **Extend the explicit launcher**: reuse immutable image/client bundles, + source checks, container labels, resource limits, reports and cleanup. Record + the ownership of every remote process and directory; cleanup only this run. + Preserve logs on startup failure and interruption. Files: + `tools/benchmark/rpc_bench_process.py`, `tools/rdma/`, `e2e-test/`. +- [ ] **Verify actual peer placement**: correlate TCP preflight/CM addresses, + QP identity and NIC port counters with the distinct hosts. Reject a result + whose endpoints accidentally connect within one host. Files: launcher and + structured transport observations. + +## Phase 2: transport correctness + +- [ ] **Run exact-byte corpus**: cover empty/small payloads, 64 KiB/256 KiB/1 MiB + classes, framing boundaries and logical controls beyond one receive window. + Run TCP, strict RDMA, single/multiple QPs and producers, packing off/on, idle + singleton and several batches in flight. Preserve client request counts and + per-request validation; SEND/CQE counts are separate observations. Files: + `tools/benchmark/rpc_regression.py`, `lib/crowdb-rpc/tests/`. +- [ ] **Exercise bounded pressure**: hold receive leases, saturate WR and byte + windows, then release capacity. Report posted receives, SQ occupancy, + registered bytes, held bytes and queue high water. All admitted owners must + remain valid until completion/release, including cancellation and shutdown. +- [ ] **Inject owned failures**: stop only this run's peer or selected service; + cover setup failure, stale generation and accepted partial delivery. Eligible + fallback preserves identity; partial reads fail terminally and ambiguous + mutations are not replayed. Do not flap a shared physical link to simulate + failure. Retain exact failing case and transport counters. + +## Phase 3: real services and embedded consumers + +- [ ] **Run distributed service corpus**: place real metadata/Chunk-KV/DiskIO + services and clients on distinct hosts, using service-discovered routing. + Verify HTTP, native Rust and unprivileged OCI results before and after owned + service restarts. Cover conditional errors, generation selection, copy, + multipart, mixed sizes and physical read integrity. Files: + `e2e-test/tests/access_s3/native/`, `tools/rdma/run-native-s3.py` and fixtures. +- [ ] **Verify caller lifetime**: retain registered read leases across stream + close and runtime shutdown while simulating asynchronous host-copy completion; + verify bounded producer/held capacity and final release. Record actual RDMA + readiness, fallback, QPs, allocation classes and copy counters per process. +- [ ] **Attach accepted Dataset corpus when available**: reuse native Dataset + acceptance without reimplementing its authority, Snapshot/Split selection or + acknowledgement. Keep this optional subcase explicit if Dataset is not yet + available; it cannot silently count as executed. + +## Phase 4: comparable performance and report + +- [ ] **Run pinned baselines**: use fixed physical cores/NUMA, the same message + distribution/seed, loader concurrency, QP count, warmup/duration/repeats and + payload-copy behavior for TCP and RDMA. Start with the checked-in mixed-size + profile, then include recorded application-size distributions when supplied. + Files: `tools/benchmark/profiles/rdma-mixed-size.json`, regression runners. +- [ ] **Report throughput and tails**: count completed client requests, bytes/s, + p50/p99/p999 by size, client+server CPU/request, memory and errors. Separate + closed-loop concurrency tests from intended-arrival open-load latency. Never + compare unrestricted cores against pinned runs as one performance gain. +- [ ] **Publish reproducible evidence**: retain source/image/binary identity, + full commands, host/fabric manifest, actual path/counters, failures and raw + trials. Explain remaining copies and hardware limitations. Do not impose an + invented TPS gate; record repeatability and latency/resource trade-offs. + +## Verification and exit + +- Unit: remote fixture selection, ownership and cleanup; incompatible/missing + prerequisites; truthful report classification and endpoint maps. +- Integration: bounded transport corpus, pre-payload fallback, partial failures, + shared-owner retention and no-device TCP regression. +- E2E: at least two physical hosts, real services, native/non-root OCI consumers, + exact results, restart/failure cases and retained-lease pressure. +- All strict-RDMA success cases require zero workload fallback, validation and + transport errors. Expected injected failures are identified separately, with + no successful partial result or premature owner reuse. +- Final report must prove cross-host execution and explain each skipped scope. + Missing environmental prerequisites remain pending with their exact evidence. + +Use the current gates through `pixi run`; launcher changes require fixture tests +and affected Rust/C++ formatting/lint/tests. Concrete remote commands are added +after inventory and endpoint selection, rather than embedding assumed hosts. diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index 9fe349b26..62b0acb40 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -13,11 +13,17 @@ semantics, bounded buffer leases, and automated OCI verification using a single-node HCA local-loop with multiple containers. Persistent-plan exception: retain this root record for accepted local-HCA -measurements and the physical multi-node handoff. Completed child plans and +measurements. Completed child plans and implementation checklists are removed; the evidence remains reproducible. ## Current implementation status +Status: implementation and local-HCA acceptance complete. Endpoint discovery +and opt-in defaults are settled by the existing implementation. Final policy +checks pass 31 native capability/configuration/preflight cases, including a +real-HCA TCP-only peer with strict and allowed fallback, plus nine Rust FFI +configuration cases. No production behavior changed during this closure. + Registered host buffers and the common RC SEND/RECV transport are accepted. The shared arena has bounded lock-free admission, retained MR/PD/device ownership, explicit leases and quarantine. TCP preflight binds CM setup to the @@ -61,8 +67,7 @@ and RPC flow analysis. The foundation, transport, datapath and Chunk-KV continuation is complete. The subsequent mixed-size send optimization also passes its local-HCA acceptance, as recorded below. The resulting shared native runtime and S3 -frontend pass their TCP and shared-HCA ownership profiles. Physical multi-node -validation remains in its separate plan. +frontend pass their TCP and shared-HCA ownership profiles. ## Accepted mixed-size send optimization @@ -149,8 +154,7 @@ Further tuning remains separate from this accepted mechanism: intended-arrival open-load latency, sampled queue-age/peer-pressure telemetry and a retained-echo benchmark mode. Transport counters already distinguish direct payload, aggregation, compatibility staging and framing/control copies; application echo -copy remains explicitly identified. Physical multi-node performance continues -under `plan-rdma-multi-node.md`. +copy remains explicitly identified. ## Fixed design decisions @@ -159,8 +163,12 @@ under `plan-rdma-multi-node.md`. and one SGE per message. The accepted datapath adds a negotiated control/data layout with separate SGEs for immutable producer payload and per-target control; its total wire admission includes both regions. -- Use service discovery for coarse RDMA capability and TCP handshake for the - current RDMA endpoint, capability generation, and data classes. +- Service discovery resolves the RPC endpoint. Live TCP preflight supplies + capability, generation, CM identity and data classes; CM binds that exact + identity. Group 0 holds no second authoritative transient capability copy. +- RDMA is opt-in through explicit cold configuration or environment enablement. + Hardware detection alone cannot activate it. Typed settings override the + environment, explicit TCP remains usable, and strict RDMA cannot fall back. - Select RDMA before the first payload; fallback is RDMA to TCP only. - Preserve existing ChunkIO and Chunk-KV aggregation. Additionally permit bounded transport coalescing of complete RPC frames without a batching timer; @@ -185,8 +193,6 @@ under `plan-rdma-multi-node.md`. Benchmark servers use `CROWDB_PORT_OFFSET`. The real-service SDK fixture assigns separate loopback endpoints and nonoverlapping RPC/CM ports, while keeping the HCA physical port unchanged. -- Physical multi-node testing is deferred to [the separate follow-up plan](plan-rdma-multi-node.md) - and is not part of the current backlog scope. ## Accepted native runtime and S3 delivery @@ -251,8 +257,8 @@ The [native SDK guide](../../lib/crowdb-sdk-native/README.md) and [architecture](../design/access-server/design-crowdb-native-sdk.md) define the supported API. [R220](../backlog/R220-dataset-fat-client-plan.md) attaches Dataset authority, Snapshot/Split binding and explicit acknowledgement to the -same runtime after its HTTP workload measurements. Dataset binding work, -GPUDirect and physical multi-node measurements remain separate follow-ups. +same runtime after its HTTP workload measurements. Dataset binding work and +GPUDirect remain separate follow-ups. ## Final verification @@ -271,9 +277,8 @@ pixi run tree-lint ``` OCI E2E verification must run the TCP-only and single-node HCA local-loop -multi-container profiles when their prerequisites are available. Physical -multi-node verification is outside this plan and is covered by the separate -follow-up plan. Namespace profiles are not part of acceptance. +multi-container profiles when their prerequisites are available. Namespace +profiles are not part of acceptance. ## Test and setup files From 026a2a422053b170f4e461ab57670534ada4ee39 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 17:40:51 +0800 Subject: [PATCH 077/258] Retire completed RDMA implementation requirement --- doc/backlog/R5-rdma-alloc.md | 93 ------------------------------------ doc/backlog/backlog.md | 5 -- doc/working/plan-rdma.md | 6 +-- 3 files changed, 2 insertions(+), 102 deletions(-) delete mode 100644 doc/backlog/R5-rdma-alloc.md diff --git a/doc/backlog/R5-rdma-alloc.md b/doc/backlog/R5-rdma-alloc.md deleted file mode 100644 index 43e3b0615..000000000 --- a/doc/backlog/R5-rdma-alloc.md +++ /dev/null @@ -1,93 +0,0 @@ - - - -### R5: RDMA — Host-buffer RPC and data transport - -## Status - -The registered host-buffer foundation, common RDMA RPC transport, -HTTP/ChunkIO/DiskIO datapath, Chunk-KV/access integration and native S3 runtime -are accepted. Final verification settles endpoint capability and opt-in -deployment policy using the existing TCP preflight and cold configuration. - -## Problem - -CROWDB needs an RDMA path for RPC and data transfer while retaining TCP as a -one-way fallback. The path must move bounded registered host buffers through -Access Server, Chunk-KV, DiskIO, and the native fat client without an -intermediate CPU payload copy. The caller must receive explicit buffer leases, -and unsupported RDMA must not affect TCP operation. The formal architecture is -defined in [`design-crowdb-rpc-rdma.md`](../design/rpc/design-crowdb-rpc-rdma.md). - -## Solution - -1. Reuse the accepted [registered-buffer and RDMA RPC contract](../design/rpc/design-crowdb-rpc-rdma.md): - bounded 64 KiB/256 KiB/1 MiB classes, explicit ownership and quarantine, - TCP preflight, per-CQ polling/event workers and one-way fallback. -2. Retain the verified transport and benchmark invariants when adding services. -3. Reuse the accepted [HTTP/ChunkIO/DiskIO stream](../design/rpc/design-crowdb-rpc-rdma.md#7-service-interactions), - preserving existing aggregation, aligned 1 MiB receive data, disk completion - ownership, caller leases and partial-result failure without a TCP suffix. -4. Reuse the accepted [Chunk-KV/access stream](../design/rpc/design-crowdb-rpc-rdma.md#7-service-interactions), - using 64 KiB metadata and 256 KiB aggregate classes, bounded logical control - assembly, shared request identity and final-caller byte admission ownership. -5. Reuse the accepted [native SDK and lease contract](../design/access-server/design-crowdb-native-sdk.md), - with host memory first and explicit host-to-GPU copy completion before final - lease release. Native and unprivileged OCI TCP/shared-HCA acceptance evidence - is retained in the [root plan](../working/plan-rdma.md). -6. Discovery supplies the RPC endpoint; the live TCP preflight supplies - `RdmaEndpointCapability` and CM binds its generation/identity. Do not add - another authoritative Group-0 copy of transient transport capability. - Disabled or incompatible peers follow configured TCP fallback before payload. -7. Keep RDMA opt-in through explicit cold configuration or environment enablement. - Hardware discovery alone cannot enable it. Typed settings override environment; - explicit TCP remains usable even with fallback disabled. Strict enabled RDMA - fails if local setup or peer capability cannot satisfy the connection. - -The base transfer mechanism is SEND/RECV. Large client buffers are split into -ordered messages with request ID, sequence, and logical offset. RDMA READ and -WRITE are outside this requirement. - -## Dependencies - -- R4 bounded memory pool principles. -- Existing common RPC framing and request-number system. -- Existing ChunkIO aggregation, Access Server, Chunk-KV, DiskIO, and FFI - ownership contracts. -- R170 remains an optional cuObject adapter and must reuse the stream contract. - -## Acceptance - -- Given a host with no RDMA capability, when the services start, TCP remains - usable and no RDMA attempt is required. Invariant: optional transport. - Integration test. -- Given RDMA capability and configured registered memory, when an access stream - runs, all buffers remain within the budget and are released only by their - owner. Invariant: bounded ownership. E2E test. -- Given an RDMA setup failure before payload submission, when an idempotent - request is retried, TCP returns the same logical result. Invariant: one-way - fallback. E2E test. -- Given a result larger than its selected class, when it is split into ordered - messages, the SDK, Chunk-KV, or DiskIO consumer reconstructs exact bytes. - Invariant: stream equivalence. E2E test. -- Given an SDK caller holds returned leases, when the pool is exhausted, new - work applies bounded backpressure and resumes after release. Invariant: - caller-controlled lifetime. Integration test. - -- Given a discovered endpoint whose RDMA identity changes after preflight, - when CM validates the offer, the stale identity fails before payload. A - TCP-only peer causes no CM attempt. Invariant: live capability authority. - Integration test. -- Given default, explicitly disabled and strict enabled configuration, when - connections start, defaults remain TCP, explicit TCP overrides environment, - and strict enabled failures cannot fall back. Invariant: explicit deployment - policy. Integration test. - -Run: - -```bash -pixi run -- cargo fmt --all -- --check -pixi run clean-env && pixi run test-cpp -pixi run -- cargo test -p crowdb-access-server --all-targets -pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings -``` diff --git a/doc/backlog/backlog.md b/doc/backlog/backlog.md index 3175f7958..cf635bd0f 100644 --- a/doc/backlog/backlog.md +++ b/doc/backlog/backlog.md @@ -97,11 +97,6 @@ live in the linked document; implementation follows - **[R4](R4-bounded-mempool.md)** — bounded memory pool for tree allocations. **Area:** crowdb-tree engine. **Complexity:** Medium. -- **[R5](R5-rdma-alloc.md)** — RDMA host-buffer RPC and data transport umbrella. - **Area:** RPC / access server / Chunk-KV / DiskIO / SDK / RDMA. - **Complexity:** High. **Status:** Foundation, transport, datapath, Chunk-KV - and native SDK accepted; coarse discovery/default-enablement policy - questions remain in the requirement. - **[R60](R60-tree-scan-sibling-leaf-readahead.md)** — sibling-leaf readahead on cold scans. **Area:** crowdb-tree / scan / DiskIO. **Complexity:** Medium. **Status:** Deferred pending cold file/block-backed measurements. diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index 62b0acb40..0d2f92048 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -3,8 +3,7 @@ # CROWDB RDMA implementation Plan -Upstream: [R5](../backlog/R5-rdma-alloc.md), -[RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md), and the +Upstream: [RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md), and the [native SDK contract](../design/access-server/design-crowdb-native-sdk.md). Goal: implement the host-memory RDMA SEND/RECV path end to end, with RDMA as @@ -13,8 +12,7 @@ semantics, bounded buffer leases, and automated OCI verification using a single-node HCA local-loop with multiple containers. Persistent-plan exception: retain this root record for accepted local-HCA -measurements. Completed child plans and -implementation checklists are removed; the evidence remains reproducible. +measurements. Completed child plans and implementation checklists are removed; the evidence remains reproducible. ## Current implementation status From 31b837edcdf586b4f860f63ce087231465b2182a Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 17:54:13 +0800 Subject: [PATCH 078/258] Add pinned Dataset HTTP workload and native integration plan --- doc/backlog/R220-dataset-fat-client-plan.md | 11 +- doc/backlog/backlog.md | 4 +- doc/working/plan-dataset-native-sdk.md | 111 +++++++++ e2e-test/image_identity.py | 2 + .../crowdb-sdk-dataset/examples/workload.rs | 212 ++++++++++++++++++ tools/sdk/dataset_workload.py | 102 +++++++++ 6 files changed, 432 insertions(+), 10 deletions(-) create mode 100644 doc/working/plan-dataset-native-sdk.md create mode 100644 sdk/crates/crowdb-sdk-dataset/examples/workload.rs create mode 100644 tools/sdk/dataset_workload.py diff --git a/doc/backlog/R220-dataset-fat-client-plan.md b/doc/backlog/R220-dataset-fat-client-plan.md index afc9efe7a..717b3aaf5 100644 --- a/doc/backlog/R220-dataset-fat-client-plan.md +++ b/doc/backlog/R220-dataset-fat-client-plan.md @@ -3,13 +3,6 @@ ### R220: Dataset — Fat-client direct access plan -Status: Deferred until dedicated CPU HTTP measurements establish latency, -throughput, memory, and digest costs. The shared native runtime, topology, -buffer leases and S3 frontend are available in `crowdb-sdk-native`; native and -unprivileged OCI consumers pass TCP and shared-HCA functional acceptance. -The public Rust HTTP SDK and Python wrapper remain independently packaged -under `sdk/`; functional container acceptance is complete. - ## Problem Trusted internal data-processing and AI systems need Dataset semantics over @@ -43,7 +36,9 @@ and [SDK API guide](../../lib/crowdb-sdk-native/README.md). including topology refresh, stale owners, failures, digest costs, and the existing 60-second active-read inactivity TTL. Keep placement, fencing, durability, and publication rules in their existing components. -5. Expose native Dataset through a Python binding to the same Rust data plane. +5. Expose native Dataset through an opt-in main-workspace Python extension + (`crowdb_dataset_native`) to the same Rust data plane. The existing + `crowdb_dataset` HTTP package remains independently installable. Preserve the portable HTTP distribution without adding mandatory native dependencies. Additional C/C++ business bindings are later work; reuse the existing lease ABI rather than making a new business ABI a prerequisite. diff --git a/doc/backlog/backlog.md b/doc/backlog/backlog.md index cf635bd0f..718249270 100644 --- a/doc/backlog/backlog.md +++ b/doc/backlog/backlog.md @@ -18,8 +18,8 @@ live in the linked document; implementation follows **Area:** SDK. **Complexity:** High. **Status:** Deferred until dedicated HTTP workload measurements. - **[R220](R220-dataset-fat-client-plan.md)** — trusted internal Dataset adapter on the existing native runtime, measurements, and Python integration. - **Area:** Dataset SDK. **Complexity:** High. **Status:** Deferred until - dedicated Dataset HTTP workload measurements. + **Area:** Dataset SDK. **Complexity:** High. **Status:** In progress; dedicated + HTTP baseline precedes native integration. ## Access server and external data diff --git a/doc/working/plan-dataset-native-sdk.md b/doc/working/plan-dataset-native-sdk.md new file mode 100644 index 000000000..99d0621e8 --- /dev/null +++ b/doc/working/plan-dataset-native-sdk.md @@ -0,0 +1,111 @@ + + + +# Native Dataset SDK Plan + +Upstream: [R220](../backlog/R220-dataset-fat-client-plan.md), +[Dataset](../design/access-server/dataset/design-crowdb-access-dataset.md), +[native runtime](../design/access-server/design-crowdb-native-sdk.md). + +Goal: attach trusted Dataset access to the accepted runtime with explicit ack, +bounded owner-backed delivery and independently packaged Python bindings. + +## Baseline and contract + +- [ ] **Measure HTTP**: add a reproducible release Rust consumer over the owned + real-service fixture. Pin one client physical core, record the server CPU + placement, immutable image/source, snapshot, mixed payload sizes, warmup, + repeats, concurrency, completed requests, per-size p50/p99, throughput, + process CPU/RSS and separately timed digest work. Preserve exact-byte checks. + Files: `sdk/crates/crowdb-sdk-dataset/examples/`, `tools/sdk/`. +- [ ] **Freeze API wiring**: use `DatasetSessions` (including Split preparation), + not legacy `DatasetDirectClient::scan` which acknowledges fetched batches. + Attach `ChunkKvDatasetStore`, `DatasetUploads` and `DatasetAuthority` to the + shared runtime's Dataset writer group. Use explicit `DatasetIdentity`; no + bearer/secret lookup or HTTP fallback. Files: `lib/crowdb-sdk-native/src/dataset/`. + +## Shared payload and native facade + +- [~] **Retain payload owners**: extend the existing payload/upload boundary + with lazy `OwnedPayload` delivery and verified lengths/digests. Keep the + byte-returning HTTP path compatible. Accept frozen producer storage without + changing publication or durable intent recovery. Files: + `lib/crowdb-access-dataset/src/payload/`, `src/upload/`. +- [ ] **Expose authority operations**: admit control calls with the existing + `OperationContext`, preserve typed Dataset errors and mutation uncertainty. + Reuse publication, upload, composition, Split and cleanup services rather + than duplicating their CAS rules. Files: native `dataset/`, `error.rs`. +- [ ] **Deliver bounded batches**: separate immutable field metadata, payload + leases, verified EOF and acknowledgement. Pull at most a bounded part ahead; + charge retained allocations, preserve pending data if a pull future is + dropped, and make terminal errors sticky. Renew the existing 60-second + active-read lease while producing; close/cancel stops production and releases + protection without acknowledging or invalidating caller-held buffers. + Files: native `dataset/read/`, shared session/lease code as needed. +- [ ] **Share topology and lifetime**: expose existing runtime observations + with Dataset locations as hints only. S3 and Dataset use independent facades + and writer policies over shared admission/transports; closing a facade or + read cannot close the runtime. Files: native `dataset/`, `tests/`. + +## Python and acceptance + +- [ ] **Add optional native binding**: main-workspace PyO3 extension + `crowdb_dataset_native`, with owned runtime, Dataset control/read operations, + retained immutable payload objects, explicit ack/cancel/close and exception + mapping. Keep `sdk/` HTTP distribution free of native storage dependencies. + Files: `lib/crowdb-sdk-native-python/`, `tools/sdk/`, workspace manifest. +- [ ] **Verify semantics**: shared authority tests and native facade tests for + projection/missing/tombstone, Split/revision/worker selection, explicit ack + replay, TTL, corruption/truncation, cancellation, dropped pulls, held memory, + stale routing, partial transport failure and S3/runtime coexistence. Files: + native and access-dataset `tests/`, access-server parity tests. +- [ ] **Run real services and compare**: use the same corpus/snapshot for HTTP + and native Rust/Python, restart storage, validate explicit resume and held + buffers, TCP and local shared-HCA policies. Run the same pinned workload and + report measured trade-offs, not an assumed throughput win. Physical multi-node + execution belongs to its independent plan and is not scheduled here. Files: + `e2e-test/`, `tools/sdk/`, local-HCA runner. +- [ ] **Document and clean up**: update supported native API and Dataset design, + retain reproducible benchmark artifacts; remove the completed requirement, + backlog entry and this plan after applicable gates. + +## Verification + +- Unit/integration: `pixi run clean-env && pixi run -- cargo test -p + crowdb-sdk-native -p crowdb-access-dataset -p crowdb-access-server --all-targets`. + Native Python binding tests run against the built installed package. +- Boundary: `pixi run check-dataset-sdk-boundary`, `pixi run test-dataset-sdk`, + `pixi run test-dataset-sdk-python`. +- E2E: `pixi run clean-env && pixi run test-e2e-dataset`; independent owned + baseline/native workload and local-HCA profile commands are recorded below + when implemented. Never count missing prerequisites as a passing case. +- Gates: `pixi run -- cargo fmt --all -- --check`, + `pixi run -- cargo clippy --workspace --all-targets --all-features -- -D warnings`, + `pixi run rs-lint`, affected fixture tests and standalone SDK fmt/clippy. + +## Evidence + +R5 is complete in `62653eef` and `f5b16896`. Its physical multi-node validation +is an independently scheduled, plan-only follow-up; no remote tests were run. + +Initial HTTP evidence: `.crowdb-runtime/artifacts/dataset-workload/6c77b1fe70df42bfbb71674b9bc5d143`. +The release client used CPU 2; all server containers used CPUs 4-15, real +file-backed storage and the unchanged accepted image. Eight sizes from 0 to +4 MiB + 17, two warmup rounds and three 128-request singleton trials completed +with exact-byte/digest checks. Singleton trials measured 18.6-29.4 requests/s, +55-58 client CPU ticks (100 Hz), and 11.3 MiB peak RSS. These short runs have only +16 observations per size/trial; tail estimates are exploratory, not a stable +performance claim. The resident-buffer MD5 kernel measured 64 MiB in 111-115 ms. + +The first multi-flight attempt correctly exceeded the HTTP SDK's default +128-MiB admission (three 48-MiB stream reservations); the driver now explicitly +records a 256-MiB budget and disables HTTP retries. The subsequent run failed +with HTTP 503 because metadata node 0 was OOM-killed at the fixture's 512-MiB +limit (Docker `OOMKilled=true`, exit 137). This is a failed baseline, not accepted +multi-flight performance. The workload runner now reserves 2 GiB for each of +its three metadata nodes and records stopped-process state without masking the +original failure. Rerun the complete comparison after rebuilding changed native +server dependencies. No production memory or retry policy was relaxed. + +The host-only runner is excluded from server-image identity checks; 19 image +identity tests and the standalone workload fmt/clippy gates pass. diff --git a/e2e-test/image_identity.py b/e2e-test/image_identity.py index 779030908..26182b555 100644 --- a/e2e-test/image_identity.py +++ b/e2e-test/image_identity.py @@ -78,6 +78,8 @@ # Independent SDK packaging builds host test consumers only; release staging # neither reads this helper nor packages its output. ":(exclude)tools/sdk/package.py", + # The pinned Dataset consumer/fixture runner never enters server staging. + ":(exclude)tools/sdk/dataset_workload.py", # This host-only RPC benchmark is not a dependency or staged server binary. ":(exclude)app/crowdb-cli/src/commands/bench/rpc.rs", ":(exclude)app/crowdb-cli/src/commands/bench/result.rs", diff --git a/sdk/crates/crowdb-sdk-dataset/examples/workload.rs b/sdk/crates/crowdb-sdk-dataset/examples/workload.rs new file mode 100644 index 000000000..9255ecd06 --- /dev/null +++ b/sdk/crates/crowdb-sdk-dataset/examples/workload.rs @@ -0,0 +1,212 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Reproducible CPU HTTP workload; the external runner owns fixtures and affinity. + +use base64::{engine::general_purpose::STANDARD, Engine}; +use bytes::Bytes; +use crowdb_sdk_dataset::{ + wire::BatchEvent, BatchRequest, DatasetClient, FieldDefinition, FieldValue, Manifest, Publication, + Sample, UploadRequest, +}; +use md5::{Digest, Md5}; +use serde_json::json; +use std::{collections::BTreeMap, path::Path, time::Instant}; + +type Failure = Box; +const SIZES: [usize; 8] = [0, 128, 513, 4096, 65_535, 262_144, 1_048_576, 4_194_321]; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Failure> { + let client = DatasetClient::with_config( + &std::env::var("CROWDB_DATASET_ENDPOINT")?, + &std::env::var("CROWDB_DATASET_NAME")?, + std::env::var("CROWDB_DATASET_READ_TOKEN").ok(), + crowdb_sdk_core::HttpConfig { + max_in_flight_bytes: 256 * 1024 * 1024, + max_attempts: 1, + ..Default::default() + }, + )?; + let root = std::env::var("CROWDB_DATASET_CONSUMER_ROOT")?; + let root = Path::new(&root); + std::fs::create_dir_all(root)?; + let snapshot = match std::fs::read_to_string(root.join("snapshot")) { + Ok(snapshot) => snapshot, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let snapshot = prepare(&client).await?; + std::fs::write(root.join("snapshot"), &snapshot)?; + snapshot + } + Err(error) => return Err(error.into()), + }; + for concurrency in [1, 3] { + trial(&client, &snapshot, concurrency, 2).await?; + for repeat in 0..3 { + let result = trial(&client, &snapshot, concurrency, 16).await?; + let report = json!({"path":"http", "snapshot":snapshot, "sizes":SIZES, + "concurrency":concurrency, "repeat":repeat, "result":result, + "digest":digest_cost(), "load":"closed-loop", "warmup_rounds":2, + "in_flight_budget_bytes":256 * 1024 * 1024, "http_max_attempts":1}); + std::fs::write( + root.join(format!("http-{concurrency}-{repeat}.json")), + serde_json::to_vec_pretty(&report)?, + )?; + println!("{report}"); + } + } + Ok(()) +} + +async fn prepare(client: &DatasetClient) -> Result { + client.create().await?; + let mut samples = Vec::new(); + for (index, size) in SIZES.into_iter().enumerate() { + let data = Bytes::from(vec![u8::try_from(index)?; size]); + let value = if size <= 4096 { + FieldValue::Inline { + data: STANDARD.encode(&data), + } + } else { + let id = uuid::Uuid::new_v4().simple().to_string(); + client + .create_upload(&UploadRequest { + id: id.clone(), + length: u64::try_from(size)?, + part_size: 1024 * 1024, + md5: hex::encode(Md5::digest(&data)), + }) + .await?; + for (part, offset) in (0..size).step_by(1024 * 1024).enumerate() { + client + .upload_part( + &id, + u32::try_from(part)?, + data.slice(offset..size.min(offset + 1024 * 1024)), + ) + .await?; + } + client.complete_upload(&id).await?; + FieldValue::Payload { upload: id } + }; + samples.push(Sample { + id: STANDARD.encode(format!("size-{index}")), + fields: BTreeMap::from([("value".into(), value)]), + }); + } + let published = client + .publish(&Publication { + operation: uuid::Uuid::new_v4().simple().to_string(), + parent: None, + manifest: Manifest { + schema: vec![FieldDefinition { + name: "value".into(), + required: true, + }], + samples, + }, + }) + .await?; + Ok(published.id) +} + +async fn read(client: &DatasetClient, snapshot: &str, index: usize) -> Result { + let start = Instant::now(); + let mut body = client + .stream_batch(&BatchRequest { + snapshot: snapshot.into(), + sample_ids: vec![STANDARD.encode(format!("size-{index}"))], + fields: vec!["value".into()], + }) + .await?; + let mut length = 0; + while let Some(event) = body.next_event().await? { + match event { + BatchEvent::Header(header) => { + if header.snapshot != snapshot || header.fields.len() != 1 { + return Err("unexpected snapshot/projection".into()); + } + } + BatchEvent::Data(data) => { + if data.iter().any(|byte| usize::from(*byte) != index) { + return Err("payload mismatch".into()); + } + length += data.len(); + } + _ => {} + } + } + if length != SIZES[index] { + return Err("payload length mismatch".into()); + } + Ok(u64::try_from(start.elapsed().as_micros())?) +} + +async fn trial( + client: &DatasetClient, + snapshot: &str, + concurrency: usize, + rounds: usize, +) -> Result { + let cpu = cpu_ticks()?; + let start = Instant::now(); + let mut tasks = tokio::task::JoinSet::new(); + for lane in 0..concurrency { + let client = client.clone(); + let snapshot = snapshot.to_owned(); + tasks.spawn(async move { + let mut observations = Vec::new(); + for round in 0..rounds { + for order in 0..SIZES.len() { + let index = (order * 5 + lane + round) % SIZES.len(); + observations.push((index, read(&client, &snapshot, index).await?)); + } + } + Ok::<_, Failure>(observations) + }); + } + let mut times: [Vec; 8] = std::array::from_fn(|_| Vec::new()); + while let Some(result) = tasks.join_next().await { + for (index, micros) in result?? { + times[index].push(micros); + } + } + let seconds = start.elapsed().as_secs_f64(); + let cpu = cpu_ticks()? - cpu; + let mut latency = Vec::new(); + for (index, values) in times.iter_mut().enumerate() { + values.sort_unstable(); + latency.push(json!({"bytes":SIZES[index], "count":values.len(), + "p50_us":values[(values.len()-1)/2], "p99_us":values[(values.len()*99).div_ceil(100)-1]})); + } + let requests = concurrency * rounds * SIZES.len(); + let bytes = concurrency * rounds * SIZES.iter().sum::(); + Ok( + json!({"seconds":seconds, "completed_requests":requests, "bytes":bytes, + "cpu_ticks":cpu, "rss_kib":rss_kib()?, "latency":latency, "errors":0}), + ) +} + +fn cpu_ticks() -> Result { + let stat = std::fs::read_to_string("/proc/self/stat")?; + let (_, fields) = stat.rsplit_once(") ").ok_or("invalid proc stat")?; + let fields: Vec<_> = fields.split_whitespace().collect(); + Ok(fields[11].parse::()? + fields[12].parse::()?) +} +fn rss_kib() -> Result { + let status = std::fs::read_to_string("/proc/self/status")?; + let line = status + .lines() + .find(|line| line.starts_with("VmHWM:")) + .ok_or("missing VmHWM")?; + Ok(line.split_whitespace().nth(1).ok_or("missing RSS")?.parse()?) +} +fn digest_cost() -> serde_json::Value { + let data = vec![93; 1024 * 1024]; + let start = Instant::now(); + for _ in 0..64 { + std::hint::black_box(Md5::digest(std::hint::black_box(&data))); + } + json!({"algorithm":"md5", "bytes":64 * data.len(), "microseconds":start.elapsed().as_micros(), + "scope":"separate resident-buffer kernel; real reads always verify integrity"}) +} diff --git a/tools/sdk/dataset_workload.py b/tools/sdk/dataset_workload.py new file mode 100644 index 000000000..e099f0fa9 --- /dev/null +++ b/tools/sdk/dataset_workload.py @@ -0,0 +1,102 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Pinned Dataset workload over an owned real-service fixture.""" + +import argparse +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import uuid + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "e2e-test")) +from access_dataset_fixture import AccessDatasetCluster +from client import bundle_client +from fixture import docker +from image_identity import verify_server_sources + + +def interrupted(_signal, _frame): + raise KeyboardInterrupt("Dataset workload interrupted") + + +def run(command, **kwargs): + print("Workload:", " ".join(map(str, command)), flush=True) + subprocess.run(command, check=True, timeout=600, **kwargs) + + +def server_usage(containers): + """Read owned cgroup v2 counters; retain CPU and peak memory separately.""" + result = {} + for container in containers: + inspected = json.loads(docker("inspect", container))[0] + state = inspected["State"] + pid = state["Pid"] + if not state["Running"]: + result[container] = {"state": state} + continue + group = Path(f"/proc/{pid}/cgroup").read_text().strip().split("::", 1)[1] + root = Path("/sys/fs/cgroup") / group.lstrip("/") + result[container] = {"cpu": (root / "cpu.stat").read_text(), + "memory_peak": (root / "memory.peak").read_text().strip()} + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--image", default="crowdb-node:dev") + parser.add_argument("--client-cpu", default="2") + parser.add_argument("--server-cpus", default="4-15") + parser.add_argument("--metadata-memory", default="2g") + args = parser.parse_args() + for number in (signal.SIGTERM, signal.SIGINT): + signal.signal(number, interrupted) + output = ROOT / ".crowdb-runtime/artifacts/dataset-workload" / uuid.uuid4().hex + output.mkdir(parents=True) + print(f"Artifacts: {output}", flush=True) + image = json.loads(docker("image", "inspect", args.image))[0] + labels = image["Config"].get("Labels") or {} + verify_server_sources(ROOT, labels.get("org.opencontainers.image.revision"), + labels.get("org.crowdb.source.tree") or None) + run(["cargo", "build", "--release", "--locked", "--manifest-path", str(ROOT / "sdk/Cargo.toml"), + "-p", "crowdb-sdk-dataset", "--example", "workload"]) + os.environ["CROWDB_RDMA_ENABLED"] = "0" + bundle = bundle_client(output / "client") + + def pinned_command(*parts, **kwargs): + if parts and parts[0] == "run": + parts = list(parts) + name = parts[parts.index("--name") + 1] + if name.endswith(("-node0", "-node1", "-node2")): + parts[parts.index("--memory") + 1] = args.metadata_memory + parts = (parts[0], "--cpuset-cpus", args.server_cpus, *parts[1:]) + return docker(*parts, **kwargs) + + with AccessDatasetCluster(image["Id"], output, bundle, command=pinned_command) as cluster: + environment = dict(os.environ, CROWDB_DATASET_ENDPOINT=f"http://{cluster.addresses[12]}:7503", + CROWDB_DATASET_NAME="workload", CROWDB_DATASET_READ_TOKEN="token", + CROWDB_DATASET_CONSUMER_ROOT=str(output), NO_PROXY="*", no_proxy="*") + manifest = {"image": image["Id"], "labels": labels, "client_cpu": args.client_cpu, + "metadata_memory": args.metadata_memory, + "server_cpus": args.server_cpus, "clock_ticks": os.sysconf("SC_CLK_TCK"), + "cpu_topology": subprocess.check_output(["lscpu", "-e=CPU,CORE,SOCKET,NODE"], text=True), + "seeds": cluster.origins[:3], "endpoints": cluster.addresses, + "source_revision": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), + "scope": "same-host pinned CPU functional workload, not isolated production hardware", + "before": server_usage(cluster.servers)} + with (output / "http.log").open("w") as log: + try: + run(["taskset", "-c", args.client_cpu, str(ROOT / "sdk/target/release/examples/workload")], + env=environment, stdout=log, stderr=subprocess.STDOUT) + finally: + print((output / "http.log").read_text(), end="", flush=True) + manifest["after"] = server_usage(cluster.servers) + (output / "manifest.json").write_text(json.dumps(manifest, indent=2)) + print("Dataset HTTP baseline complete", flush=True) + + +if __name__ == "__main__": + main() From ef2a64c8a077d580fd537b8fb3207e3f990178d4 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 18:42:12 +0800 Subject: [PATCH 079/258] Add owner-backed native Dataset access and Python bindings --- Cargo.lock | 86 +++++++ Cargo.toml | 1 + .../access-server/design-crowdb-native-sdk.md | 29 ++- doc/working/plan-dataset-native-sdk.md | 24 +- lib/crowdb-access-dataset/Cargo.toml | 1 + lib/crowdb-access-dataset/src/lib.rs | 5 +- lib/crowdb-access-dataset/src/payload.rs | 54 ++++- .../src/payload/stream.rs | 33 +++ lib/crowdb-access-dataset/src/upload.rs | 2 + lib/crowdb-access-dataset/src/upload/parts.rs | 65 ++++- .../src/upload/stream.rs | 98 ++++++++ .../tests/common/payload_stream.rs | 87 +++++++ .../tests/upload_stream_test.rs | 70 ++++++ lib/crowdb-sdk-native-python/Cargo.toml | 30 +++ lib/crowdb-sdk-native-python/README.md | 80 +++++++ lib/crowdb-sdk-native-python/pyproject.toml | 17 ++ lib/crowdb-sdk-native-python/src/client.rs | 164 +++++++++++++ lib/crowdb-sdk-native-python/src/control.rs | 199 +++++++++++++++ lib/crowdb-sdk-native-python/src/error.rs | 52 ++++ lib/crowdb-sdk-native-python/src/lease.rs | 47 ++++ lib/crowdb-sdk-native-python/src/lib.rs | 21 ++ lib/crowdb-sdk-native-python/src/runtime.rs | 202 ++++++++++++++++ lib/crowdb-sdk-native-python/src/stream.rs | 124 ++++++++++ .../tests/hca_consumer.py | 57 +++++ .../tests/real_consumer.py | 220 +++++++++++++++++ .../tests/workload.py | 86 +++++++ lib/crowdb-sdk-native/Cargo.toml | 4 +- lib/crowdb-sdk-native/README.md | 35 ++- .../examples/dataset_workload.rs | 178 ++++++++++++++ lib/crowdb-sdk-native/src/dataset.rs | 114 +++++++++ .../src/dataset/composition.rs | 226 ++++++++++++++++++ lib/crowdb-sdk-native/src/dataset/control.rs | 78 ++++++ lib/crowdb-sdk-native/src/dataset/error.rs | 109 +++++++++ lib/crowdb-sdk-native/src/dataset/read.rs | 162 +++++++++++++ .../src/dataset/read/body.rs | 140 +++++++++++ .../src/dataset/read/field.rs | 128 ++++++++++ .../src/dataset/read/protection.rs | 119 +++++++++ lib/crowdb-sdk-native/src/dataset/session.rs | 68 ++++++ lib/crowdb-sdk-native/src/dataset/upload.rs | 99 ++++++++ lib/crowdb-sdk-native/src/error.rs | 5 + lib/crowdb-sdk-native/src/lib.rs | 1 + lib/crowdb-sdk-native/tests/common/dataset.rs | 89 +++++++ .../tests/common/dataset_composition.rs | 113 +++++++++ .../tests/common/metadata.rs | 20 ++ lib/crowdb-sdk-native/tests/common/storage.rs | 9 +- .../tests/common/write_storage.rs | 24 +- .../tests/dataset_payload_test.rs | 102 ++++++++ .../tests/dataset_read_test.rs | 156 ++++++++++++ tools/rdma/run-native-dataset.py | 63 +++++ tools/sdk/dataset_workload.py | 41 +++- tools/sdk/native_package.py | 38 +++ 51 files changed, 3938 insertions(+), 37 deletions(-) create mode 100644 lib/crowdb-access-dataset/src/payload/stream.rs create mode 100644 lib/crowdb-access-dataset/src/upload/stream.rs create mode 100644 lib/crowdb-access-dataset/tests/common/payload_stream.rs create mode 100644 lib/crowdb-access-dataset/tests/upload_stream_test.rs create mode 100644 lib/crowdb-sdk-native-python/Cargo.toml create mode 100644 lib/crowdb-sdk-native-python/README.md create mode 100644 lib/crowdb-sdk-native-python/pyproject.toml create mode 100644 lib/crowdb-sdk-native-python/src/client.rs create mode 100644 lib/crowdb-sdk-native-python/src/control.rs create mode 100644 lib/crowdb-sdk-native-python/src/error.rs create mode 100644 lib/crowdb-sdk-native-python/src/lease.rs create mode 100644 lib/crowdb-sdk-native-python/src/lib.rs create mode 100644 lib/crowdb-sdk-native-python/src/runtime.rs create mode 100644 lib/crowdb-sdk-native-python/src/stream.rs create mode 100644 lib/crowdb-sdk-native-python/tests/hca_consumer.py create mode 100644 lib/crowdb-sdk-native-python/tests/real_consumer.py create mode 100644 lib/crowdb-sdk-native-python/tests/workload.py create mode 100644 lib/crowdb-sdk-native/examples/dataset_workload.rs create mode 100644 lib/crowdb-sdk-native/src/dataset.rs create mode 100644 lib/crowdb-sdk-native/src/dataset/composition.rs create mode 100644 lib/crowdb-sdk-native/src/dataset/control.rs create mode 100644 lib/crowdb-sdk-native/src/dataset/error.rs create mode 100644 lib/crowdb-sdk-native/src/dataset/read.rs create mode 100644 lib/crowdb-sdk-native/src/dataset/read/body.rs create mode 100644 lib/crowdb-sdk-native/src/dataset/read/field.rs create mode 100644 lib/crowdb-sdk-native/src/dataset/read/protection.rs create mode 100644 lib/crowdb-sdk-native/src/dataset/session.rs create mode 100644 lib/crowdb-sdk-native/src/dataset/upload.rs create mode 100644 lib/crowdb-sdk-native/tests/common/dataset.rs create mode 100644 lib/crowdb-sdk-native/tests/common/dataset_composition.rs create mode 100644 lib/crowdb-sdk-native/tests/dataset_payload_test.rs create mode 100644 lib/crowdb-sdk-native/tests/dataset_read_test.rs create mode 100644 tools/rdma/run-native-dataset.py create mode 100644 tools/sdk/native_package.py diff --git a/Cargo.lock b/Cargo.lock index cf58da0ed..64de298e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -618,6 +618,7 @@ dependencies = [ "crowdb-chunk-client", "crowdb-chunk-kv-client", "crowdb-protocol", + "crowdb-rpc-ffi", "futures", "md-5", "reqwest", @@ -1283,6 +1284,7 @@ dependencies = [ "bincode", "bytes", "crossbeam-skiplist", + "crowdb-access-dataset", "crowdb-access-multipart", "crowdb-access-s3", "crowdb-chunk-client", @@ -1294,12 +1296,27 @@ dependencies = [ "crowdb-sdk-native", "flatbuffers", "futures", + "md-5", "serde_json", "thiserror 2.0.18", "tokio", "tracing", ] +[[package]] +name = "crowdb-sdk-native-python" +version = "0.3.0" +dependencies = [ + "crowdb-access-dataset", + "crowdb-protocol", + "crowdb-rpc-ffi", + "crowdb-sdk-native", + "pyo3", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "crowdb-test-harness" version = "0.3.0" @@ -3031,6 +3048,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + [[package]] name = "potential_utf" version = "0.1.5" @@ -3083,6 +3106,63 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pyo3" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "700d18fa267b73b9b521fd7e13580e2f446916f176cacee1ab63fcc8191f1655" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b3fc0c4d08f6bb10e71fe39dfb9e2f59c6eb6854e22ec8092f50c69a4499adb" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc0b8e19df29aad7086cf977bb0c2a2f143e30567eb113e9cf72b62ca698330" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6100e8a4b5eba53afaa5ed078364851a0b2499c553a44c31026b929049b49dc6" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6143877a16e82b5a727b7127ff4cd86858a24a28d745d72f43e6f227c7b1bdb3" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "quick-xml" version = "0.38.4" @@ -4000,6 +4080,12 @@ dependencies = [ "libc", ] +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + [[package]] name = "tempfile" version = "3.27.0" diff --git a/Cargo.toml b/Cargo.toml index e79f46297..737475838 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ members = [ "lib/crowdb-access-dataset", "lib/crowdb-access-multipart", "lib/crowdb-sdk-native", + "lib/crowdb-sdk-native-python", ] exclude = ["third-party/hyper", "sdk"] # : crowdb-tree/ffi moved from `exclude` into `members` now that diff --git a/doc/design/access-server/design-crowdb-native-sdk.md b/doc/design/access-server/design-crowdb-native-sdk.md index d58aef150..b54cda21e 100644 --- a/doc/design/access-server/design-crowdb-native-sdk.md +++ b/doc/design/access-server/design-crowdb-native-sdk.md @@ -4,7 +4,7 @@ # CROWDB - Design: Trusted native storage SDK The native SDK embeds internal storage access in the application. Its shared -runtime, topology and host-buffer delivery support a typed S3 frontend. +runtime, topology and host-buffer delivery support typed S3 and Dataset frontends. Depends on: [Access Server](design-crowdb-access-server.md), [RDMA RPC](../rpc/design-crowdb-rpc-rdma.md), @@ -38,13 +38,19 @@ preserve S3 generation, checksum, visibility and uncertain-publication rules. Copy captures a source generation. Multipart completion binds an exact selected part set; a repeated completed request returns its recorded outcome. +`DatasetClient` attaches `DatasetAuthority`, `DatasetUploads` and +`DatasetSessions` to these same clients. Shared composition helpers enroll +upload ownership before edits and bind candidate ownership before publication. +Snapshot/Read Plan/Split selection, worker partition, CAS, durable intent +recovery and cleanup remain in the existing Dataset services. + ## 2. Shared runtime `ClientRuntime` owns or borrows control, metadata and data clients. Business handles share transport groups, discovery, topology observations and resource budgets. Compatible writer policies share one bounded small-writer pool; incompatible policies use distinct pools while sharing the transport owners. -S3 chunks retain type `S3`; native embedding introduces no generic user-data +S3 and Dataset chunks retain their respective types; native embedding introduces no generic user-data chunk type. `connect` creates owned groups. `adopt_clients` transfers shutdown @@ -126,14 +132,25 @@ The independent Dataset HTTP distribution has no dependency on this storage/FFI runtime. A Dataset facade attaches to the common runtime and delivery contracts while retaining Dataset authority, Snapshot/Split binding and explicit acknowledgement. Buffer release, download completion and processing -acknowledgement remain separate events. +acknowledgement remain separate events. `BatchBody` delivers field metadata and +lazy payload leases with per-part and whole-field integrity checks. Active +reads renew the existing 60-second protection while live; cancellation stops +renewal and releases the active handle, with TTL expiry bounding interrupted +cleanup. Durable session close is explicit and never acknowledges processing. + +The optional native Python extension owns a shared runtime and exposes retained +immutable leases. Slicing shares allocation ownership; conversion to Python +bytes explicitly copies. Pulls are serialized per batch with a bounded request +channel and no automatic payload prefetch. Runtime shutdown preserves already +delivered leases. Forked processes create new runtimes. Python control commands +reuse the Rust facade and shared authority schema without another storage path. ## 7. Correctness invariants - **NS-I1 — Trusted embedding:** native tenant/context execute inside the internal-service trust domain without a per-operation HTTP authentication hop. -- **NS-I2 — Shared semantics:** HTTP and native S3 use the same selection, - durability, publication, multipart and cleanup authorities. +- **NS-I2 — Shared semantics:** HTTP and native frontends use the same selection, + durability, publication, multipart and Dataset cleanup authorities. - **NS-I3 — Non-authoritative observation:** topology inspection never changes placement or bypasses owner, generation or layout validation. - **NS-I4 — Verified completion:** only successful EOF establishes a full read; @@ -144,3 +161,5 @@ acknowledgement remain separate events. invalidate bytes still held by a caller or asynchronous copy. - **NS-I7 — Bounded transport failure:** accepted RDMA failures never append TCP data or replay an ambiguous mutation. +- **NS-I8 — Explicit Dataset progress:** fetching, verified EOF and final buffer + release cannot advance a read cursor; acknowledgement alone records progress. diff --git a/doc/working/plan-dataset-native-sdk.md b/doc/working/plan-dataset-native-sdk.md index 99d0621e8..2f778ced3 100644 --- a/doc/working/plan-dataset-native-sdk.md +++ b/doc/working/plan-dataset-native-sdk.md @@ -18,7 +18,7 @@ bounded owner-backed delivery and independently packaged Python bindings. repeats, concurrency, completed requests, per-size p50/p99, throughput, process CPU/RSS and separately timed digest work. Preserve exact-byte checks. Files: `sdk/crates/crowdb-sdk-dataset/examples/`, `tools/sdk/`. -- [ ] **Freeze API wiring**: use `DatasetSessions` (including Split preparation), +- [x] **Freeze API wiring**: use `DatasetSessions` (including Split preparation), not legacy `DatasetDirectClient::scan` which acknowledges fetched batches. Attach `ChunkKvDatasetStore`, `DatasetUploads` and `DatasetAuthority` to the shared runtime's Dataset writer group. Use explicit `DatasetIdentity`; no @@ -26,30 +26,30 @@ bounded owner-backed delivery and independently packaged Python bindings. ## Shared payload and native facade -- [~] **Retain payload owners**: extend the existing payload/upload boundary +- [x] **Retain payload owners**: extend the existing payload/upload boundary with lazy `OwnedPayload` delivery and verified lengths/digests. Keep the byte-returning HTTP path compatible. Accept frozen producer storage without changing publication or durable intent recovery. Files: `lib/crowdb-access-dataset/src/payload/`, `src/upload/`. -- [ ] **Expose authority operations**: admit control calls with the existing +- [x] **Expose authority operations**: admit control calls with the existing `OperationContext`, preserve typed Dataset errors and mutation uncertainty. Reuse publication, upload, composition, Split and cleanup services rather than duplicating their CAS rules. Files: native `dataset/`, `error.rs`. -- [ ] **Deliver bounded batches**: separate immutable field metadata, payload +- [x] **Deliver bounded batches**: separate immutable field metadata, payload leases, verified EOF and acknowledgement. Pull at most a bounded part ahead; charge retained allocations, preserve pending data if a pull future is dropped, and make terminal errors sticky. Renew the existing 60-second active-read lease while producing; close/cancel stops production and releases protection without acknowledging or invalidating caller-held buffers. Files: native `dataset/read/`, shared session/lease code as needed. -- [ ] **Share topology and lifetime**: expose existing runtime observations +- [x] **Share topology and lifetime**: expose existing runtime observations with Dataset locations as hints only. S3 and Dataset use independent facades and writer policies over shared admission/transports; closing a facade or read cannot close the runtime. Files: native `dataset/`, `tests/`. ## Python and acceptance -- [ ] **Add optional native binding**: main-workspace PyO3 extension +- [~] **Add optional native binding**: main-workspace PyO3 extension `crowdb_dataset_native`, with owned runtime, Dataset control/read operations, retained immutable payload objects, explicit ack/cancel/close and exception mapping. Keep `sdk/` HTTP distribution free of native storage dependencies. @@ -109,3 +109,15 @@ server dependencies. No production memory or retry policy was relaxed. The host-only runner is excluded from server-image identity checks; 19 image identity tests and the standalone workload fmt/clippy gates pass. + +Native implementation evidence: all native SDK target tests pass, including +Dataset explicit-ack replay, dropped capacity waits, retained owners after +cancellation/shutdown, and shared S3/Dataset publication. The production framed +writer test retains the original producer pointer through durable storage; +composition cleanup retains published upload ownership. Shared upload-stream +tests validate length/digest and terminal partial failures. + +Native/Python all-target, all-feature clippy passes. The release native wheel +builds, bundles its native shared libraries and imports from a separate installed +consumer. Full Python semantics, common-snapshot performance and real-service +restart are pending; the wheel build/import alone is not binding acceptance. diff --git a/lib/crowdb-access-dataset/Cargo.toml b/lib/crowdb-access-dataset/Cargo.toml index 5d1adc344..401bbabb1 100644 --- a/lib/crowdb-access-dataset/Cargo.toml +++ b/lib/crowdb-access-dataset/Cargo.toml @@ -19,6 +19,7 @@ futures = "0.3" crowdb-chunk-client = { path = "../crowdb-chunk-client" } crowdb-chunk-kv-client = { path = "../crowdb-chunk-kv-client" } crowdb-protocol = { path = "../crowdb-protocol" } +crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi" } md-5 = "0.10" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } serde = { version = "1", features = ["derive"] } diff --git a/lib/crowdb-access-dataset/src/lib.rs b/lib/crowdb-access-dataset/src/lib.rs index 4dad0f7b8..c6f48bc6d 100644 --- a/lib/crowdb-access-dataset/src/lib.rs +++ b/lib/crowdb-access-dataset/src/lib.rs @@ -32,7 +32,10 @@ mod transport; mod upload; mod wire; -pub use payload::{ChunkPayloadStorage, PayloadDurability, PayloadStorage, UploadedChunkReader}; +pub use payload::{ + ChunkPayloadStorage, PayloadDurability, PayloadStorage, PayloadStream, UploadedChunkReader, +}; +pub use upload::VerifiedUploadPart; pub use upload::{ DatasetUploads, SnapshotUploadCleanup, UploadError, UploadPart, UploadReclaimError, UploadRecord, UploadState, MAX_UPLOAD_PART_BYTES, diff --git a/lib/crowdb-access-dataset/src/payload.rs b/lib/crowdb-access-dataset/src/payload.rs index 2dbe59744..a3b85c5af 100644 --- a/lib/crowdb-access-dataset/src/payload.rs +++ b/lib/crowdb-access-dataset/src/payload.rs @@ -3,12 +3,16 @@ use async_trait::async_trait; use bytes::Bytes; -use crowdb_chunk_client::{ChunkIoClient, ChunkIoWriter, ReclaimOutcome, SmallWriteIntent}; +use crowdb_chunk_client::{ + ChunkIoClient, ChunkIoWriter, FramedWriteBuffer, ReclaimOutcome, SmallWriteIntent, +}; use crowdb_protocol::chunkdb::rpc::Location; use std::sync::Arc; mod reader; +mod stream; pub use reader::UploadedChunkReader; +pub use stream::PayloadStream; use crate::{UploadError, MAX_UPLOAD_PART_BYTES}; @@ -29,6 +33,21 @@ pub trait PayloadStorage: Send + Sync { intent: Arc, ) -> Result, UploadError>; async fn read(&self, locations: &[Location]) -> Result; + /// Submits caller-prepared immutable storage frames, retaining their owner. + /// Implementations without this path reject it instead of staging a copy. + async fn write_framed( + &self, + _key: &[u8], + _data: Box, + _intent: Arc, + ) -> Result, UploadError> { + Err(UploadError::Invalid) + } + /// Retains physical allocation owners for bounded native delivery. + /// The compatibility default preserves the existing byte reader. + async fn read_stream(&self, locations: &[Location]) -> Result, UploadError> { + Ok(Box::new(stream::ByteStream(Some(self.read(locations).await?)))) + } async fn durability(&self, locations: &[Location]) -> Result; async fn discard(&self, locations: &[Location]) -> Result; } @@ -47,6 +66,39 @@ impl ChunkPayloadStorage { #[async_trait] impl PayloadStorage for ChunkPayloadStorage { + async fn write_framed( + &self, + key: &[u8], + data: Box, + intent: Arc, + ) -> Result, UploadError> { + let length = usize::try_from(data.logical_len()).map_err(|_| UploadError::Invalid)?; + if length == 0 || length > MAX_UPLOAD_PART_BYTES { + return Err(UploadError::Invalid); + } + let mut writer = self + .chunks + .prepare_shared_object_write_for_key(length, key) + .await + .map_err(|error| UploadError::Storage(error.to_string()))?; + if let Err(error) = writer.on_framed_data(data).await { + let _ = writer.on_error().await; + return Err(UploadError::Storage(error.to_string())); + } + writer + .finish_durable_with_intent(intent) + .await + .map_err(|error| UploadError::Storage(error.to_string())) + } + + async fn read_stream(&self, locations: &[Location]) -> Result, UploadError> { + let stream = self + .chunks + .read_stream(locations) + .map_err(|error| UploadError::Storage(error.to_string()))?; + Ok(Box::new(stream::ChunkStream(stream))) + } + async fn write( &self, key: &[u8], diff --git a/lib/crowdb-access-dataset/src/payload/stream.rs b/lib/crowdb-access-dataset/src/payload/stream.rs new file mode 100644 index 000000000..36a9d97ec --- /dev/null +++ b/lib/crowdb-access-dataset/src/payload/stream.rs @@ -0,0 +1,33 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use crowdb_rpc_ffi::OwnedPayload; + +use crate::UploadError; + +/// Pull-based payload storage. Successful EOF is independent of processing ack. +#[async_trait] +pub trait PayloadStream: Send { + async fn next(&mut self) -> Result, UploadError>; +} + +pub(super) struct ByteStream(pub Option); +#[async_trait] +impl PayloadStream for ByteStream { + async fn next(&mut self) -> Result, UploadError> { + Ok(self.0.take().map(OwnedPayload::from)) + } +} + +pub(super) struct ChunkStream(pub crowdb_chunk_client::ChunkReadStream); +#[async_trait] +impl PayloadStream for ChunkStream { + async fn next(&mut self) -> Result, UploadError> { + self.0 + .next_payload() + .await + .transpose() + .map_err(|error| UploadError::Storage(error.to_string())) + } +} diff --git a/lib/crowdb-access-dataset/src/upload.rs b/lib/crowdb-access-dataset/src/upload.rs index fcf7c6205..6bf7eb183 100644 --- a/lib/crowdb-access-dataset/src/upload.rs +++ b/lib/crowdb-access-dataset/src/upload.rs @@ -8,6 +8,8 @@ mod ownership; mod parts; mod registry; mod retirement; +mod stream; +pub use stream::VerifiedUploadPart; use bytes::Bytes; use md5::{Digest, Md5}; diff --git a/lib/crowdb-access-dataset/src/upload/parts.rs b/lib/crowdb-access-dataset/src/upload/parts.rs index 493a573dd..ff5e39778 100644 --- a/lib/crowdb-access-dataset/src/upload/parts.rs +++ b/lib/crowdb-access-dataset/src/upload/parts.rs @@ -8,7 +8,7 @@ use super::{ use crate::{CasOutcome, OperationId}; use async_trait::async_trait; use bytes::Bytes; -use crowdb_chunk_client::{IoError, SmallWriteIntent}; +use crowdb_chunk_client::{FramedWriteBuffer, IoError, SmallWriteIntent}; use crowdb_protocol::chunkdb::rpc::Location; use md5::{Digest, Md5}; use std::sync::Arc; @@ -22,9 +22,36 @@ impl DatasetUploads { index: u32, data: Bytes, now: u64, + ) -> Result { + self.put_input(location, index, PartInput::Bytes(data), now).await + } + + /// Writes final storage frames with a caller-computed logical digest. + /// Completion verifies stored bytes against this digest before publication. + /// # Errors + /// Rejects divergent identity/length, unsupported storage or failed durability. + pub async fn put_framed_part( + &self, + location: &[u8], + index: u32, + data: Box, + md5: [u8; 16], + now: u64, + ) -> Result { + self.put_input(location, index, PartInput::Framed(data, md5), now) + .await + } + + async fn put_input( + &self, + location: &[u8], + index: u32, + data: PartInput, + now: u64, ) -> Result { let session = self.session(location).await?; - validate(&session, index, &data, now)?; + let length = data.length(); + validate(&session, index, length, now)?; if session.state == UploadState::Open { self.register(location).await?; } @@ -36,8 +63,8 @@ impl DatasetUploads { state: PartState::Reserved, part: UploadPart { index, - length: data.len() as u64, - md5: digest(&data), + length, + md5: data.digest(), locations: Vec::new(), }, }; @@ -90,7 +117,10 @@ impl DatasetUploads { key: key.clone(), pending: pending.clone(), }); - let locations = self.payloads.write(&key, data, intent).await?; + let locations = match data { + PartInput::Bytes(bytes) => self.payloads.write(&key, bytes, intent).await?, + PartInput::Framed(buffer, _) => self.payloads.write_framed(&key, buffer, intent).await?, + }; let current = self.store.get(&key).await?.ok_or(UploadError::Integrity)?; let entry = decode_part(¤t.bytes)?; if entry.owner != pending.owner @@ -230,16 +260,33 @@ impl PartIntent { Ok(()) } } -fn validate(session: &UploadRecord, index: u32, data: &[u8], now: u64) -> Result<(), UploadError> { +enum PartInput { + Bytes(Bytes), + Framed(Box, [u8; 16]), +} +impl PartInput { + fn length(&self) -> u64 { + match self { + Self::Bytes(bytes) => bytes.len() as u64, + Self::Framed(buffer, _) => buffer.logical_len(), + } + } + fn digest(&self) -> [u8; 16] { + match self { + Self::Bytes(bytes) => digest(bytes), + Self::Framed(_, digest) => *digest, + } + } +} + +fn validate(session: &UploadRecord, index: u32, length: u64, now: u64) -> Result<(), UploadError> { if session.state == UploadState::Aborted || (session.state != UploadState::Claimed && now >= session.expires_at) { return Err(UploadError::Conflict); } let offset = u64::from(index) * u64::from(session.part_size); - if offset >= session.length - || data.len() as u64 != (session.length - offset).min(u64::from(session.part_size)) - { + if offset >= session.length || length != (session.length - offset).min(u64::from(session.part_size)) { return Err(UploadError::Invalid); } Ok(()) diff --git a/lib/crowdb-access-dataset/src/upload/stream.rs b/lib/crowdb-access-dataset/src/upload/stream.rs new file mode 100644 index 000000000..a184fefa2 --- /dev/null +++ b/lib/crowdb-access-dataset/src/upload/stream.rs @@ -0,0 +1,98 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_rpc_ffi::OwnedPayload; +use md5::{Digest, Md5}; + +use super::{decode_part, part_key, DatasetUploads, PartState, UploadError, UploadPart, UploadState}; +use crate::PayloadStream; + +/// An accepted upload part with incremental integrity checks. Emitted owners +/// remain valid independently; only successful EOF certifies the whole part. +pub struct VerifiedUploadPart { + stream: Option>, + expected: UploadPart, + delivered: u64, + digest: Md5, + failed: bool, +} + +impl DatasetUploads { + /// Opens one immutable accepted part without concatenating receive buffers. + /// # Errors + /// Rejects uncommitted uploads, invalid part boundaries or unavailable storage. + pub async fn read_part_stream( + &self, + location: &[u8], + index: u32, + ) -> Result { + let session = self.session(location).await?; + if !matches!(session.state, UploadState::Complete | UploadState::Claimed) { + return Err(UploadError::Conflict); + } + let entry = self + .store + .get(&part_key(location, index)) + .await? + .ok_or(UploadError::NotFound)?; + let entry = decode_part(&entry.bytes)?; + let start = u64::from(index) * u64::from(session.part_size); + if entry.state != PartState::Accepted + || entry.part.index != index + || start >= session.length + || entry.part.length != (session.length - start).min(u64::from(session.part_size)) + { + return Err(UploadError::Integrity); + } + let stream = self.payloads.read_stream(&entry.part.locations).await?; + Ok(VerifiedUploadPart { + stream: Some(stream), + expected: entry.part, + delivered: 0, + digest: Md5::new(), + failed: false, + }) + } +} + +impl VerifiedUploadPart { + /// Failure is terminal; subsequent pulls cannot report successful EOF. + /// # Errors + /// Returns storage failures, truncated/oversized payloads or digest mismatch. + pub async fn next(&mut self) -> Result, UploadError> { + if self.failed { + return Err(UploadError::Integrity); + } + let Some(stream) = self.stream.as_mut() else { + return Ok(None); + }; + let result = match stream.next().await { + Ok(Some(payload)) => { + let length = self.delivered.saturating_add(payload.len() as u64); + if payload.is_empty() || length > self.expected.length { + Err(UploadError::Integrity) + } else { + self.delivered = length; + self.digest.update(payload.as_ref()); + Ok(Some(payload)) + } + } + Ok(None) => { + if self.delivered != self.expected.length + || <[u8; 16]>::from(self.digest.clone().finalize()) != self.expected.md5 + { + Err(UploadError::Integrity) + } else { + self.stream = None; + Ok(None) + } + } + Err(error) => Err(error), + }; + if result.is_err() { + self.failed = true; + self.stream = None; + } + result + } +} diff --git a/lib/crowdb-access-dataset/tests/common/payload_stream.rs b/lib/crowdb-access-dataset/tests/common/payload_stream.rs new file mode 100644 index 000000000..ec865cf30 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/common/payload_stream.rs @@ -0,0 +1,87 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "upload.rs"] +mod upload; +pub use upload::TestStore; + +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_access_dataset::{PayloadDurability, PayloadStorage, PayloadStream, UploadError}; +use crowdb_chunk_client::SmallWriteIntent; +use crowdb_protocol::chunkdb::rpc::Location; +use crowdb_rpc_ffi::OwnedPayload; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +#[derive(Default)] +pub struct TestPayloadStreamStorage { + inner: upload::TestPayload, + pub mode: AtomicUsize, + pub pulls: Arc, +} +#[async_trait] +impl PayloadStorage for TestPayloadStreamStorage { + async fn write( + &self, + key: &[u8], + data: Bytes, + intent: Arc, + ) -> Result, UploadError> { + self.inner.write(key, data, intent).await + } + async fn read(&self, locations: &[Location]) -> Result { + self.inner.read(locations).await + } + async fn durability(&self, locations: &[Location]) -> Result { + self.inner.durability(locations).await + } + async fn discard(&self, locations: &[Location]) -> Result { + self.inner.discard(locations).await + } + async fn read_stream(&self, locations: &[Location]) -> Result, UploadError> { + let mut bytes = self.inner.read(locations).await?.to_vec(); + let mode = self.mode.load(Ordering::Acquire); + if mode == 1 { + bytes.pop(); + } + if mode == 2 { + bytes[0] ^= 1; + } + if mode == 4 { + bytes.push(0); + } + bytes.reserve(4096); + Ok(Box::new(TestPayloadStream { + payload: OwnedPayload::from(bytes), + offset: 0, + mode, + pulls: Arc::clone(&self.pulls), + })) + } +} + +struct TestPayloadStream { + payload: OwnedPayload, + offset: usize, + mode: usize, + pulls: Arc, +} +#[async_trait] +impl PayloadStream for TestPayloadStream { + async fn next(&mut self) -> Result, UploadError> { + self.pulls.fetch_add(1, Ordering::Relaxed); + if self.mode == 3 && self.offset > 0 { + return Err(UploadError::Storage("accepted partial read failed".into())); + } + if self.offset == self.payload.len() { + return Ok(None); + } + let end = (self.offset + 2).min(self.payload.len()); + let payload = self.payload.slice(self.offset..end); + self.offset = end; + Ok(Some(payload)) + } +} diff --git a/lib/crowdb-access-dataset/tests/upload_stream_test.rs b/lib/crowdb-access-dataset/tests/upload_stream_test.rs new file mode 100644 index 000000000..37a481899 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/upload_stream_test.rs @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/payload_stream.rs"] +mod payload_stream; + +use bytes::Bytes; +use crowdb_access_dataset::{DatasetIdentity, DatasetUploads, NamespacePath, OperationId}; +use md5::{Digest, Md5}; +use payload_stream::{TestPayloadStreamStorage, TestStore}; +use std::sync::{atomic::Ordering, Arc}; + +async fn fixture() -> (DatasetUploads, Vec, Arc) { + let payloads = Arc::new(TestPayloadStreamStorage::default()); + let uploads = DatasetUploads::new(Arc::new(TestStore::default()), payloads.clone()); + let identity = DatasetIdentity::new(NamespacePath::root(), "owned").unwrap(); + let location = DatasetUploads::location(&identity, OperationId::random()).unwrap(); + uploads + .create(&location, 4, 4, Md5::digest(b"data").into(), 100) + .await + .unwrap(); + uploads + .put_part(&location, 0, Bytes::from_static(b"data"), 1) + .await + .unwrap(); + (uploads, location, payloads) +} + +#[tokio::test] +async fn stream_retains_allocation_without_collecting_and_verifies_at_eof() { + let (uploads, location, payloads) = fixture().await; + assert!(uploads.read_part_stream(&location, 0).await.is_err()); + uploads.complete(&location, 2).await.unwrap(); + let mut stream = uploads.read_part_stream(&location, 0).await.unwrap(); + assert_eq!(payloads.pulls.load(Ordering::Relaxed), 0); + let first = stream.next().await.unwrap().unwrap(); + let second = stream.next().await.unwrap().unwrap(); + assert!(Arc::ptr_eq( + first.allocation().unwrap(), + second.allocation().unwrap() + )); + assert!(first.allocation().unwrap().capacity() >= 4100); + assert_eq!(first.as_ref(), b"da"); + assert_eq!(second.as_ref(), b"ta"); + assert!(stream.next().await.unwrap().is_none()); + assert!(stream.next().await.unwrap().is_none()); + drop(stream); + drop(uploads); + assert_eq!(second.as_ref(), b"ta"); +} + +#[tokio::test] +async fn corruption_truncation_overrun_and_partial_transport_failure_stay_terminal() { + for mode in 1..=4 { + let (uploads, location, payloads) = fixture().await; + uploads.complete(&location, 2).await.unwrap(); + payloads.mode.store(mode, Ordering::Release); + let mut stream = uploads.read_part_stream(&location, 0).await.unwrap(); + loop { + match stream.next().await { + Ok(Some(_)) => {} + Ok(None) => panic!("mode {mode} incorrectly completed"), + Err(_) => break, + } + } + let pulls = payloads.pulls.load(Ordering::Relaxed); + assert!(stream.next().await.is_err()); + assert_eq!(payloads.pulls.load(Ordering::Relaxed), pulls); + } +} diff --git a/lib/crowdb-sdk-native-python/Cargo.toml b/lib/crowdb-sdk-native-python/Cargo.toml new file mode 100644 index 000000000..a13ff8068 --- /dev/null +++ b/lib/crowdb-sdk-native-python/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "crowdb-sdk-native-python" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish = false +description = "Optional trusted native Dataset Python binding" + +[lib] +name = "crowdb_dataset_native" +crate-type = ["cdylib", "rlib"] + +[lints] +workspace = true + +[features] +extension-module = ["pyo3/extension-module"] + +[dependencies] +crowdb-sdk-native = { path = "../crowdb-sdk-native" } +crowdb-access-dataset = { path = "../crowdb-access-dataset" } +crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi" } +crowdb-protocol = { path = "../crowdb-protocol" } +pyo3 = { version = "=0.29.3", features = ["auto-initialize"] } +tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time", "macros"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" + diff --git a/lib/crowdb-sdk-native-python/README.md b/lib/crowdb-sdk-native-python/README.md new file mode 100644 index 000000000..075abd659 --- /dev/null +++ b/lib/crowdb-sdk-native-python/README.md @@ -0,0 +1,80 @@ + + + +# Native Dataset Python + +`crowdb_dataset_native` embeds the trusted main-workspace storage runtime. +Internal management and storage endpoints must be reachable. It accepts no +HTTP token and performs no automatic HTTP fallback. The independently packaged +`crowdb_dataset` remains the portable HTTP SDK. + +Build and install a wheel in an isolated consumer: + +```sh +pixi run -- python tools/sdk/native_package.py --destination /tmp/crowdb-native-python +``` + +The wheel bundles linked native libraries. RDMA additionally requires a working +host HCA, provider and device access. `NativeRuntime(seeds)` explicitly selects +TCP; `options` accepts JSON containing `data_rdma`, `metadata_rdma` and +`aggregate_rdma` overrides using the Rust `RdmaConfig` schema. Set `enabled` and +`allow_tcp_fallback` separately per group; metadata RDMA requires its control/data +class. Options also accept `held_bytes`, `producer_bytes` and `requests`. Runtime lifetime +belongs to the caller. Create a new runtime in each worker process; inherited +clients are rejected after fork. + +```python +import json +import crowdb_dataset_native as native + +runtime = native.NativeRuntime(["http://127.0.0.1:7001"]) +dataset = runtime.dataset("training") +metadata = json.loads(dataset.call('{"op":"open"}')) +# IDs passed to read/upload methods are immutable bytes; JSON IDs are arrays +# of 16 bytes, matching the shared Rust authority schema. +batch = dataset.read_batch(snapshot_id, [list(b"sample")], ["image"]) +while (lease := batch.next()) is not None: + consume(lease.field, lease.logical_offset, lease.to_bytes()) +batch.close() +dataset.close() +runtime.close() +``` + +- `Batch.header` is immutable JSON. `next()` yields a `Lease`, or `None` after + verified EOF. It never acknowledges processing. A batch supports one concurrent + consumer; independent batches can run concurrently. Blocking calls release + Python execution ownership so completion workers and other callers progress. +- `Lease.slice(start, end)` retains the same immutable allocation. Capacity and + registration identity are inspectable. `to_bytes()` explicitly copies into + Python-owned memory; that new allocation is outside SDK held-buffer accounting. + No Python buffer-protocol pointer or new unsafe lifetime adapter is exposed. +- `Batch.cancel()`/`close()` interrupts production without acknowledging. Held + leases remain readable after cancellation, facade close and runtime close. + Closing a facade rejects new calls only on that facade; existing batches and + other facades retain their independently owned lifetimes. +- `runtime.status()` reports request and memory admission, high water and + compatibility-copy bytes. `runtime.topology(refresh=True)` observes metadata + and data transport state; these hints cannot override routing authority. +- `dataset.call(json, timeout=30)` uses a tagged `op` command and the existing + Rust Dataset JSON schema. Commands cover create/open/snapshot/head/commit + status, composition begin/status/edit/evaluate/prepare/publish/abandon/cleanup, + prepare/resume/acknowledge/close read, and upload create/complete/status/abort. + `upload_location` returns the authority's opaque location for a composition + field. No command is retried automatically. +- `prepare_read` accepts `spec: SessionSpecV2` and `page`; continue its returned + page until complete. `next_batch(read_id)` fetches the issued batch. Call + `acknowledge` with the previously acknowledged cursor and `Batch.header.next` + only after application processing succeeds. Split/worker binding uses the + same shared session service as Rust and HTTP. +- `upload_part(upload_id, index, bytes)` copies Python input once into final + transport frame slots and freezes it before submission. Publication uses + upload-managed compositions and preserves durable attempt identity. +- Typed exceptions include capacity, integrity, cancellation, closed runtime, + deadline, conflict and uncertain mutation. An uncertain mutation does not + authorize replay with a new operation identity. + +`tests/real_consumer.py` validates the installed binding against real services; +`tests/workload.py` includes explicit Python byte-copy costs. The comparison is +run with `pixi run -- python tools/sdk/dataset_workload.py` after building the +current shared service image. It uses one snapshot for HTTP and native clients, +then validates native publication after storage restart. diff --git a/lib/crowdb-sdk-native-python/pyproject.toml b/lib/crowdb-sdk-native-python/pyproject.toml new file mode 100644 index 000000000..e72d80ff2 --- /dev/null +++ b/lib/crowdb-sdk-native-python/pyproject.toml @@ -0,0 +1,17 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +[build-system] +requires = ["maturin>=1.15,<2"] +build-backend = "maturin" + +[project] +name = "crowdb-dataset-native" +version = "0.3.0" +description = "Trusted native CROWDB Dataset runtime" +requires-python = ">=3.10" +license = "Apache-2.0" + +[tool.maturin] +features = ["extension-module"] +module-name = "crowdb_dataset_native" + diff --git a/lib/crowdb-sdk-native-python/src/client.rs b/lib/crowdb-sdk-native-python/src/client.rs new file mode 100644 index 000000000..2f838b338 --- /dev/null +++ b/lib/crowdb-sdk-native-python/src/client.rs @@ -0,0 +1,164 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + control, + error::{failure, ClosedError, NativeError}, + runtime::Resources, + stream::Batch, +}; +use crowdb_access_dataset::{DatasetIdentity, NamespacePath, OperationId, SnapshotId}; +use crowdb_sdk_native::{ + dataset::{BatchRequest, DatasetClient}, + OperationContext, +}; +use pyo3::prelude::*; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; + +#[pyclass(frozen, module = "crowdb_dataset_native")] +pub struct Dataset { + pub resources: Arc, + pub client: DatasetClient, + closed: AtomicBool, +} + +impl Dataset { + pub fn new( + py: Python<'_>, + resources: Arc, + name: String, + namespace: Vec, + ) -> PyResult { + let namespace = NamespacePath::new(namespace).map_err(|e| NativeError::new_err(e.to_string()))?; + let identity = + DatasetIdentity::new(namespace, name).map_err(|e| NativeError::new_err(e.to_string()))?; + let client = py + .detach(|| { + let _entered = resources.executor().enter(); + DatasetClient::new(resources.native(), identity) + }) + .map_err(|e| failure(&e))?; + Ok(Self { + resources, + client, + closed: AtomicBool::new(false), + }) + } + fn context(&self, seconds: f64) -> PyResult { + if self.closed.load(Ordering::Acquire) { + return Err(ClosedError::new_err("Dataset facade is closed")); + } + self.resources.context(seconds) + } +} + +#[pymethods] +impl Dataset { + /// Execute a typed control command encoded as JSON with the shared Rust schema. + #[pyo3(signature=(command, timeout=30.0))] + fn call(&self, py: Python<'_>, command: &str, timeout: f64) -> PyResult { + let context = self.context(timeout)?; + if command.len() > 1024 * 1024 { + return Err(NativeError::new_err("control input exceeds 1 MiB")); + } + let command: control::Command = + serde_json::from_str(command).map_err(|e| NativeError::new_err(e.to_string()))?; + py.detach(|| { + self.resources + .executor() + .block_on(control::execute(&self.client, context, command)) + }) + .map(|v| v.to_string()) + .map_err(|e| failure(&e)) + } + + #[pyo3(signature=(snapshot, sample_ids, fields, timeout=30.0))] + fn read_batch( + &self, + py: Python<'_>, + snapshot: &[u8], + sample_ids: Vec>, + fields: Vec, + timeout: f64, + ) -> PyResult { + let context = self.context(timeout)?; + let cancel = context.cancellation.clone(); + let snapshot = SnapshotId::from_bytes(snapshot).map_err(|e| NativeError::new_err(e.to_string()))?; + let read = py + .detach(|| { + self.resources.executor().block_on(self.client.read_batch( + context, + BatchRequest { + snapshot, + sample_ids, + fields, + }, + )) + }) + .map_err(|e| failure(&e))?; + Ok(Batch::new(self.resources.clone(), read, cancel)) + } + + #[pyo3(signature=(read_id, timeout=30.0))] + fn next_batch(&self, py: Python<'_>, read_id: &[u8], timeout: f64) -> PyResult { + let context = self.context(timeout)?; + let cancel = context.cancellation.clone(); + let id = OperationId::from_bytes(read_id).map_err(|e| NativeError::new_err(e.to_string()))?; + let read = py + .detach(|| { + self.resources + .executor() + .block_on(self.client.next_batch(context, id)) + }) + .map_err(|e| failure(&e))?; + Ok(Batch::new(self.resources.clone(), read, cancel)) + } + + /// Copies Python bytes once into the final transport frame allocation. + #[pyo3(signature=(upload, index, data, timeout=30.0))] + fn upload_part( + &self, + py: Python<'_>, + upload: &[u8], + index: u32, + data: &[u8], + timeout: f64, + ) -> PyResult { + let context = self.context(timeout)?; + let id = OperationId::from_bytes(upload).map_err(|e| NativeError::new_err(e.to_string()))?; + if data.is_empty() || data.len() > crowdb_access_dataset::MAX_UPLOAD_PART_BYTES { + return Err(NativeError::new_err("invalid upload part size")); + } + let capacity = crowdb_protocol::frame::MAX_FRAME_BYTES; + let frames = data + .len() + .div_ceil(crowdb_protocol::frame::MAX_FRAME_PAYLOAD_BYTES); + let capacity = + u32::try_from(frames * capacity).map_err(|_| NativeError::new_err("frame size overflow"))?; + let buffer = self + .resources + .native() + .allocate_producer(capacity) + .map_err(|e| failure(&e))? + .initialize_framed(data.len(), 0, |offset, output| { + output.copy_from_slice(&data[offset..offset + output.len()]); + }) + .map_err(|e| failure(&e))?; + let result = py + .detach(|| { + self.resources + .executor() + .block_on(self.client.upload_framed_part(context, id, index, buffer)) + }) + .map_err(|e| failure(&e))?; + serde_json::to_string(&result).map_err(|e| NativeError::new_err(e.to_string())) + } + + /// Stops new operations on this facade; existing streams own their lifetimes. + fn close(&self) { + self.closed.store(true, Ordering::Release); + } +} diff --git a/lib/crowdb-sdk-native-python/src/control.rs b/lib/crowdb-sdk-native-python/src/control.rs new file mode 100644 index 000000000..dc8f081d4 --- /dev/null +++ b/lib/crowdb-sdk-native-python/src/control.rs @@ -0,0 +1,199 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_access_dataset::{ + composition::{CompositionEdit, CompositionInput}, + CommitAttempt, DatasetUploads, OperationId, SessionSpecV2, SnapshotId, +}; +use crowdb_sdk_native::{ + dataset::{CompositionStatus, DatasetClient}, + Error, OperationContext, Result, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; + +#[derive(Deserialize)] +#[serde(tag = "op", rename_all = "snake_case", deny_unknown_fields)] +pub enum Command { + Create, + Open, + Snapshot { + snapshot: SnapshotId, + }, + Head, + CommitStatus { + attempt: OperationId, + }, + BeginComposition { + operation: OperationId, + input: CompositionInput, + }, + CompositionStatus { + operation: OperationId, + }, + EditComposition { + operation: OperationId, + edit: CompositionEdit, + }, + EvaluateComposition { + operation: OperationId, + revision: u64, + page: u64, + }, + PrepareComposition { + operation: OperationId, + revision: u64, + }, + PublishComposition { + operation: OperationId, + attempt: CommitAttempt, + }, + AbandonComposition { + operation: OperationId, + revision: u64, + }, + CleanupComposition { + operation: OperationId, + }, + PrepareRead { + spec: SessionSpecV2, + page: u64, + }, + ResumeRead { + read: OperationId, + }, + Acknowledge { + read: OperationId, + expected: String, + next: String, + }, + CloseRead { + read: OperationId, + }, + CreateUpload { + upload: OperationId, + length: u64, + part_size: u32, + md5: [u8; 16], + expires_at: u64, + }, + CompleteUpload { + upload: OperationId, + }, + UploadStatus { + upload: OperationId, + }, + UploadLocation { + upload: OperationId, + }, + AbortUpload { + upload: OperationId, + }, +} + +fn encoded(value: impl Serialize) -> Result { + serde_json::to_value(value).map_err(|e| Error::Storage(e.to_string())) +} +fn composition(status: &CompositionStatus) -> Value { + json!({"draft":status.draft,"uploads":status.uploads}) +} + +pub async fn execute(client: &DatasetClient, context: OperationContext, command: Command) -> Result { + match command { + Command::UploadLocation { upload } => encoded(DatasetUploads::location(client.identity(), upload)?), + Command::Create => encoded(client.create(context).await?), + Command::Open => encoded(client.open(context).await?), + Command::Snapshot { snapshot } => encoded(client.snapshot(context, snapshot).await?), + Command::Head => { + client + .control(context, |s| async move { + { + let head = s.authority.inspect_head(&s.identity).await?; + Ok(json!({"latest":head.latest,"generation":head.generation})) + } + }) + .await + } + Command::CommitStatus { attempt } => { + client + .control(context, |s| async move { + encoded(s.authority.commit_status(&s.identity, attempt).await?) + }) + .await + } + Command::BeginComposition { operation, input } => Ok(composition( + &client.begin_composition(context, operation, &input).await?, + )), + Command::CompositionStatus { operation } => { + Ok(composition(&client.composition_status(context, operation).await?)) + } + Command::EditComposition { operation, edit } => Ok(composition( + &client.edit_composition(context, operation, &edit).await?, + )), + Command::EvaluateComposition { + operation, + revision, + page, + } => encoded( + client + .evaluate_composition(context, operation, revision, page) + .await?, + ), + Command::PrepareComposition { operation, revision } => Ok(composition( + &client.prepare_composition(context, operation, revision).await?, + )), + Command::PublishComposition { operation, attempt } => { + encoded(client.publish_composition(context, operation, &attempt).await?) + } + Command::AbandonComposition { operation, revision } => Ok(composition( + &client.abandon_composition(context, operation, revision).await?, + )), + Command::CleanupComposition { operation } => { + let status = client.cleanup_composition(context, operation).await?; + Ok( + json!({"complete":status.complete,"retired_upload_owners":status.retired_upload_owners, + "physical_cleanup_deferred":status.physical_cleanup_deferred}), + ) + } + Command::PrepareRead { spec, page } => encoded(client.prepare_read(context, &spec, page).await?), + Command::ResumeRead { read } => encoded(client.resume_read(context, read).await?), + Command::Acknowledge { read, expected, next } => { + encoded(client.acknowledge(context, read, &expected, &next).await?) + } + Command::CloseRead { read } => encoded(client.close_read(context, read).await?), + Command::CreateUpload { + upload, + length, + part_size, + md5, + expires_at, + } => encoded( + client + .create_upload(context, upload, length, part_size, md5, expires_at) + .await?, + ), + Command::CompleteUpload { upload } => encoded(client.complete_upload(context, upload).await?), + Command::UploadStatus { upload } => { + client + .control(context, |s| async move { + encoded( + s.uploads + .get(&DatasetUploads::location(&s.identity, upload)?) + .await?, + ) + }) + .await + } + Command::AbortUpload { upload } => { + client + .control(context, |s| async move { + encoded( + s.uploads + .abort(&DatasetUploads::location(&s.identity, upload)?) + .await?, + ) + }) + .await + } + } +} diff --git a/lib/crowdb-sdk-native-python/src/error.rs b/lib/crowdb-sdk-native-python/src/error.rs new file mode 100644 index 000000000..7655d7613 --- /dev/null +++ b/lib/crowdb-sdk-native-python/src/error.rs @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_sdk_native::{dataset::FailureKind, Error}; +use pyo3::{create_exception, exceptions::PyException, prelude::*}; + +create_exception!(crowdb_dataset_native, NativeError, PyException); +create_exception!(crowdb_dataset_native, CapacityError, NativeError); +create_exception!(crowdb_dataset_native, IntegrityError, NativeError); +create_exception!(crowdb_dataset_native, CancelledError, NativeError); +create_exception!(crowdb_dataset_native, ClosedError, NativeError); +create_exception!(crowdb_dataset_native, DeadlineError, NativeError); +create_exception!(crowdb_dataset_native, ConflictError, NativeError); +create_exception!(crowdb_dataset_native, UncertainMutationError, NativeError); + +pub fn failure(error: &Error) -> PyErr { + let detail = error.to_string(); + match error { + Error::Capacity(_) + | Error::Dataset { + kind: FailureKind::Capacity, + .. + } => CapacityError::new_err(detail), + Error::Integrity + | Error::Length { .. } + | Error::Dataset { + kind: FailureKind::Integrity, + .. + } => IntegrityError::new_err(detail), + Error::Cancelled => CancelledError::new_err(detail), + Error::Shutdown => ClosedError::new_err(detail), + Error::Deadline => DeadlineError::new_err(detail), + Error::Dataset { + kind: FailureKind::Conflict, + .. + } => ConflictError::new_err(detail), + Error::UncertainMutation { .. } => UncertainMutationError::new_err(detail), + _ => NativeError::new_err(detail), + } +} + +pub fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py = module.py(); + module.add("NativeError", py.get_type::())?; + module.add("CapacityError", py.get_type::())?; + module.add("IntegrityError", py.get_type::())?; + module.add("CancelledError", py.get_type::())?; + module.add("ClosedError", py.get_type::())?; + module.add("DeadlineError", py.get_type::())?; + module.add("ConflictError", py.get_type::())?; + module.add("UncertainMutationError", py.get_type::()) +} diff --git a/lib/crowdb-sdk-native-python/src/lease.rs b/lib/crowdb-sdk-native-python/src/lease.rs new file mode 100644 index 000000000..f54ff1185 --- /dev/null +++ b/lib/crowdb-sdk-native-python/src/lease.rs @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_sdk_native::buffer::BufferLease; +use pyo3::{prelude::*, types::PyBytes}; + +/// Immutable owner; slicing retains storage and `to_bytes` explicitly copies. +#[pyclass(frozen, module = "crowdb_dataset_native")] +pub struct Lease { + pub buffer: BufferLease, + #[pyo3(get)] + pub field: usize, +} + +#[pymethods] +impl Lease { + fn __len__(&self) -> usize { + self.buffer.as_slice().len() + } + + fn to_bytes<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> { + PyBytes::new(py, self.buffer.as_slice()) + } + + fn slice(&self, start: usize, end: usize) -> PyResult { + Ok(Self { + buffer: self + .buffer + .slice(start..end) + .map_err(|e| super::error::failure(&e))?, + field: self.field, + }) + } + + #[getter] + fn logical_offset(&self) -> u64 { + self.buffer.metadata().logical_offset + } + #[getter] + fn allocation_capacity(&self) -> usize { + self.buffer.metadata().allocation_capacity + } + #[getter] + fn registration_id(&self) -> u32 { + self.buffer.metadata().registration_id + } +} diff --git a/lib/crowdb-sdk-native-python/src/lib.rs b/lib/crowdb-sdk-native-python/src/lib.rs new file mode 100644 index 000000000..b84a85891 --- /dev/null +++ b/lib/crowdb-sdk-native-python/src/lib.rs @@ -0,0 +1,21 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Explicitly selected native Dataset bindings; no HTTP or implicit replay. +mod client; +mod control; +mod error; +mod lease; +mod runtime; +mod stream; + +use pyo3::prelude::*; + +#[pymodule(gil_used = false)] +fn crowdb_dataset_native(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + error::register(module) +} diff --git a/lib/crowdb-sdk-native-python/src/runtime.rs b/lib/crowdb-sdk-native-python/src/runtime.rs new file mode 100644 index 000000000..28eaf7061 --- /dev/null +++ b/lib/crowdb-sdk-native-python/src/runtime.rs @@ -0,0 +1,202 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + client::Dataset, + error::{failure, NativeError}, +}; +use crowdb_sdk_native::{ClientRuntime, OperationContext, RuntimeConfig}; +use pyo3::prelude::*; +use std::{ + num::NonZeroU64, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; + +pub struct Resources { + executor: Option, + native: Option, + pid: u32, + sequence: AtomicU64, +} + +impl Resources { + pub fn check_process(&self) -> PyResult<()> { + if self.pid != std::process::id() { + return Err(NativeError::new_err( + "create a native runtime in each worker process", + )); + } + Ok(()) + } + pub fn native(&self) -> &ClientRuntime { + self.native.as_ref().expect("owned runtime") + } + pub fn executor(&self) -> &tokio::runtime::Runtime { + self.executor.as_ref().expect("owned executor") + } + pub fn context(&self, seconds: f64) -> PyResult { + self.check_process()?; + let duration = + Duration::try_from_secs_f64(seconds).map_err(|_| NativeError::new_err("invalid timeout"))?; + let deadline = Instant::now() + .checked_add(duration) + .ok_or_else(|| NativeError::new_err("timeout overflow"))?; + let number = self + .sequence + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |n| n.checked_add(1)) + .map_err(|_| NativeError::new_err("request sequence exhausted"))?; + Ok(OperationContext::new( + NonZeroU64::new(number).expect("starts at one"), + deadline, + )) + } +} + +impl Drop for Resources { + fn drop(&mut self) { + let executor = self.executor.take().expect("owned executor"); + let native = self.native.take().expect("owned runtime"); + if self.pid != std::process::id() { + std::mem::forget(native); + std::mem::forget(executor); + return; + } + // Final release may run on an executor worker. Join owned RPC workers + // from a separate cleanup thread before destroying that executor. + std::thread::spawn(move || { + if let Err(error) = executor.block_on(native.shutdown()) { + eprintln!("native Python runtime cleanup failed: {error}"); + } + drop(native); + executor.shutdown_background(); + }); + } +} + +#[pyclass(frozen, module = "crowdb_dataset_native")] +pub struct NativeRuntime { + pub resources: Arc, +} + +#[pymethods] +impl NativeRuntime { + #[new] + #[pyo3(signature=(seeds, options="{}"))] + fn new(py: Python<'_>, seeds: Vec, options: &str) -> PyResult { + if options.len() > 64 * 1024 { + return Err(NativeError::new_err("runtime options exceed 64 KiB")); + } + let options: Options = + serde_json::from_str(options).map_err(|e| NativeError::new_err(e.to_string()))?; + py.detach(move || { + let executor = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .map_err(|e| NativeError::new_err(e.to_string()))?; + let mut config = RuntimeConfig::new(seeds); + config.limits.held_bytes = options.held_bytes; + config.limits.producer_bytes = options.producer_bytes; + config.limits.requests = options.requests; + config.data_rdma = Some(options.data_rdma); + config.chunk_kv_transport.metadata_rdma = Some(options.metadata_rdma); + config.chunk_kv_transport.aggregate_rdma = Some(options.aggregate_rdma); + let native = executor + .block_on(ClientRuntime::connect(config)) + .map_err(|e| failure(&e))?; + Ok(Self { + resources: Arc::new(Resources { + executor: Some(executor), + native: Some(native), + pid: std::process::id(), + sequence: AtomicU64::new(1), + }), + }) + }) + } + + #[pyo3(signature=(name, namespace=None))] + fn dataset(&self, py: Python<'_>, name: String, namespace: Option>) -> PyResult { + self.resources.check_process()?; + Dataset::new(py, self.resources.clone(), name, namespace.unwrap_or_default()) + } + + fn close(&self, py: Python<'_>) -> PyResult<()> { + self.resources.check_process()?; + py.detach(|| { + self.resources + .executor() + .block_on(self.resources.native().shutdown()) + }) + .map_err(|e| failure(&e)) + } + + fn status(&self) -> PyResult { + self.resources.check_process()?; + let s = self.resources.native().resources().status(); + Ok( + serde_json::json!({"closed":s.closed,"requests":s.requests,"producer_bytes":s.producer_bytes, + "held_bytes":s.held_bytes,"held_bytes_high_water":s.held_bytes_high_water, + "compatibility_copy_bytes":s.compatibility_copy_bytes}) + .to_string(), + ) + } + + /// Retained observations are hints; execution always validates current routing. + #[pyo3(signature=(refresh=false, timeout=30.0))] + fn topology(&self, py: Python<'_>, refresh: bool, timeout: f64) -> PyResult { + let context = self.resources.context(timeout)?; + let view = py + .detach(|| { + if refresh { + self.resources + .executor() + .block_on(self.resources.native().refresh_topology(context)) + } else { + self.resources.native().topology() + } + }) + .map_err(|e| failure(&e))?; + let metadata: serde_json::Value = serde_json::from_slice(&view.metadata_transport_json) + .map_err(|e| NativeError::new_err(e.to_string()))?; + let data: Option = view + .data_transport_json + .as_ref() + .map(|bytes| serde_json::from_slice(bytes)) + .transpose() + .map_err(|e| NativeError::new_err(e.to_string()))?; + Ok( + serde_json::json!({"catalog_age_ms":view.catalog_age().map(|age|age.as_millis()), + "metadata":metadata,"data":data,"authoritative":false}) + .to_string(), + ) + } +} + +#[derive(serde::Deserialize)] +#[serde(default, deny_unknown_fields)] +struct Options { + held_bytes: usize, + producer_bytes: usize, + requests: usize, + data_rdma: crowdb_rpc_ffi::RdmaConfig, + metadata_rdma: crowdb_rpc_ffi::RdmaConfig, + aggregate_rdma: crowdb_rpc_ffi::RdmaConfig, +} +impl Default for Options { + fn default() -> Self { + let limits = crowdb_sdk_native::RuntimeLimits::default(); + Self { + held_bytes: limits.held_bytes, + producer_bytes: limits.producer_bytes, + requests: limits.requests, + data_rdma: crowdb_rpc_ffi::RdmaConfig::default(), + metadata_rdma: crowdb_rpc_ffi::RdmaConfig::default(), + aggregate_rdma: crowdb_rpc_ffi::RdmaConfig::default(), + } + } +} diff --git a/lib/crowdb-sdk-native-python/src/stream.rs b/lib/crowdb-sdk-native-python/src/stream.rs new file mode 100644 index 000000000..ac023dfb2 --- /dev/null +++ b/lib/crowdb-sdk-native-python/src/stream.rs @@ -0,0 +1,124 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + error::{failure, NativeError}, + lease::Lease, + runtime::Resources, +}; +use crowdb_sdk_native::{ + dataset::{BatchBody, FieldBuffer, ReadBatch}, + Cancellation, Error, Result, +}; +use pyo3::prelude::*; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; +use tokio::sync::{mpsc, oneshot}; + +type Pull = oneshot::Sender>>; + +#[pyclass(frozen, module = "crowdb_dataset_native")] +pub struct Batch { + resources: Arc, + sender: mpsc::Sender, + cancel: Cancellation, + pulling: AtomicBool, + #[pyo3(get)] + header: String, +} + +impl Batch { + pub fn new(resources: Arc, read: ReadBatch, cancel: Cancellation) -> Self { + let h = read.header; + let fields: Vec<_> = h + .fields + .iter() + .map(|f| { + serde_json::json!({"sample_id":f.sample_id, + "name":f.name,"state":format!("{:?}", f.state).to_lowercase(),"length":f.length,"md5":f.md5}) + }) + .collect(); + let header = serde_json::json!({"snapshot":h.snapshot,"sample_ids":h.sample_ids,"fields":fields, + "next":h.next,"end":h.end}) + .to_string(); + let (sender, receiver) = mpsc::channel(1); + resources + .executor() + .spawn(produce(read.body, receiver, cancel.clone())); + Self { + resources, + sender, + cancel, + pulling: AtomicBool::new(false), + header, + } + } +} + +struct PullGuard<'a>(&'a AtomicBool); +impl Drop for PullGuard<'_> { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } +} + +#[pymethods] +impl Batch { + /// Pulls one retained buffer; None certifies verified EOF, never acknowledgement. + fn next(&self, py: Python<'_>) -> PyResult> { + self.resources.check_process()?; + if self.pulling.swap(true, Ordering::AcqRel) { + return Err(NativeError::new_err("one consumer may pull a batch at a time")); + } + let _guard = PullGuard(&self.pulling); + py.detach(|| { + self.resources.executor().block_on(async { + if self.cancel.is_cancelled() { + return Err(Error::Cancelled); + } + let (sender, receiver) = oneshot::channel(); + self.sender.send(sender).await.map_err(|_| Error::Cancelled)?; + receiver.await.map_err(|_| Error::Cancelled)?.map(|item| { + item.map(|f| Lease { + field: f.field, + buffer: f.buffer, + }) + }) + }) + }) + .map_err(|e| failure(&e)) + } + + fn close(&self) { + self.cancel.cancel(); + } + fn cancel(&self) { + self.cancel.cancel(); + } +} + +impl Drop for Batch { + fn drop(&mut self) { + self.cancel.cancel(); + } +} + +async fn produce(mut body: BatchBody, mut receiver: mpsc::Receiver, cancel: Cancellation) { + loop { + tokio::select! { + biased; + () = cancel.cancelled() => { body.cancel(); return; } + request = receiver.recv() => { + let Some(reply) = request else { return; }; + let result = tokio::select! { + biased; + () = cancel.cancelled() => { body.cancel(); Err(Error::Cancelled) } + result = body.next() => result, + }; + if reply.send(result).is_err() { return; } + } + } + } +} diff --git a/lib/crowdb-sdk-native-python/tests/hca_consumer.py b/lib/crowdb-sdk-native-python/tests/hca_consumer.py new file mode 100644 index 000000000..4a701a412 --- /dev/null +++ b/lib/crowdb-sdk-native-python/tests/hca_consumer.py @@ -0,0 +1,57 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Strict local-HCA Dataset publication, restart, owner and transport checks.""" +import gc +import json +import os +from pathlib import Path +import crowdb_dataset_native as native +from real_consumer import publish_native, verify_native + + +def main(): + root = Path(os.environ["CROWDB_DATASET_CONSUMER_ROOT"]) + transport = dict(enabled=True, allow_tcp_fallback=False, listen_enabled=False, + device=os.environ["CROWDB_RDMA_DEVICE"], source_address=os.environ["CROWDB_RDMA_SOURCE_ADDR"], + cq_count=1, cq_depth=512, sq_depth=16, rq_depth=16, posted_receive_target=16, + registered_memory_budget_bytes=512 << 20, aggregation_enabled=False, + wire_layout="control_data", wire_slot_capacity=(1 << 20) + 1024, + send_arena_bytes=16 << 20, receive_arena_bytes=16 * ((1 << 20) + 1024)) + runtime = native.NativeRuntime(os.environ["CROWDB_MANAGEMENT_SEEDS"].split(","), + options=json.dumps(dict(data_rdma=transport))) + phase = os.environ["CROWDB_DATASET_PHASE"] + try: + if phase == "write": + publish_native(runtime, root) + else: + verify_native(runtime, root) + client = runtime.dataset("native-python-write") + snapshot = bytes(json.loads(root.joinpath("python-snapshot.json").read_text())) + batch = client.read_batch(snapshot, [list(b"one")], ["value"]) + held = batch.next() + assert held.registration_id != 0 + first = held.to_bytes() + while batch.next() is not None: + pass + topology = json.loads(runtime.topology()) + status = topology["data"] + assert status["rdma"]["ready"] and not status["allow_tcp_fallback"] + for key in ("fallback_count", "protocol_errors", "cq_errors", "send_errors", "receive_post_errors"): + assert status["rdma"][key] == 0, (key, status) + resources = json.loads(runtime.status()) + assert resources["held_bytes"] >= held.allocation_capacity + runtime.close() + assert held.to_bytes() == first + del held + gc.collect() + assert json.loads(runtime.status())["held_bytes"] == 0 + root.joinpath(f"dataset-hca-{phase}.json").write_text(json.dumps(dict( + phase=phase, topology=topology, resources=resources, errors=0, + scope="same-host HCA functional acceptance"), indent=2)) + print(f"Dataset HCA {phase}: registered receive ownership, strict transport and shutdown passed") + finally: + runtime.close() + + +if __name__ == "__main__": + main() diff --git a/lib/crowdb-sdk-native-python/tests/real_consumer.py b/lib/crowdb-sdk-native-python/tests/real_consumer.py new file mode 100644 index 000000000..c85834c1b --- /dev/null +++ b/lib/crowdb-sdk-native-python/tests/real_consumer.py @@ -0,0 +1,220 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Installed native binding acceptance over the HTTP workload's exact snapshot.""" +import gc +import hashlib +import json +import os +from pathlib import Path +import uuid +import time +from concurrent.futures import ThreadPoolExecutor + +import crowdb_dataset_native as native + +SIZES = (0, 128, 513, 4096, 65535, 262144, 1048576, 4194321) + + +def call(client, op, **kwargs): + return json.loads(client.call(json.dumps(dict(op=op, **kwargs)), timeout=60)) + + +def drain(batch): + values = {} + while (lease := batch.next()) is not None: + values.setdefault(lease.field, bytearray()).extend(lease.to_bytes()) + return values + + +def identity(): + return list(uuid.uuid4().bytes) + + +def native_data(): + return (b"native-producer" * 300000)[:(4 << 20) + 17] + + +def prepare(client, snapshot, worker=0, workers=1, ids=None, split=None): + read = identity() + spec = {"read": {"id": read, "snapshot": snapshot, + "selection": {"Explicit": [list(item) for item in ids] if ids is not None else + [list(f"size-{i}".encode()) for i in range(8)]}, + "fields": ["value"], "batch_size": 2, + "partition": {"worker": worker, "workers": workers, "rank": 0, "world_size": 1}, + "shuffle": None}, "split": split} + page = 0 + for _ in range(32): + result = call(client, "prepare_read", spec=spec, page=page) + if result["complete"]: + return read, result["session"] + page = result["next_page"] + raise AssertionError("read preparation did not complete") + + +def session_parity(client, snapshot): + selections = [] + for worker in range(2): + read, session = prepare(client, snapshot, worker, 2) + cursor = session["acknowledged"] + batch = client.next_batch(bytes(read), timeout=60) + header = json.loads(batch.header) + first = drain(batch) + assert call(client, "resume_read", read=read)["acknowledged"] == cursor + replay = client.next_batch(bytes(read), timeout=60) + assert json.loads(replay.header) == header + assert drain(replay) == first + selected = [] + while True: + selected.extend(bytes(item) for item in header["sample_ids"]) + session = call(client, "acknowledge", read=read, expected=cursor, next=header["next"]) + cursor = session["acknowledged"] + if header["end"]: + break + batch = client.next_batch(bytes(read), timeout=60) + header = json.loads(batch.header) + drain(batch) + assert call(client, "close_read", read=read)["closed"] + selections.append(set(selected)) + assert not selections[0].intersection(selections[1]) + assert selections[0] | selections[1] == {f"size-{i}".encode() for i in range(8)} + + +def pressure_and_idle_renewal(client, snapshot): + read, session = prepare(client, snapshot) + batch = client.next_batch(bytes(read), timeout=120) + cursor = session["acknowledged"] + # Cross the existing inactivity TTL while a live producer is idle. + print("Holding a native batch across the 60-second active-read TTL", flush=True) + time.sleep(65) + drain(batch) + assert call(client, "resume_read", read=read)["acknowledged"] == cursor + call(client, "close_read", read=read) + runtime = native.NativeRuntime(os.environ["CROWDB_MANAGEMENT_SEEDS"].split(","), + options=json.dumps(dict(held_bytes=128))) + try: + bounded = runtime.dataset(os.environ["CROWDB_DATASET_NAME"]) + first = bounded.read_batch(bytes.fromhex(snapshot), [list(b"size-1")], ["value"]) + held = first.next() + pending = bounded.read_batch(bytes.fromhex(snapshot), [list(b"size-1")], ["value"]) + with ThreadPoolExecutor(max_workers=1) as worker: + waiting = worker.submit(pending.next) + time.sleep(0.1) + assert not waiting.done(), "held capacity must stop production" + pending.cancel() + try: + waiting.result(timeout=5) + except native.CancelledError: + pass + else: + raise AssertionError("cancel did not interrupt a capacity wait") + assert json.loads(runtime.status())["held_bytes"] == 128 + runtime.close() + assert held.to_bytes() == bytes([1]) * 128 + finally: + runtime.close() + + +def publish_native(runtime, root): + client = runtime.dataset("native-python-write") + call(client, "create") + upload, operation = identity(), identity() + data = native_data() + digest = list(hashlib.md5(data).digest()) + call(client, "create_upload", upload=upload, length=len(data), part_size=4 << 20, + md5=digest, expires_at=2**63) + for index, offset in enumerate(range(0, len(data), 4 << 20)): + client.upload_part(bytes(upload), index, data[offset:offset + (4 << 20)]) + call(client, "complete_upload", upload=upload) + location = call(client, "upload_location", upload=upload) + begin = {"parent": None, "source": "All", "schema": {"version": 1, + "fields": [{"name": "value", "required": True}]}, + "inherit_parent_splits": False, "parent_split_revision": None, + "splits": [{"name": "train", "definition": {"Selection": {"Explicit": [list(b"one")]}}, + "origin": "Evaluated"}], "policy": None} + call(client, "begin_composition", operation=operation, input=begin) + edit = {"id": identity(), "expected_revision": 1, "schema": None, "splits": None, "policy": None, + "samples": [{"Upsert": {"sample_id": list(b"one"), "fields": [{"name": "value", + "value": {"Chunk": {"location": location, "length": len(data), "md5": digest}}}]}}]} + call(client, "edit_composition", operation=operation, edit=edit) + for page in range(32): + if call(client, "evaluate_composition", operation=operation, revision=2, page=page)["complete"]: + break + else: + raise AssertionError("composition evaluation did not complete") + for _ in range(32): + state = call(client, "prepare_composition", operation=operation, revision=2) + if state["uploads"]["publication_ready"]: + break + else: + raise AssertionError("composition ownership did not complete") + snapshot = state["uploads"]["candidate"] + call(client, "publish_composition", operation=operation, attempt={"version": 1, "id": identity(), + "target": {"Snapshot": {"snapshot": snapshot}}, "expected_head": None}) + split_read, _ = prepare(client, bytes(snapshot).hex(), ids=[b"one"], split={"revision": None, "name": "train"}) + split_batch = client.next_batch(bytes(split_read)) + assert json.loads(split_batch.header)["sample_ids"] == [list(b"one")] + assert drain(split_batch)[0] == data + call(client, "close_read", read=split_read) + for _ in range(32): + if call(client, "cleanup_composition", operation=operation)["complete"]: + break + else: + raise AssertionError("composition cleanup did not complete") + try: + call(client, "abort_upload", upload=upload) + except native.ConflictError: + pass + else: + raise AssertionError("published payload lost its durable owner") + root.joinpath("python-snapshot.json").write_text(json.dumps(snapshot)) + verify_native(runtime, root) + + +def verify_native(runtime, root): + client = runtime.dataset("native-python-write") + snapshot = bytes(json.loads(root.joinpath("python-snapshot.json").read_text())) + batch = client.read_batch(snapshot, [list(b"one")], ["value"]) + assert drain(batch)[0] == native_data() + + +def main(): + root = Path(os.environ["CROWDB_DATASET_CONSUMER_ROOT"]) + runtime = native.NativeRuntime(os.environ["CROWDB_MANAGEMENT_SEEDS"].split(",")) + try: + if os.environ.get("CROWDB_DATASET_PHASE") == "verify": + verify_native(runtime, root) + print("native Python payload survives storage restart") + return + client = runtime.dataset(os.environ["CROWDB_DATASET_NAME"]) + snapshot = root.joinpath("snapshot").read_text() + for i, size in enumerate(SIZES): + batch = client.read_batch(bytes.fromhex(snapshot), [list(f"size-{i}".encode())], ["value"]) + assert drain(batch).get(0, b"") == bytes([i]) * size + session_parity(client, snapshot) + pressure_and_idle_renewal(client, snapshot) + publish_native(runtime, root) + batch = client.read_batch(bytes.fromhex(snapshot), [list(b"size-1")], ["value"]) + held = batch.next() + view = held.slice(3, 11) + batch.cancel() + try: + batch.next() + except native.CancelledError: + pass + else: + raise AssertionError("cancelled producer continued") + client.close() + assert call(runtime.dataset(os.environ["CROWDB_DATASET_NAME"]), "open") + runtime.close() + assert held.to_bytes() == bytes([1]) * 128 and view.to_bytes() == bytes([1]) * 8 + assert json.loads(runtime.status())["held_bytes"] >= 128 + del held, view, batch + gc.collect() + assert json.loads(runtime.status())["held_bytes"] == 0 + print("native Python same-snapshot bytes, workers, ack replay, publication, cancellation and lifetime passed") + finally: + runtime.close() + + +if __name__ == "__main__": + main() diff --git a/lib/crowdb-sdk-native-python/tests/workload.py b/lib/crowdb-sdk-native-python/tests/workload.py new file mode 100644 index 000000000..85cfc6678 --- /dev/null +++ b/lib/crowdb-sdk-native-python/tests/workload.py @@ -0,0 +1,86 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Native Python comparison; to_bytes copies are included in measured CPU.""" +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import os +from pathlib import Path +import resource +import time + +import crowdb_dataset_native as native +from real_consumer import SIZES + + +def read(client, snapshot, index): + start = time.perf_counter_ns() + batch = client.read_batch(snapshot, [list(f"size-{index}".encode())], ["value"]) + header = json.loads(batch.header) + assert bytes(header["snapshot"]) == snapshot and len(header["fields"]) == 1 + length = 0 + while (lease := batch.next()) is not None: + data = lease.to_bytes() + assert data == bytes([index]) * len(data) + length += len(data) + assert length == SIZES[index] + return (time.perf_counter_ns() - start) // 1000 + + +def trial(client, snapshot, concurrency, rounds): + def lane(number): + observations = [] + for repeat in range(rounds): + for order in range(len(SIZES)): + index = (order * 5 + number + repeat) % len(SIZES) + observations.append((index, read(client, snapshot, index))) + return observations + + cpu = time.process_time() + start = time.perf_counter() + with ThreadPoolExecutor(max_workers=concurrency) as workers: + results = list(workers.map(lane, range(concurrency))) + seconds = time.perf_counter() - start + cpu = time.process_time() - cpu + times = [[] for _ in SIZES] + for result in results: + for index, micros in result: + times[index].append(micros) + latencies = [] + for size, values in zip(SIZES, times): + values.sort() + latencies.append(dict(bytes=size, count=len(values), p50_us=values[(len(values)-1)//2], + p99_us=values[(len(values)*99+99)//100-1])) + return dict(seconds=seconds, completed_requests=concurrency * rounds * len(SIZES), + bytes=concurrency * rounds * sum(SIZES), cpu_ticks=cpu * os.sysconf("SC_CLK_TCK"), + rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, latency=latencies, errors=0) + + +def main(): + root = Path(os.environ["CROWDB_DATASET_CONSUMER_ROOT"]) + snapshot = bytes.fromhex(root.joinpath("snapshot").read_text()) + runtime = native.NativeRuntime(os.environ["CROWDB_MANAGEMENT_SEEDS"].split(",")) + client = runtime.dataset(os.environ["CROWDB_DATASET_NAME"]) + try: + for concurrency in (1, 3): + trial(client, snapshot, concurrency, 2) + for repeat in range(3): + result = trial(client, snapshot, concurrency, 16) + data = bytes([93]) * 1024 * 1024 + start = time.perf_counter_ns() + for _ in range(64): + hashlib.md5(data).digest() + digest = dict(algorithm="md5", implementation="Python hashlib", bytes=64 * len(data), + microseconds=(time.perf_counter_ns()-start)//1000) + report = dict(path="native-python", snapshot=snapshot.hex(), sizes=SIZES, concurrency=concurrency, + repeat=repeat, result=result, digest=digest, load="closed-loop", warmup_rounds=2, + in_flight_budget_bytes=256 * 1024 * 1024, python_bytes_copy="included") + root.joinpath(f"native-python-{concurrency}-{repeat}.json").write_text(json.dumps(report, indent=2)) + print(json.dumps(report), flush=True) + root.joinpath("native-python-resources.json").write_text(runtime.status()) + finally: + runtime.close() + + +if __name__ == "__main__": + main() diff --git a/lib/crowdb-sdk-native/Cargo.toml b/lib/crowdb-sdk-native/Cargo.toml index a33d4be11..8c6cd98ad 100644 --- a/lib/crowdb-sdk-native/Cargo.toml +++ b/lib/crowdb-sdk-native/Cargo.toml @@ -5,7 +5,7 @@ edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true -description = "Trusted embedded CROWDB runtime and native S3 client" +description = "Trusted embedded CROWDB runtime and native S3/Dataset clients" [lints] workspace = true @@ -19,6 +19,7 @@ crossbeam-skiplist = "0.1" arc-swap = "1.9" bincode = "1" crowdb-access-s3 = { path = "../crowdb-access-s3" } +crowdb-access-dataset = { path = "../crowdb-access-dataset" } crowdb-access-multipart = { path = "../crowdb-access-multipart" } crowdb-chunk-client = { path = "../crowdb-chunk-client" } crowdb-chunk-kv-client = { path = "../crowdb-chunk-kv-client" } @@ -31,6 +32,7 @@ thiserror = { workspace = true } tokio = { workspace = true, features = ["time", "sync", "macros"] } tracing = { workspace = true } serde_json = "1" +md-5 = "0.10" [dev-dependencies] async-trait = "0.1" diff --git a/lib/crowdb-sdk-native/README.md b/lib/crowdb-sdk-native/README.md index 683b9b55b..bfa37b762 100644 --- a/lib/crowdb-sdk-native/README.md +++ b/lib/crowdb-sdk-native/README.md @@ -3,7 +3,7 @@ # CrowDB native SDK -`crowdb-sdk-native` embeds trusted internal S3 access through Chunk-KV and +`crowdb-sdk-native` embeds trusted internal S3 and Dataset access through Chunk-KV and ChunkIO/DiskIO. It belongs to the main Rust workspace and links the native RPC stack. Supply management seeds, a tenant and per-operation context. The embedding process has the same internal-service trust as Access Server; this @@ -84,6 +84,39 @@ async fn read_object(seeds: Vec) -> Result<(), Box +// Licensed under the Apache License, Version 2.0. + +//! Reproducible native workload over the HTTP-prepared immutable snapshot; the external runner owns fixtures and affinity. + +use crowdb_access_dataset::{DatasetIdentity, NamespacePath, SnapshotId}; +use crowdb_sdk_native::{ + dataset::{BatchRequest, DatasetClient}, + ClientRuntime, OperationContext, RuntimeConfig, +}; +use md5::{Digest, Md5}; +use serde_json::json; +use std::sync::atomic::{AtomicU64, Ordering}; +static REQUEST: AtomicU64 = AtomicU64::new(1); +use std::{ + num::NonZeroU64, + path::Path, + time::{Duration, Instant}, +}; + +type Failure = Box; +const SIZES: [usize; 8] = [0, 128, 513, 4096, 65_535, 262_144, 1_048_576, 4_194_321]; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Failure> { + let mut config = RuntimeConfig::new( + std::env::var("CROWDB_MANAGEMENT_SEEDS")? + .split(',') + .map(str::to_owned) + .collect(), + ); + config.data_rdma = Some(crowdb_rpc_ffi::RdmaConfig { + enabled: false, + ..Default::default() + }); + let runtime = ClientRuntime::connect(config).await?; + let client = DatasetClient::new( + &runtime, + DatasetIdentity::new(NamespacePath::root(), std::env::var("CROWDB_DATASET_NAME")?)?, + )?; + let root = std::env::var("CROWDB_DATASET_CONSUMER_ROOT")?; + let root = Path::new(&root); + let snapshot = std::fs::read_to_string(root.join("snapshot"))?; + for concurrency in [1, 3] { + trial(&client, &snapshot, concurrency, 2).await?; + for repeat in 0..3 { + let result = trial(&client, &snapshot, concurrency, 16).await?; + let report = json!({"path":"native-rust", "snapshot":snapshot, "sizes":SIZES, + "concurrency":concurrency, "repeat":repeat, "result":result, + "digest":digest_cost(), "load":"closed-loop", "warmup_rounds":2, + "in_flight_budget_bytes":256 * 1024 * 1024, "implicit_http_retry":false}); + std::fs::write( + root.join(format!("native-rust-{concurrency}-{repeat}.json")), + serde_json::to_vec_pretty(&report)?, + )?; + println!("{report}"); + } + } + let resources = runtime.resources().status(); + std::fs::write( + root.join("native-rust-resources.json"), + serde_json::to_vec_pretty(&json!({ + "held_bytes":resources.held_bytes,"high_water":resources.held_bytes_high_water, + "compatibility_copy_bytes":resources.compatibility_copy_bytes}))?, + )?; + runtime.shutdown().await?; + Ok(()) +} + +fn snapshot_id(value: &str) -> Result { + let bytes: Result, _> = (0..value.len()) + .step_by(2) + .map(|index| u8::from_str_radix(&value[index..index + 2], 16)) + .collect(); + Ok(SnapshotId::from_bytes(&bytes?)?) +} + +async fn read(client: &DatasetClient, snapshot: &str, index: usize) -> Result { + let start = Instant::now(); + let context = OperationContext::new( + NonZeroU64::new(REQUEST.fetch_add(1, Ordering::Relaxed)).unwrap(), + Instant::now() + Duration::from_secs(30), + ); + let mut batch = client + .read_batch( + context, + BatchRequest { + snapshot: snapshot_id(snapshot)?, + sample_ids: vec![format!("size-{index}").into_bytes()], + fields: vec!["value".into()], + }, + ) + .await?; + if batch.header.snapshot.to_string() != snapshot || batch.header.fields.len() != 1 { + return Err("unexpected snapshot/projection".into()); + } + let mut length = 0; + while let Some(field) = batch.body.next().await? { + let data = field.buffer.as_slice(); + if data.iter().any(|byte| usize::from(*byte) != index) { + return Err("payload mismatch".into()); + } + length += data.len(); + } + if length != SIZES[index] { + return Err("payload length mismatch".into()); + } + Ok(u64::try_from(start.elapsed().as_micros())?) +} + +async fn trial( + client: &DatasetClient, + snapshot: &str, + concurrency: usize, + rounds: usize, +) -> Result { + let cpu = cpu_ticks()?; + let start = Instant::now(); + let mut tasks = tokio::task::JoinSet::new(); + for lane in 0..concurrency { + let client = client.clone(); + let snapshot = snapshot.to_owned(); + tasks.spawn(async move { + let mut observations = Vec::new(); + for round in 0..rounds { + for order in 0..SIZES.len() { + let index = (order * 5 + lane + round) % SIZES.len(); + observations.push((index, read(&client, &snapshot, index).await?)); + } + } + Ok::<_, Failure>(observations) + }); + } + let mut times: [Vec; 8] = std::array::from_fn(|_| Vec::new()); + while let Some(result) = tasks.join_next().await { + for (index, micros) in result?? { + times[index].push(micros); + } + } + let seconds = start.elapsed().as_secs_f64(); + let cpu = cpu_ticks()? - cpu; + let mut latency = Vec::new(); + for (index, values) in times.iter_mut().enumerate() { + values.sort_unstable(); + latency.push(json!({"bytes":SIZES[index], "count":values.len(), + "p50_us":values[(values.len()-1)/2], "p99_us":values[(values.len()*99).div_ceil(100)-1]})); + } + let requests = concurrency * rounds * SIZES.len(); + let bytes = concurrency * rounds * SIZES.iter().sum::(); + Ok( + json!({"seconds":seconds, "completed_requests":requests, "bytes":bytes, + "cpu_ticks":cpu, "rss_kib":rss_kib()?, "latency":latency, "errors":0}), + ) +} + +fn cpu_ticks() -> Result { + let stat = std::fs::read_to_string("/proc/self/stat")?; + let (_, fields) = stat.rsplit_once(") ").ok_or("invalid proc stat")?; + let fields: Vec<_> = fields.split_whitespace().collect(); + Ok(fields[11].parse::()? + fields[12].parse::()?) +} +fn rss_kib() -> Result { + let status = std::fs::read_to_string("/proc/self/status")?; + let line = status + .lines() + .find(|line| line.starts_with("VmHWM:")) + .ok_or("missing VmHWM")?; + Ok(line.split_whitespace().nth(1).ok_or("missing RSS")?.parse()?) +} +fn digest_cost() -> serde_json::Value { + let data = vec![93; 1024 * 1024]; + let start = Instant::now(); + for _ in 0..64 { + std::hint::black_box(Md5::digest(std::hint::black_box(&data))); + } + json!({"algorithm":"md5", "bytes":64 * data.len(), "microseconds":start.elapsed().as_micros(), + "scope":"separate resident-buffer kernel; real reads always verify integrity"}) +} diff --git a/lib/crowdb-sdk-native/src/dataset.rs b/lib/crowdb-sdk-native/src/dataset.rs new file mode 100644 index 000000000..b22bc8595 --- /dev/null +++ b/lib/crowdb-sdk-native/src/dataset.rs @@ -0,0 +1,114 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Trusted Dataset access over the shared native runtime and existing authority. + +mod composition; +mod control; +mod error; +mod read; +mod session; +mod upload; + +pub use composition::{CompositionCleanup, CompositionStatus}; +pub use control::DatasetControl; +pub use error::FailureKind; +pub use read::{BatchBody, BatchField, BatchHeader, BatchRequest, FieldBuffer, FieldState, ReadBatch}; + +use crate::{ClientRuntime, Result}; +use crowdb_access_dataset::{ + ChunkKvDatasetStore, ChunkPayloadStorage, DatasetAuthority, DatasetIdentity, DatasetUploads, + UploadedChunkReader, +}; +use crowdb_chunk_client::SmallWritePolicy; +use crowdb_protocol::chunkdb::rpc::ChunkType; +use std::sync::Arc; + +#[derive(Clone)] +pub struct DatasetClient { + runtime: ClientRuntime, + authority: Arc, + uploads: Arc, + identity: DatasetIdentity, +} + +impl DatasetClient { + /// Attaches a Dataset writer group without another runtime or transport. + /// # Errors + /// Returns invalid identity/policy, closed runtime or writer group capacity. + pub fn new(runtime: &ClientRuntime, identity: DatasetIdentity) -> Result { + identity.key()?; + let mut policy = SmallWritePolicy::new(ChunkType::Dataset); + policy.object_limit = crowdb_access_dataset::MAX_UPLOAD_PART_BYTES; + policy.max_batch_bytes = policy.object_limit; + policy.max_pipelines = 8; + policy.memory_budget = 128 * 1024 * 1024; + Self::with_writer_policy(runtime, identity, &policy) + } + + /// Attaches an explicitly tuned Dataset writer group to the shared runtime. + /// # Errors + /// Rejects other chunk purposes, undersized part limits or invalid policy. + pub fn with_writer_policy( + runtime: &ClientRuntime, + identity: DatasetIdentity, + policy: &SmallWritePolicy, + ) -> Result { + identity.key()?; + if policy.chunk_type != ChunkType::Dataset + || policy.object_limit < crowdb_access_dataset::MAX_UPLOAD_PART_BYTES + { + return Err(crate::Error::Configuration( + "Dataset writer must admit complete upload parts".into(), + )); + } + let runtime = runtime.with_writer_policy(policy)?; + let store = Arc::new(ChunkKvDatasetStore::new(Arc::clone(&runtime.storage.metadata))); + let uploads = Arc::new(DatasetUploads::new( + store.clone(), + Arc::new(ChunkPayloadStorage::new((*runtime.storage.chunks).clone())), + )); + let authority = Arc::new(DatasetAuthority::with_chunk_reader( + store, + Arc::new(UploadedChunkReader::new((*uploads).clone())), + )); + Ok(Self { + runtime, + authority, + uploads, + identity, + }) + } + + #[must_use] + pub fn identity(&self) -> &DatasetIdentity { + &self.identity + } + + #[must_use] + pub fn runtime(&self) -> &ClientRuntime { + &self.runtime + } + + #[cfg(feature = "test-util")] + #[must_use] + pub fn from_services_for_tests( + runtime: ClientRuntime, + identity: DatasetIdentity, + authority: Arc, + uploads: Arc, + ) -> Self { + Self { + runtime, + authority, + uploads, + identity, + } + } +} + +fn now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |time| time.as_secs()) +} diff --git a/lib/crowdb-sdk-native/src/dataset/composition.rs b/lib/crowdb-sdk-native/src/dataset/composition.rs new file mode 100644 index 000000000..f2429a8e7 --- /dev/null +++ b/lib/crowdb-sdk-native/src/dataset/composition.rs @@ -0,0 +1,226 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{DatasetClient, DatasetControl}; +use crate::{OperationContext, Result}; +use crowdb_access_dataset::{ + composition::{ + CompositionDraft, CompositionEdit, CompositionInput, CompositionPreview, CompositionUploadStatus, + DraftProgress, + }, + CommitAttempt, OperationId, ReadCancellation, SuccessfulCommit, +}; + +#[derive(Debug)] +pub struct CompositionStatus { + pub draft: CompositionDraft, + pub uploads: CompositionUploadStatus, +} + +#[derive(Debug)] +pub struct CompositionCleanup { + pub complete: bool, + pub retired_upload_owners: u64, + pub physical_cleanup_deferred: bool, +} + +impl DatasetControl { + async fn composition_status(&self, operation: OperationId) -> Result { + Ok(CompositionStatus { + draft: self + .authority + .composition_status(&self.identity, operation) + .await?, + uploads: self + .authority + .composition_upload_status(&self.identity, operation) + .await?, + }) + } +} + +impl DatasetClient { + /// Starts an upload-managed composition before any edits can acquire payloads. + /// # Errors + /// Returns divergent identity/input or uncertain storage outcomes. + pub async fn begin_composition( + &self, + context: OperationContext, + operation: OperationId, + input: &CompositionInput, + ) -> Result { + Box::pin(self.control(context, |scope| async move { + scope + .authority + .begin_composition(&scope.identity, operation, input) + .await?; + scope + .authority + .manage_composition_uploads(&scope.identity, operation) + .await?; + scope.composition_status(operation).await + })) + .await + } + + /// # Errors + /// Returns missing/corrupt composition or unavailable metadata. + pub async fn composition_status( + &self, + context: OperationContext, + operation: OperationId, + ) -> Result { + Box::pin(self.control(context, |scope| async move { + scope.composition_status(operation).await + })) + .await + } + + /// Applies one revision-bound edit with the same durable upload owners as HTTP. + /// # Errors + /// Returns revision conflict, invalid payloads or uncertain metadata writes. + pub async fn edit_composition( + &self, + context: OperationContext, + operation: OperationId, + edit: &CompositionEdit, + ) -> Result { + Box::pin(self.control(context, |scope| async move { + scope + .authority + .edit_composition_with_uploads(&scope.identity, operation, edit, &scope.uploads) + .await?; + scope.composition_status(operation).await + })) + .await + } + + /// Advances one bounded evaluation page; callers carry the returned cursor. + /// # Errors + /// Returns stale revision/page, invalid selection or uncertain metadata writes. + pub async fn evaluate_composition( + &self, + context: OperationContext, + operation: OperationId, + revision: u64, + page: u64, + ) -> Result { + Box::pin(self.control(context, |scope| async move { + Ok(scope + .authority + .evaluate_composition( + &scope.identity, + operation, + revision, + page, + &ReadCancellation::new(), + ) + .await?) + })) + .await + } + + /// Advances preparation and upload ownership in bounded, resumable windows. + /// # Errors + /// Returns stale revision, invalid preparation or uncertain metadata writes. + pub async fn prepare_composition( + &self, + context: OperationContext, + operation: OperationId, + revision: u64, + ) -> Result { + Box::pin(self.control(context, |scope| async move { + let cancel = ReadCancellation::new(); + let draft = scope + .authority + .prepare_composition(&scope.identity, operation, revision, &cancel) + .await?; + if matches!(draft.record.progress, DraftProgress::Prepared { .. }) { + scope + .authority + .prepare_composition_uploads(&scope.identity, operation, &scope.uploads, &cancel) + .await?; + } + scope.composition_status(operation).await + })) + .await + } + + /// Publishes using the caller's durable attempt identity, without replay. + /// # Errors + /// Returns incomplete ownership, conflicting head or uncertain publication. + pub async fn publish_composition( + &self, + context: OperationContext, + operation: OperationId, + attempt: &CommitAttempt, + ) -> Result { + Box::pin(self.control(context, |scope| async move { + Ok(scope + .authority + .publish_composition(&scope.identity, operation, attempt) + .await?) + })) + .await + } + + /// # Errors + /// Returns stale revision, publication conflict or uncertain metadata writes. + pub async fn abandon_composition( + &self, + context: OperationContext, + operation: OperationId, + revision: u64, + ) -> Result { + Box::pin(self.control(context, |scope| async move { + scope + .authority + .abandon_composition(&scope.identity, operation, revision) + .await?; + scope.composition_status(operation).await + })) + .await + } + + /// Advances logical cleanup only; shared authority retains physical fences. + /// # Errors + /// Returns conflicting live state or uncertain cleanup progress. + pub async fn cleanup_composition( + &self, + context: OperationContext, + operation: OperationId, + ) -> Result { + Box::pin(self.control(context, |scope| async move { + let cancel = ReadCancellation::new(); + let authority = &scope.authority; + let identity = &scope.identity; + let candidates = authority + .cleanup_composition_candidates_step(identity, operation, &cancel) + .await?; + let owners = authority + .cleanup_composition_uploads_step(identity, operation, &scope.uploads) + .await?; + let scratch = authority + .cleanup_composition_scratch_step(identity, operation, &cancel) + .await?; + let draft = authority.composition_status(identity, operation).await?; + let edits_complete = if matches!( + draft.record.progress, + DraftProgress::Published { .. } | DraftProgress::Abandoned + ) { + authority + .cleanup_composition_edits_step(identity, operation, &cancel) + .await? + .complete + } else { + true + }; + Ok(CompositionCleanup { + complete: candidates.complete && owners.complete && scratch.complete && edits_complete, + retired_upload_owners: owners.retired_owners, + physical_cleanup_deferred: true, + }) + })) + .await + } +} diff --git a/lib/crowdb-sdk-native/src/dataset/control.rs b/lib/crowdb-sdk-native/src/dataset/control.rs new file mode 100644 index 000000000..7ea0392f8 --- /dev/null +++ b/lib/crowdb-sdk-native/src/dataset/control.rs @@ -0,0 +1,78 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::DatasetClient; +use crate::{OperationContext, Result}; +use crowdb_access_dataset::{ + DatasetAuthority, DatasetId, DatasetIdentity, DatasetRecord, DatasetUploads, SnapshotId, SnapshotRecord, +}; +use std::{future::Future, sync::Arc}; + +/// Existing authority services scoped by an explicitly supplied trusted identity. +/// Composition, Split and cleanup use their original typed contracts and CAS rules. +pub struct DatasetControl { + pub authority: Arc, + pub uploads: Arc, + pub identity: DatasetIdentity, +} + +impl DatasetClient { + /// Runs one caller-selected control operation under shared request admission. + /// Use original authority operation/attempt IDs to inspect uncertain outcomes; + /// this method never retries the callback or switches to HTTP. + /// # Errors + /// Returns admission, domain failures or interrupted mutation uncertainty. + pub async fn control(&self, context: OperationContext, operation: F) -> Result + where + F: FnOnce(DatasetControl) -> R, + R: Future>, + { + let id = context.request_id.get(); + let request = self.runtime.resources.admit(context)?; + let scope = DatasetControl { + authority: Arc::clone(&self.authority), + uploads: Arc::clone(&self.uploads), + identity: self.identity.clone(), + }; + request + .run_mutation(async { operation(scope).await }) + .await + .map_err(|error| match error { + crate::Error::Dataset { + kind: super::FailureKind::Storage, + .. + } => crate::Error::UncertainMutation { + request_id: id, + reason: crate::error::MutationInterruption::Transport, + }, + other => other, + }) + } + + /// # Errors + /// Returns duplicate identity, admission or storage failure. + pub async fn create(&self, context: OperationContext) -> Result { + self.control(context, |scope| async move { + Ok(scope.authority.create_dataset(scope.identity).await?) + }) + .await + } + + /// # Errors + /// Returns missing identity, admission or storage failure. + pub async fn open(&self, context: OperationContext) -> Result { + let request = self.runtime.resources.admit(context)?; + request + .run(async { Ok(self.authority.get_dataset(&self.identity).await?) }) + .await + } + + /// # Errors + /// Returns missing snapshot, admission or storage failure. + pub async fn snapshot(&self, context: OperationContext, id: SnapshotId) -> Result { + let request = self.runtime.resources.admit(context)?; + request + .run(async { Ok(self.authority.get_snapshot(&self.identity, id).await?) }) + .await + } +} diff --git a/lib/crowdb-sdk-native/src/dataset/error.rs b/lib/crowdb-sdk-native/src/dataset/error.rs new file mode 100644 index 000000000..1a2167c86 --- /dev/null +++ b/lib/crowdb-sdk-native/src/dataset/error.rs @@ -0,0 +1,109 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::Error; +use crowdb_access_dataset::{AuthorityError, DatasetError, UploadError}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FailureKind { + Invalid, + NotFound, + Conflict, + Integrity, + Capacity, + Pending, + Storage, +} + +fn error(kind: FailureKind, detail: impl std::fmt::Display) -> Error { + Error::Dataset { + kind, + detail: detail.to_string(), + } +} + +impl From for Error { + fn from(value: DatasetError) -> Self { + if matches!(value, DatasetError::ReadCancelled) { + Self::Cancelled + } else { + error(FailureKind::Invalid, value) + } + } +} +impl From for Error { + fn from(value: AuthorityError) -> Self { + let kind = match value { + AuthorityError::Invalid(value) => return value.into(), + AuthorityError::NotFound | AuthorityError::SnapshotNotFound => FailureKind::NotFound, + AuthorityError::AncestryCycle | AuthorityError::Corrupt => FailureKind::Integrity, + AuthorityError::Store(_) => FailureKind::Storage, + _ => FailureKind::Conflict, + }; + error(kind, value) + } +} +impl From for Error { + fn from(value: UploadError) -> Self { + let kind = match &value { + UploadError::Invalid => FailureKind::Invalid, + UploadError::NotFound => FailureKind::NotFound, + UploadError::Conflict => FailureKind::Conflict, + UploadError::Integrity => FailureKind::Integrity, + UploadError::Pending => FailureKind::Pending, + UploadError::Capacity => FailureKind::Capacity, + UploadError::Storage(_) | UploadError::Store(_) => FailureKind::Storage, + }; + error(kind, value) + } +} +impl From for Error { + fn from(value: crowdb_access_dataset::composition::CompositionError) -> Self { + use crowdb_access_dataset::composition::CompositionError as E; + let kind = match value { + E::Authority(value) => return value.into(), + E::Upload(value) => return value.into(), + E::Split(value) => return value.into(), + E::Inspection(value) => return value.into(), + E::NotFound => FailureKind::NotFound, + E::Conflict | E::Frozen => FailureKind::Conflict, + E::Corrupt => FailureKind::Integrity, + E::Limit => FailureKind::Capacity, + E::Store(_) => FailureKind::Storage, + E::Record(_) => FailureKind::Invalid, + }; + error(kind, value) + } +} +impl From for Error { + fn from(value: crowdb_access_dataset::split::SplitLifecycleError) -> Self { + use crowdb_access_dataset::split::SplitLifecycleError as E; + let kind = match value { + E::Authority(value) => return value.into(), + E::Inspection(value) => return value.into(), + E::NotFound => FailureKind::NotFound, + E::Conflict | E::Frozen => FailureKind::Conflict, + E::Limit => FailureKind::Capacity, + E::Corrupt => FailureKind::Integrity, + E::Store(_) => FailureKind::Storage, + _ => FailureKind::Invalid, + }; + error(kind, value) + } +} + +impl From for Error { + fn from(value: crowdb_access_dataset::inspection::InspectionError) -> Self { + use crowdb_access_dataset::inspection::InspectionError as E; + let kind = match value { + E::Authority(value) => return value.into(), + E::Cancelled => return Self::Cancelled, + E::Store(_) => FailureKind::Storage, + E::Limit => FailureKind::Capacity, + E::NotPrepared => FailureKind::Pending, + E::Unsupported(_) => FailureKind::Invalid, + E::Corrupt => FailureKind::Integrity, + }; + error(kind, value) + } +} diff --git a/lib/crowdb-sdk-native/src/dataset/read.rs b/lib/crowdb-sdk-native/src/dataset/read.rs new file mode 100644 index 000000000..d65a3d619 --- /dev/null +++ b/lib/crowdb-sdk-native/src/dataset/read.rs @@ -0,0 +1,162 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +mod body; +mod field; +mod protection; +pub use body::{BatchBody, FieldBuffer}; + +use super::{now, DatasetClient}; +use crate::runtime::RequestLease; +use crate::{OperationContext, Result}; +use crowdb_access_dataset::{ + DatasetSessions, FieldLocator, OperationId, ReadCancellation, SampleView, SnapshotId, +}; +use std::sync::Arc; + +pub struct BatchRequest { + pub snapshot: SnapshotId, + pub sample_ids: Vec>, + pub fields: Vec, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FieldState { + Value, + Missing, + Tombstone, +} + +#[derive(Clone, Debug)] +pub struct BatchField { + pub sample_id: Vec, + pub name: String, + pub state: FieldState, + pub length: u64, + pub md5: Option<[u8; 16]>, +} + +#[derive(Clone, Debug)] +pub struct BatchHeader { + pub snapshot: SnapshotId, + pub sample_ids: Vec>, + pub fields: Vec, + pub next: Option, + pub end: bool, +} + +pub struct ReadBatch { + pub header: Arc, + pub body: BatchBody, +} + +impl DatasetClient { + /// Reads an explicit immutable snapshot/projection without creating a cursor. + /// # Errors + /// Returns missing data, invalid projection, admission or storage failure. + pub async fn read_batch(&self, context: OperationContext, batch: BatchRequest) -> Result { + let request = self.runtime.resources.admit(context)?; + self.batch(request, batch, None, true, None).await + } + + /// Fetches the issued batch; only `acknowledge` advances durable progress. + /// # Errors + /// Returns expired/closed sessions, selection, admission or storage failure. + pub async fn next_batch(&self, context: OperationContext, id: OperationId) -> Result { + let request = self.runtime.resources.admit(context)?; + let sessions = DatasetSessions::new(Arc::clone(&self.authority)); + let batch = request + .run(async { Ok(sessions.next(&self.identity, id, now()).await?) }) + .await?; + self.batch( + request, + BatchRequest { + snapshot: batch.snapshot, + sample_ids: batch.ids, + fields: batch.fields, + }, + Some(batch.next), + batch.end, + Some(id), + ) + .await + } + + async fn batch( + &self, + request: RequestLease, + batch: BatchRequest, + next: Option, + end: bool, + session: Option, + ) -> Result { + let protection = protection::ReadProtection::acquire(self, &request, batch.snapshot, session).await?; + let samples = if batch.sample_ids.is_empty() { + Vec::new() + } else { + request + .run(async { + Ok(self + .authority + .resolve_inspection_batch_locators( + &self.identity, + batch.snapshot, + &batch.sample_ids, + &batch.fields, + &ReadCancellation::new(), + ) + .await?) + }) + .await? + }; + let (fields, locators) = descriptors(&batch, samples); + let header = Arc::new(BatchHeader { + snapshot: batch.snapshot, + sample_ids: batch.sample_ids, + fields, + next, + end, + }); + let body = BatchBody::new( + Arc::clone(&header), + locators, + request, + Arc::clone(&self.runtime.resources), + Arc::clone(&self.uploads), + protection, + ); + Ok(ReadBatch { header, body }) + } +} + +fn descriptors( + batch: &BatchRequest, + samples: Vec, +) -> (Vec, Vec<(usize, FieldLocator)>) { + let mut fields = Vec::new(); + let mut locators = Vec::new(); + for mut sample in samples { + for name in &batch.fields { + let locator = sample.fields.remove(name); + let (state, length, md5) = match &locator { + Some(FieldLocator::Inline { value, md5 }) => { + (FieldState::Value, value.len() as u64, Some(*md5)) + } + Some(FieldLocator::Chunk { length, md5, .. }) => (FieldState::Value, *length, Some(*md5)), + Some(FieldLocator::Tombstone) => (FieldState::Tombstone, 0, None), + None => (FieldState::Missing, 0, None), + }; + if let Some(value @ (FieldLocator::Inline { .. } | FieldLocator::Chunk { .. })) = locator { + locators.push((fields.len(), value)); + } + fields.push(BatchField { + sample_id: sample.sample_id.clone(), + name: name.clone(), + state, + length, + md5, + }); + } + } + (fields, locators) +} diff --git a/lib/crowdb-sdk-native/src/dataset/read/body.rs b/lib/crowdb-sdk-native/src/dataset/read/body.rs new file mode 100644 index 000000000..f1776cc68 --- /dev/null +++ b/lib/crowdb-sdk-native/src/dataset/read/body.rs @@ -0,0 +1,140 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{field::FieldReader, protection::ReadProtection, BatchHeader}; +use crate::{buffer::BufferLease, runtime::RequestLease, Error, Result, RuntimeResources}; +use crowdb_access_dataset::{DatasetUploads, FieldLocator}; +use crowdb_rpc_ffi::OwnedPayload; +use std::{collections::VecDeque, sync::Arc}; + +pub struct FieldBuffer { + pub field: usize, + pub buffer: BufferLease, +} + +/// One bounded pull ahead; caller-held allocations stay charged until released. +/// Dropping an individual pull future preserves pending data for the next pull. +pub struct BatchBody { + header: Arc, + locators: VecDeque<(usize, FieldLocator)>, + field: Option, + pending: Option, + request: Option, + resources: Arc, + uploads: Arc, + protection: Option, + sequence: u64, + complete: bool, + failure: Option, +} + +impl BatchBody { + pub(super) fn new( + header: Arc, + locators: Vec<(usize, FieldLocator)>, + request: RequestLease, + resources: Arc, + uploads: Arc, + protection: ReadProtection, + ) -> Self { + Self { + header, + locators: locators.into(), + field: None, + pending: None, + request: Some(request), + resources, + uploads, + protection: Some(protection), + sequence: 0, + complete: false, + failure: None, + } + } + + #[must_use] + pub fn header(&self) -> &BatchHeader { + &self.header + } + #[must_use] + pub fn is_complete(&self) -> bool { + self.complete + } + + /// Successful EOF certifies all field and part digests, never processing ack. + /// # Errors + /// Returns sticky integrity/transport/cancellation failures or expired admission. + pub async fn next(&mut self) -> Result> { + if let Some(error) = &self.failure { + return Err(error.clone()); + } + if self.complete { + return Ok(None); + } + match self.pull().await { + Ok(Some(data)) => Ok(Some(data)), + Ok(None) => { + self.clear(); + self.complete = true; + Ok(None) + } + Err(error) => { + self.clear(); + self.failure = Some(error.clone()); + Err(error) + } + } + } + + pub fn cancel(&mut self) { + if !self.complete && self.failure.is_none() { + self.clear(); + self.failure = Some(Error::Cancelled); + } + } + + fn clear(&mut self) { + self.field = None; + self.pending = None; + self.locators.clear(); + self.protection = None; + self.request = None; + } + + async fn pull(&mut self) -> Result> { + let request = self.request.as_ref().ok_or(Error::Cancelled)?; + let protection = self.protection.as_ref().ok_or(Error::Cancelled)?; + request.run(protection.check()).await?; + loop { + if self.field.is_none() { + let Some((index, locator)) = self.locators.pop_front() else { + return Ok(None); + }; + self.field = Some(FieldReader::new(index, locator)?); + } + let field = self.field.as_mut().ok_or(Error::Integrity)?; + if self.pending.is_none() { + self.pending = field.next(&self.uploads, request).await?; + if self.pending.is_none() { + self.field = None; + continue; + } + } + let payload = self.pending.as_ref().ok_or(Error::Integrity)?.clone(); + let id = request.context().request_id.get(); + let buffer = request + .run( + self.resources + .retain_payload(payload, id, id, self.sequence, field.offset), + ) + .await?; + field.delivered(buffer.as_slice())?; + self.pending = None; + self.sequence = self.sequence.checked_add(1).ok_or(Error::BufferRange)?; + return Ok(Some(FieldBuffer { + field: field.index, + buffer, + })); + } + } +} diff --git a/lib/crowdb-sdk-native/src/dataset/read/field.rs b/lib/crowdb-sdk-native/src/dataset/read/field.rs new file mode 100644 index 000000000..8feede6bd --- /dev/null +++ b/lib/crowdb-sdk-native/src/dataset/read/field.rs @@ -0,0 +1,128 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::{runtime::RequestLease, Error, Result}; +use crowdb_access_dataset::{DatasetUploads, FieldLocator, UploadRecord, VerifiedUploadPart}; +use crowdb_rpc_ffi::OwnedPayload; +use md5::{Digest, Md5}; + +enum Source { + Inline(Option), + Upload { + location: Vec, + metadata: Option, + part: u32, + stream: Option>, + }, +} + +pub(super) struct FieldReader { + pub(super) index: usize, + pub(super) offset: u64, + source: Source, + length: u64, + expected: [u8; 16], + digest: Md5, +} + +impl FieldReader { + pub(super) fn new(index: usize, locator: FieldLocator) -> Result { + let (source, length, expected) = match locator { + FieldLocator::Inline { value, md5 } => { + let length = value.len() as u64; + ( + Source::Inline((!value.is_empty()).then(|| OwnedPayload::from(value))), + length, + md5, + ) + } + FieldLocator::Chunk { + location, + length, + md5, + } => ( + Source::Upload { + location, + metadata: None, + part: 0, + stream: None, + }, + length, + md5, + ), + FieldLocator::Tombstone => return Err(Error::Integrity), + }; + Ok(Self { + index, + offset: 0, + source, + length, + expected, + digest: Md5::new(), + }) + } + + pub(super) async fn next( + &mut self, + uploads: &DatasetUploads, + request: &RequestLease, + ) -> Result> { + let payload = match &mut self.source { + Source::Inline(value) => value.take(), + Source::Upload { + location, + metadata, + part, + stream, + } => { + if metadata.is_none() { + let record = request + .run(async { Ok(uploads.metadata(location).await?) }) + .await?; + if record.length != self.length || record.md5 != self.expected { + return Err(Error::Integrity); + } + *metadata = Some(record); + } + let record = metadata.as_ref().ok_or(Error::Integrity)?; + loop { + if u64::from(*part) >= record.length.div_ceil(u64::from(record.part_size)) { + break None; + } + if stream.is_none() { + *stream = Some( + request + .run(async { Ok(Box::new(uploads.read_part_stream(location, *part).await?)) }) + .await?, + ); + } + let reader = stream.as_mut().ok_or(Error::Integrity)?; + if let Some(payload) = request.run(async { Ok(reader.next().await?) }).await? { + break Some(payload); + } + *stream = None; + *part += 1; + } + } + }; + if payload.is_none() + && (self.offset != self.length + || <[u8; 16]>::from(self.digest.clone().finalize()) != self.expected) + { + return Err(Error::Integrity); + } + Ok(payload) + } + + pub(super) fn delivered(&mut self, bytes: &[u8]) -> Result<()> { + self.offset = self + .offset + .checked_add(bytes.len() as u64) + .ok_or(Error::BufferRange)?; + if self.offset > self.length { + return Err(Error::Integrity); + } + self.digest.update(bytes); + Ok(()) + } +} diff --git a/lib/crowdb-sdk-native/src/dataset/read/protection.rs b/lib/crowdb-sdk-native/src/dataset/read/protection.rs new file mode 100644 index 000000000..52c3d2082 --- /dev/null +++ b/lib/crowdb-sdk-native/src/dataset/read/protection.rs @@ -0,0 +1,119 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::super::{now, DatasetClient}; +use crate::{runtime::RequestLease, Cancellation, Error, Result}; +use crowdb_access_dataset::{DatasetAuthority, DatasetIdentity, DatasetSessions, OperationId, SnapshotId}; +use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; + +struct Protection { + authority: Arc, + identity: DatasetIdentity, + snapshot: SnapshotId, + handle: OperationId, + session: Option, + failed: AtomicBool, +} + +pub(super) struct ReadProtection { + state: Arc, + stop: Cancellation, + worker: tokio::task::JoinHandle<()>, +} + +impl ReadProtection { + pub(super) async fn acquire( + client: &DatasetClient, + request: &RequestLease, + snapshot: SnapshotId, + session: Option, + ) -> Result { + let state = Arc::new(Protection { + authority: Arc::clone(&client.authority), + identity: client.identity.clone(), + snapshot, + handle: OperationId::random(), + session, + failed: AtomicBool::new(false), + }); + request.run(state.renew()).await?; + let stop = Cancellation::default(); + let closing = client.runtime.resources.closing.clone(); + let worker_stop = stop.clone(); + let worker_state = Arc::clone(&state); + let context = request.context().clone(); + let worker = tokio::spawn(async move { + let renewal = async { + loop { + tokio::time::sleep(Duration::from_secs(15)).await; + worker_state.renew().await?; + } + }; + let result = tokio::select! { + () = worker_stop.cancelled() => Ok(()), + () = closing.cancelled() => Err(Error::Shutdown), + result = context.run(renewal) => result, + }; + if result.is_err() { + worker_state.failed.store(true, Ordering::Release); + } + }); + Ok(Self { state, stop, worker }) + } + + pub(super) async fn check(&self) -> Result<()> { + if self.state.failed.load(Ordering::Acquire) { + return Err(Error::Cancelled); + } + if let Some(session) = self.state.session { + DatasetSessions::new(Arc::clone(&self.state.authority)) + .check_open(&self.state.identity, session) + .await?; + } + Ok(()) + } +} + +impl Protection { + async fn renew(&self) -> Result<()> { + if let Some(session) = self.session { + DatasetSessions::new(Arc::clone(&self.authority)) + .resume(&self.identity, session, now()) + .await?; + } + self.authority + .touch_read_handle(&self.identity, self.snapshot, self.handle, now(), 60) + .await?; + Ok(()) + } +} + +impl Drop for ReadProtection { + fn drop(&mut self) { + self.stop.cancel(); + self.worker.abort(); + let state = Arc::clone(&self.state); + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + runtime.spawn(async move { + let release = + state + .authority + .release_read_handle(&state.identity, state.snapshot, state.handle); + if !matches!( + tokio::time::timeout(Duration::from_secs(5), release).await, + Ok(Ok(())) + ) { + tracing::error!( + "Dataset read release failed; inactivity expiry retains cleanup responsibility" + ); + } + }); + } + } +} diff --git a/lib/crowdb-sdk-native/src/dataset/session.rs b/lib/crowdb-sdk-native/src/dataset/session.rs new file mode 100644 index 000000000..8ae5c957a --- /dev/null +++ b/lib/crowdb-sdk-native/src/dataset/session.rs @@ -0,0 +1,68 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{now, DatasetClient}; +use crate::{OperationContext, Result}; +use crowdb_access_dataset::{ + DatasetSessions, OperationId, ReadCancellation, ReadPreparation, ReadSession, SessionSpecV2, +}; + +impl DatasetClient { + /// Advances one bounded preparation window; alias and Split binding are durable. + /// # Errors + /// Returns divergent preparation, selection/worker limits or storage failure. + pub async fn prepare_read( + &self, + context: OperationContext, + spec: &SessionSpecV2, + page: u64, + ) -> Result { + self.control(context, |scope| async move { + Ok(DatasetSessions::new(scope.authority) + .prepare_split_read(&scope.identity, spec, page, now(), &ReadCancellation::new()) + .await?) + }) + .await + } + + /// Resumes the acknowledged cursor. Buffer release never changes this cursor. + /// # Errors + /// Returns closed/missing session, cancellation or storage failure. + pub async fn resume_read(&self, context: OperationContext, id: OperationId) -> Result { + self.control(context, |scope| async move { + Ok(DatasetSessions::new(scope.authority) + .resume(&scope.identity, id, now()) + .await?) + }) + .await + } + + /// # Errors + /// Rejects forged/divergent cursors, expired sessions or storage failure. + pub async fn acknowledge( + &self, + context: OperationContext, + id: OperationId, + expected: &str, + next: &str, + ) -> Result { + self.control(context, |scope| async move { + Ok(DatasetSessions::new(scope.authority) + .acknowledge(&scope.identity, id, expected, next, now()) + .await?) + }) + .await + } + + /// Closes only this read and its active storage protection, without ack. + /// # Errors + /// Returns missing preparation, admission or storage failure. + pub async fn close_read(&self, context: OperationContext, id: OperationId) -> Result { + self.control(context, |scope| async move { + Ok(DatasetSessions::new(scope.authority) + .cancel_split_read(&scope.identity, id) + .await?) + }) + .await + } +} diff --git a/lib/crowdb-sdk-native/src/dataset/upload.rs b/lib/crowdb-sdk-native/src/dataset/upload.rs new file mode 100644 index 000000000..c2b3d0cf8 --- /dev/null +++ b/lib/crowdb-sdk-native/src/dataset/upload.rs @@ -0,0 +1,99 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{now, DatasetClient}; +use crate::{ + buffer::{BufferLease, FrozenObjectBuffer}, + Error, OperationContext, Result, +}; +use crowdb_access_dataset::{DatasetUploads, OperationId, UploadPart, UploadRecord}; +use md5::{Digest, Md5}; + +impl DatasetClient { + /// # Errors + /// Returns invalid size/digest/expiry, divergent identity or storage failure. + pub async fn create_upload( + &self, + context: OperationContext, + id: OperationId, + length: u64, + part_size: u32, + md5: [u8; 16], + expires_at: u64, + ) -> Result { + let location = DatasetUploads::location(&self.identity, id)?; + self.control(context, |scope| async move { + Ok(scope + .uploads + .create(&location, length, part_size, md5, expires_at) + .await?) + }) + .await + } + + /// Keeps the admitted caller owner until the durable part write completes. + /// # Errors + /// Rejects foreign owners, invalid parts or ambiguous storage outcomes. + pub async fn upload_part( + &self, + context: OperationContext, + id: OperationId, + index: u32, + data: BufferLease, + ) -> Result { + if !data.belongs_to(&self.runtime.resources) { + return Err(Error::BufferRange); + } + let location = DatasetUploads::location(&self.identity, id)?; + self.control(context, |scope| async move { + Ok(scope + .uploads + .put_part(&location, index, data.into_bytes(), now()) + .await?) + }) + .await + } + + /// Writes payload prepared in final frame slots, preserving its registered owner. + /// # Errors + /// Rejects foreign owners, invalid sizes, conflict or failed durability. + pub async fn upload_framed_part( + &self, + context: OperationContext, + id: OperationId, + index: u32, + data: FrozenObjectBuffer, + ) -> Result { + if !data.belongs_to(&self.runtime.resources) { + return Err(Error::BufferRange); + } + let location = DatasetUploads::location(&self.identity, id)?; + self.control(context, |scope| async move { + let mut digest = Md5::new(); + for payload in data.payload_views() { + digest.update(&payload); + } + Ok(scope + .uploads + .put_framed_part( + &location, + index, + Box::new(data.owner), + digest.finalize().into(), + now(), + ) + .await?) + }) + .await + } + + /// # Errors + /// Rejects incomplete/corrupt parts, expired uploads or uncertain storage. + pub async fn complete_upload(&self, context: OperationContext, id: OperationId) -> Result { + let location = DatasetUploads::location(&self.identity, id)?; + self.control(context, |scope| async move { + Ok(scope.uploads.complete(&location, now()).await?) + }) + .await + } +} diff --git a/lib/crowdb-sdk-native/src/error.rs b/lib/crowdb-sdk-native/src/error.rs index 5fe758abc..e8ea98672 100644 --- a/lib/crowdb-sdk-native/src/error.rs +++ b/lib/crowdb-sdk-native/src/error.rs @@ -70,6 +70,11 @@ pub enum Error { BufferRange, #[error("S3 operation failed: {0:?}")] S3(crowdb_access_s3::S3ErrorCode), + #[error("Dataset operation failed: {kind:?}: {detail}")] + Dataset { + kind: crate::dataset::FailureKind, + detail: String, + }, #[error("object was not modified")] NotModified, #[error("object checksum does not match its published metadata")] diff --git a/lib/crowdb-sdk-native/src/lib.rs b/lib/crowdb-sdk-native/src/lib.rs index e6ccb926e..72b01b933 100644 --- a/lib/crowdb-sdk-native/src/lib.rs +++ b/lib/crowdb-sdk-native/src/lib.rs @@ -9,6 +9,7 @@ mod admission; pub mod buffer; pub mod context; +pub mod dataset; pub mod error; pub mod runtime; pub mod s3; diff --git a/lib/crowdb-sdk-native/tests/common/dataset.rs b/lib/crowdb-sdk-native/tests/common/dataset.rs new file mode 100644 index 000000000..4d047ea32 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/common/dataset.rs @@ -0,0 +1,89 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_access_dataset::{ + ChunkKvDatasetStore, DatasetAuthority, DatasetIdentity, FieldDefinition, FieldLocator, FieldRecord, + ManifestRecord, NamespacePath, SampleRecord, SchemaRecord, SnapshotId, +}; +use crowdb_sdk_native::{dataset::DatasetClient, ClientRuntime, OperationContext, RuntimeLimits}; +use md5::{Digest, Md5}; +use std::{ + num::NonZeroU64, + sync::Arc, + time::{Duration, Instant}, +}; + +pub struct TestFixture { + pub runtime: ClientRuntime, + pub client: DatasetClient, + pub snapshot: SnapshotId, +} + +pub fn context(id: u64) -> OperationContext { + OperationContext::new( + NonZeroU64::new(id).unwrap(), + Instant::now() + Duration::from_secs(10), + ) +} + +impl TestFixture { + pub async fn new(held: usize) -> Self { + let metadata = super::metadata::client().await; + let authority = DatasetAuthority::new(Arc::new(ChunkKvDatasetStore::new(metadata.clone()))); + let runtime = super::storage::runtime_with_limits( + metadata, + RuntimeLimits { + held_bytes: held, + ..RuntimeLimits::default() + }, + ); + let identity = DatasetIdentity::new(NamespacePath::root(), "native").unwrap(); + let client = DatasetClient::new(&runtime, identity.clone()).unwrap(); + client.create(context(1)).await.unwrap(); + let snapshot = authority + .publish_snapshot(&identity, None, manifest()) + .await + .unwrap(); + Self { + runtime, + client, + snapshot, + } + } +} + +fn manifest() -> Vec { + let manifest = ManifestRecord { + version: 1, + snapshot: SnapshotId::random(), + schema: SchemaRecord { + version: 1, + fields: vec![ + FieldDefinition { + name: "value".into(), + required: true, + }, + FieldDefinition { + name: "optional".into(), + required: false, + }, + ], + }, + samples: (0..3) + .map(|index| { + let value = vec![index; 16]; + SampleRecord { + sample_id: vec![index + 1], + fields: vec![FieldRecord { + name: "value".into(), + value: FieldLocator::Inline { + md5: Md5::digest(&value).into(), + value, + }, + }], + } + }) + .collect(), + }; + bincode::serialize(&manifest).unwrap() +} diff --git a/lib/crowdb-sdk-native/tests/common/dataset_composition.rs b/lib/crowdb-sdk-native/tests/common/dataset_composition.rs new file mode 100644 index 000000000..a1ede9f61 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/common/dataset_composition.rs @@ -0,0 +1,113 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_access_dataset::{composition::*, *}; +use crowdb_sdk_native::dataset::DatasetClient; + +pub async fn publish(client: &DatasetClient, location: Vec, digest: [u8; 16]) -> SnapshotId { + let operation = OperationId::random(); + client + .begin_composition( + super::context(20), + operation, + &CompositionInput { + parent: None, + source: CompositionSource::All, + schema: Some(SchemaRecord { + version: 1, + fields: vec![FieldDefinition { + name: "value".into(), + required: true, + }], + }), + inherit_parent_splits: false, + parent_split_revision: None, + splits: vec![], + policy: None, + }, + ) + .await + .unwrap(); + let edited = client + .edit_composition( + super::context(21), + operation, + &CompositionEdit { + id: OperationId::random(), + expected_revision: 1, + schema: None, + splits: None, + policy: None, + samples: vec![SampleChange::Upsert(SampleRecord { + sample_id: b"one".to_vec(), + fields: vec![FieldRecord { + name: "value".into(), + value: FieldLocator::Chunk { + location, + length: 8, + md5: digest, + }, + }], + })], + }, + ) + .await + .unwrap(); + assert_eq!(edited.uploads.staged_uploads, 1); + let snapshot = prepare(client, operation).await; + client + .publish_composition( + super::context(24), + operation, + &CommitAttempt { + version: 1, + id: OperationId::random(), + target: CommitTarget::Snapshot { snapshot }, + expected_head: None, + }, + ) + .await + .unwrap(); + let mut cleaned = false; + for _ in 0..32 { + if client + .cleanup_composition(super::context(25), operation) + .await + .unwrap() + .complete + { + cleaned = true; + break; + } + } + assert!(cleaned); + snapshot +} + +async fn prepare(client: &DatasetClient, operation: OperationId) -> SnapshotId { + let mut evaluated = false; + for page in 0..32 { + if client + .evaluate_composition(super::context(22), operation, 2, page) + .await + .unwrap() + .complete + { + evaluated = true; + break; + } + } + assert!(evaluated); + let mut snapshot = None; + for _ in 0..32 { + let status = client + .prepare_composition(super::context(23), operation, 2) + .await + .unwrap(); + if status.uploads.publication_ready { + snapshot = status.uploads.candidate; + break; + } + } + snapshot.expect("bounded preparation and ownership complete") +} diff --git a/lib/crowdb-sdk-native/tests/common/metadata.rs b/lib/crowdb-sdk-native/tests/common/metadata.rs index 07f7e3f93..278e07a17 100644 --- a/lib/crowdb-sdk-native/tests/common/metadata.rs +++ b/lib/crowdb-sdk-native/tests/common/metadata.rs @@ -119,6 +119,26 @@ impl ChunkKvTransport for TestMetadataTransport { .unwrap(); Ok(receiver.await.unwrap()) } + + async fn scan_bounded( + &self, + endpoint: &str, + request: &ScanRequest, + max_bytes: usize, + ) -> Result { + let response = self.scan(endpoint, request).await?; + if let Ok(OperationResult::Scan { items, .. }) = &response.result { + if items + .iter() + .map(|value| value.key.len() + value.value.len()) + .sum::() + > max_bytes + { + return Err(crowdb_chunk_kv_client::ClientError::TooLarge); + } + } + Ok(response) + } } fn matches(observed: Option<&RpcValue>, condition: &RpcCompareCondition) -> bool { diff --git a/lib/crowdb-sdk-native/tests/common/storage.rs b/lib/crowdb-sdk-native/tests/common/storage.rs index 6400e7dfe..5bf35cef5 100644 --- a/lib/crowdb-sdk-native/tests/common/storage.rs +++ b/lib/crowdb-sdk-native/tests/common/storage.rs @@ -53,6 +53,13 @@ pub async fn runtime() -> ClientRuntime { } pub fn runtime_with_metadata(metadata: Arc) -> ClientRuntime { + runtime_with_limits(metadata, RuntimeLimits::default()) +} + +pub fn runtime_with_limits( + metadata: Arc, + limits: RuntimeLimits, +) -> ClientRuntime { let metadata_transport = Arc::new(ChunkKvRpcTransport::with_config(8, 1, 1, &ChunkKvRpcTransportConfig::default()).unwrap()); ClientRuntime::from_clients( @@ -66,7 +73,7 @@ pub fn runtime_with_metadata(metadata: Arc ChunkStrip { impl ChunkAllocator for TestWriteStorage { async fn allocate_chunk(&self, request: AllocateChunkRequest) -> Result { assert_eq!(request.strip_type, StripType::Mirror as i32); - let id = request.chunk_id.unwrap(); + let id = request.chunk_id.unwrap_or_else(|| ChunkId { + high: 91, + low: self.allocations.fetch_add(1, Ordering::Relaxed) as u64 + 1, + }); let chunk = Chunk { id: Some(id), modify_ts: 1, state: ChunkState::Active as i32, capacity: request.write_granularity * request.strip_count, chunk_type: request.chunk_type, + writer_epoch: request.writer_epoch, strips: (0..request.strip_count) .map(|index| strip(id, index, request.write_granularity, request.copy_count)) .collect(), @@ -165,7 +171,7 @@ impl ChunkAllocator for TestWriteStorage { }); } let start = u32::try_from(previous.strips.len()).unwrap(); - let size = previous.strips[0].unit_kb * request.strip_size; + let size = previous.strips.first().map_or(1024, |strip| strip.unit_kb) * request.strip_size; let strips: Vec<_> = (0..request.strip_count) .map(|index| strip(id, start + index, size, request.copy_count)) .collect(); @@ -187,6 +193,20 @@ impl ChunkAllocator for TestWriteStorage { }); Ok(SealChunkResponse { chunk: Some(chunk) }) } + + async fn advance_chunk_write( + &self, + request: AdvanceChunkWriteRequest, + ) -> Result { + let id = request.chunk_id.unwrap(); + let previous = self.chunk(id); + assert_eq!(previous.writer_epoch, request.writer_epoch); + assert_eq!(previous.modify_ts, request.expected_modify_ts); + let chunk = self.update(id, |chunk| { + chunk.acknowledged_cursor = request.acknowledged_cursor; + }); + Ok(AdvanceChunkWriteResponse { chunk: Some(chunk) }) + } async fn delete_chunk(&self, request: DeleteChunkRequest) -> Result { self.deletes.fetch_add(1, Ordering::AcqRel); let chunk = self.update(request.chunk_id.unwrap(), |chunk| { diff --git a/lib/crowdb-sdk-native/tests/dataset_payload_test.rs b/lib/crowdb-sdk-native/tests/dataset_payload_test.rs new file mode 100644 index 000000000..f5c08900e --- /dev/null +++ b/lib/crowdb-sdk-native/tests/dataset_payload_test.rs @@ -0,0 +1,102 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/dataset_composition.rs"] +mod composition; +#[path = "common/metadata.rs"] +mod metadata; +#[path = "common/write_storage.rs"] +mod write_storage; + +use crowdb_access_dataset::{DatasetIdentity, DatasetUploads, NamespacePath, OperationId}; +use crowdb_access_s3::metadata::TenantId; +use crowdb_protocol::frame::MAX_FRAME_BYTES; +use crowdb_sdk_native::{ + dataset::{BatchRequest, DatasetClient}, + s3::S3Client, + OperationContext, +}; +use md5::{Digest, Md5}; +use std::{ + num::NonZeroU64, + time::{Duration, Instant}, +}; + +fn context(id: u64) -> OperationContext { + OperationContext::new( + NonZeroU64::new(id).unwrap(), + Instant::now() + Duration::from_secs(10), + ) +} + +#[tokio::test] +async fn framed_dataset_upload_and_s3_share_runtime_but_preserve_independent_authority() { + let backend = write_storage::TestWriteStorage::new(); + let runtime = backend.runtime(metadata::client().await); + let identity = DatasetIdentity::new(NamespacePath::root(), "payload").unwrap(); + let mut policy = + crowdb_chunk_client::SmallWritePolicy::new(crowdb_protocol::chunkdb::rpc::ChunkType::Dataset); + policy.conversion_enabled = false; + policy.max_pipelines = 1; + let dataset = DatasetClient::with_writer_policy(&runtime, identity.clone(), &policy).unwrap(); + let s3 = S3Client::new(runtime.clone(), TenantId::new(b"tenant".to_vec()).unwrap()).unwrap(); + s3.create_bucket(context(1), b"bucket").await.unwrap(); + dataset.create(context(2)).await.unwrap(); + let upload = OperationId::random(); + let digest: [u8; 16] = Md5::digest(b"retained").into(); + dataset + .create_upload(context(3), upload, 8, 8, digest, u64::MAX) + .await + .unwrap(); + let buffer = runtime + .allocate_producer(u32::try_from(MAX_FRAME_BYTES).unwrap()) + .unwrap() + .initialize_framed(8, 0, |_, output| output.copy_from_slice(b"retained")) + .unwrap(); + let views = buffer.payload_views(); + let pointer = views[0].as_ptr(); + dataset + .upload_framed_part(context(4), upload, 0, buffer) + .await + .unwrap(); + assert!( + backend.data.iter().any(|entry| { + let bytes = entry.value(); + let start = bytes.as_ptr() as usize; + (start..start + bytes.len()).contains(&(pointer as usize)) + }), + "storage must retain the original framed producer allocation" + ); + dataset.complete_upload(context(5), upload).await.unwrap(); + let location = DatasetUploads::location(&identity, upload).unwrap(); + let snapshot = composition::publish(&dataset, location.clone(), digest).await; + dataset + .control(context(60), |scope| async move { + assert!(matches!( + scope.uploads.abort(&location).await, + Err(crowdb_access_dataset::UploadError::Conflict) + )); + Ok(()) + }) + .await + .unwrap(); + let mut read = dataset + .read_batch( + context(7), + BatchRequest { + snapshot, + sample_ids: vec![b"one".to_vec()], + fields: vec!["value".into()], + }, + ) + .await + .unwrap(); + let held = read.body.next().await.unwrap().unwrap().buffer; + assert_eq!(held.as_slice(), b"retained"); + assert!(read.body.next().await.unwrap().is_none()); + drop(dataset); + assert!(s3.head_bucket(context(8), b"bucket").await.is_ok()); + runtime.shutdown().await.unwrap(); + assert_eq!(held.as_slice(), b"retained"); + assert_eq!(views[0].as_ref(), b"retained"); +} diff --git a/lib/crowdb-sdk-native/tests/dataset_read_test.rs b/lib/crowdb-sdk-native/tests/dataset_read_test.rs new file mode 100644 index 000000000..745a91b05 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/dataset_read_test.rs @@ -0,0 +1,156 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/dataset.rs"] +mod dataset; +#[path = "common/metadata.rs"] +mod metadata; +#[path = "common/storage.rs"] +mod storage; + +use crowdb_access_dataset::{OperationId, Selection, SessionPartition, SessionSpec, SessionSpecV2}; +use crowdb_sdk_native::{ + dataset::{BatchRequest, FieldState}, + Error, +}; +use dataset::{context, TestFixture}; +use std::time::Duration; + +fn spec() -> SessionSpecV2 { + SessionSpecV2 { + read: SessionSpec { + id: OperationId::random(), + snapshot: "latest".into(), + selection: Selection::Explicit(vec![vec![1], vec![2], vec![3]]), + fields: vec!["value".into(), "optional".into()], + batch_size: 1, + partition: SessionPartition { + worker: 0, + workers: 1, + rank: 0, + world_size: 1, + }, + shuffle: None, + }, + split: None, + } +} + +#[tokio::test] +async fn fetching_and_releasing_owners_never_acknowledge_progress() { + let fixture = TestFixture::new(1024).await; + let spec = spec(); + let mut preparation = fixture.client.prepare_read(context(2), &spec, 0).await.unwrap(); + for _ in 0..16 { + if preparation.complete { + break; + } + preparation = fixture + .client + .prepare_read(context(2), &spec, preparation.next_page) + .await + .unwrap(); + } + assert!(preparation.complete); + let cursor = preparation.session.unwrap().acknowledged; + let mut first = fixture.client.next_batch(context(3), spec.read.id).await.unwrap(); + assert_eq!(first.header.snapshot, fixture.snapshot); + assert_eq!(first.header.fields[1].state, FieldState::Missing); + let held = first.body.next().await.unwrap().unwrap().buffer; + assert_eq!(held.as_slice(), &[0; 16]); + assert!(first.body.next().await.unwrap().is_none()); + drop(held); + assert_eq!( + fixture + .client + .resume_read(context(4), spec.read.id) + .await + .unwrap() + .acknowledged, + cursor + ); + let mut replay = fixture.client.next_batch(context(5), spec.read.id).await.unwrap(); + assert_eq!(replay.header.next, first.header.next); + assert_eq!( + replay.body.next().await.unwrap().unwrap().buffer.as_slice(), + &[0; 16] + ); + assert!(replay.body.next().await.unwrap().is_none()); + fixture + .client + .acknowledge( + context(6), + spec.read.id, + &cursor, + first.header.next.as_ref().unwrap(), + ) + .await + .unwrap(); + let mut second = fixture.client.next_batch(context(7), spec.read.id).await.unwrap(); + assert_eq!(second.header.sample_ids, vec![vec![2]]); + let retained = second.body.next().await.unwrap().unwrap().buffer; + fixture.client.close_read(context(8), spec.read.id).await.unwrap(); + assert!(second.body.next().await.is_err()); + fixture.runtime.shutdown().await.unwrap(); + assert_eq!(retained.as_slice(), &[1; 16]); + drop(retained); + assert_eq!(fixture.runtime.resources().status().held_bytes, 0); +} + +#[tokio::test] +async fn dropped_wait_resumes_pending_data_without_copy_or_unbounded_prefetch() { + let fixture = TestFixture::new(16).await; + let mut read = fixture + .client + .read_batch( + context(2), + BatchRequest { + snapshot: fixture.snapshot, + sample_ids: vec![vec![1], vec![2]], + fields: vec!["value".into()], + }, + ) + .await + .unwrap(); + let first = read.body.next().await.unwrap().unwrap().buffer; + assert!(tokio::time::timeout(Duration::from_millis(10), read.body.next()) + .await + .is_err()); + assert_eq!(fixture.runtime.resources().status().held_bytes, 16); + assert_eq!(fixture.runtime.resources().status().compatibility_copy_bytes, 0); + drop(first); + let second = read.body.next().await.unwrap().unwrap(); + assert_eq!(second.field, 1); + assert_eq!(second.buffer.as_slice(), &[1; 16]); + assert!(read.body.next().await.unwrap().is_none()); + assert!(read.body.is_complete()); + assert_eq!(fixture.runtime.resources().status().requests, 0); +} + +#[tokio::test] +async fn cancellation_and_runtime_shutdown_preserve_delivered_owners() { + let fixture = TestFixture::new(32).await; + let context = context(2); + let cancel = context.cancellation.clone(); + let mut read = fixture + .client + .read_batch( + context, + BatchRequest { + snapshot: fixture.snapshot, + sample_ids: vec![vec![1], vec![2]], + fields: vec!["value".into()], + }, + ) + .await + .unwrap(); + let held = read.body.next().await.unwrap().unwrap().buffer; + cancel.cancel(); + assert!(matches!(read.body.next().await, Err(Error::Cancelled))); + assert!(matches!(read.body.next().await, Err(Error::Cancelled))); + assert!(!read.body.is_complete()); + fixture.runtime.shutdown().await.unwrap(); + assert_eq!(held.as_slice(), &[0; 16]); + drop(held); + assert_eq!(fixture.runtime.resources().status().held_bytes, 0); +} diff --git a/tools/rdma/run-native-dataset.py b/tools/rdma/run-native-dataset.py new file mode 100644 index 000000000..89c7ea36f --- /dev/null +++ b/tools/rdma/run-native-dataset.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Native Dataset against owned real services and one local HCA; no remote nodes.""" +import argparse +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import uuid + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "e2e-test")) +sys.path.insert(0, str(ROOT / "tools/sdk")) +from client import bundle_client +from fixture import docker +from image_identity import verify_server_sources +from native_package import package +from native_s3_hca import NativeS3HcaCluster + + +def interrupted(_signal, _frame): + raise KeyboardInterrupt("Dataset local HCA acceptance interrupted") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--address", default="10.200.1.1") + parser.add_argument("--device", default="rocep35s0f0") + parser.add_argument("--image", default="crowdb-node:dev") + args = parser.parse_args() + for number in (signal.SIGTERM, signal.SIGINT): + signal.signal(number, interrupted) + output = ROOT / ".crowdb-runtime/artifacts/native-dataset-hca" / uuid.uuid4().hex + output.mkdir(parents=True) + print(f"Artifacts: {output}", flush=True) + image = json.loads(docker("image", "inspect", args.image))[0] + labels = image["Config"].get("Labels") or {} + verify_server_sources(ROOT, labels.get("org.opencontainers.image.revision"), + labels.get("org.crowdb.source.tree") or None) + os.environ["CROWDB_RDMA_ENABLED"] = "0" + python = package(output / "python") + bundle = bundle_client(output / "client") + with NativeS3HcaCluster(image["Id"], output, bundle, address=args.address, device=args.device) as cluster: + environment = dict(os.environ, CROWDB_MANAGEMENT_SEEDS=",".join(cluster.origins[:3]), + CROWDB_RDMA_SOURCE_ADDR=args.address, CROWDB_RDMA_DEVICE=args.device, + CROWDB_DATASET_CONSUMER_ROOT=str(output), NO_PROXY="*", no_proxy="*") + for phase in ("write", "verify"): + result = subprocess.run([str(python), str(ROOT / "lib/crowdb-sdk-native-python/tests/hca_consumer.py")], + env=dict(environment, CROWDB_DATASET_PHASE=phase), text=True, capture_output=True, timeout=600) + log = result.stdout + result.stderr + output.joinpath(f"{phase}.log").write_text(log) + print(log, end="", flush=True) + result.check_returncode() + if phase == "write": + cluster.crash_restart() + print("Native Dataset local HCA acceptance passed", flush=True) + + +if __name__ == "__main__": + main() diff --git a/tools/sdk/dataset_workload.py b/tools/sdk/dataset_workload.py index e099f0fa9..9af2f2899 100644 --- a/tools/sdk/dataset_workload.py +++ b/tools/sdk/dataset_workload.py @@ -17,6 +17,7 @@ from client import bundle_client from fixture import docker from image_identity import verify_server_sources +from native_package import package def interrupted(_signal, _frame): @@ -63,6 +64,8 @@ def main(): labels.get("org.crowdb.source.tree") or None) run(["cargo", "build", "--release", "--locked", "--manifest-path", str(ROOT / "sdk/Cargo.toml"), "-p", "crowdb-sdk-dataset", "--example", "workload"]) + run(["cargo", "build", "--release", "--locked", "-p", "crowdb-sdk-native", "--example", "dataset_workload"]) + native_python = package(output / "native-python") os.environ["CROWDB_RDMA_ENABLED"] = "0" bundle = bundle_client(output / "client") @@ -77,7 +80,7 @@ def pinned_command(*parts, **kwargs): with AccessDatasetCluster(image["Id"], output, bundle, command=pinned_command) as cluster: environment = dict(os.environ, CROWDB_DATASET_ENDPOINT=f"http://{cluster.addresses[12]}:7503", - CROWDB_DATASET_NAME="workload", CROWDB_DATASET_READ_TOKEN="token", + CROWDB_DATASET_NAME="workload", CROWDB_MANAGEMENT_SEEDS=",".join(cluster.origins[:3]), CROWDB_DATASET_READ_TOKEN="token", CROWDB_DATASET_CONSUMER_ROOT=str(output), NO_PROXY="*", no_proxy="*") manifest = {"image": image["Id"], "labels": labels, "client_cpu": args.client_cpu, "metadata_memory": args.metadata_memory, @@ -87,15 +90,33 @@ def pinned_command(*parts, **kwargs): "source_revision": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), "scope": "same-host pinned CPU functional workload, not isolated production hardware", "before": server_usage(cluster.servers)} - with (output / "http.log").open("w") as log: - try: - run(["taskset", "-c", args.client_cpu, str(ROOT / "sdk/target/release/examples/workload")], - env=environment, stdout=log, stderr=subprocess.STDOUT) - finally: - print((output / "http.log").read_text(), end="", flush=True) - manifest["after"] = server_usage(cluster.servers) - (output / "manifest.json").write_text(json.dumps(manifest, indent=2)) - print("Dataset HTTP baseline complete", flush=True) + tests = ROOT / "lib/crowdb-sdk-native-python/tests" + commands = [ + ("http", [ROOT / "sdk/target/release/examples/workload"]), + ("native-rust", [ROOT / "target/release/examples/dataset_workload"]), + ("native-python", [native_python, tests / "workload.py"]), + ("native-parity", [native_python, tests / "real_consumer.py"]), + ] + try: + for name, command in commands: + before = server_usage(cluster.servers) + with (output / f"{name}.log").open("w") as log: + try: + run(["taskset", "-c", args.client_cpu, *map(str, command)], env=environment, + stdout=log, stderr=subprocess.STDOUT) + finally: + print((output / f"{name}.log").read_text(), end="", flush=True) + manifest[name] = {"before": before, "after": server_usage(cluster.servers)} + for node in (12, 11): + cluster.control({"action": "kill", "node": node}) + for node in (11, 12): + cluster.control({"action": "start", "node": node}) + run([str(native_python), str(tests / "real_consumer.py")], + env=dict(environment, CROWDB_DATASET_PHASE="verify")) + finally: + manifest["after"] = server_usage(cluster.servers) + (output / "manifest.json").write_text(json.dumps(manifest, indent=2)) + print("Dataset HTTP/native comparison and restart acceptance complete", flush=True) if __name__ == "__main__": diff --git a/tools/sdk/native_package.py b/tools/sdk/native_package.py new file mode 100644 index 000000000..cfe142785 --- /dev/null +++ b/tools/sdk/native_package.py @@ -0,0 +1,38 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Build and install the optional main-workspace native Dataset wheel.""" +import argparse +import os +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[2] + + +def run(command, **kwargs): + print("Native packaging:", " ".join(map(str, command)), flush=True) + subprocess.run(command, check=True, timeout=900, **kwargs) + + +def package(destination): + destination = Path(destination).resolve() + destination.mkdir(parents=True, exist_ok=True) + project = ROOT / "lib/crowdb-sdk-native-python" + run(["maturin", "build", "--release", "--locked", "--interpreter", sys.executable, + "--out", str(destination)], cwd=project) + wheel, = destination.glob("crowdb_dataset_native-*.whl") + environment = destination / "consumer" + run([sys.executable, "-m", "venv", "--without-pip", str(environment)]) + python = environment / "bin/python" + env = dict(os.environ, PATH=str(environment / "bin")) + run([sys.executable, "-m", "pip", "--python", str(python), "install", "--no-index", + "--no-deps", "--force-reinstall", str(wheel)], env=env, cwd=destination) + run([str(python), "-c", "import crowdb_dataset_native as n; print(n.NativeRuntime, n.Lease)"], env=env, cwd=destination) + return python + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--destination", required=True) + package(parser.parse_args().destination) From 7ecf09716654a85ff3f1f2aa3fe8a46cf1135586 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 18:53:22 +0800 Subject: [PATCH 080/258] Verify native Dataset routing and projection semantics --- doc/working/plan-dataset-native-sdk.md | 12 +++ lib/crowdb-sdk-native/tests/common/dataset.rs | 21 +++-- .../tests/dataset_read_test.rs | 1 + lib/crowdb-sdk-native/tests/topology_test.rs | 81 +++++++++++++++++++ 4 files changed, 108 insertions(+), 7 deletions(-) diff --git a/doc/working/plan-dataset-native-sdk.md b/doc/working/plan-dataset-native-sdk.md index 2f778ced3..1492f7855 100644 --- a/doc/working/plan-dataset-native-sdk.md +++ b/doc/working/plan-dataset-native-sdk.md @@ -121,3 +121,15 @@ Native/Python all-target, all-feature clippy passes. The release native wheel builds, bundles its native shared libraries and imports from a separate installed consumer. Full Python semantics, common-snapshot performance and real-service restart are pending; the wheel build/import alone is not binding acceptance. + +The shared Dataset/Access Server all-target tests, standalone HTTP SDK boundary +and Rust tests, isolated HTTP sdist/wheel tests (10 Python cases), and 163 fixture +unit tests pass. Workspace all-feature clippy, rs-lint, version consistency and +fmt pass. Environment-gated HCA/installed-HTTP-Python cases in the default Rust +suite remain separate acceptance steps, not implicit passes. + +The implementation is committed in `f5fefdfe`. Additional native tests confirm +Dataset snapshot selection survives stale-owner refresh and that projected +missing fields and tombstones remain distinct. The service image is being +rebuilt from the committed source before running the new workload and local-HCA +scripts. Both scripts own and remove only their own fixture resources. diff --git a/lib/crowdb-sdk-native/tests/common/dataset.rs b/lib/crowdb-sdk-native/tests/common/dataset.rs index 4d047ea32..05afb0f97 100644 --- a/lib/crowdb-sdk-native/tests/common/dataset.rs +++ b/lib/crowdb-sdk-native/tests/common/dataset.rs @@ -72,15 +72,22 @@ fn manifest() -> Vec { samples: (0..3) .map(|index| { let value = vec![index; 16]; + let mut fields = vec![FieldRecord { + name: "value".into(), + value: FieldLocator::Inline { + md5: Md5::digest(&value).into(), + value, + }, + }]; + if index == 1 { + fields.push(FieldRecord { + name: "optional".into(), + value: FieldLocator::Tombstone, + }); + } SampleRecord { sample_id: vec![index + 1], - fields: vec![FieldRecord { - name: "value".into(), - value: FieldLocator::Inline { - md5: Md5::digest(&value).into(), - value, - }, - }], + fields, } }) .collect(), diff --git a/lib/crowdb-sdk-native/tests/dataset_read_test.rs b/lib/crowdb-sdk-native/tests/dataset_read_test.rs index 745a91b05..7fdc8913e 100644 --- a/lib/crowdb-sdk-native/tests/dataset_read_test.rs +++ b/lib/crowdb-sdk-native/tests/dataset_read_test.rs @@ -88,6 +88,7 @@ async fn fetching_and_releasing_owners_never_acknowledge_progress() { .unwrap(); let mut second = fixture.client.next_batch(context(7), spec.read.id).await.unwrap(); assert_eq!(second.header.sample_ids, vec![vec![2]]); + assert_eq!(second.header.fields[1].state, FieldState::Tombstone); let retained = second.body.next().await.unwrap().unwrap().buffer; fixture.client.close_read(context(8), spec.read.id).await.unwrap(); assert!(second.body.next().await.is_err()); diff --git a/lib/crowdb-sdk-native/tests/topology_test.rs b/lib/crowdb-sdk-native/tests/topology_test.rs index 1c9c36af0..44d0c47f3 100644 --- a/lib/crowdb-sdk-native/tests/topology_test.rs +++ b/lib/crowdb-sdk-native/tests/topology_test.rs @@ -105,3 +105,84 @@ async fn retained_view_cannot_pin_s3_operations_to_a_moved_owner() { ); runtime.shutdown().await.unwrap(); } + +#[tokio::test] +async fn dataset_keeps_snapshot_selection_when_stale_owner_refreshes() { + use crowdb_access_dataset::{ + DatasetIdentity, FieldDefinition, FieldLocator, FieldRecord, ManifestRecord, NamespacePath, + SampleRecord, SchemaRecord, SnapshotId, + }; + use crowdb_sdk_native::dataset::{BatchRequest, DatasetClient}; + use md5::{Digest, Md5}; + + let source = Arc::new(topology::TestCatalogSource { + generation: 1.into(), + unavailable: false.into(), + }); + let route = Arc::new(metadata::TestRouteChange { + generation: 1.into(), + ..Default::default() + }); + let metadata = metadata::client_with_route(source.clone(), route.clone()).await; + let runtime = storage::runtime_with_metadata(metadata); + let client = DatasetClient::new( + &runtime, + DatasetIdentity::new(NamespacePath::root(), "routed").unwrap(), + ) + .unwrap(); + client.create(context(20)).await.unwrap(); + let snapshot = client + .control(context(21), |s| async move { + let manifest = ManifestRecord { + version: 1, + snapshot: SnapshotId::random(), + schema: SchemaRecord { + version: 1, + fields: vec![FieldDefinition { + name: "v".into(), + required: true, + }], + }, + samples: vec![SampleRecord { + sample_id: b"id".to_vec(), + fields: vec![FieldRecord { + name: "v".into(), + value: FieldLocator::Inline { + value: b"selected".to_vec(), + md5: Md5::digest(b"selected").into(), + }, + }], + }], + }; + Ok(s.authority + .publish_snapshot(&s.identity, None, bincode::serialize(&manifest).unwrap()) + .await?) + }) + .await + .unwrap(); + let old = runtime.topology().unwrap(); + source.generation.store(2, Ordering::Release); + route.generation.store(2, Ordering::Release); + let mut read = client + .read_batch( + context(22), + BatchRequest { + snapshot, + sample_ids: vec![b"id".to_vec()], + fields: vec!["v".into()], + }, + ) + .await + .unwrap(); + assert_eq!(read.header.snapshot, snapshot); + assert_eq!( + read.body.next().await.unwrap().unwrap().buffer.as_slice(), + b"selected" + ); + assert!(read.body.next().await.unwrap().is_none()); + assert!(route.rejected.load(Ordering::Acquire) >= 1); + assert_eq!(route.accepted.load(Ordering::Acquire), 2); + assert_eq!(runtime.topology().unwrap().catalog().generation(), 2); + assert_eq!(old.catalog().generation(), 1); + runtime.shutdown().await.unwrap(); +} From 4700cf38d88b6c3c206ce5e9078ea1630d251814 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 19:23:30 +0800 Subject: [PATCH 081/258] Prevent Dataset read lease updates from starving under contention --- .../src/dataset/api/batch.rs | 3 +- doc/working/plan-dataset-native-sdk.md | 16 ++++ .../src/authority/access.rs | 18 +++-- .../tests/access_contention_test.rs | 78 +++++++++++++++++++ .../tests/common/access_store.rs | 66 ++++++++++++++++ 5 files changed, 172 insertions(+), 9 deletions(-) create mode 100644 lib/crowdb-access-dataset/tests/access_contention_test.rs create mode 100644 lib/crowdb-access-dataset/tests/common/access_store.rs diff --git a/app/crowdb-access-server/src/dataset/api/batch.rs b/app/crowdb-access-server/src/dataset/api/batch.rs index 003542ddf..b40b7d6e0 100644 --- a/app/crowdb-access-server/src/dataset/api/batch.rs +++ b/app/crowdb-access-server/src/dataset/api/batch.rs @@ -89,7 +89,8 @@ impl DatasetHttpService { let lease = Lease::acquire(authority.clone(), identity, id, session).await?; let weak = Arc::downgrade(&lease); tokio::spawn(async move { - let mut timer = tokio::time::interval(std::time::Duration::from_secs(15)); + let period = std::time::Duration::from_secs(15); + let mut timer = tokio::time::interval_at(tokio::time::Instant::now() + period, period); loop { timer.tick().await; let Some(lease) = weak.upgrade() else { break }; diff --git a/doc/working/plan-dataset-native-sdk.md b/doc/working/plan-dataset-native-sdk.md index 1492f7855..0d05108d1 100644 --- a/doc/working/plan-dataset-native-sdk.md +++ b/doc/working/plan-dataset-native-sdk.md @@ -133,3 +133,19 @@ Dataset snapshot selection survives stale-owner refresh and that projected missing fields and tombstones remain distinct. The service image is being rebuilt from the committed source before running the new workload and local-HCA scripts. Both scripts own and remove only their own fixture resources. + +Real-service comparison exposed a shared read-lease contention bug, independently +reproduced in runs `3258f05c74d84ecfb76bca7f6065bc27` and +`f79a47a4449b4035a7c8761f830a9020`. HTTP singleton trials passed; concurrency three +returned 409 after a lease update lost all 64 CAS attempts. All containers stayed +alive without OOM. Temporary diagnostics showed another lease update advancing +the record between each GET and CAS (final expected revision 2925, observed 2930). +These runs are failed acceptance, not comparable native performance. + +The shared authority now recomputes from the CAS conflict observation, preserving +other handles and reclaim fences without reopening a point-read race each time. +HTTP heartbeats begin after 15 seconds instead of immediately duplicating lease +acquisition. A deterministic competing-renewal test fails before this fix and +passes afterward, including independent release and terminal reclaim protection. +Temporary diagnostics are removed. Full affected tests and rebuilt real-service +acceptance are running before this item can close. diff --git a/lib/crowdb-access-dataset/src/authority/access.rs b/lib/crowdb-access-dataset/src/authority/access.rs index e0df1c41a..965189942 100644 --- a/lib/crowdb-access-dataset/src/authority/access.rs +++ b/lib/crowdb-access-dataset/src/authority/access.rs @@ -76,8 +76,8 @@ impl DatasetAuthority { mut change: impl FnMut(&mut Access) -> Result<(), AuthorityError>, ) -> Result<(), AuthorityError> { let key = key::read_lease(identity, snapshot)?; + let mut old = self.store.get(&key).await?; for _ in 0..64 { - let old = self.store.get(&key).await?; let mut access = old .as_ref() .map(|v| decode_access(&v.bytes)) @@ -85,13 +85,15 @@ impl DatasetAuthority { .unwrap_or_default(); change(&mut access)?; let bytes = encode_access(access)?; - if matches!( - self.store - .compare_exchange(&key, old.as_ref().map(|v| v.bytes.as_slice()), &bytes) - .await?, - CasOutcome::Applied(_) - ) { - return Ok(()); + match self + .store + .compare_exchange(&key, old.as_ref().map(|v| v.bytes.as_slice()), &bytes) + .await? + { + CasOutcome::Applied(_) => return Ok(()), + // Recompute against the sequencer's observation without opening another + // point-read race against concurrent acquisitions and releases. + CasOutcome::Conflict(observed) => old = observed, } } Err(AuthorityError::HeadConflict) diff --git a/lib/crowdb-access-dataset/tests/access_contention_test.rs b/lib/crowdb-access-dataset/tests/access_contention_test.rs new file mode 100644 index 000000000..5cc1fa937 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/access_contention_test.rs @@ -0,0 +1,78 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/access_store.rs"] +mod access_store; +#[path = "common/publication.rs"] +mod publication; +use crowdb_access_dataset::*; +use std::sync::{atomic::Ordering, Arc}; + +#[tokio::test] +async fn competing_renewal_does_not_starve_acquisition_or_release_another_reader() { + let identity = DatasetIdentity::new(NamespacePath::root(), "lease-contention").unwrap(); + let inner = access_store::store(); + let setup = DatasetAuthority::new(inner.clone()); + setup.create_dataset(identity.clone()).await.unwrap(); + let snapshot = setup + .publish_snapshot(&identity, None, publication::publication_manifest(1)) + .await + .unwrap(); + let store = Arc::new(access_store::TestAccessStore { + inner, + race: true.into(), + renewals: 0.into(), + identity: identity.clone(), + snapshot, + other: OperationId::random(), + }); + let authority = DatasetAuthority::new(store.clone()); + let reader = OperationId::random(); + authority + .touch_read_handle(&identity, snapshot, reader, u64::MAX / 2, 60) + .await + .unwrap(); + authority + .release_read_handle(&identity, snapshot, reader) + .await + .unwrap(); + assert_eq!(store.renewals.load(Ordering::Relaxed), 2); + store.race.store(false, Ordering::Relaxed); + // Remove head protection so only the competing reader prevents reclamation. + let next = setup + .prepare_snapshot_candidate( + &identity, + None, + publication::publication_manifest(2), + OperationId::random(), + ) + .await + .unwrap(); + setup + .publish_snapshot_candidate( + &identity, + &CommitAttempt { + version: 1, + id: OperationId::random(), + target: CommitTarget::Snapshot { snapshot: next }, + expected_head: setup.inspect_head(&identity).await.unwrap().generation, + }, + ) + .await + .unwrap(); + assert!(matches!( + authority.begin_reclaim(&identity, snapshot).await, + Err(AuthorityError::SnapshotProtected) + )); + authority + .release_read_handle(&identity, snapshot, store.other) + .await + .unwrap(); + authority.begin_reclaim(&identity, snapshot).await.unwrap(); + assert!(matches!( + authority + .touch_read_handle(&identity, snapshot, reader, u64::MAX / 2, 60) + .await, + Err(AuthorityError::SnapshotProtected) + )); +} diff --git a/lib/crowdb-access-dataset/tests/common/access_store.rs b/lib/crowdb-access-dataset/tests/common/access_store.rs new file mode 100644 index 000000000..f069de29d --- /dev/null +++ b/lib/crowdb-access-dataset/tests/common/access_store.rs @@ -0,0 +1,66 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "dataset_store.rs"] +#[allow(dead_code)] +mod dataset_store; +use crowdb_access_dataset::*; +use std::sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, +}; + +pub struct TestAccessStore { + pub inner: Arc, + pub race: AtomicBool, + pub renewals: AtomicU64, + pub identity: DatasetIdentity, + pub snapshot: SnapshotId, + pub other: OperationId, +} + +pub fn store() -> Arc { + Arc::new(dataset_store::TestDatasetStore::default()) +} + +#[async_trait::async_trait] +impl DatasetStore for TestAccessStore { + async fn get_bounded( + &self, + key: &[u8], + budget: MetadataScanBudget, + cancel: &ReadCancellation, + ) -> Result, StoreError> { + self.inner.get_bounded(key, budget, cancel).await + } + + async fn scan_page( + &self, + request: &MetadataScan, + cancel: &ReadCancellation, + ) -> Result { + self.inner.scan_page(request, cancel).await + } + + async fn get(&self, key: &[u8]) -> Result, StoreError> { + let value = self.inner.get(key).await?; + if self.race.load(Ordering::Relaxed) { + // A different reader renews between our point read and conditional write. + let time = u64::MAX / 2 + self.renewals.fetch_add(1, Ordering::Relaxed); + DatasetAuthority::new(self.inner.clone()) + .touch_read_handle(&self.identity, self.snapshot, self.other, time, 3600) + .await + .unwrap(); + } + Ok(value) + } + + async fn compare_exchange( + &self, + key: &[u8], + expected: Option<&[u8]>, + value: &[u8], + ) -> Result { + self.inner.compare_exchange(key, expected, value).await + } +} From 1c56e7ad62184b851b407ecf6e6a8e8698169a9a Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 19:35:16 +0800 Subject: [PATCH 082/258] Desynchronize contended Dataset read lease updates --- doc/working/plan-dataset-native-sdk.md | 11 ++++++ .../src/authority/access.rs | 15 ++++++- .../tests/access_contention_test.rs | 39 +++++++++++++++++++ .../tests/common/access_store.rs | 4 ++ 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/doc/working/plan-dataset-native-sdk.md b/doc/working/plan-dataset-native-sdk.md index 0d05108d1..3b0010c53 100644 --- a/doc/working/plan-dataset-native-sdk.md +++ b/doc/working/plan-dataset-native-sdk.md @@ -149,3 +149,14 @@ acquisition. A deterministic competing-renewal test fails before this fix and passes afterward, including independent release and terminal reclaim protection. Temporary diagnostics are removed. Full affected tests and rebuilt real-service acceptance are running before this item can close. + +Run `9b155417702648d88850778d946737df` still exhausted the same lease CAS bound +at concurrency three after that first fix. Reusing the conflict observation +alone does not prevent competing acquisitions and asynchronous releases from +repeatedly colliding. A second regression synchronizes 96 independent readers: +the previous implementation fails acquisition; bounded randomized contention +windows allow all acquisitions and releases to finish. The CAS budget remains +64; only repeated conflicts wait 1-16 ms, then refresh the observation. No HTTP +retry, lock, or lease/reclaim invariant is changed. Focused contention and reclaim +tests, formatting and affected all-feature clippy pass. Rebuilt real-service +acceptance remains necessary before claiming this resolves the load failure. diff --git a/lib/crowdb-access-dataset/src/authority/access.rs b/lib/crowdb-access-dataset/src/authority/access.rs index 965189942..bb5429434 100644 --- a/lib/crowdb-access-dataset/src/authority/access.rs +++ b/lib/crowdb-access-dataset/src/authority/access.rs @@ -77,7 +77,7 @@ impl DatasetAuthority { ) -> Result<(), AuthorityError> { let key = key::read_lease(identity, snapshot)?; let mut old = self.store.get(&key).await?; - for _ in 0..64 { + for attempt in 0..64 { let mut access = old .as_ref() .map(|v| decode_access(&v.bytes)) @@ -93,7 +93,18 @@ impl DatasetAuthority { CasOutcome::Applied(_) => return Ok(()), // Recompute against the sequencer's observation without opening another // point-read race against concurrent acquisitions and releases. - CasOutcome::Conflict(observed) => old = observed, + CasOutcome::Conflict(observed) => { + old = observed; + if attempt % 2 == 1 && attempt < 63 { + // Concurrent acquisitions and asynchronous releases must not + // repeatedly rejoin the same CAS wave and starve each other. + let window = 1_u64 << (attempt / 2).min(4); + let delay = 1 + u64::from(uuid::Uuid::new_v4().as_bytes()[0]) % window; + tokio::time::sleep(std::time::Duration::from_millis(delay)).await; + // The conflict observation predates the contention window. + old = self.store.get(&key).await?; + } + } } } Err(AuthorityError::HeadConflict) diff --git a/lib/crowdb-access-dataset/tests/access_contention_test.rs b/lib/crowdb-access-dataset/tests/access_contention_test.rs index 5cc1fa937..e40b68ea5 100644 --- a/lib/crowdb-access-dataset/tests/access_contention_test.rs +++ b/lib/crowdb-access-dataset/tests/access_contention_test.rs @@ -21,6 +21,7 @@ async fn competing_renewal_does_not_starve_acquisition_or_release_another_reader let store = Arc::new(access_store::TestAccessStore { inner, race: true.into(), + interleave: false, renewals: 0.into(), identity: identity.clone(), snapshot, @@ -76,3 +77,41 @@ async fn competing_renewal_does_not_starve_acquisition_or_release_another_reader Err(AuthorityError::SnapshotProtected) )); } + +#[tokio::test] +async fn concurrent_readers_acquire_and_release_without_exhausting_cas_budget() { + let identity = DatasetIdentity::new(NamespacePath::root(), "concurrent-leases").unwrap(); + let inner = access_store::store(); + let setup = DatasetAuthority::new(inner.clone()); + setup.create_dataset(identity.clone()).await.unwrap(); + let snapshot = setup + .publish_snapshot(&identity, None, publication::publication_manifest(1)) + .await + .unwrap(); + let store = Arc::new(access_store::TestAccessStore { + inner, + race: false.into(), + interleave: true, + renewals: 0.into(), + identity: identity.clone(), + snapshot, + other: OperationId::random(), + }); + let authority = DatasetAuthority::new(store); + let handles: Vec<_> = (0..96).map(|_| OperationId::random()).collect(); + for release in [false, true] { + let results = futures::future::join_all(handles.iter().map(|handle| async { + if release { + authority.release_read_handle(&identity, snapshot, *handle).await + } else { + authority + .touch_read_handle(&identity, snapshot, *handle, u64::MAX / 2, 60) + .await + } + })) + .await; + for (index, result) in results.into_iter().enumerate() { + result.unwrap_or_else(|error| panic!("reader {index}, release={release}: {error}")); + } + } +} diff --git a/lib/crowdb-access-dataset/tests/common/access_store.rs b/lib/crowdb-access-dataset/tests/common/access_store.rs index f069de29d..690666522 100644 --- a/lib/crowdb-access-dataset/tests/common/access_store.rs +++ b/lib/crowdb-access-dataset/tests/common/access_store.rs @@ -13,6 +13,7 @@ use std::sync::{ pub struct TestAccessStore { pub inner: Arc, pub race: AtomicBool, + pub interleave: bool, pub renewals: AtomicU64, pub identity: DatasetIdentity, pub snapshot: SnapshotId, @@ -61,6 +62,9 @@ impl DatasetStore for TestAccessStore { expected: Option<&[u8]>, value: &[u8], ) -> Result { + if self.interleave { + tokio::task::yield_now().await; + } self.inner.compare_exchange(key, expected, value).await } } From 40cf003e306afa98783e6d615cbb42a4e4036439 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 19:56:25 +0800 Subject: [PATCH 083/258] Scale lease contention windows with durable write latency --- doc/working/plan-dataset-native-sdk.md | 16 ++++++++++++++++ .../src/authority/access.rs | 9 +++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/doc/working/plan-dataset-native-sdk.md b/doc/working/plan-dataset-native-sdk.md index 3b0010c53..070f29a34 100644 --- a/doc/working/plan-dataset-native-sdk.md +++ b/doc/working/plan-dataset-native-sdk.md @@ -160,3 +160,19 @@ windows allow all acquisitions and releases to finish. The CAS budget remains retry, lock, or lease/reclaim invariant is changed. Focused contention and reclaim tests, formatting and affected all-feature clippy pass. Rebuilt real-service acceptance remains necessary before claiming this resolves the load failure. + +The fixed short window still failed in `ec575a0475a541e782ac269756cda8f5`. +Temporary complete conflict histories in diagnostic run +`cf24af1bf19a4c8c8307fd35d64e00e2` show other handles advancing the record at +every attempt, with durable CAS round trips around 28-35 ms. The contention +window now scales with that measured round trip (capped at 256 ms), refreshes +after waiting and preserves the 64-attempt bound. All diagnostics are removed. +Dataset all-target tests, fmt and workspace rs-lint pass. + +The exact HTTP reproduction `387d9466a9814362b11494353f2d605e` now completes +all six measured trials (1,536 requests plus warmups) with exact bytes and zero +errors. This is correctness evidence, not a throughput win: singleton rates +are 30.6-36.0 requests/s, while concurrency-three repeats degrade from 17.7 to +11.5 to 6.1 requests/s and tails reach seconds. The full common-snapshot +HTTP/native/Python comparison must retain this elapsed-run/state limitation; +sequential phase results cannot establish a transport speedup by themselves. diff --git a/lib/crowdb-access-dataset/src/authority/access.rs b/lib/crowdb-access-dataset/src/authority/access.rs index bb5429434..02f3aa054 100644 --- a/lib/crowdb-access-dataset/src/authority/access.rs +++ b/lib/crowdb-access-dataset/src/authority/access.rs @@ -85,6 +85,7 @@ impl DatasetAuthority { .unwrap_or_default(); change(&mut access)?; let bytes = encode_access(access)?; + let submitted = std::time::Instant::now(); match self .store .compare_exchange(&key, old.as_ref().map(|v| v.bytes.as_slice()), &bytes) @@ -98,8 +99,12 @@ impl DatasetAuthority { if attempt % 2 == 1 && attempt < 63 { // Concurrent acquisitions and asynchronous releases must not // repeatedly rejoin the same CAS wave and starve each other. - let window = 1_u64 << (attempt / 2).min(4); - let delay = 1 + u64::from(uuid::Uuid::new_v4().as_bytes()[0]) % window; + // Include the durable store round trip: a fixed sub-RTT + // window keeps slow writers in the same contention wave. + let round_trip = submitted.elapsed().as_millis().clamp(1, 32); + let window = round_trip * (1_u128 << (attempt / 2).min(3)); + let delay = 1 + u128::from(uuid::Uuid::new_v4().as_bytes()[0]) % window; + let delay = u64::try_from(delay).expect("contention window is at most 256 ms"); tokio::time::sleep(std::time::Duration::from_millis(delay)).await; // The conflict observation predates the contention window. old = self.store.get(&key).await?; From 1a4833fb6370f3c0d73734fa77fc5f42a80464bc Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 20:29:08 +0800 Subject: [PATCH 084/258] Fix native Python packaging and bounded multipart producers --- doc/working/plan-dataset-native-sdk.md | 39 +++++++ e2e-test/image_identity.py | 1 + lib/crowdb-sdk-native-python/README.md | 14 ++- lib/crowdb-sdk-native-python/src/client.rs | 12 +-- lib/crowdb-sdk-native-python/src/runtime.rs | 4 +- lib/crowdb-sdk-native/README.md | 6 ++ lib/crowdb-sdk-native/src/buffer.rs | 1 + .../src/buffer/framed_chain.rs | 84 +++++++++++++++ lib/crowdb-sdk-native/src/buffer/producer.rs | 102 ++++++++++++++++-- .../tests/dataset_payload_test.rs | 43 ++++++++ .../tests/framed_producer_test.rs | 42 ++++++++ pixi.lock | 24 +++++ pixi.toml | 7 ++ .../crowdb-sdk-dataset/examples/workload.rs | 4 + tools/sdk/dataset_workload.py | 9 +- tools/sdk/native_package.py | 15 ++- 16 files changed, 383 insertions(+), 24 deletions(-) create mode 100644 lib/crowdb-sdk-native/src/buffer/framed_chain.rs diff --git a/doc/working/plan-dataset-native-sdk.md b/doc/working/plan-dataset-native-sdk.md index 070f29a34..a2f696887 100644 --- a/doc/working/plan-dataset-native-sdk.md +++ b/doc/working/plan-dataset-native-sdk.md @@ -176,3 +176,42 @@ are 30.6-36.0 requests/s, while concurrency-three repeats degrade from 17.7 to 11.5 to 6.1 requests/s and tails reach seconds. The full common-snapshot HTTP/native/Python comparison must retain this elapsed-run/state limitation; sequential phase results cannot establish a transport speedup by themselves. + +Full comparison `8cf0653d34f74e0c955f5ac5642d1316` completes the six HTTP trials +without lease errors and three native Rust singleton trials with exact bytes. +The next native multi-flight phase fails with a Chunk-KV deadline because +metadata replicas 1 and 2 are OOM-killed at their 2-GiB limits (exit 137). +The metadata group's tree snapshots and immutable extent pages grow throughout +the run; `ChunkStream::publish_state` rewrites every historical extent page on +every append, and no production caller currently drives +`Partition::reclaim_published_checkpoint`. This is unfinished shared-storage +diagnosis, not accepted native performance. Do not increase memory, shorten +the workload, or retry the failed requests to count the comparison as passed. + +Ordinary `pixi run clean-env && pixi run test-e2e-dataset` passes in +`container-e2e/run-97e635e2d2054f1b8f6ff8d44e7a475c`, including HTTP publication, +restart/recovery and installed Rust/Python Split consumers. + +Repeated native wheel packaging exposed a cached extension hardlinked to +Maturin's repaired staging artifact: the second build references wheel-local +hashed SONAMEs unavailable to its linker search. Packaging now emits an +unrepaired Linux wheel, then runs external auditwheel repair on that wheel; +two consecutive build/install/import passes preserve the Cargo cache. Host +`libibverbs` and `librdmacm` remain external so the installed host providers +register devices with the same library instance. Bundling a renamed verbs +library had produced zero devices despite two available HCAs. + +The Python default namespace now matches HTTP/Rust `NamespacePath::root()`; +an empty namespace previously made the HTTP-published snapshot invisible. +The new `--semantics-only` workload mode preserves real publication, Python +semantics and restart while independently diagnosing the long-run OOM; it is +not performance acceptance. + +HCA run `abc5aeec5cea4ced80f79ea3392aa521` establishes native connections but +fails its first 4-MiB upload: producer allocations support at most 1 MiB. +The native facade now concatenates independently retained final-frame owners; +Python fills each bounded allocation once. The maximum-part regression +verifies every original payload address survives the durable writer, while +partial allocation failure, short interior frames and mixed budgets release +their credits correctly. Six focused producer/upload tests pass. Real HCA +and common-snapshot semantics are still pending. diff --git a/e2e-test/image_identity.py b/e2e-test/image_identity.py index 26182b555..03a0c8e70 100644 --- a/e2e-test/image_identity.py +++ b/e2e-test/image_identity.py @@ -78,6 +78,7 @@ # Independent SDK packaging builds host test consumers only; release staging # neither reads this helper nor packages its output. ":(exclude)tools/sdk/package.py", + ":(exclude)tools/sdk/native_package.py", # The pinned Dataset consumer/fixture runner never enters server staging. ":(exclude)tools/sdk/dataset_workload.py", # This host-only RPC benchmark is not a dependency or staged server binary. diff --git a/lib/crowdb-sdk-native-python/README.md b/lib/crowdb-sdk-native-python/README.md index 075abd659..41b3490b8 100644 --- a/lib/crowdb-sdk-native-python/README.md +++ b/lib/crowdb-sdk-native-python/README.md @@ -14,8 +14,10 @@ Build and install a wheel in an isolated consumer: pixi run -- python tools/sdk/native_package.py --destination /tmp/crowdb-native-python ``` -The wheel bundles linked native libraries. RDMA additionally requires a working -host HCA, provider and device access. `NativeRuntime(seeds)` explicitly selects +The Linux wheel bundles native dependencies except `libibverbs.so.1` and +`librdmacm.so.1`; install those host libraries with matching RDMA providers. +It retains the host-specific `linux` wheel tag. RDMA additionally requires a +working host HCA and device access. `NativeRuntime(seeds)` explicitly selects TCP; `options` accepts JSON containing `data_rdma`, `metadata_rdma` and `aggregate_rdma` overrides using the Rust `RdmaConfig` schema. Set `enabled` and `allow_tcp_fallback` separately per group; metadata RDMA requires its control/data @@ -23,6 +25,9 @@ class. Options also accept `held_bytes`, `producer_bytes` and `requests`. Runtim belongs to the caller. Create a new runtime in each worker process; inherited clients are rejected after fork. +`runtime.dataset(name)` uses the shared default `dataset-ns` namespace, matching +HTTP and Rust. Pass `namespace=[...]` to select an explicit namespace. + ```python import json import crowdb_dataset_native as native @@ -67,7 +72,8 @@ runtime.close() only after application processing succeeds. Split/worker binding uses the same shared session service as Rust and HTTP. - `upload_part(upload_id, index, bytes)` copies Python input once into final - transport frame slots and freezes it before submission. Publication uses + transport frame slots across allocations of at most 1 MiB and freezes them + before submission. A logical upload part can still contain 4 MiB. Publication uses upload-managed compositions and preserves durable attempt identity. - Typed exceptions include capacity, integrity, cancellation, closed runtime, deadline, conflict and uncertain mutation. An uncertain mutation does not @@ -78,3 +84,5 @@ runtime.close() run with `pixi run -- python tools/sdk/dataset_workload.py` after building the current shared service image. It uses one snapshot for HTTP and native clients, then validates native publication after storage restart. +Use `--semantics-only` to run installed-binding semantics and restart separately; +that mode produces no performance comparison. diff --git a/lib/crowdb-sdk-native-python/src/client.rs b/lib/crowdb-sdk-native-python/src/client.rs index 2f838b338..ffe37afac 100644 --- a/lib/crowdb-sdk-native-python/src/client.rs +++ b/lib/crowdb-sdk-native-python/src/client.rs @@ -132,20 +132,10 @@ impl Dataset { if data.is_empty() || data.len() > crowdb_access_dataset::MAX_UPLOAD_PART_BYTES { return Err(NativeError::new_err("invalid upload part size")); } - let capacity = crowdb_protocol::frame::MAX_FRAME_BYTES; - let frames = data - .len() - .div_ceil(crowdb_protocol::frame::MAX_FRAME_PAYLOAD_BYTES); - let capacity = - u32::try_from(frames * capacity).map_err(|_| NativeError::new_err("frame size overflow"))?; let buffer = self .resources .native() - .allocate_producer(capacity) - .map_err(|e| failure(&e))? - .initialize_framed(data.len(), 0, |offset, output| { - output.copy_from_slice(&data[offset..offset + output.len()]); - }) + .prepare_framed(data) .map_err(|e| failure(&e))?; let result = py .detach(|| { diff --git a/lib/crowdb-sdk-native-python/src/runtime.rs b/lib/crowdb-sdk-native-python/src/runtime.rs index 28eaf7061..7c4a96a57 100644 --- a/lib/crowdb-sdk-native-python/src/runtime.rs +++ b/lib/crowdb-sdk-native-python/src/runtime.rs @@ -122,7 +122,9 @@ impl NativeRuntime { #[pyo3(signature=(name, namespace=None))] fn dataset(&self, py: Python<'_>, name: String, namespace: Option>) -> PyResult { self.resources.check_process()?; - Dataset::new(py, self.resources.clone(), name, namespace.unwrap_or_default()) + let namespace = + namespace.unwrap_or_else(|| crowdb_access_dataset::NamespacePath::root().segments().to_vec()); + Dataset::new(py, self.resources.clone(), name, namespace) } fn close(&self, py: Python<'_>) -> PyResult<()> { diff --git a/lib/crowdb-sdk-native/README.md b/lib/crowdb-sdk-native/README.md index bfa37b762..6c40c6a30 100644 --- a/lib/crowdb-sdk-native/README.md +++ b/lib/crowdb-sdk-native/README.md @@ -76,6 +76,12 @@ async fn read_object(seeds: Vec) -> Result<(), Box +// Licensed under the Apache License, Version 2.0. + +//! Logical frame offsets over independently retained producer allocations. + +use bytes::Bytes; +use crowdb_access_s3::native_buffer::NativeFramedOwner; +use crowdb_chunk_client::FramedWriteBuffer; +use crowdb_protocol::{ + common::ChunkId, + frame::{FrameError, FrameMagic, MAX_FRAME_BYTES}, +}; +use std::ops::Range; + +pub(crate) struct FramedChain(pub Vec); + +impl FramedWriteBuffer for FramedChain { + fn logical_len(&self) -> u64 { + self.0.iter().map(FramedWriteBuffer::logical_len).sum() + } + + fn frame_count(&self) -> usize { + self.0.iter().map(FramedWriteBuffer::frame_count).sum() + } + + fn frame_payload_len(&self, mut index: usize) -> Option { + for owner in &self.0 { + if index < owner.frame_count() { + return owner.frame_payload_len(index); + } + index -= owner.frame_count(); + } + None + } + + fn prepare_frames(&mut self, magic: FrameMagic, write_time_ms: u64) -> Result<(), FrameError> { + for owner in &mut self.0 { + owner.prepare_frames(magic, write_time_ms)?; + } + Ok(()) + } + + fn finalize_frame( + &mut self, + mut index: usize, + magic: FrameMagic, + chunk_id: ChunkId, + write_time_ms: u64, + ) -> Result, FrameError> { + let mut offset = 0; + for owner in &mut self.0 { + if index < owner.frame_count() { + let range = owner.finalize_frame(index, magic, chunk_id, write_time_ms)?; + return Ok(offset + range.start..offset + range.end); + } + index -= owner.frame_count(); + offset += owner.frame_count() * MAX_FRAME_BYTES; + } + Err(FrameError::InvalidLocationRange) + } + + fn views(&self, range: Range) -> Result, FrameError> { + if range.is_empty() { + return Err(FrameError::InvalidLocationRange); + } + let mut offset = 0; + let mut covered = range.start; + let mut views = Vec::new(); + for owner in &self.0 { + let end = offset + owner.frame_count() * MAX_FRAME_BYTES; + let start = range.start.max(offset); + let stop = range.end.min(end); + if start < stop { + views.extend(owner.views(start - offset..stop - offset)?); + covered = stop; + } + offset = end; + } + if covered != range.end { + return Err(FrameError::InvalidLocationRange); + } + Ok(views) + } +} diff --git a/lib/crowdb-sdk-native/src/buffer/producer.rs b/lib/crowdb-sdk-native/src/buffer/producer.rs index 5ae8aedb5..d97cc15cb 100644 --- a/lib/crowdb-sdk-native/src/buffer/producer.rs +++ b/lib/crowdb-sdk-native/src/buffer/producer.rs @@ -3,14 +3,54 @@ //! Immutable framed producers retain runtime credits through physical completion. -use super::WritableLease; +use super::{framed_chain::FramedChain, WritableLease}; use crate::{Error, Result}; use crowdb_access_s3::native_buffer::NativeFramedOwner; use crowdb_chunk_client::FramedWriteBuffer; use crowdb_protocol::frame::{MAX_FRAME_BYTES, MAX_FRAME_PAYLOAD_BYTES}; +impl crate::ClientRuntime { + /// Copies caller bytes once into final frames across bounded allocations. + /// # Errors + /// Rejects empty input, shutdown or exhausted producer capacity. Partial + /// allocation failures release every admitted owner before returning. + pub fn prepare_framed(&self, data: &[u8]) -> Result { + let allocator = + self.storage.chunks.producer_allocator().ok_or_else(|| { + Error::Configuration("data transport has no native producer allocator".into()) + })?; + self.resources.prepare_framed(&allocator, data) + } +} + +impl crate::RuntimeResources { + /// Initialize final frame storage using the transport's 1-MiB size limit. + /// # Errors + /// Returns invalid input or capacity exhaustion, releasing partial work. + pub fn prepare_framed( + &self, + allocator: &crowdb_rpc_ffi::ProducerBufferAllocator, + data: &[u8], + ) -> Result { + let logical_capacity = (1 << 20) / MAX_FRAME_BYTES * MAX_FRAME_PAYLOAD_BYTES; + let mut buffers = Vec::new(); + for payload in data.chunks(logical_capacity) { + let physical = payload.len().div_ceil(MAX_FRAME_PAYLOAD_BYTES) * MAX_FRAME_BYTES; + let capacity = u32::try_from(physical).map_err(|_| Error::BufferRange)?; + buffers.push(self.allocate(allocator, capacity)?.initialize_framed( + payload.len(), + 0, + |offset, output| { + output.copy_from_slice(&payload[offset..offset + output.len()]); + }, + )?); + } + FrozenObjectBuffer::concatenate(buffers) + } +} + pub struct FrozenObjectBuffer { - pub(crate) owner: NativeFramedOwner, + pub(crate) owner: FramedChain, budget: std::sync::Arc, } @@ -25,17 +65,64 @@ impl FrozenObjectBuffer { #[must_use] pub fn allocation_capacity(&self) -> usize { - self.owner.allocation_capacity() + self.owner + .0 + .iter() + .map(NativeFramedOwner::allocation_capacity) + .sum() } #[must_use] pub fn registration_id(&self) -> u32 { - self.owner.registration_id() + let first = self.owner.0[0].registration_id(); + if self.owner.0.iter().all(|owner| owner.registration_id() == first) { + first + } else { + 0 + } } #[must_use] pub fn payload_views(&self) -> Vec { - self.owner.payload_views() + self.owner + .0 + .iter() + .flat_map(NativeFramedOwner::payload_views) + .collect() + } + + /// Joins final frame allocations without copying their payloads. + /// # Errors + /// Rejects empty input, mixed runtime budgets, overflowing lengths, or a + /// partial interior frame. Only the final allocation may have a short tail. + pub fn concatenate(buffers: Vec) -> Result { + let first = buffers.first().ok_or(Error::BufferRange)?; + let budget = std::sync::Arc::clone(&first.budget); + let mut owners = Vec::new(); + let mut frames = 0usize; + let mut capacity = 0usize; + for (index, buffer) in buffers.iter().enumerate() { + if !std::sync::Arc::ptr_eq(&budget, &buffer.budget) + || (index + 1 < buffers.len() + && buffer.length() != (buffer.owner.frame_count() * MAX_FRAME_PAYLOAD_BYTES) as u64) + { + return Err(Error::BufferRange); + } + frames = frames + .checked_add(buffer.owner.frame_count()) + .ok_or(Error::BufferRange)?; + capacity = capacity + .checked_add(buffer.allocation_capacity()) + .ok_or(Error::BufferRange)?; + } + frames.checked_mul(MAX_FRAME_BYTES).ok_or(Error::BufferRange)?; + for buffer in buffers { + owners.extend(buffer.owner.0); + } + Ok(Self { + owner: FramedChain(owners), + budget, + }) } } @@ -65,6 +152,9 @@ impl WritableLease { initialize, ) .map_err(|_| Error::BufferRange)?; - Ok(FrozenObjectBuffer { owner, budget }) + Ok(FrozenObjectBuffer { + owner: FramedChain(vec![owner]), + budget, + }) } } diff --git a/lib/crowdb-sdk-native/tests/dataset_payload_test.rs b/lib/crowdb-sdk-native/tests/dataset_payload_test.rs index f5c08900e..bffea1cd4 100644 --- a/lib/crowdb-sdk-native/tests/dataset_payload_test.rs +++ b/lib/crowdb-sdk-native/tests/dataset_payload_test.rs @@ -100,3 +100,46 @@ async fn framed_dataset_upload_and_s3_share_runtime_but_preserve_independent_aut assert_eq!(held.as_slice(), b"retained"); assert_eq!(views[0].as_ref(), b"retained"); } + +#[tokio::test] +async fn maximum_upload_part_keeps_every_producer_allocation_through_durable_write() { + let backend = write_storage::TestWriteStorage::new(); + let runtime = backend.runtime(metadata::client().await); + let identity = DatasetIdentity::new(NamespacePath::root(), "large-part").unwrap(); + let dataset = DatasetClient::new(&runtime, identity).unwrap(); + dataset.create(context(1)).await.unwrap(); + let data: Vec = (0..4 * 1024 * 1024) + .map(|i| u8::try_from(i % 251).unwrap()) + .collect(); + let upload = OperationId::random(); + dataset + .create_upload( + context(2), + upload, + data.len() as u64, + 4 << 20, + Md5::digest(&data).into(), + u64::MAX, + ) + .await + .unwrap(); + let buffer = runtime.prepare_framed(&data).unwrap(); + let views = buffer.payload_views(); + dataset + .upload_framed_part(context(3), upload, 0, buffer) + .await + .unwrap(); + for view in &views { + assert!( + backend.data.iter().any(|entry| { + let bytes = entry.value(); + let start = bytes.as_ptr() as usize; + (start..start + bytes.len()).contains(&(view.as_ptr() as usize)) + }), + "every physical write must retain its original producer allocation" + ); + } + dataset.complete_upload(context(4), upload).await.unwrap(); + assert_eq!(views.concat(), data); + runtime.shutdown().await.unwrap(); +} diff --git a/lib/crowdb-sdk-native/tests/framed_producer_test.rs b/lib/crowdb-sdk-native/tests/framed_producer_test.rs index 89c4c3eec..ecfc93a42 100644 --- a/lib/crowdb-sdk-native/tests/framed_producer_test.rs +++ b/lib/crowdb-sdk-native/tests/framed_producer_test.rs @@ -71,3 +71,45 @@ fn invalid_lengths_and_initializer_panics_release_native_storage_before_returnin .allocate(&allocator, u32::try_from(MAX_FRAME_BYTES).unwrap()) .is_ok()); } + +#[test] +fn multi_allocation_producers_release_partial_work_and_keep_external_views_charged() { + let allocator = ProducerBufferAllocator::new(Arc::new(RpcServer::new(None)), false); + let resources = RuntimeResources::new(RuntimeLimits { + producer_bytes: (1 << 20) + MAX_FRAME_BYTES, + ..RuntimeLimits::default() + }) + .unwrap(); + let large = vec![0x57; 2 << 20]; + assert!(resources.prepare_framed(&allocator, &large).is_err()); + assert_eq!(resources.status().producer_bytes, 0); + let length = 16 * MAX_FRAME_PAYLOAD_BYTES + 7; + let frozen = resources.prepare_framed(&allocator, &large[..length]).unwrap(); + assert_eq!(frozen.length(), length as u64); + let views = frozen.payload_views(); + assert_eq!(views.len(), 17); + assert!(views.iter().flatten().all(|byte| *byte == 0x57)); + assert_eq!(views.last().unwrap().len(), 7); + drop(frozen); + assert_eq!(resources.status().producer_bytes, (1 << 20) + MAX_FRAME_BYTES); + drop(views); + assert_eq!(resources.status().producer_bytes, 0); +} + +#[test] +fn concatenation_rejects_partial_interior_frames_and_foreign_budgets() { + use crowdb_sdk_native::buffer::FrozenObjectBuffer; + let allocator = ProducerBufferAllocator::new(Arc::new(RpcServer::new(None)), false); + let first = resources(); + let second = resources(); + let full = vec![0; MAX_FRAME_PAYLOAD_BYTES]; + let partial = first.prepare_framed(&allocator, b"short").unwrap(); + let next = first.prepare_framed(&allocator, &full).unwrap(); + assert!(FrozenObjectBuffer::concatenate(vec![partial, next]).is_err()); + assert_eq!(first.status().producer_bytes, 0); + let one = first.prepare_framed(&allocator, &full).unwrap(); + let two = second.prepare_framed(&allocator, b"tail").unwrap(); + assert!(FrozenObjectBuffer::concatenate(vec![one, two]).is_err()); + assert_eq!(first.status().producer_bytes, 0); + assert_eq!(second.status().producer_bytes, 0); +} diff --git a/pixi.lock b/pixi.lock index 89230649e..8750f97ec 100644 --- a/pixi.lock +++ b/pixi.lock @@ -126,6 +126,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/rust-std-x86_64-unknown-linux-gnu-1.98.0-h2c6d0dc_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sysroot_linux-64-2.28-h4ee821c_9.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda + - pypi: https://files.pythonhosted.org/packages/46/2a/f9697576603dae937727827505a6126a066affb227034e77e6f9068910da/pyelftools-0.33-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/78/33/475398f5eb069aa20910ec8486bdc62a87deadd8229494cbc80a1ddbf740/auditwheel-6.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fe/91/b495e19e2f5c503b540452b2039115e7b2363867e8c5ad4179eb752fa92c/maturin-1.15.0-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl osx-arm64: @@ -226,6 +229,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xz-tools-5.8.3-h8088a28_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zlib-1.3.2-h8088a28_3.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda + - pypi: https://files.pythonhosted.org/packages/46/2a/f9697576603dae937727827505a6126a066affb227034e77e6f9068910da/pyelftools-0.33-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/78/33/475398f5eb069aa20910ec8486bdc62a87deadd8229494cbc80a1ddbf740/auditwheel-6.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/eb/1f/2b431554e11687cdb1077e0cdadcc118c53f611086b3af00c8545a67c6a5/maturin-1.15.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl - pypi: https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl iceberg-e2e: @@ -9220,6 +9226,24 @@ packages: - zstd >=1.5.7,<1.6.0a0 size: 433687 timestamp: 1786599629846 +- pypi: https://files.pythonhosted.org/packages/46/2a/f9697576603dae937727827505a6126a066affb227034e77e6f9068910da/pyelftools-0.33-py3-none-any.whl + name: pyelftools + version: '0.33' + sha256: f215ad5f47d3f1373a21496a6c9e0707c622840d0622f23ff7ce08678b020036 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl + name: packaging + version: '26.3' + sha256: d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/78/33/475398f5eb069aa20910ec8486bdc62a87deadd8229494cbc80a1ddbf740/auditwheel-6.8.2-py3-none-any.whl + name: auditwheel + version: 6.8.2 + sha256: 05fd2dd4b743217cd5f66066adc95b03a079424b1f0a878ad1e40fe2cdea4704 + requires_dist: + - packaging>=20.9 + - pyelftools>=0.24 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/eb/1f/2b431554e11687cdb1077e0cdadcc118c53f611086b3af00c8545a67c6a5/maturin-1.15.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl name: maturin version: 1.15.0 diff --git a/pixi.toml b/pixi.toml index 4f95c28b4..9c218005c 100644 --- a/pixi.toml +++ b/pixi.toml @@ -201,6 +201,7 @@ tree-ubsan = { cmd = "bash tools/pixi-tasks/tree-ubsan.sh" } coverage = { cmd = "cargo tarpaulin --workspace --out Html --output-dir target/coverage --exclude-files '*/tests/*'", depends-on = ["install-deps"] } profile-write = { cmd = "bash tools/profiling/profile-write.sh", depends-on = ["build"] } +<<<<<<< HEAD ################################################################################ # OCI containers — build once, run isolated clusters, stop or clean owned resources ################################################################################ @@ -224,3 +225,9 @@ test-boto3-copy-error = { cmd = "python tools/pixi-tasks/test.py unit boto3" } [feature.iceberg-ecosystem.tasks] test-iceberg-container-ecosystem = "bash e2e-test/clients/ecosystem/run.sh" +======= +[pypi-dependencies] +maturin = ">=1.15, <2" +pip = ">=25, <27" +auditwheel = ">=6, <7" +>>>>>>> d3e285b2 (Fix native Python packaging and bounded multipart producers) diff --git a/sdk/crates/crowdb-sdk-dataset/examples/workload.rs b/sdk/crates/crowdb-sdk-dataset/examples/workload.rs index 9255ecd06..12dfafb6f 100644 --- a/sdk/crates/crowdb-sdk-dataset/examples/workload.rs +++ b/sdk/crates/crowdb-sdk-dataset/examples/workload.rs @@ -40,6 +40,10 @@ async fn main() -> Result<(), Failure> { } Err(error) => return Err(error.into()), }; + if std::env::args().any(|argument| argument == "--publish-only") { + println!("Published mixed-size corpus: {snapshot}; no workload measured"); + return Ok(()); + } for concurrency in [1, 3] { trial(&client, &snapshot, concurrency, 2).await?; for repeat in 0..3 { diff --git a/tools/sdk/dataset_workload.py b/tools/sdk/dataset_workload.py index 9af2f2899..893607499 100644 --- a/tools/sdk/dataset_workload.py +++ b/tools/sdk/dataset_workload.py @@ -52,6 +52,8 @@ def main(): parser.add_argument("--client-cpu", default="2") parser.add_argument("--server-cpus", default="4-15") parser.add_argument("--metadata-memory", default="2g") + parser.add_argument("--semantics-only", action="store_true", + help="Run installed-binding parity and restart without performance trials") args = parser.parse_args() for number in (signal.SIGTERM, signal.SIGINT): signal.signal(number, interrupted) @@ -88,6 +90,7 @@ def pinned_command(*parts, **kwargs): "cpu_topology": subprocess.check_output(["lscpu", "-e=CPU,CORE,SOCKET,NODE"], text=True), "seeds": cluster.origins[:3], "endpoints": cluster.addresses, "source_revision": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), + "semantics_only": args.semantics_only, "scope": "same-host pinned CPU functional workload, not isolated production hardware", "before": server_usage(cluster.servers)} tests = ROOT / "lib/crowdb-sdk-native-python/tests" @@ -97,6 +100,9 @@ def pinned_command(*parts, **kwargs): ("native-python", [native_python, tests / "workload.py"]), ("native-parity", [native_python, tests / "real_consumer.py"]), ] + if args.semantics_only: + commands = [("http-publication", [ROOT / "sdk/target/release/examples/workload", "--publish-only"]), + commands[-1]] try: for name, command in commands: before = server_usage(cluster.servers) @@ -116,7 +122,8 @@ def pinned_command(*parts, **kwargs): finally: manifest["after"] = server_usage(cluster.servers) (output / "manifest.json").write_text(json.dumps(manifest, indent=2)) - print("Dataset HTTP/native comparison and restart acceptance complete", flush=True) + print("Dataset installed-binding semantics and restart acceptance complete" if args.semantics_only else + "Dataset HTTP/native comparison and restart acceptance complete", flush=True) if __name__ == "__main__": diff --git a/tools/sdk/native_package.py b/tools/sdk/native_package.py index cfe142785..04bc33a1e 100644 --- a/tools/sdk/native_package.py +++ b/tools/sdk/native_package.py @@ -6,6 +6,7 @@ from pathlib import Path import subprocess import sys +import tempfile ROOT = Path(__file__).resolve().parents[2] @@ -19,8 +20,18 @@ def package(destination): destination = Path(destination).resolve() destination.mkdir(parents=True, exist_ok=True) project = ROOT / "lib/crowdb-sdk-native-python" - run(["maturin", "build", "--release", "--locked", "--interpreter", sys.executable, - "--out", str(destination)], cwd=project) + build = ["maturin", "build", "--release", "--locked", "--interpreter", sys.executable] + if sys.platform == "linux": + # Repair an extracted wheel, leaving Cargo's cached extension untouched. + # Verbs providers must register with the host's canonical libibverbs. + with tempfile.TemporaryDirectory(prefix="crowdb-native-wheel-") as directory: + run([*build, "--auditwheel", "skip", "--compatibility", "linux", + "--out", directory], cwd=project) + raw, = Path(directory).glob("*.whl") + run(["auditwheel", "repair", "--no-update-tags", "--exclude", "libibverbs.so.1", + "--exclude", "librdmacm.so.1", "--wheel-dir", str(destination), str(raw)]) + else: + run([*build, "--out", str(destination)], cwd=project) wheel, = destination.glob("crowdb_dataset_native-*.whl") environment = destination / "consumer" run([sys.executable, "-m", "venv", "--without-pip", str(environment)]) From e929474bced54822a9709ea894f5bbee32afb7b8 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 20:51:07 +0800 Subject: [PATCH 085/258] Reclaim snapshot-backed Paxos history with retained readers --- doc/design/kv/design-crowdb-kv-slot.md | 321 +++---- doc/design/kv/design-crowdb-kv-wal.md | 17 +- doc/doc_index.md | 2 +- doc/working/plan-dataset-native-sdk.md | 146 ++- doc/working/plan-rdma.md | 34 +- lib/crowdb-kv/src/cluster.rs | 4 + lib/crowdb-kv/src/cluster/group.rs | 3 + .../src/cluster/group_election_leader.rs | 21 +- .../src/cluster/group_maintenance.rs | 34 +- lib/crowdb-kv/src/cluster/group_membership.rs | 4 + lib/crowdb-kv/src/cluster/group_retention.rs | 148 +++ lib/crowdb-kv/src/cluster/group_tests.rs | 14 + lib/crowdb-kv/src/cluster/local_replica.rs | 4 + lib/crowdb-kv/src/cluster/replica.rs | 3 + lib/crowdb-kv/src/paxos/acceptor.rs | 79 +- lib/crowdb-kv/src/rpc/px_rpc_service.rs | 856 +++++++++++++++++- .../src/rpc/px_rpc_service/heartbeat.rs | 117 +++ lib/crowdb-kv/src/rpc/px_rpc_transport.rs | 22 +- .../src/rpc/px_rpc_transport/heartbeat.rs | 69 ++ .../tests/election_test/apply_loop_test.rs | 2 + .../tests/election_test/heartbeat_test.rs | 2 + lib/crowdb-kv/tests/group_test.rs | 4 + .../tests/group_test/retention_test.rs | 217 +++++ .../tests/paxos_test/acceptor_test.rs | 66 ++ lib/crowdb-kv/tests/store_test/status_test.rs | 2 + lib/crowdb-protocol/src/fbs/kv_consensus.fbs | 2 + lib/crowdb-protocol/src/mgmt.rs | 10 + lib/crowdb-protocol/src/types/kv_consensus.rs | 2 + pixi.toml | 4 +- 29 files changed, 1889 insertions(+), 320 deletions(-) create mode 100644 lib/crowdb-kv/src/cluster/group_retention.rs create mode 100644 lib/crowdb-kv/src/rpc/px_rpc_service/heartbeat.rs create mode 100644 lib/crowdb-kv/src/rpc/px_rpc_transport/heartbeat.rs create mode 100644 lib/crowdb-kv/tests/group_test/retention_test.rs diff --git a/doc/design/kv/design-crowdb-kv-slot.md b/doc/design/kv/design-crowdb-kv-slot.md index c423d2512..da3130fc4 100644 --- a/doc/design/kv/design-crowdb-kv-slot.md +++ b/doc/design/kv/design-crowdb-kv-slot.md @@ -1,7 +1,7 @@ -# CROWDB - Design: Slots — Parallel Pipelining & Concurrent Slot List +# CROWDB - Design: Slots — Parallel Pipelining & Acceptor Retention Depends on: [`design-crowdb-kv.md`](design-crowdb-kv.md) §5.1, §6.5, §7.3 Satisfies: [`design-crowdb-kv.md`](design-crowdb-kv.md) §5.1, §6.5, §7.3 @@ -9,7 +9,7 @@ Satisfies: [`design-crowdb-kv.md`](design-crowdb-kv.md) §5.1, §6.5, §7.3 This document covers two aspects of CROWDB's slot mechanism: - **Part A (§1–§14): Parallel Slot Pipelining** — the consensus protocol that distinguishes CROWDB from Raft-based KV systems. How the leader pipelines proposals, how gaps are detected and repaired, how the safe-slot is maintained, and how the system stays correct under concurrent in-flight slots. -- **Part B (§15–§22): Concurrent Sparse Slot List** — the `SlotList` data structure that backs the Acceptor's per-slot state. Chunked array, lock-free reads, trim/GC, reclamation. +- **Part B (§15–§22): Acceptor Retention** — guarded per-slot state, concurrent trim/reclamation and snapshot-backed retention certificates. ## Table of Contents @@ -28,14 +28,14 @@ This document covers two aspects of CROWDB's slot mechanism: - [12. Tunables and Defaults](#12-tunables-and-defaults) - [13. Correctness Analysis for Parallel Slot Writes](#13-correctness-analysis-for-parallel-slot-writes) - [14. Parallel-Slot Linearizability Analysis](#14-parallel-slot-linearizability-analysis) -- [15. Slot List: Problem Statement](#15-slot-list-problem-statement) -- [16. Slot List: Design Overview](#16-slot-list-design-overview) -- [17. Slot List: PxSlotNode](#17-slot-list-pxslotnode) -- [18. Slot List: Algorithms](#18-slot-list-algorithms) -- [19. Slot List: API Surface](#19-slot-list-api-surface) -- [20. Slot List: Correctness Invariants](#20-slot-list-correctness-invariants) -- [21. Slot List: Performance Model](#21-slot-list-performance-model) -- [22. Slot List: Risks, Reclamation & Open Questions](#22-slot-list-risks-reclamation--open-questions) +- [15. Acceptor Storage Requirements](#15-acceptor-storage-requirements) +- [16. Retained Slot Entries](#16-retained-slot-entries) +- [17. Per-Slot State](#17-per-slot-state) +- [18. Lookup and Reclamation](#18-lookup-and-reclamation) +- [19. Acceptor API](#19-acceptor-api) +- [20. Retention Invariants](#20-retention-invariants) +- [21. Storage Cost](#21-storage-cost) +- [22. Retention Limits](#22-retention-limits) - [23. Server-side Proposal Coalescing](#23-server-side-proposal-coalescing) --- @@ -78,7 +78,7 @@ The blind-ops premise from design-crowdb-kv.md §5.2](design-crowdb-kv.md) makes **When does the ack happen?** When the slot reaches `Chosen` *and* the leader's own learner has applied it. The leader's own learner application is required so that an immediately-following `Get` on the leader sees the new value (see [§6.1 of design-crowdb-kv.md](design-crowdb-kv.md#61-linearizable-leader-read)). -**Eviction.** Once a slot's record is `Applied` *and* its slot number is below the safe-slot, it can be dropped from the in-memory map. The WAL record stays until WAL GC catches up. +**Eviction.** Acceptor entries and WAL segments become eligible below the snapshot-backed reclamation watermark in §11. Local application alone does not authorize eviction. --- @@ -286,14 +286,30 @@ This ensures accepted-but-not-chosen slots are never applied, while allowing out ## 11. Interaction with Snapshot and WAL GC -Parallel slots interact with WAL GC through two watermarks: - -- `safe_slot` — every slot ≤ `safe_slot` is chosen and applied on every learner. -- `snapshot_slot` — the engine state at `snapshot_slot` is durably snapshotted on at least the leader and one peer. - -**WAL GC rule:** discard WAL records with `slot < min(safe_slot, snapshot_slot)`. Both must advance past a slot before its WAL record can be GC'd. Repair never needs GC'd slots; every slot ≤ safe_slot is already chosen on every learner, so it cannot be a gap. - -Detailed further in [`design-crowdb-kv-wal.md`](design-crowdb-kv-wal.md) §4. +Acceptor and WAL reclamation share a certificate distinct from the commit +frontier and voting-member `group_safe_slot`: + +- The leader takes the minimum contiguous-applied frontier of itself and every + configured remote learner, including non-voting learners. Unknown progress + is zero; replies arriving after quorum still update progress. +- It caps that frontier by its local durable snapshot and the greatest durable + snapshot of a voting peer. A singleton voting group needs only its own durable + snapshot, while any attached non-voting learners still constrain progress. +- Heartbeats carry `gc_before_slot` and `gc_membership_epoch`. A follower accepts + the certificate only from its known voting leader in the current term and + membership epoch. The retained certificate binds these identities and the + watermark atomically; a later epoch or term cannot relabel an old certificate. +- Each replica caps its certificate by its own contiguous-applied and durable + snapshot frontiers before reclaiming. A missing WAL/snapshot yields zero. +- Maintenance trims acceptor entries with `slot < gc_slot` and unlinks only WAL + segments wholly below that boundary. It retains the boundary slot. Outstanding + readers retain node/payload owners until they finish. + +A new group or leader tenure clears progress evidence. Membership replacement +also clears evidence, including when it adds a non-voting learner. A joining +replica that needs a prefix already reclaimed uses snapshot catch-up. + +See [WAL garbage collection](design-crowdb-kv-wal.md#7-garbage-collection). --- @@ -350,195 +366,84 @@ This would NOT be safe for `CAS` or `Increment`, which read current state. Not s --- -# Part B: Concurrent Sparse Slot List - -## 15. Slot List: Problem Statement - -The Acceptor's per-slot state (`promised` ballot + `accepted` entry) needs a data structure optimized for: - -- **O(1) insert by slot** — no re-allocation per slot; chunk created lazily. -- **O(1) hot-slot access** — tail-first lookup hits the newest chunk. -- **O(1) batch front-trim** — drop whole chunks, not individual slots. -- **Wait-free / lock-free reads** — readers never block each other. -- **Safe reclamation** — a chunk is freed only after all concurrent readers have passed it. - -`PxSlot` is a monotonically increasing `u64` assigned by an external sequencer. Access is strongly tail-biased (latest-slot prepare/accept/replay), while allocation may be sparse. - ---- - -## 16. Slot List: Design Overview - -A **chunked, reader-pinned concurrent sparse list** (`SlotList`): - -``` -SlotList -├─ head ──► Chunk { start: 1024, entries: [1024..2047] } (partially filled) -│ ⇅ -│ Chunk { start: 4096, entries: [4096..5119] } (sparse gap 2048..4095) -│ ⇅ -│ Chunk { start: 5120, entries: [5120..6143] } ◄── tail -│ -├─ trim_slot: AtomicU64 (slots below this are logically invalid) -└─ retired: AtomicPtr> -``` - -Each chunk is a fixed-size array (`SLOT_CHUNK_SIZE = 1024` slots). A slot index maps to a chunk via `chunk = start / SLOT_CHUNK_SIZE` and an intra-chunk offset. Chunks are doubly-linked so the hot path walks backward from `tail` while the general path walks forward from `head`. - -### Chunk layout - -```rust -const SLOT_CHUNK_SIZE: usize = 1024; - -struct Chunk { - start_slot: PxSlot, - entries: [AtomicPtr; SLOT_CHUNK_SIZE], - next: AtomicPtr>, - prev: AtomicPtr>, - live_count: AtomicUsize, - reader_refs: AtomicU32, - retired: AtomicBool, - _pad: [u8; 64], // cache-line padded to prevent false sharing -} -``` - -**Why `AtomicPtr` per slot?** Writers CAS from `null → Box::into_raw(Box::new(value))`. Readers pin the containing chunk (`reader_refs += 1`), then load the pointer. Retired chunks are freed only after `reader_refs == 0`. - -### List header - -```rust -pub struct SlotList { - head: AtomicPtr>, - tail: AtomicPtr>, - trim_slot: AtomicU64, - retired: AtomicPtr>, -} -``` - ---- - -## 17. Slot List: PxSlotNode - -For the `PxAcceptor` we store **both** the promised ballot and the accepted entry in a single node, eliminating the double-map indirection: - -```rust -pub struct PxSlotNode { - promised: AtomicPtr, - accepted: AtomicPtr, - retired_promised: AtomicPtr>, - retired_accepted: AtomicPtr>, -} - -pub struct PxAcceptor { - log: SlotList, -} -``` - -Paxos operations use `get_tail_ptr` / `get_ptr` to obtain the `AtomicPtr` for a slot, then CAS the node pointer itself (installing a default node if absent) before mutating fields inside the stable node. `PxSlotNode` uses deferred reclamation: when `cas_promised` / `cas_accepted` swaps an existing pointer, the old pointer goes into a per-field retired list, drained on `PxSlotNode::drop`. - ---- - -## 18. Slot List: Algorithms - -### 18.1 Insert (by external slot) - -The caller decides the exact `slot`; the list only stores the value. Slots may be non-consecutive (sparse). `insert` locates or lazily creates the chunk covering the slot, pins the chunk (`reader_refs += 1`), then CAS-installs the slot object from `null → ptr`. If another writer raced, the existing object is kept. `insert` never replaces an existing slot pointer; callers needing field-level CAS use `get_ptr` / `get_tail_ptr` directly. - -Chunk lookup (`find_or_create_chunk`) walks the doubly-linked list to find the predecessor/successor window around the aligned chunk start, then splices a new chunk in via CAS. Sparse gaps are cheap: no chunk is allocated for empty ranges. - -### 18.2 Get (head-first, general path) - -`get(slot)` walks from `head`, checking `trim_slot` first. When the covering chunk is found, it pins the chunk, loads the slot pointer, and returns a `SlotReadGuard`. Used for any already-known slot. - -### 18.3 Get Tail (tail-first, hot path) - -`get_tail(slot)` walks backward from `tail`. This is the normal Paxos read path: latest-slot prepare/accept/replay almost always hit the last chunk or one of its immediate predecessors. - -### 18.4 Trim (front GC) - -`trim(before_slot)` advances `trim_slot` via `fetch_max`, then walks from `head` unlinking chunks whose `end ≤ before_slot`. Unlinked chunks are marked `retired = true` and pushed onto the retired list. **Single-caller by contract.** Each `get` / `get_tail` checks `trim_slot` before touching any chunk, so newly arriving readers immediately reject trimmed slots. - -### 18.5 Chunk-Level Reclamation - -A background `reclaim()` walks the retired list and frees chunks whose `reader_refs == 0`. **Single-caller by contract.** This is safe because `trim_slot` is advanced before chunk unlinking; only readers that pinned the chunk before trim can still hold a reference. - -**Current limitation:** there is a narrow race between "reader has observed a chunk pointer" and "reader has incremented `reader_refs`". The current manual scheme is safe under a restricted envelope (single GC caller, disciplined reclaim timing). For fully general lock-free safety, use epoch/hazard pointers or `Arc>` ownership. - ---- - -## 19. Slot List: API Surface - -```rust -impl SlotList { - pub fn new() -> Self; - pub fn get(&self, slot: PxSlot) -> Option>; - pub fn get_tail(&self, slot: PxSlot) -> Option>; - pub fn get_ptr(&self, slot: PxSlot) -> Option>; - pub fn get_tail_ptr(&self, slot: PxSlot) -> Option>; - pub fn iter_range(&self, start: PxSlot, end_exclusive: PxSlot) -> SlotIter<'_, T>; - pub fn insert(&self, slot: PxSlot, value: T) -> SlotReadGuard<'_, T>; - pub fn trim(&self, before_slot: PxSlot); - pub fn reclaim(&self) -> usize; - pub fn trim_slot(&self) -> PxSlot; - pub fn len(&self) -> usize; -} -``` - -`SlotReadGuard` derefs to `&T` and decrements `reader_refs` on drop. `SlotPtrGuard` derefs to `&AtomicPtr` for caller-controlled CAS. - ---- - -## 20. Slot List: Correctness Invariants - -| Invariant | Why it matters | Enforced by | -|---|---|---| -| **I1 — External slot assignment** | Slot number chosen by caller; list only stores at that index | `insert` validates slot is inside the chunk range it targets | -| **I2 — Stable slot object** | Once installed, readers keep seeing the same pointer | `insert` / `get_ptr` only CASes `null → ptr`; later updates mutate fields inside the object | -| **I3 — Trim coherence** | Readers never observe a trimmed slot | `trim_slot` advanced before chunks are unlinked; every `get` checks it first | -| **I4 — No dangling reads** | A retired chunk is freed only after all pinned readers have dropped | Chunk-level `reader_refs` + retired list | -| **I5 — Sparse ordering** | Chunks stay ordered by `start_slot` | `find_window` and `link_between` preserve sorted doubly linked order | - ---- - -## 21. Slot List: Performance Model - -| Operation | Cost | Notes | -|---|---|---| -| `insert` (existing chunk) | 1 slot CAS + 1 reader-ref increment | Reuses existing slot object if present | -| `insert` (new chunk) | 1 allocation + chunk-link CAS | Amortised only when touching a previously empty chunk range | -| `get_tail` | 1 trim check + 1 chunk pin + 1 atomic load | Tail chunk is cache-hot; covers most reads | -| `get` (head-first) | 1 trim check + ≤ N chunk hops + 1 atomic load | N = number of live sparse chunks | -| `trim` | 1 watermark advance + O(retired chunks) unlink | Drops whole chunks, never individual slots | -| `reclaim` | O(retired chunks) | Frees only chunks whose `reader_refs == 0` | -| Memory overhead | ~8 bytes / slot (on 64-bit) | `AtomicPtr` per slot; chunk metadata negligible | - -Compared to `BTreeMap`: ~10× faster insert, ~5× faster latest-slot access, ~3× lower per-slot overhead. - ---- - -## 22. Slot List: Risks, Reclamation & Open Questions - -### 22.1 Risks - -| Risk | Mitigation | -|---|---| -| `unsafe` in raw-pointer linking | Review with `miri`; maintain `Arc` fallback for comparison | -| `reader_refs` leak due to forgotten guard drop | Keep all public reads guard-based; no bare `&T` escapes | -| Prev/next inconsistency under sparse insertion | Centralise linking in `find_window` / `link_between`; stress tests | -| Reader late-pin race | Short-term: single GC caller + disciplined reclaim. Long-term: epoch/hazard pointers or `Arc>` | - -### 22.2 PxSlotNode Reclamation Evolution - -**Current:** Replaced `promised`/`accepted` pointers are retired and reclaimed on `PxSlotNode::drop`. No replacement leak, no dangling-reference regression. - -**Future triggers:** Per-node retired-chain depth growing across GC cycles, RSS growth from field churn, or tail latency regression from large node-drop reclamation bursts. - -**Options:** (1) Epoch/hazard-based early reclaim (preferred long-term). (2) Guarded/owned return API replacing raw references. (3) `Arc` payload fields (simplest, higher overhead). - -### 22.3 Open Questions - -1. **Manual `reader_refs` vs `Arc>`:** Keep manual as target; validate against `Arc` prototype if risk grows. -2. **Chunk size:** Start with 1K; make `SLOT_CHUNK_SIZE` a `const` generic for benchmarking. -3. **Per-slot `AtomicPtr` vs inline storage:** Start with `AtomicPtr`; optimise if profiles show bottleneck. +# Part B: Acceptor Retention + +## 15. Acceptor Storage Requirements + +Per-slot promised ballots and accepted entries must support sparse, concurrent +Prepare/Accept operations while maintenance reclaims a proven-safe prefix. +Lookups must retain the node before accessing its fields. Reclamation must +support overlapping maintenance callers during group replacement. + +## 16. Retained Slot Entries + +`PxAcceptor` owns a `crossbeam_skiplist::SkipMap`, a +monotonic `trim_slot`, and a monotonic `highest_seen_slot`. Each operation keeps +the returned map `Entry` alive throughout field access. Removing an entry does +not invalidate an outstanding retained entry. Sparse slots allocate individual +nodes; absent slot ranges allocate nothing. + +## 17. Per-Slot State + +`PxSlotNode` contains atomic promised and accepted pointers. Replaced pointers +enter per-node retired lists and are freed only when the node is dropped. This +keeps a field read valid for the entire retained node lifetime. Returned log +entries clone their `Bytes` owner, so a caller may keep a payload after trim. + +## 18. Lookup and Reclamation + +- `open_slot` rejects a slot below the trim boundary, obtains or inserts a + retained node, then checks the boundary again. An insertion racing trim is + removed and rejected instead of resurrecting old history. +- Prepare/Accept keep the entry guard while reading or changing node fields. + Queries and ordered range scans clone values while retaining their entries. +- `trim` advances the boundary with `fetch_max`. `reclaim` removes entries below + that boundary; concurrent removers can safely encounter the same entry. +- A lookup already in progress may complete with its retained state. A newly + admitted request below the boundary is rejected. The boundary is exclusive. + +## 19. Acceptor API + +- `prepare`, `accept`, `accepted_at`, `promised_at`: per-slot protocol access. +- `accepted_iter_range`: ordered retained-history view; missing/trimmed slots + are absent. `JournalScan` reports a GC gap for a start below `trim_slot`. +- `accepted_log_tip`: highest retained accepted slot and its term. +- `highest_seen_slot`: highest slot ever opened, independent of reclamation. +- `trim`, `reclaim`, `trim_slot`, `retained_slots`: reclamation and observability. + +`ReplicaStatus.acceptor` exposes the exclusive trim boundary and retained slot +count. The count describes map entries, not payload bytes or process RSS. + +## 20. Retention Invariants + +- **R1 — Retained access:** every node field access holds a live map entry. +- **R2 — No resurrection:** a request admitted below the current trim boundary + cannot create visible accepted history. +- **R3 — Durable local recovery:** reclamation cannot exceed the local durable + snapshot or contiguous-applied frontier. +- **R4 — Learner coverage:** the leader certificate cannot exceed the least + applied frontier of any configured learner, including non-voting learners. +- **R5 — Certificate identity:** followers use only the certificate bound to + their current membership epoch, term and known voting leader. +- **R6 — Independent payload lifetime:** reclaiming a node does not invalidate + a log entry already cloned by a caller. + +## 21. Storage Cost + +The ordered map adds per-node allocation and lookup overhead. Reclamation walks +only the eligible prefix and releases historical payload owners as readers +finish. No throughput improvement follows from this representation alone; +measure request throughput, tails, retained slots and RSS under sustained load. + +## 22. Retention Limits + +A lagging configured learner or unavailable durable witness can hold retention +at zero. An operator must restore/catch up that learner or change membership; +maintenance cannot discard history merely to satisfy a memory target. A slot +with repeated ballot/value changes retains its old field versions until that +slot is reclaimed. Long-lived readers may delay physical release after logical +removal. The generic `PxSlotList` remains a separate utility and is not the +acceptor's reclamation mechanism. ## 23. Server-side Proposal Coalescing diff --git a/doc/design/kv/design-crowdb-kv-wal.md b/doc/design/kv/design-crowdb-kv-wal.md index cfc7212bd..8ecb3261b 100644 --- a/doc/design/kv/design-crowdb-kv-wal.md +++ b/doc/design/kv/design-crowdb-kv-wal.md @@ -478,13 +478,16 @@ WAL GC is the mechanism that keeps disk usage bounded. It interacts with snapsho ### 7.1 GC watermark -The watermark for a group's WAL GC is: +The local watermark is `min(reclamation_certificate, local_contiguous_applied, +local_durable_snapshot)`. The leader derives the certificate from every +configured learner's applied progress and durable snapshots on itself plus a +voting peer (local durability alone for a singleton voting group). Followers +receive the certificate in a heartbeat bound to the current membership epoch, +term and known voting leader. A missing certificate or local snapshot gives zero. -``` - gc_slot = min(safe_slot, snapshot_slot) -``` - -Records with `slot < gc_slot` are eligible for GC. The two-watermark rule is justified in [`design-crowdb-kv-slot.md`](design-crowdb-kv-slot.md) §11. +Only records with `slot < gc_slot` are eligible. The same boundary trims +in-memory acceptor history; retained readers keep their owners alive. The full +contract is in [slot retention](design-crowdb-kv-slot.md#11-interaction-with-snapshot-and-wal-gc). ### 7.2 GC granularity @@ -499,7 +502,7 @@ GC is triggered by: ### 7.4 Interaction with leader change -After leader change, the new leader may temporarily not know other acceptors' `contiguous_applied`, so `safe_slot` is conservatively low. GC pauses until safe-slot has stabilized. This is fine: GC is asynchronous to correctness. +After a leader or membership change, fresh progress and a matching certificate must be established. Until then reclamation pauses. Snapshot creation remains local and can continue independently. ### 7.5 Force-retain window diff --git a/doc/doc_index.md b/doc/doc_index.md index 7b8f689c2..ce604673f 100644 --- a/doc/doc_index.md +++ b/doc/doc_index.md @@ -81,7 +81,7 @@ Temporary plans live under `doc/working/`; flow analyses live under | ----------------------------------------------------------------------------- | ------------------------------------------------------- | | `doc/design/deploy/design-crowdb-oci-image.md` | Pixi OCI build, Docker/containerd, CI and macOS slot. | | `doc/design/kv/design-crowdb-kv-leader-election.md` | Election, lease, ReadIndex, step-down. | -| `doc/design/kv/design-crowdb-kv-slot.md` | Slot pipelining, gap repair, catch-up, coalescing. | +| `doc/design/kv/design-crowdb-kv-slot.md` | Slot pipelining, catch-up, coalescing, safe retention. | | `doc/design/kv/design-crowdb-kv-rpc.md` | Paxos RPC transport, schema, and errors. | | `doc/design/kv/design-crowdb-kv-rpc-client.md` | Client KV RPC, watch push, rollout, topology cache. | | `doc/design/kv/design-crowdb-kv-reconfiguration.md` | Membership change, leader transfer, epoch fence. | diff --git a/doc/working/plan-dataset-native-sdk.md b/doc/working/plan-dataset-native-sdk.md index a2f696887..3ed87f8b3 100644 --- a/doc/working/plan-dataset-native-sdk.md +++ b/doc/working/plan-dataset-native-sdk.md @@ -49,16 +49,22 @@ bounded owner-backed delivery and independently packaged Python bindings. ## Python and acceptance -- [~] **Add optional native binding**: main-workspace PyO3 extension +- [x] **Add optional native binding**: main-workspace PyO3 extension `crowdb_dataset_native`, with owned runtime, Dataset control/read operations, retained immutable payload objects, explicit ack/cancel/close and exception mapping. Keep `sdk/` HTTP distribution free of native storage dependencies. Files: `lib/crowdb-sdk-native-python/`, `tools/sdk/`, workspace manifest. -- [ ] **Verify semantics**: shared authority tests and native facade tests for +- [x] **Verify semantics**: shared authority tests and native facade tests for projection/missing/tombstone, Split/revision/worker selection, explicit ack replay, TTL, corruption/truncation, cancellation, dropped pulls, held memory, stale routing, partial transport failure and S3/runtime coexistence. Files: native and access-dataset `tests/`, access-server parity tests. +- [~] **Bound shared KV retention**: retain acceptor nodes through concurrent + lookup/reclamation, reclaim only below an all-learner and durable-snapshot + certificate, and propagate that certificate to followers with membership + and term fencing. Verify RPC, maintenance, replay and concurrent access; + prove the unchanged full workload no longer OOMs before closing this task. + Files: `lib/crowdb-kv/src/{paxos,cluster,rpc}/`, related tests, protocol schema. - [ ] **Run real services and compare**: use the same corpus/snapshot for HTTP and native Rust/Python, restart storage, validate explicit resume and held buffers, TCP and local shared-HCA policies. Run the same pinned workload and @@ -215,3 +221,139 @@ verifies every original payload address survives the durable writer, while partial allocation failure, short interior frames and mixed budgets release their credits correctly. Six focused producer/upload tests pass. Real HCA and common-snapshot semantics are still pending. + +The packaging, namespace and producer-chain fixes are committed in `d3e285b2`. +All native SDK target tests, native/Python all-feature clippy, fmt, rs-lint and +19 image-identity tests pass. A first unprivileged suite invocation could not +open the TCP fixture listener; the authorized socket-capable rerun passes. + +Strict same-host HCA acceptance passes in +`native-dataset-hca/3d1ddb03a4f148d2917300cb6f97a2c1`: 4-MiB multipart +publication, Split read, durable owner cleanup, restart, registered receive +ownership and held buffers after runtime shutdown. Fallback, protocol, CQ, +send and receive-post error counters stay zero. This is local HCA functional +evidence, not a physical multi-host throughput claim. + +Long-run OOM diagnosis also finds that no production caller trims/reclaims +Paxos acceptor history. Simply enabling `PxSlotList` reclamation is unsafe: +its documented late-pin race and `get_or_prepare_slot`'s unpinned returned +reference permit a reader to outlive its node. The bounded retention fix must +first give each acceptor access a retained node, then reclaim only below all +learners' applied frontier and durable snapshots. Heartbeat progress from +peers completing after the first quorum must be retained, and followers need +an explicit membership-fenced reclamation watermark distinct from the commit +frontier. Keep the exact failed workload and 2-GiB limits for verification. + +Installed Python semantics and restart acceptance passes in +`dataset-workload/988b49756a1841e0862b63b75977a222`, snapshot +`afc3c2c1c2be4960aa78b894f924d11c`. HTTP publishes all eight payload sizes; +Python verifies exact bytes, disjoint workers, explicit ack/unacked replay, +65-second idle across the 60-second lease TTL, held-budget pressure/cancel, +multipart publication, Split, owner cleanup and retained buffers after close. +This `--semantics-only` run is functional evidence, not performance acceptance. + +## Handoff 2026-10-23 + +### Scope and pause + +- User requested a handoff, commit of completed work, and deferred long-running + acceptance. Stop after the commit; do not start another image build or workload. +- Branch: `task-rdma`. R5 is complete; retain its accepted evidence in + [RDMA acceptance record](plan-rdma.md). Physical multi-node testing is only a + refined [independent plan](plan-rdma-multi-node.md), not a remaining R5 gate. +- R220 is not complete. Keep its requirement/backlog entry and this plan until + full workload acceptance and final cleanup. Do not restart R242 or R5 work. + +### Completed implementation and evidence + +- Native Dataset implementation: `f5fefdfe`; common workload: `982c1809`. + Shared read-lease CAS fixes: `93aefa43`, `4127a966`, `a8e1571e` with focused + competing-renewal regressions. HTTP contention no longer returns 409 in the + recorded full HTTP runs; the subsequent failure is metadata process OOM. +- Packaging/default namespace/bounded multi-owner producer fixes: `d3e285b2`. + Python fills bounded final-frame owners once; multi-part producer chains + preserve original pointers through the durable writer. Ordinary Python input + still needs that initial copy; do not claim a completely copy-free upload. +- Ordinary Dataset E2E passes in + `container-e2e/run-97e635e2d2054f1b8f6ff8d44e7a475c`. +- Strict local HCA passes in + `native-dataset-hca/3d1ddb03a4f148d2917300cb6f97a2c1`, using + `rocep35s0f0` / `10.200.1.1`. Python TCP semantics passes in the run above. +- Artifact paths are relative to `.crowdb-runtime/artifacts/`. Prior images and + artifacts remain evidence for their own source versions, not the new KV fix. + +### KV retention change in this handoff commit + +- `PxAcceptor` uses the existing lock-free `SkipMap`, retaining its `Entry` + throughout node access. The old `PxSlotList` late-pin/unpinned-reference paths + are no longer used by the acceptor. Late requests below trim are rejected; + outstanding cloned payload owners survive reclamation. +- `group_retention` derives a certificate capped by every configured learner's + applied progress, local durable snapshot and a durable voting witness. + Singleton voting groups need only local durability but still wait for their + non-voting learners. Replies completing after quorum now update progress. +- Heartbeats carry an exclusive GC boundary and membership epoch. Followers + retain epoch, term, leader and watermark together in `ArcSwapOption`, then + cap reclamation by local applied/snapshot state. Epoch adoption cannot relabel + an old certificate. Membership replacement and new tenure reset evidence. +- Maintenance drives acceptor trim/reclaim and WAL segment GC. Status exposes + `acceptor.trim_before_slot` and `acceptor.retained_slots`. RPC heartbeat + implementation is split into child modules; protocol fields are appended. +- Focused verification passes: four group retention tests, thirteen acceptor + tests including concurrent readers/removers and retained payload lifetime. + `pixi run clean-env && pixi run cargo test -p crowdb-kv -p crowdb-protocol + --all-targets` passes, including election, group, WAL/replay and RPC tests. + Scoped KV/protocol all-feature clippy, workspace `rs-lint`, Rust formatting + and `git diff --check` pass. No C++ source changed in this commit. +- No new lock, dependency or unsafe exception is introduced. Generic slot-list + reclamation remains a separate utility; this change does not establish that + utility's general concurrent-reclamation safety. +- Long-run memory/performance acceptance is still pending. The ordered map + adds per-slot allocation/lookup overhead; do not describe it as a measured + throughput improvement. A stalled configured learner can still hold GC back. + +### Next work, in order + +1. Resume from the committed branch and check `git status` before editing. + Rebuild/import the server image from the current source before real-service + acceptance; the currently accepted image predates the retention change. +2. Run the unchanged complete HTTP/native Rust/native Python workload. Keep + client CPU 2, server CPUs 4-15, metadata memory 2 GiB, all eight payload sizes, + warmups/repeats and exact-byte checks. The runner pins one common snapshot. + Do not use `--semantics-only` as the performance gate; do not raise memory, + shorten the workload, extend deadlines or add request retries to obtain a pass. +3. Inspect retained slots, trim progress, per-container RSS/CPU and exit status + alongside throughput and per-size tails. Compare phase/state effects: an + HTTP phase earlier in the same accumulating fixture is not an isolated + transport-speed baseline. Report trade-offs even if native TPS does not win. +4. If memory is still unbounded, diagnose the first growing owner. Separate + known remaining amplification: `ChunkStream::publish_state` rewrites all + historical extent pages per append, and `Partition::reclaim_published_checkpoint` + has no production caller. These paths are not changed by this commit. +5. After final behavior changes, run affected real-service restart/TCP/local-HCA + acceptance and applicable gates. Only then remove the R220 detail, backlog + entry and this plan in a cleanup commit, moving final accepted evidence to + the retained record and fixing its links. + +Commands from the repository root (run long commands asynchronously and poll): + +```bash +pixi run image-build --privileged +pixi run image-import target/crowdb.oci.tar --image crowdb-node:dev +pixi run -- python tools/sdk/dataset_workload.py +pixi run -- python tools/rdma/run-native-dataset.py --address 10.200.1.1 --device rocep35s0f0 +pixi run clean-env && pixi run test-e2e-dataset +``` + +The currently imported pre-retention image is +`sha256:6a840db3f06d2247d5827b2ed59e75fca3f547bb6427d8f6460db6831469f242` +with revision `d3e285b2`; the runner must reject it after the server-source change. +Docker, HCA and socket-based tests require the existing authorized execution +environment. Do not run heavy builds concurrently with measured trials. + +The failed full comparison to reproduce is +`dataset-workload/8cf0653d34f74e0c955f5ac5642d1316`: HTTP six trials and native +Rust singleton trials completed, then native multi-flight failed with a Chunk-KV +deadline; metadata replicas 1 and 2 exited 137 with `OOMKilled=true` at 2 GiB. +Those partial rates are not accepted comparative performance. There is no +remaining user design decision and no requirement `Blocked` condition. diff --git a/doc/working/plan-rdma.md b/doc/working/plan-rdma.md index 0d2f92048..485b1371b 100644 --- a/doc/working/plan-rdma.md +++ b/doc/working/plan-rdma.md @@ -1,19 +1,22 @@ -# CROWDB RDMA implementation Plan +# CROWDB RDMA Acceptance Record Upstream: [RDMA RPC design](../design/rpc/design-crowdb-rpc-rdma.md), and the [native SDK contract](../design/access-server/design-crowdb-native-sdk.md). -Goal: implement the host-memory RDMA SEND/RECV path end to end, with RDMA as -the preferred connection path, TCP as the one-way fallback, shared aggregation -semantics, bounded buffer leases, and automated OCI verification using a -single-node HCA local-loop with multiple containers. +Scope: accepted host-memory RDMA SEND/RECV, bounded mixed-size aggregation, +registered buffer ownership, and single-node shared-HCA verification. Persistent-plan exception: retain this root record for accepted local-HCA measurements. Completed child plans and implementation checklists are removed; the evidence remains reproducible. +R5 is complete. Active Dataset work and its pause/resume handoff live in the +[native Dataset plan](plan-dataset-native-sdk.md#handoff-2026-10-23). +[Physical multi-node verification](plan-rdma-multi-node.md) is an independent, +plan-only follow-up and is outside the completed R5 acceptance. + ## Current implementation status Status: implementation and local-HCA acceptance complete. Endpoint discovery @@ -255,10 +258,11 @@ The [native SDK guide](../../lib/crowdb-sdk-native/README.md) and [architecture](../design/access-server/design-crowdb-native-sdk.md) define the supported API. [R220](../backlog/R220-dataset-fat-client-plan.md) attaches Dataset authority, Snapshot/Split binding and explicit acknowledgement to the -same runtime after its HTTP workload measurements. Dataset binding work and -GPUDirect remain separate follow-ups. +same runtime. Its implementation and real-service Python/local-HCA semantics +pass; complete workload acceptance and shared KV retention verification remain +in the linked Dataset plan. GPUDirect remains a separate follow-up. -## Final verification +## Verification commands Unit and integration gates: @@ -274,20 +278,12 @@ pixi run rs-lint pixi run tree-lint ``` -OCI E2E verification must run the TCP-only and single-node HCA local-loop -multi-container profiles when their prerequisites are available. Namespace -profiles are not part of acceptance. +Revalidation uses the TCP-only and single-node HCA local-loop multi-container +profiles with their declared prerequisites. These commands reproduce checks; +they are not outstanding R5 tasks. Namespace profiles are outside acceptance. ## Test and setup files - `tools/rdma/setup-rdma.sh` - `tools/rdma/run-local-loop.sh` - OCI image and multi-container runner configuration - -## Handoff and blocking rules - -Each child plan is marked `[x]` only after its own acceptance cases and gates -pass. If a child discovers a protocol ambiguity, stop that child, record the -question here, and resolve it before changing the formal design. If the host -cannot provide RDMA, continue portable SDK/TCP checks and mark only hardware -E2E as pending. diff --git a/lib/crowdb-kv/src/cluster.rs b/lib/crowdb-kv/src/cluster.rs index af6c57e8c..fe5e4a33e 100644 --- a/lib/crowdb-kv/src/cluster.rs +++ b/lib/crowdb-kv/src/cluster.rs @@ -19,7 +19,11 @@ pub mod group_operations; pub(crate) mod group_owner_fence; pub mod group_prepare; pub mod group_propose; +<<<<<<< HEAD mod group_routes; +======= +mod group_retention; +>>>>>>> 0f5d94a8 (Reclaim snapshot-backed Paxos history with retained readers) #[cfg(feature = "test-util")] mod group_tests; pub mod kv_server; diff --git a/lib/crowdb-kv/src/cluster/group.rs b/lib/crowdb-kv/src/cluster/group.rs index 3105f9493..5f887262d 100644 --- a/lib/crowdb-kv/src/cluster/group.rs +++ b/lib/crowdb-kv/src/cluster/group.rs @@ -197,6 +197,8 @@ pub struct PxGroup { /// local WAL to report its own snapshot progress). Recomputed at the /// end of each quorum heartbeat round. pub(crate) group_snapshot_slot: AtomicU64, + /// Membership-fenced certificate received from the accepted leader. + pub(super) leader_gc_certificate: arc_swap::ArcSwapOption, /// In-flight proposal admission gate. Holds N semaphores (one per /// queue), each sized to `ceil(max_inflight / N)` permits. Each /// in-flight `propose` call holds one permit for its duration. @@ -406,6 +408,7 @@ impl PxGroup { group_safe_slot: AtomicU64::new(0), peer_durable: parking_lot::Mutex::new(HashMap::new()), group_snapshot_slot: AtomicU64::new(0), + leader_gc_certificate: arc_swap::ArcSwapOption::empty(), inflight: InflightAdmission::new( PaxosConfig::DEFAULT.max_inflight_proposals, PaxosConfig::DEFAULT.reject_on_inflight_window_full, diff --git a/lib/crowdb-kv/src/cluster/group_election_leader.rs b/lib/crowdb-kv/src/cluster/group_election_leader.rs index a20f7df10..a4d0afba7 100644 --- a/lib/crowdb-kv/src/cluster/group_election_leader.rs +++ b/lib/crowdb-kv/src/cluster/group_election_leader.rs @@ -121,6 +121,8 @@ impl PxGroup { committed_safe_slot: replica.contiguous_chosen(), lease_grant_until_ms_mono: t_send_ms_mono.saturating_add(cfg.lease_duration_ms), t_send_ms_mono, + gc_before_slot: self.reclamation_certificate(), + gc_membership_epoch: self.membership_epoch(), }; let mut acks: usize = 1; @@ -146,10 +148,7 @@ impl PxGroup { // A singleton still sends the commit frontier to joining learners. if acks >= quorum { replica.renew_lease(t_send, cfg); - tokio::spawn( - async move { while joinset.join_next().await.is_some() {} } - .instrument(tracing::Span::current()), - ); + self.drain_heartbeat_progress(joinset, leader_term); return HeartbeatOutcome::Continued { quorum_acked: true }; } @@ -179,11 +178,7 @@ impl PxGroup { // renew lease, note peer applied. No catch-up. // Refresh this peer's applied watermark and recompute // the group safe-slot used by bounded/safe-slot reads. - self.note_peer_applied(peer_id, hb.contiguous_applied); - // Refresh this peer's durable-snapshot watermark and - // recompute the group's real "durable on leader + - // >=1 peer" snapshot-slot. - self.note_peer_durable(peer_id, hb.durable_snapshot_slot); + self.record_heartbeat_progress(peer_id, leader_term, hb); if acks >= quorum { replica.renew_lease(t_send, cfg); // Quorum reached — renew lease and return @@ -195,12 +190,8 @@ impl PxGroup { // peer, leaving it in Unknown state // indefinitely. Spawn a background drain so // the remaining tasks complete and deliver - // the heartbeat; their results are ignored - // (a higher-term self-heals next round). - tokio::spawn( - async move { while joinset.join_next().await.is_some() {} } - .instrument(tracing::Span::current()), - ); + // the heartbeat and update retention progress. + self.drain_heartbeat_progress(joinset, leader_term); return HeartbeatOutcome::Continued { quorum_acked: true }; } } diff --git a/lib/crowdb-kv/src/cluster/group_maintenance.rs b/lib/crowdb-kv/src/cluster/group_maintenance.rs index dc423d256..c603e0aa1 100644 --- a/lib/crowdb-kv/src/cluster/group_maintenance.rs +++ b/lib/crowdb-kv/src/cluster/group_maintenance.rs @@ -25,25 +25,12 @@ //! ([`KVEngine::set_gc_watermark`](crate::kv::KVEngine::set_gc_watermark)), //! conditionally runs cadence-driven block compaction //! ([`KVEngine::compact_sparse_blocks`](crate::kv::KVEngine::compact_sparse_blocks)), -//! and runs a WAL segment GC pass -//! ([`crate::wal::gc::run_gc_with_watermark`]) -- these *are* -//! cross-replica-safety sensitive, so they stay gated on -//! [`PxGroup::group_safe_slot`] (`0` — not yet established, or this -//! replica is a follower that doesn't track it; see that method's own -//! doc — means "nothing yet provably safe to reclaim"). -//! -//! `set_gc_watermark`'s two inputs are `snapshot_slot` (state -//! durable on the leader plus at least one peer) and `safe_slot` (state every learner has applied). `snapshot_slot` is -//! [`PxGroup::group_snapshot_slot`]: each replica's own -//! `WalEngine::snapshot_slot` (updated below, right after `persist_snapshot` -//! advances it) is gossiped to the leader piggybacked on the same heartbeat -//! round as `contiguous_applied` (see `HeartbeatReply::durable_snapshot_slot` -//! / `PxGroup::note_peer_durable`), which aggregates it into `min(leader's -//! own durable slot, max(voting peer durable slots))` -- always `<=` what -//! `group_safe_slot` would have allowed (a slot cannot be locally applied -//! everywhere without having first been durably chosen), so this can only -//! let reclaim run *more* conservatively than the old `safe_slot` -//! approximation, never less. +//! using [`PxGroup::group_safe_slot`] and [`PxGroup::group_snapshot_slot`]. +//! 5. Reclaims acceptor entries and WAL segments below a separate certificate +//! capped by local applied and durable-snapshot progress. The leader needs +//! every learner's applied progress and a durable voting witness; followers +//! consume its membership/term-fenced certificate. Unknown progress blocks +//! reclamation without preventing local snapshots. use std::sync::atomic::Ordering; use std::sync::{Arc, Weak}; @@ -55,6 +42,7 @@ use tracing::{debug, error, info, info_span, warn, Instrument, Span}; use super::group::PxGroup; use super::group_election::LeaderElection; +use crate::paxos::roles::Acceptor; use crate::wal::gc::run_gc_with_watermark; /// Start the per-group maintenance loop on `group`, unless already running @@ -362,17 +350,19 @@ pub(crate) async fn run_pass(group: &PxGroup, do_flush: bool) { } } - // 5. WAL GC: only advance snapshot_slot when we actually persisted. + // 5. Acceptor and WAL GC: advance local durability only after persistence. let Some(wal) = group.local_replica().wal() else { return; }; if engine_snapshot_at > 0 && engine_snapshot_at > wal.snapshot_slot() { wal.set_snapshot_slot(engine_snapshot_at); } - if safe_slot == 0 { + let gc_slot = group.local_reclamation_slot(); + if gc_slot == 0 { return; } - let gc_slot = wal.snapshot_slot().min(safe_slot); + group.local_replica().acceptor.trim(gc_slot); + group.local_replica().acceptor.reclaim(); if let Err(e) = run_gc_with_watermark(wal, gc_slot).await { warn!( error = %e, diff --git a/lib/crowdb-kv/src/cluster/group_membership.rs b/lib/crowdb-kv/src/cluster/group_membership.rs index 7996e7761..aed010ef6 100644 --- a/lib/crowdb-kv/src/cluster/group_membership.rs +++ b/lib/crowdb-kv/src/cluster/group_membership.rs @@ -166,6 +166,7 @@ impl PxGroup { /// re-establish them, which is the conservative guarantee their docs /// promise. pub(crate) fn reset_safe_slot_tracking(&self) { + self.leader_gc_certificate.store(None); self.peer_applied.lock().clear(); self.group_safe_slot.store(0, Ordering::Release); self.peer_durable.lock().clear(); @@ -254,6 +255,7 @@ impl PxGroup { /// Set up the group with a list of remote replicas. pub fn set_remote_replicas(&mut self, remote_replicas: Vec) { + self.reset_safe_slot_tracking(); let max_node_id = remote_replicas.iter().map(|r| r.node_id).max().unwrap_or(0); let vec_len = (max_node_id + 1) as usize; self.remote_replicas = (0..vec_len).map(|_| RemoteReplicaKind::Placeholder).collect(); @@ -874,6 +876,7 @@ impl PxGroup { } self.remote_replicas[idx] = RemoteReplicaKind::Real(Box::new(remote)); + self.reset_safe_slot_tracking(); self.recompute_quorum(); // Bump the epoch iff this mutation actually changes the voting @@ -898,6 +901,7 @@ impl PxGroup { "removed remote replica from group" ); self.remote_replicas[idx] = RemoteReplicaKind::Placeholder; + self.reset_safe_slot_tracking(); self.valid_replica_count -= 1; self.recompute_quorum(); // Removing a non-voting member never changes the voting set. diff --git a/lib/crowdb-kv/src/cluster/group_retention.rs b/lib/crowdb-kv/src/cluster/group_retention.rs new file mode 100644 index 000000000..5cc648591 --- /dev/null +++ b/lib/crowdb-kv/src/cluster/group_retention.rs @@ -0,0 +1,148 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Snapshot-backed reclamation certificates, separate from commit progress. + +use super::{ + group::{PxGroup, RemoteReplicaKind}, + local_replica::PxLocalReplicaRole, + replica::{HeartbeatReply, HeartbeatRequestPayload, PxReplicaError}, +}; +use crate::paxos::PxNodeId; +use std::sync::Arc; +use tokio::task::JoinSet; +use tracing::Instrument; + +pub(super) type HeartbeatTasks = JoinSet<(PxNodeId, bool, Result)>; + +pub(super) struct ReclamationCertificate { + epoch: u64, + term: u64, + leader: PxNodeId, + before_slot: u64, +} + +impl PxGroup { + /// Every learner has applied this prefix, and snapshots retain it on the + /// leader plus a voting peer (or locally for a singleton voting group). + pub(crate) fn reclamation_certificate(&self) -> u64 { + let local = self.local_replica(); + if self.tenure_cancel.is_cancelled() || local.role() != PxLocalReplicaRole::Leader { + return 0; + } + let local_durable = local.wal().map_or(0, |wal| wal.snapshot_slot()); + let mut safe = local.contiguous_applied().min(local_durable); + let applied = self.peer_applied.lock(); + for remote in &self.remote_replicas { + if let RemoteReplicaKind::Real(remote) = remote { + safe = safe.min(applied.get(&remote.node_id).copied().unwrap_or(0)); + } + } + drop(applied); + if self.quorum() > 1 { + let durable = self.peer_durable.lock(); + let witness = self + .remote_replicas + .iter() + .filter_map(|remote| match remote { + RemoteReplicaKind::Real(remote) if remote.voting => { + Some(durable.get(&remote.node_id).copied().unwrap_or(0)) + } + _ => None, + }) + .max() + .unwrap_or(0); + safe = safe.min(witness); + } + safe + } + + pub(crate) fn local_reclamation_slot(&self) -> u64 { + let local = self.local_replica(); + let certificate = if local.role() == PxLocalReplicaRole::Leader { + self.reclamation_certificate() + } else { + self.leader_gc_certificate + .load() + .as_ref() + .map_or(0, |certificate| { + if certificate.epoch == self.membership_epoch() + && certificate.term == local.current_term_snapshot() + && Some(certificate.leader) == local.believed_leader_id() + && !self.tenure_cancel.is_cancelled() + { + certificate.before_slot + } else { + 0 + } + }) + }; + certificate + .min(local.contiguous_applied()) + .min(local.wal().map_or(0, |wal| wal.snapshot_slot())) + } + + pub(crate) fn record_reclamation_heartbeat(&self, request: &HeartbeatRequestPayload) { + let local = self.local_replica(); + if !self.tenure_cancel.is_cancelled() + && request.gc_membership_epoch == self.membership_epoch() + && local.role() == PxLocalReplicaRole::Follower + && local.current_term_snapshot() == request.term + && local.believed_leader_id() == Some(request.leader_id) + && self + .get_remote_replica(request.leader_id) + .is_some_and(|peer| peer.voting) + { + // Keep the epoch and watermark in one retained object: an epoch + // change racing this publication must not relabel an old value. + self.leader_gc_certificate.rcu(|previous| { + let before_slot = previous + .as_ref() + .filter(|previous| { + previous.epoch == request.gc_membership_epoch + && previous.term == request.term + && previous.leader == request.leader_id + }) + .map_or(request.gc_before_slot, |previous| { + previous.before_slot.max(request.gc_before_slot) + }); + Some(Arc::new(ReclamationCertificate { + epoch: request.gc_membership_epoch, + term: request.term, + leader: request.leader_id, + before_slot, + })) + }); + } + } + + pub(crate) fn record_heartbeat_progress(&self, peer: PxNodeId, term: u64, reply: HeartbeatReply) { + let local = self.local_replica(); + if reply.success + && reply.term == term + && local.current_term_snapshot() == term + && local.role() == PxLocalReplicaRole::Leader + && !self.tenure_cancel.is_cancelled() + { + self.note_peer_applied(peer, reply.contiguous_applied); + self.note_peer_durable(peer, reply.durable_snapshot_slot); + } + } + + /// Quorum latency is independent of the slowest peer, but its progress is + /// still required before reclaiming a prefix shared by every learner. + pub(super) fn drain_heartbeat_progress(self: &Arc, mut tasks: HeartbeatTasks, term: u64) { + let weak = Arc::downgrade(self); + tokio::spawn( + async move { + while let Some(result) = tasks.join_next().await { + let Some(group) = weak.upgrade() else { return }; + if let Ok((peer, _, Ok(reply))) = result { + group.record_heartbeat_progress(peer, term, reply); + } + } + } + .instrument(tracing::Span::current()), + ); + } +} diff --git a/lib/crowdb-kv/src/cluster/group_tests.rs b/lib/crowdb-kv/src/cluster/group_tests.rs index 7c402b8fa..31f00c39c 100644 --- a/lib/crowdb-kv/src/cluster/group_tests.rs +++ b/lib/crowdb-kv/src/cluster/group_tests.rs @@ -14,12 +14,26 @@ use crate::paxos::PxNodeId; /// repair step, and peer-applied injection — to integration tests under /// `tests/` without permanently widening the production public API. impl PxGroup { +<<<<<<< HEAD /// Whether a conditional mutation currently owns one key's admission. #[must_use] pub fn conditional_write_pending_for_tests(&self, key: &[u8]) -> bool { self.cas_transient_map.get(key).is_some() } +======= + pub fn reclamation_certificate_for_tests(&self) -> u64 { + self.reclamation_certificate() + } + + pub fn local_reclamation_slot_for_tests(&self) -> u64 { + self.local_reclamation_slot() + } + + pub fn record_reclamation_heartbeat_for_tests(&self, request: &super::replica::HeartbeatRequestPayload) { + self.record_reclamation_heartbeat(request); + } +>>>>>>> 0f5d94a8 (Reclaim snapshot-backed Paxos history with retained readers) /// Run one heartbeat round without depending on the election timer. pub async fn heartbeat_round_for_tests(self: &std::sync::Arc) -> bool { matches!( diff --git a/lib/crowdb-kv/src/cluster/local_replica.rs b/lib/crowdb-kv/src/cluster/local_replica.rs index f6f57d37d..e807d29c5 100644 --- a/lib/crowdb-kv/src/cluster/local_replica.rs +++ b/lib/crowdb-kv/src/cluster/local_replica.rs @@ -956,6 +956,10 @@ impl PxLocalReplica { crowtree_stats, }, election, + acceptor: Some(crowdb_protocol::mgmt::AcceptorStateView { + trim_before_slot: self.acceptor.trim_slot(), + retained_slots: self.acceptor.retained_slots(), + }), } } } diff --git a/lib/crowdb-kv/src/cluster/replica.rs b/lib/crowdb-kv/src/cluster/replica.rs index 7ecf9a7cf..54a68e5ce 100644 --- a/lib/crowdb-kv/src/cluster/replica.rs +++ b/lib/crowdb-kv/src/cluster/replica.rs @@ -65,6 +65,9 @@ pub struct HeartbeatRequestPayload { pub committed_safe_slot: SlotIndex, pub lease_grant_until_ms_mono: u64, pub t_send_ms_mono: u64, + /// Reclamation certificate; never inferred from the chosen commit frontier. + pub gc_before_slot: SlotIndex, + pub gc_membership_epoch: u64, } /// `Heartbeat` reply from a follower. diff --git a/lib/crowdb-kv/src/paxos/acceptor.rs b/lib/crowdb-kv/src/paxos/acceptor.rs index ca7010895..5b52811ba 100644 --- a/lib/crowdb-kv/src/paxos/acceptor.rs +++ b/lib/crowdb-kv/src/paxos/acceptor.rs @@ -4,7 +4,7 @@ //! Acceptor state machine for one consensus group. //! //! Implements P1 M1 consensus logic. Holds per-slot promised and accepted -//! state in a lock-free `SlotList`; persistence is added in P2 (WAL). +//! state in a lock-free ordered map with retained entries during reclamation. //! //! Invariants enforced: //! - **C2 — Ballot monotonic per slot.** `prepare`/`accept` reject any ballot strictly @@ -13,8 +13,8 @@ #![allow(unsafe_code)] use crate::paxos::roles::{Acceptor, PxAcceptReply, PxBallot, PxLogEntry, PxPrepareReply, SlotIndex}; -use crate::paxos::slot_list::PxSlotList; -use crate::paxos::slot_node::{get_or_prepare_slot, PxSlotNode}; +use crate::paxos::slot_node::PxSlotNode; +use crossbeam_skiplist::{map::Entry, SkipMap}; use std::sync::atomic::{AtomicU64, Ordering}; #[derive(Debug, PartialEq, Eq)] @@ -25,7 +25,8 @@ enum PxAcceptResult { #[derive(Default)] pub struct PxAcceptor { - slot_list: PxSlotList, + slots: SkipMap, + trim_slot: AtomicU64, /// Highest slot index ever opened on this acceptor via `prepare` or /// `accept`. Used as the bulk-Phase-1 ceiling input so a new leader /// can re-Prepare every slot a previous leader may have touched here, @@ -47,13 +48,11 @@ impl PxAcceptor { #[must_use] pub fn accepted_log_tip(&self) -> Option<(SlotIndex, u64)> { - let highest = self.highest_seen_slot(); - for slot in (1..=highest).rev() { - if let Some(entry) = self.slot_list.get(slot).and_then(|node| node.accepted_cloned()) { - return Some((entry.slot, entry.term)); - } - } - None + self.slots.range(self.trim_slot()..).rev().find_map(|node| { + node.value() + .accepted_cloned() + .map(|entry| (entry.slot, entry.term)) + }) } /// Slot-ordered iteration over accepted entries in @@ -66,19 +65,20 @@ impl PxAcceptor { /// slot-ordered op log bounded by `contiguous_applied`. /// /// Lock-free and safe under concurrent writers: each yielded entry - /// is a clone taken under a chunk read pin. + /// is a clone taken while retaining the map entry. pub fn accepted_iter_range( &self, start_slot: SlotIndex, end_slot_exclusive: SlotIndex, ) -> Vec<(SlotIndex, PxLogEntry)> { - let mut out = Vec::new(); - for (slot, guard) in self.slot_list.iter_range(start_slot, end_slot_exclusive) { - if let Some(entry) = guard.accepted_cloned() { - out.push((slot, entry)); - } + let start = start_slot.max(self.trim_slot()); + if start >= end_slot_exclusive { + return Vec::new(); } - out + self.slots + .range(start..end_slot_exclusive) + .filter_map(|node| node.value().accepted_cloned().map(|entry| (*node.key(), entry))) + .collect() } /// Current trim slot — slots below this have been GC'd and are no @@ -86,7 +86,26 @@ impl PxAcceptor { /// when `min_slot < trim_slot`. #[must_use] pub fn trim_slot(&self) -> SlotIndex { - self.slot_list.trim_slot() + self.trim_slot.load(Ordering::Acquire) + } + + #[must_use] + pub fn retained_slots(&self) -> usize { + self.slots.len() + } + + fn open_slot(&self, slot: SlotIndex) -> Option> { + if slot < self.trim_slot() { + return None; + } + let node = self.slots.get_or_insert_with(slot, PxSlotNode::default); + // A concurrent trim may have passed the initial admission check. + // Retaining this entry keeps all node fields alive through rejection. + if slot < self.trim_slot() { + node.remove(); + return None; + } + Some(node) } fn bump_highest_seen(&self, slot: SlotIndex) { @@ -108,12 +127,13 @@ impl PxAcceptor { #[allow(clippy::unused_async, clippy::unused_async_trait_impl)] async fn prepare(&self, slot: SlotIndex, ballot: PxBallot) -> PxPrepareReply { - let Some(node) = get_or_prepare_slot(&self.slot_list, slot) else { + let Some(guard) = self.open_slot(slot) else { return PxPrepareReply::Rejected { slot, current_promised: PxBallot::new(0, 0), }; }; + let node = guard.value(); self.bump_highest_seen(slot); loop { let current_ptr = node.promised.load(Ordering::Acquire); @@ -137,7 +157,8 @@ impl PxAcceptor { fn inner_accept(&self, entry: &PxLogEntry) -> Option { let slot = entry.slot; let ballot = entry.ballot; - let node = get_or_prepare_slot(&self.slot_list, slot)?; + let guard = self.open_slot(slot)?; + let node = guard.value(); self.bump_highest_seen(slot); loop { let promised_ptr = node.promised.load(Ordering::Acquire); @@ -189,18 +210,24 @@ impl Acceptor for PxAcceptor { } fn accepted_at(&self, slot: SlotIndex) -> Option { - self.slot_list.get(slot)?.accepted_cloned() + if slot < self.trim_slot() { + return None; + } + self.slots.get(&slot)?.value().accepted_cloned() } fn promised_at(&self, slot: SlotIndex) -> Option { - self.slot_list.get(slot)?.promised_cloned() + if slot < self.trim_slot() { + return None; + } + self.slots.get(&slot)?.value().promised_cloned() } fn reclaim(&self) -> usize { - self.slot_list.reclaim() + self.slots.range(..self.trim_slot()).filter(Entry::remove).count() } fn trim(&self, before_slot: SlotIndex) { - self.slot_list.trim(before_slot); + self.trim_slot.fetch_max(before_slot, Ordering::AcqRel); } fn trim_slot(&self) -> SlotIndex { - self.slot_list.trim_slot() + self.trim_slot() } } diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service.rs b/lib/crowdb-kv/src/rpc/px_rpc_service.rs index a04944799..ba69313bf 100644 --- a/lib/crowdb-kv/src/rpc/px_rpc_service.rs +++ b/lib/crowdb-kv/src/rpc/px_rpc_service.rs @@ -23,6 +23,8 @@ //! `Arc` so it can submit responses from either the dispatch //! thread (sync error path) or the spawned task (async success path). +mod heartbeat; + use std::future::Future; use std::sync::Arc; @@ -30,11 +32,11 @@ use crowdb_protocol::fb::FBMsgType; use crowdb_protocol::kv_consensus_fb::{ FBAcceptRequest, FBAcceptedResponse, FBAcceptedResponseArgs, FBAcceptedValue, FBAcceptedValueArgs, FBBatchChosenNotification, FBChosenNotification, FBFetchGapRequest, FBFetchGapResponse, - FBFetchGapResponseArgs, FBHeartbeatRequest, FBHeartbeatResponse, FBHeartbeatResponseArgs, FBKvRetCode, - FBPreVoteRequest, FBPreVoteResponse, FBPreVoteResponseArgs, FBPrepareRequest, FBPromiseResponse, - FBPromiseResponseArgs, FBRequestVoteRequest, FBRequestVoteResponse, FBRequestVoteResponseArgs, - FBSnapshotAbortRequest, FBSnapshotAbortResponse, FBSnapshotAbortResponseArgs, FBSnapshotBeginRequest, - FBSnapshotBeginResponse, FBSnapshotBeginResponseArgs, FBSnapshotFinishRequest, FBSnapshotFinishResponse, + FBFetchGapResponseArgs, FBHeartbeatResponse, FBHeartbeatResponseArgs, FBKvRetCode, FBPreVoteRequest, + FBPreVoteResponse, FBPreVoteResponseArgs, FBPrepareRequest, FBPromiseResponse, FBPromiseResponseArgs, + FBRequestVoteRequest, FBRequestVoteResponse, FBRequestVoteResponseArgs, FBSnapshotAbortRequest, + FBSnapshotAbortResponse, FBSnapshotAbortResponseArgs, FBSnapshotBeginRequest, FBSnapshotBeginResponse, + FBSnapshotBeginResponseArgs, FBSnapshotFinishRequest, FBSnapshotFinishResponse, FBSnapshotFinishResponseArgs, FBSnapshotReadRequest, FBSnapshotReadResponse, FBSnapshotReadResponseArgs, FBStepDownRequest, FBStepDownResponse, FBStepDownResponseArgs, }; @@ -45,9 +47,7 @@ use tracing::{debug, field, info_span, warn, Instrument, Span}; use crate::cluster::local_replica::PxLocalReplica; use crate::cluster::px_kv_store::PxKvStore; -use crate::cluster::replica::{ - HeartbeatRequestPayload, PxReplicaError, ReplicaHandler, StepDownRequestPayload, VoteRequestPayload, -}; +use crate::cluster::replica::{PxReplicaError, ReplicaHandler, StepDownRequestPayload, VoteRequestPayload}; use crate::paxos::roles::{PxAcceptReply, PxBallot, PxLogEntry, PxPrepareReply}; use crate::rpc::snapshot_registry::{ SnapshotRegistration, SnapshotRegistry, SnapshotSessionError, SnapshotSessionId, SnapshotSourceMetrics, @@ -203,6 +203,7 @@ impl PxRpcService { } } +<<<<<<< HEAD fn target_matches(&self, req: &ServerRequest) -> bool { let consensus = [ FBMsgType::EPrepareRequest, @@ -220,6 +221,845 @@ impl PxRpcService { .any(|kind| u16::try_from(kind.0).ok() == Some(req.msg_type)) { return true; +======= + // ── Prepare ────────────────────────────────────────────────── + + #[allow(clippy::too_many_lines)] + fn handle_prepare(&self, req: ServerRequest, server: &Arc) { + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle_usize = req.conn_handle as usize; + let msg_type = FBMsgType::EPromiseResponse.0 as u16; + let store = Arc::clone(&self.store); + let server = Arc::clone(server); + spawn_request(&self.rt, "prepare", store.store_id, async move { + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::InvalidArgument, + "invalid request flatbuffer", + ); + return; + }; + let group_id = fb_req.group_id(); + record_group_context(&store, group_id); + let slot = fb_req.slot(); + let round = fb_req.round(); + let leader_id = fb_req.leader_id(); + let term = fb_req.term(); + let membership_epoch = fb_req.membership_epoch(); + + let Some(group) = store.get_group(group_id) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::NotFound, + "px group not found", + ); + return; + }; + let responder_epoch = group.membership_epoch(); + + // Membership-epoch fence (same as px_service.rs L131-147). + if membership_epoch != responder_epoch { + let converged_epoch = group.adopt_membership_epoch(membership_epoch); + warn!( + group_id, + slot, + round, + leader_id, + proposer_epoch = membership_epoch, + responder_epoch, + converged_epoch, + "prepare rejected by membership-epoch fence; adopting higher epoch from proposer" + ); + let term = group.local_replica().current_term_snapshot(); + let ctrl = build_promise_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + slot, + round, + leader_id, + None, + false, + 0, + 0, + term, + false, + responder_epoch, + true, + ); + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + return; + } + + let ballot = PxBallot { round, leader_id }; + let replica = group.local_replica(); + let reply = + ::on_prepare(replica, slot, ballot, term, group_id).await; + let ctrl = match reply { + Ok(PxPrepareReply::Promised { slot, accepted }) => { + let mut builder = FlatBufferBuilder::new(); + let prev_accepted_off = accepted.as_ref().map(|entry| { + let payload = builder.create_vector(&entry.payload); + FBAcceptedValue::create( + &mut builder, + &FBAcceptedValueArgs { + slot: entry.slot, + round: entry.ballot.round, + leader_id: entry.ballot.leader_id, + term: entry.term, + payload: Some(payload), + }, + ) + }); + let term = replica.current_term_snapshot(); + finish_promise_response( + builder, + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + slot, + round, + leader_id, + prev_accepted_off, + false, + 0, + 0, + term, + false, + responder_epoch, + false, + ) + } + Ok(PxPrepareReply::TermStale { slot, new_term }) => build_promise_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + slot, + round, + leader_id, + None, + false, + 0, + 0, + new_term, + true, + responder_epoch, + false, + ), + Ok(PxPrepareReply::Rejected { + slot, + current_promised, + }) => { + warn!( + group_id, + slot, + current_round = current_promised.round, + current_leader_id = current_promised.leader_id, + "prepare rejected; next step: proposer should retry with a higher ballot" + ); + let term = replica.current_term_snapshot(); + build_promise_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + slot, + round, + leader_id, + None, + true, + current_promised.round, + current_promised.leader_id, + term, + false, + responder_epoch, + false, + ) + } + Ok(PxPrepareReply::EpochMismatch { .. }) => { + unreachable!("on_prepare does not produce EpochMismatch") + } + Err(e) => { + let (code, msg) = px_error_to_ret_code(&e); + build_promise_response( + req_id, + create_nano, + code, + Some(&msg), + 1, + slot, + round, + leader_id, + None, + false, + 0, + 0, + replica.current_term_snapshot(), + false, + responder_epoch, + false, + ) + } + }; + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + // req dropped here, frame released + }); + } + + // ── Accept ─────────────────────────────────────────────────── + + #[allow(clippy::too_many_lines)] + fn handle_accept(&self, req: ServerRequest, server: &Arc) { + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle_usize = req.conn_handle as usize; + let msg_type = FBMsgType::EAcceptedResponse.0 as u16; + let store = Arc::clone(&self.store); + let server = Arc::clone(server); + spawn_request(&self.rt, "accept", store.store_id, async move { + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::InvalidArgument, + "invalid request flatbuffer", + ); + return; + }; + let group_id = fb_req.group_id(); + record_group_context(&store, group_id); + let slot = fb_req.slot(); + let round = fb_req.round(); + let leader_id = fb_req.leader_id(); + let term = fb_req.term(); + let membership_epoch = fb_req.membership_epoch(); + + let Some(fb_value) = fb_req.value() else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::InvalidArgument, + "missing value", + ); + return; + }; + let payload_bytes: Vec = fb_value.payload().map(|v| v.bytes().to_vec()).unwrap_or_default(); + let entry = PxLogEntry { + slot, + ballot: PxBallot { round, leader_id }, + term, + payload: payload_bytes.into(), + }; + + let Some(group) = store.get_group(group_id) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::NotFound, + "px group not found", + ); + return; + }; + let responder_epoch = group.membership_epoch(); + + if membership_epoch != responder_epoch { + let converged_epoch = group.adopt_membership_epoch(membership_epoch); + warn!( + group_id, + slot, + round, + leader_id, + proposer_epoch = membership_epoch, + responder_epoch, + converged_epoch, + "accept rejected by membership-epoch fence; adopting higher epoch from proposer" + ); + let term = group.local_replica().current_term_snapshot(); + let ctrl = build_accepted_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + slot, + round, + leader_id, + false, + 0, + 0, + term, + false, + responder_epoch, + true, + ); + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + return; + } + + let replica = group.local_replica(); + let reply = ::on_accept(replica, &entry, group_id).await; + let (rejected, rejected_round, rejected_leader_id, term_stale, reply_term) = match reply { + Ok(PxAcceptReply::Accepted { .. }) => (false, 0, 0, false, replica.current_term_snapshot()), + Ok(PxAcceptReply::Rejected { current_promised, .. }) => { + warn!( + group_id, + slot, + current_round = current_promised.round, + current_leader_id = current_promised.leader_id, + "accept rejected; next step: proposer should run prepare with a higher ballot" + ); + ( + true, + current_promised.round, + current_promised.leader_id, + false, + replica.current_term_snapshot(), + ) + } + Ok(PxAcceptReply::TermStale { new_term, .. }) => { + warn!( + group_id, + slot, new_term, "accept rejected by term fence; proposer should step down" + ); + (false, 0, 0, true, new_term) + } + Ok(PxAcceptReply::EpochMismatch { .. }) => { + unreachable!("on_accept does not produce EpochMismatch") + } + Err(e) => { + let (code, msg) = px_error_to_ret_code(&e); + let ctrl = build_accepted_response( + req_id, + create_nano, + code, + Some(&msg), + 1, + slot, + round, + leader_id, + false, + 0, + 0, + replica.current_term_snapshot(), + false, + responder_epoch, + false, + ); + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + return; + } + }; + let ctrl = build_accepted_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + slot, + round, + leader_id, + rejected, + rejected_round, + rejected_leader_id, + reply_term, + term_stale, + responder_epoch, + false, + ); + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + // req dropped here, frame released + }); + } + + // ── PreVote ────────────────────────────────────────────────── + + fn handle_pre_vote(&self, req: ServerRequest, server: &Arc) { + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle_usize = req.conn_handle as usize; + let msg_type = FBMsgType::EPreVoteResponse.0 as u16; + let store = Arc::clone(&self.store); + let server = Arc::clone(server); + spawn_request(&self.rt, "pre_vote", store.store_id, async move { + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::InvalidArgument, + "invalid request flatbuffer", + ); + return; + }; + let group_id = fb_req.group_id(); + record_group_context(&store, group_id); + let Some(group) = store.get_group(group_id) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::NotFound, + "px group not found", + ); + return; + }; + let payload = VoteRequestPayload { + term: fb_req.term(), + candidate_id: fb_req.candidate_id(), + accepted_log_tip_slot: fb_req.accepted_log_tip_slot(), + accepted_log_tip_term: fb_req.accepted_log_tip_term(), + }; + let replica = group.local_replica(); + let reply = ::on_pre_vote(replica, payload, group_id).await; + let ctrl = match reply { + Ok(r) => build_pre_vote_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + group_id, + r.term, + r.granted, + r.contiguous_chosen, + r.last_chosen_term, + r.highest_seen_slot, + ), + Err(e) => { + let (code, msg) = px_error_to_ret_code(&e); + build_pre_vote_response( + req_id, + create_nano, + code, + Some(&msg), + 1, + group_id, + 0, + false, + 0, + 0, + 0, + ) + } + }; + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + // req dropped here, frame released + }); + } + + // ── RequestVote ────────────────────────────────────────────── + + fn handle_request_vote(&self, req: ServerRequest, server: &Arc) { + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle_usize = req.conn_handle as usize; + let msg_type = FBMsgType::ERequestVoteResponse.0 as u16; + let store = Arc::clone(&self.store); + let server = Arc::clone(server); + spawn_request(&self.rt, "request_vote", store.store_id, async move { + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::InvalidArgument, + "invalid request flatbuffer", + ); + return; + }; + let group_id = fb_req.group_id(); + record_group_context(&store, group_id); + let Some(group) = store.get_group(group_id) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::NotFound, + "px group not found", + ); + return; + }; + let payload = VoteRequestPayload { + term: fb_req.term(), + candidate_id: fb_req.candidate_id(), + accepted_log_tip_slot: fb_req.accepted_log_tip_slot(), + accepted_log_tip_term: fb_req.accepted_log_tip_term(), + }; + let replica = group.local_replica(); + let reply = ::on_request_vote(replica, payload, group_id).await; + let ctrl = match reply { + Ok(r) => build_request_vote_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + group_id, + r.term, + r.granted, + r.contiguous_chosen, + r.last_chosen_term, + r.highest_seen_slot, + ), + Err(e) => { + let (code, msg) = px_error_to_ret_code(&e); + build_request_vote_response( + req_id, + create_nano, + code, + Some(&msg), + 1, + group_id, + 0, + false, + 0, + 0, + 0, + ) + } + }; + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + // req dropped here, frame released + }); + } + + // ── StepDown ───────────────────────────────────────────────── + + fn handle_step_down(&self, req: ServerRequest, server: &Arc) { + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle_usize = req.conn_handle as usize; + let msg_type = FBMsgType::EStepDownResponse.0 as u16; + let store = Arc::clone(&self.store); + let server = Arc::clone(server); + spawn_request(&self.rt, "step_down", store.store_id, async move { + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::InvalidArgument, + "invalid request flatbuffer", + ); + return; + }; + let group_id = fb_req.group_id(); + record_group_context(&store, group_id); + let Some(group) = store.get_group(group_id) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::NotFound, + "px group not found", + ); + return; + }; + let reason = fb_req.reason().unwrap_or("").to_string(); + let payload = StepDownRequestPayload { + term: fb_req.term(), + target_leader_id: fb_req.target_leader_id(), + reason, + }; + let replica = group.local_replica(); + let reply = ::on_step_down(replica, &payload, group_id).await; + let ctrl = match reply { + Ok(r) => build_step_down_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + group_id, + r.accepted, + r.current_term, + r.current_leader_id, + ), + Err(e) => { + let (code, msg) = px_error_to_ret_code(&e); + build_step_down_response(req_id, create_nano, code, Some(&msg), 1, group_id, false, 0, 0) + } + }; + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + // req dropped here, frame released + }); + } + + // ── ChosenNotification (fire-and-forget) ───────────────────── + + #[allow(clippy::needless_pass_by_value, reason = "make_handler uniform signature")] + fn handle_chosen_notice(&self, req: ServerRequest, _server: &Arc) { + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + debug!("chosen notice: invalid flatbuffer"); + return; + }; + let group_id = fb_req.group_id(); + let slot = fb_req.slot(); + let term = fb_req.term(); + let leader_id = fb_req.leader_id(); + let ballot_round = fb_req.ballot_round(); + + let Some(group) = self.store.get_group(group_id) else { + debug!(slot, term, "chosen notice dropped (group not found)"); + return; + }; + let chosen_ballot = PxBallot { + round: ballot_round, + leader_id, + }; + spawn_group_request( + &self.rt, + "chosen_notice", + self.store.store_id, + group_id, + group.local_replica().id, + async move { + let replica = group.local_replica(); + let accepted = replica.accepted_at(slot).await; + let ballot_matches = accepted.as_ref().is_some_and(|e| e.ballot == chosen_ballot); + if ballot_matches { + replica.learner.update_chosen_frontier(slot, term); + replica.wake_apply_loop(); + debug!( + slot, + term, leader_id, ballot_round, "chosen notification applied (ballot match)" + ); + } else { + replica.note_chosen(slot, term); + replica.record_gap(slot); + if let Some(ref entry) = accepted { + replica.incr_chosen_notice_stale(); + debug!( + slot, + term, + leader_id, + chosen_ballot_round = ballot_round, + accepted_ballot_round = entry.ballot.round, + accepted_ballot_leader = entry.ballot.leader_id, + "chosen notification stale ballot (gap recorded)" + ); + } else { + replica.incr_chosen_notice_missing(); + debug!( + slot, + term, leader_id, "chosen notification missing value (gap recorded)" + ); + } + } + }, + ); + } + + // ── BatchChosenNotification (fire-and-forget) ──────────────── + + #[allow(clippy::needless_pass_by_value, reason = "make_handler uniform signature")] + fn handle_batch_chosen(&self, req: ServerRequest, _server: &Arc) { + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + debug!("batch chosen: invalid flatbuffer"); + return; + }; + let group_id = fb_req.group_id(); + let start_slot = fb_req.start_slot(); + let end_slot = fb_req.end_slot(); + let term = fb_req.term(); + let leader_id = fb_req.leader_id(); + let ballot_round = fb_req.ballot_round(); + + let Some(group) = self.store.get_group(group_id) else { + debug!("batch chosen dropped (group not found)"); + return; + }; + let chosen_ballot = PxBallot { + round: ballot_round, + leader_id, + }; + spawn_group_request( + &self.rt, + "batch_chosen", + self.store.store_id, + group_id, + group.local_replica().id, + async move { + let replica = group.local_replica(); + let mut advanced_count = 0u64; + for slot in start_slot..=end_slot { + let accepted = replica.accepted_at(slot).await; + if accepted.as_ref().is_some_and(|e| e.ballot == chosen_ballot) { + replica.learner.update_chosen_frontier(slot, term); + advanced_count += 1; + } else { + replica.note_chosen(slot, term); + replica.record_gap(slot); + } + } + replica.wake_apply_loop(); + debug!( + group_id, + start_slot, + end_slot, + term, + leader_id, + ballot_round, + advanced_count, + "batch chosen notification applied" + ); + }, + ); + } + + // ── FetchGap ───────────────────────────────────────────────── + + #[allow(clippy::needless_pass_by_value, reason = "make_handler uniform signature")] + fn handle_fetch_gap(&self, req: ServerRequest, server: &Arc) { + let span = request_span("fetch_gap", self.store.store_id); + let _entered = span.enter(); + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle = req.conn_handle; + let msg_type = FBMsgType::EFetchGapResponse.0 as u16; + + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + submit_error( + server, + conn_handle, + req_id, + create_nano, + msg_type, + FBKvRetCode::InvalidArgument, + "invalid request flatbuffer", + ); + return; + }; + let group_id = fb_req.group_id(); + record_group_context(&self.store, group_id); + let slot = fb_req.slot(); + + let Some(group) = self.store.get_group(group_id) else { + submit_error( + server, + conn_handle, + req_id, + create_nano, + msg_type, + FBKvRetCode::NotFound, + "px group not found", + ); + return; + }; + if let Some(resp) = group.handle_fetch_gap(slot) { + let payload_vec = resp.payload.clone(); + let ctrl = build_fetch_gap_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + group_id, + resp.slot, + resp.term, + resp.ballot_round, + resp.leader_id, + &payload_vec, + ); + submit_fb_response(server, conn_handle, ctrl, msg_type, req_id); + } else { + debug!(slot, "fetch_gap no value (not yet chosen)"); + submit_error( + server, + conn_handle, + req_id, + create_nano, + msg_type, + FBKvRetCode::NotFound, + "slot not yet chosen", + ); +>>>>>>> 0f5d94a8 (Reclaim snapshot-backed Paxos history with retained readers) } let Ok(target) = flatbuffers::root::(req.control()) diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service/heartbeat.rs b/lib/crowdb-kv/src/rpc/px_rpc_service/heartbeat.rs new file mode 100644 index 000000000..6392b33a1 --- /dev/null +++ b/lib/crowdb-kv/src/rpc/px_rpc_service/heartbeat.rs @@ -0,0 +1,117 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + build_heartbeat_response, px_error_to_ret_code, record_group_context, spawn_request, submit_error, + submit_fb_response, PxRpcService, +}; +use crate::cluster::{ + local_replica::PxLocalReplica, + replica::{HeartbeatRequestPayload, ReplicaHandler}, +}; +use crowdb_protocol::{ + fb::FBMsgType, + kv_consensus_fb::{FBHeartbeatRequest, FBKvRetCode}, +}; +use crowdb_rpc_ffi::{RpcServer, ServerRequest}; +use std::sync::Arc; + +impl PxRpcService { + // ── Heartbeat ──────────────────────────────────────────────── + + pub(super) fn handle_heartbeat(&self, req: ServerRequest, server: &Arc) { + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle_usize = req.conn_handle as usize; + let msg_type = FBMsgType::EHeartbeatResponse.0 as u16; + let store = Arc::clone(&self.store); + let server = Arc::clone(server); + spawn_request(&self.rt, "heartbeat", store.store_id, async move { + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::InvalidArgument, + "invalid request flatbuffer", + ); + return; + }; + let group_id = fb_req.group_id(); + record_group_context(&store, group_id); + let Some(group) = store.get_group(group_id) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::NotFound, + "px group not found", + ); + return; + }; + let payload = HeartbeatRequestPayload { + term: fb_req.term(), + leader_id: fb_req.leader_id(), + prev_log_slot: fb_req.prev_log_slot(), + prev_log_term: fb_req.prev_log_term(), + committed_safe_slot: fb_req.committed_safe_slot(), + lease_grant_until_ms_mono: fb_req.lease_grant_until_ms_mono(), + t_send_ms_mono: fb_req.t_send_ms_mono(), + gc_before_slot: fb_req.gc_before_slot(), + gc_membership_epoch: fb_req.gc_membership_epoch(), + }; + let replica = group.local_replica(); + let reply = ::on_heartbeat(replica, payload, group_id).await; + if reply.as_ref().is_ok_and(|reply| reply.success) { + group.record_reclamation_heartbeat(&payload); + } + let ctrl = match reply { + Ok(r) => build_heartbeat_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + group_id, + r.term, + r.success, + r.contiguous_chosen, + r.last_chosen_term, + r.contiguous_applied, + r.highest_seen_slot, + r.durable_snapshot_slot, + ), + Err(e) => { + let (code, msg) = px_error_to_ret_code(&e); + build_heartbeat_response( + req_id, + create_nano, + code, + Some(&msg), + 1, + group_id, + 0, + false, + 0, + 0, + 0, + 0, + 0, + ) + } + }; + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + // req dropped here, frame released + }); + } +} diff --git a/lib/crowdb-kv/src/rpc/px_rpc_transport.rs b/lib/crowdb-kv/src/rpc/px_rpc_transport.rs index 1a74e93b6..d0b7b473e 100644 --- a/lib/crowdb-kv/src/rpc/px_rpc_transport.rs +++ b/lib/crowdb-kv/src/rpc/px_rpc_transport.rs @@ -7,6 +7,8 @@ //! awaits `CallFuture`, and parses flatbuffer responses via the //! zero-copy `Ref` wrappers. +mod heartbeat; + use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -14,18 +16,18 @@ use flatbuffers::FlatBufferBuilder; use crowdb_protocol::fb::FBMsgType; use crowdb_protocol::fb_wrappers::kv_consensus::{ - FBAcceptedResponseRef, FBFetchGapResponseRef, FBHeartbeatResponseRef, FBPreVoteResponseRef, - FBPromiseResponseRef, FBRequestVoteResponseRef, FBSnapshotAbortResponseRef, FBSnapshotBeginResponseRef, + FBAcceptedResponseRef, FBFetchGapResponseRef, FBPreVoteResponseRef, FBPromiseResponseRef, + FBRequestVoteResponseRef, FBSnapshotAbortResponseRef, FBSnapshotBeginResponseRef, FBSnapshotFinishResponseRef, FBSnapshotReadResponseRef, FBStepDownResponseRef, }; use crowdb_protocol::kv_consensus_fb::{ FBAcceptRequest, FBAcceptRequestArgs, FBAcceptedValue, FBAcceptedValueArgs, FBBatchChosenNotification, FBBatchChosenNotificationArgs, FBChosenNotification, FBChosenNotificationArgs, FBFetchGapRequest, - FBFetchGapRequestArgs, FBHeartbeatRequest, FBHeartbeatRequestArgs, FBKvRetCode, FBPreVoteRequest, - FBPreVoteRequestArgs, FBPrepareRequest, FBPrepareRequestArgs, FBRequestVoteRequest, - FBRequestVoteRequestArgs, FBSnapshotAbortRequest, FBSnapshotAbortRequestArgs, FBSnapshotBeginRequest, - FBSnapshotBeginRequestArgs, FBSnapshotFinishRequest, FBSnapshotFinishRequestArgs, FBSnapshotReadRequest, - FBSnapshotReadRequestArgs, FBStepDownRequest, FBStepDownRequestArgs, + FBFetchGapRequestArgs, FBKvRetCode, FBPreVoteRequest, FBPreVoteRequestArgs, FBPrepareRequest, + FBPrepareRequestArgs, FBRequestVoteRequest, FBRequestVoteRequestArgs, FBSnapshotAbortRequest, + FBSnapshotAbortRequestArgs, FBSnapshotBeginRequest, FBSnapshotBeginRequestArgs, FBSnapshotFinishRequest, + FBSnapshotFinishRequestArgs, FBSnapshotReadRequest, FBSnapshotReadRequestArgs, FBStepDownRequest, + FBStepDownRequestArgs, }; use crowdb_rpc_ffi::{ noop_completion, Buffer, ConnectionPoolError, ConnectionPoolIndex, RpcClient, RpcError, RpcServer, @@ -33,8 +35,7 @@ use crowdb_rpc_ffi::{ }; use crate::cluster::replica::{ - FetchGapReply, HeartbeatReply, HeartbeatRequestPayload, PxReplicaError, StepDownReply, - StepDownRequestPayload, VoteReply, VoteRequestPayload, + FetchGapReply, PxReplicaError, StepDownReply, StepDownRequestPayload, VoteReply, VoteRequestPayload, }; use crate::paxos::roles::{PxAcceptReply, PxBallot, PxLogEntry, PxPrepareReply}; @@ -405,6 +406,7 @@ impl PxRpcTransport { }) } +<<<<<<< HEAD /// Send a `Heartbeat` request via crowdb-rpc. pub async fn send_heartbeat( &self, @@ -463,6 +465,8 @@ impl PxRpcTransport { }) } +======= +>>>>>>> 0f5d94a8 (Reclaim snapshot-backed Paxos history with retained readers) /// Send a `StepDown` request via crowdb-rpc. pub async fn send_step_down( &self, diff --git a/lib/crowdb-kv/src/rpc/px_rpc_transport/heartbeat.rs b/lib/crowdb-kv/src/rpc/px_rpc_transport/heartbeat.rs new file mode 100644 index 000000000..b99835d5d --- /dev/null +++ b/lib/crowdb-kv/src/rpc/px_rpc_transport/heartbeat.rs @@ -0,0 +1,69 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{check_ret_code, PxRpcTransport}; +use crate::cluster::replica::{HeartbeatReply, HeartbeatRequestPayload, PxReplicaError}; +use crowdb_protocol::{ + fb::FBMsgType, + fb_wrappers::kv_consensus::FBHeartbeatResponseRef, + kv_consensus_fb::{FBHeartbeatRequest, FBHeartbeatRequestArgs}, +}; +use crowdb_rpc_ffi::Buffer; +use flatbuffers::FlatBufferBuilder; + +impl PxRpcTransport { + /// Send a `Heartbeat` request via crowdb-rpc. + pub async fn send_heartbeat( + &self, + rpc_endpoint: &str, + req: HeartbeatRequestPayload, + group_id: u64, + ) -> Result { + let req_id = self.next_id(); + let conn = self.conn_for(rpc_endpoint)?; + let mut builder = FlatBufferBuilder::new(); + let args = FBHeartbeatRequestArgs { + id: req_id, + rpc_create_nano: 0, + version: 1, + group_id, + term: req.term, + leader_id: req.leader_id, + prev_log_slot: req.prev_log_slot, + prev_log_term: req.prev_log_term, + committed_safe_slot: req.committed_safe_slot, + lease_grant_until_ms_mono: req.lease_grant_until_ms_mono, + t_send_ms_mono: req.t_send_ms_mono, + gc_before_slot: req.gc_before_slot, + gc_membership_epoch: req.gc_membership_epoch, + }; + let fb_req = FBHeartbeatRequest::create(&mut builder, &args); + builder.finish(fb_req, None); + let control = Buffer::from_bytes(builder.finished_data()); + let msg_type = FBMsgType::EHeartbeatRequest.0 as u16; + let fut = self + .rpc + .call(&self.server, &conn, req_id, control, None, msg_type) + .map_err(|e| self.map_rpc_err(e, rpc_endpoint, conn.generation()))?; + let resp = fut + .await + .map_err(|e| self.map_rpc_err(e, rpc_endpoint, conn.generation()))?; + let ctrl = resp + .control + .ok_or_else(|| PxReplicaError::Internal("heartbeat response missing control buffer".into()))?; + let r = FBHeartbeatResponseRef::new(ctrl.bytes()); + if !r.valid() { + return Err(PxReplicaError::Internal("heartbeat response malformed".into())); + } + check_ret_code(r.ret_code(), r.error_msg())?; + Ok(HeartbeatReply { + term: r.term(), + success: r.success(), + contiguous_chosen: r.contiguous_chosen(), + last_chosen_term: r.last_chosen_term(), + contiguous_applied: r.contiguous_applied(), + highest_seen_slot: r.highest_seen_slot(), + durable_snapshot_slot: r.durable_snapshot_slot(), + }) + } +} diff --git a/lib/crowdb-kv/tests/election_test/apply_loop_test.rs b/lib/crowdb-kv/tests/election_test/apply_loop_test.rs index 6a8b09c4c..cabeda31d 100644 --- a/lib/crowdb-kv/tests/election_test/apply_loop_test.rs +++ b/lib/crowdb-kv/tests/election_test/apply_loop_test.rs @@ -20,6 +20,8 @@ fn heartbeat(term: u64, leader_id: u64, commit_slot: u64) -> HeartbeatRequestPay committed_safe_slot: commit_slot, lease_grant_until_ms_mono: 0, t_send_ms_mono: 0, + gc_before_slot: 0, + gc_membership_epoch: 0, } } diff --git a/lib/crowdb-kv/tests/election_test/heartbeat_test.rs b/lib/crowdb-kv/tests/election_test/heartbeat_test.rs index 98fc3925b..27e41b926 100644 --- a/lib/crowdb-kv/tests/election_test/heartbeat_test.rs +++ b/lib/crowdb-kv/tests/election_test/heartbeat_test.rs @@ -22,6 +22,8 @@ fn heartbeat(term: u64, leader_id: u64, commit_slot: u64) -> HeartbeatRequestPay committed_safe_slot: commit_slot, lease_grant_until_ms_mono: 0, t_send_ms_mono: 0, + gc_before_slot: 0, + gc_membership_epoch: 0, } } diff --git a/lib/crowdb-kv/tests/group_test.rs b/lib/crowdb-kv/tests/group_test.rs index 776da272e..7518d2d51 100644 --- a/lib/crowdb-kv/tests/group_test.rs +++ b/lib/crowdb-kv/tests/group_test.rs @@ -119,6 +119,10 @@ mod safe_slot; #[path = "group_test/snapshot_slot_test.rs"] mod snapshot_slot; +#[cfg(feature = "test-util")] +#[path = "group_test/retention_test.rs"] +mod retention; + #[cfg(feature = "test-util")] #[path = "group_test/maintenance_test.rs"] mod maintenance; diff --git a/lib/crowdb-kv/tests/group_test/retention_test.rs b/lib/crowdb-kv/tests/group_test/retention_test.rs new file mode 100644 index 000000000..ebf2a921a --- /dev/null +++ b/lib/crowdb-kv/tests/group_test/retention_test.rs @@ -0,0 +1,217 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use crowdb_kv::cluster::group::PxGroup; +use crowdb_kv::cluster::replica::{HeartbeatRequestPayload, ReplicaHandler}; +use crowdb_kv::cluster::{KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; +use crowdb_kv::paxos::roles::{Acceptor, Learner, PxBallot, PxLogEntry}; +use crowdb_kv::wal::wal_engine::WalEngine; +use crowdb_kv::wal::{IoBackend, MemBlockDevice, WalConfig}; + +async fn local(id: u64, role: PxLocalReplicaRole, applied: u64, durable: u64) -> PxLocalReplica { + let mut replica = PxLocalReplica::new(id, role); + for slot in 1..=applied { + let entry = PxLogEntry { + slot, + ballot: PxBallot::new(0, 0), + term: 0, + payload: bytes::Bytes::new(), + }; + replica.acceptor.accept(&entry).await; + replica.learner.learn(entry, &[]).await; + } + let wal = WalEngine::create( + Arc::new(IoBackend::MemBlock(MemBlockDevice::new())), + WalConfig { + wal_disks: vec![PathBuf::from(format!("/retention-{id}"))], + ..Default::default() + }, + id, + ) + .await + .unwrap(); + wal.set_snapshot_slot(durable); + replica.set_wal(wal); + replica +} + +fn remote(id: u64, voting: bool) -> PxRemoteReplica { + PxRemoteReplica::new(id, format!("127.0.0.1:{id}")).with_voting(voting) +} + +#[tokio::test] +async fn retention_waits_for_every_learner_and_a_durable_voter() { + let mut group = PxGroup::new(1, local(1, PxLocalReplicaRole::Leader, 10, 8).await); + group.set_remote_replicas(vec![remote(2, true), remote(3, true), remote(4, false)]); + group.note_peer_applied_for_tests(2, 9); + group.note_peer_applied_for_tests(3, 7); + group.note_peer_durable_for_tests(2, 9); + assert_eq!(group.reclamation_certificate_for_tests(), 0, "unheard learner"); + group.note_peer_applied_for_tests(4, 3); + assert_eq!(group.reclamation_certificate_for_tests(), 3, "slow learner"); + group.note_peer_applied_for_tests(4, 10); + assert_eq!(group.reclamation_certificate_for_tests(), 7, "slow voter"); + group.note_peer_applied_for_tests(3, 10); + assert_eq!(group.reclamation_certificate_for_tests(), 8, "local snapshot"); + + group.reset_safe_slot_tracking_for_tests(); + for peer in 2..=4 { + group.note_peer_applied_for_tests(peer, 10); + } + group.note_peer_durable_for_tests(4, 10); + assert_eq!( + group.reclamation_certificate_for_tests(), + 0, + "learner is no durable voter" + ); + group.note_peer_durable_for_tests(3, 6); + assert_eq!(group.reclamation_certificate_for_tests(), 6); + group.add_remote_replica(remote(5, false)); + assert_eq!( + group.reclamation_certificate_for_tests(), + 0, + "membership replacement resets evidence" + ); +} + +#[tokio::test] +async fn singleton_retention_requires_local_durability_and_learner_progress() { + let mut group = PxGroup::new(1, local(1, PxLocalReplicaRole::Leader, 7, 5).await); + assert_eq!(group.reclamation_certificate_for_tests(), 5); + group.add_remote_replica(remote(2, false)); + assert_eq!(group.reclamation_certificate_for_tests(), 0); + group.note_peer_applied_for_tests(2, 6); + assert_eq!(group.reclamation_certificate_for_tests(), 5); + group.run_maintenance_pass_for_tests().await; + assert_eq!(group.local_replica().acceptor.trim_slot(), 5); + assert_eq!(group.local_replica().acceptor.retained_slots(), 3); + + let without_wal = PxGroup::new(2, PxLocalReplica::new(1, PxLocalReplicaRole::Leader)); + assert_eq!(without_wal.reclamation_certificate_for_tests(), 0); +} + +fn heartbeat(epoch: u64) -> HeartbeatRequestPayload { + HeartbeatRequestPayload { + term: 3, + leader_id: 2, + prev_log_slot: 10, + prev_log_term: 0, + committed_safe_slot: 10, + lease_grant_until_ms_mono: 0, + t_send_ms_mono: 0, + gc_before_slot: 9, + gc_membership_epoch: epoch, + } +} + +#[tokio::test] +async fn follower_retention_fences_epoch_term_leader_and_local_snapshot() { + let mut group = PxGroup::new(1, local(1, PxLocalReplicaRole::Follower, 10, 6).await); + group.set_remote_replicas(vec![remote(2, true), remote(3, false)]); + group.set_membership_epoch(7); + let valid = heartbeat(7); + let reply = group.local_replica().on_heartbeat(valid, 1).await.unwrap(); + assert!(reply.success); + for invalid in [ + HeartbeatRequestPayload { + gc_membership_epoch: 6, + ..valid + }, + HeartbeatRequestPayload { term: 2, ..valid }, + HeartbeatRequestPayload { + leader_id: 3, + ..valid + }, + HeartbeatRequestPayload { + leader_id: 99, + ..valid + }, + ] { + group.record_reclamation_heartbeat_for_tests(&invalid); + assert_eq!(group.local_reclamation_slot_for_tests(), 0); + } + group.record_reclamation_heartbeat_for_tests(&valid); + assert_eq!(group.local_reclamation_slot_for_tests(), 6); + group.run_maintenance_pass_for_tests().await; + assert_eq!(group.local_replica().acceptor.trim_slot(), 6); + assert_eq!(group.local_replica().acceptor.retained_slots(), 5); + + group.adopt_membership_epoch(8); + assert_eq!( + group.local_reclamation_slot_for_tests(), + 0, + "stored certificate has its own epoch" + ); + group.record_reclamation_heartbeat_for_tests(&valid); + assert_eq!(group.local_reclamation_slot_for_tests(), 0); + group.record_reclamation_heartbeat_for_tests(&heartbeat(8)); + assert_eq!(group.local_reclamation_slot_for_tests(), 6); + group.local_replica().become_follower(4); + assert_eq!( + group.local_reclamation_slot_for_tests(), + 0, + "new term must establish a leader" + ); +} + +#[tokio::test] +async fn heartbeat_wire_carries_certificate_and_drains_non_quorum_progress() { + let _network = crate::common::net_lock::lock().await; + let follower = Arc::new(PxKvStore::new(2, "127.0.0.1:0".parse().unwrap())); + let mut fg = PxGroup::new( + 1, + local(2, PxLocalReplicaRole::Follower, 10, 7) + .await + .with_voting(false), + ); + fg.set_remote_replicas(vec![remote(1, true)]); + fg.set_membership_epoch(1); + follower.add_group_without_election(fg); + follower.start().await.unwrap(); + + let leader = Arc::new(PxKvStore::new(1, "127.0.0.1:0".parse().unwrap())); + let mut lg = PxGroup::new(1, local(1, PxLocalReplicaRole::Leader, 10, 8).await); + lg.set_remote_replicas(vec![PxRemoteReplica::new( + 2, + follower.listen_addr().unwrap().to_string(), + ) + .with_voting(false)]); + lg.set_membership_epoch(1); + leader.add_group_without_election(lg); + leader.start().await.unwrap(); + leader.wire_rpc_transport(); + let lg = leader.get_group(1).unwrap(); + assert_eq!(lg.reclamation_certificate_for_tests(), 0); + assert!(lg.heartbeat_round_for_tests().await); + let progress = tokio::time::timeout(Duration::from_secs(1), async { + while lg.reclamation_certificate_for_tests() != 8 { + tokio::task::yield_now().await; + } + }) + .await; + assert!(lg.heartbeat_round_for_tests().await); + let fg = follower.get_group(1).unwrap(); + let certificate = tokio::time::timeout(Duration::from_secs(1), async { + while fg.local_reclamation_slot_for_tests() != 7 { + tokio::task::yield_now().await; + } + }) + .await; + for node in [&leader, &follower] { + let _ = node.get_group(1).unwrap().shutdown(Duration::from_secs(1)).await; + node.stop(); + node.join().await; + } + assert!( + progress.is_ok(), + "non-voter reply after quorum must update retention progress" + ); + assert!( + certificate.is_ok(), + "leader certificate must survive wire encoding and local snapshot cap" + ); +} diff --git a/lib/crowdb-kv/tests/paxos_test/acceptor_test.rs b/lib/crowdb-kv/tests/paxos_test/acceptor_test.rs index caffa8228..ffe0c3c33 100644 --- a/lib/crowdb-kv/tests/paxos_test/acceptor_test.rs +++ b/lib/crowdb-kv/tests/paxos_test/acceptor_test.rs @@ -183,3 +183,69 @@ async fn accepted_log_tip_returns_highest_accepted_slot() { let tip = acc.accepted_log_tip(); assert_eq!(tip, Some((5, 1))); } + +#[tokio::test] +async fn reclaimed_prefix_rejects_late_requests_and_preserves_owned_payloads() { + let acc = PxAcceptor::new(); + for slot in 1..=2049 { + acc.accept(&entry(slot, PxBallot::new(1, 1), b"retained")).await; + } + let held = acc.accepted_at(1).unwrap(); + acc.trim(2048); + acc.reclaim(); + assert_eq!(acc.retained_slots(), 2); + assert!(acc.accepted_at(1).is_none()); + assert!(acc.accepted_at(2048).is_some()); + assert_eq!(acc.accepted_log_tip(), Some((2049, 1))); + assert_eq!(held.payload.as_ref(), b"retained"); + assert!(matches!( + acc.prepare(1, PxBallot::new(9, 9)).await, + PxPrepareReply::Rejected { .. } + )); + assert!(matches!( + acc.accept(&entry(1, PxBallot::new(9, 9), b"late")).await, + PxAcceptReply::Rejected { .. } + )); + assert!(acc.accepted_at(1).is_none()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_readers_and_reclaimers_keep_live_slots_and_owned_payloads() { + let acc = std::sync::Arc::new(PxAcceptor::new()); + for slot in 1..=2049 { + acc.accept(&entry(slot, PxBallot::new(1, 1), b"retained")).await; + } + let held = acc.accepted_at(1).unwrap(); + let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(5)); + let mut workers = tokio::task::JoinSet::new(); + for worker in 0..4 { + let acc = acc.clone(); + let barrier = barrier.clone(); + workers.spawn(async move { + barrier.wait().await; + for slot in 1..2049 { + if worker < 2 { + acc.trim(slot); + acc.reclaim(); + } else { + if let Some(value) = acc.accepted_at(slot) { + assert_eq!(value.payload.as_ref(), b"retained"); + } + acc.prepare(slot, PxBallot::new(2, worker)).await; + acc.accept(&entry(slot, PxBallot::new(2, worker), b"retained")) + .await; + } + tokio::task::yield_now().await; + } + }); + } + barrier.wait().await; + while let Some(result) = workers.join_next().await { + result.unwrap(); + } + acc.reclaim(); + assert_eq!(acc.trim_slot(), 2048); + assert_eq!(acc.retained_slots(), 2); + assert_eq!(acc.accepted_log_tip(), Some((2049, 1))); + assert_eq!(held.payload.as_ref(), b"retained"); +} diff --git a/lib/crowdb-kv/tests/store_test/status_test.rs b/lib/crowdb-kv/tests/store_test/status_test.rs index 2d8c1a886..6932bb668 100644 --- a/lib/crowdb-kv/tests/store_test/status_test.rs +++ b/lib/crowdb-kv/tests/store_test/status_test.rs @@ -104,6 +104,8 @@ async fn remote_status_reports_rpc_success_and_error_totals_by_exact_name() { committed_safe_slot: 0, lease_grant_until_ms_mono: 0, t_send_ms_mono: 0, + gc_before_slot: 0, + gc_membership_epoch: 0, }; for _ in 0..2 { live.send_heartbeat(hb, 1).await.expect("heartbeat to live peer"); diff --git a/lib/crowdb-protocol/src/fbs/kv_consensus.fbs b/lib/crowdb-protocol/src/fbs/kv_consensus.fbs index 1ccc3b399..79f65a629 100644 --- a/lib/crowdb-protocol/src/fbs/kv_consensus.fbs +++ b/lib/crowdb-protocol/src/fbs/kv_consensus.fbs @@ -186,6 +186,8 @@ table FBHeartbeatRequest { committed_safe_slot:uint64; lease_grant_until_ms_mono:uint64; t_send_ms_mono:uint64; + gc_before_slot:uint64; + gc_membership_epoch:uint64; } table FBHeartbeatResponse { diff --git a/lib/crowdb-protocol/src/mgmt.rs b/lib/crowdb-protocol/src/mgmt.rs index 294df8e1d..1ded4567a 100644 --- a/lib/crowdb-protocol/src/mgmt.rs +++ b/lib/crowdb-protocol/src/mgmt.rs @@ -422,6 +422,8 @@ pub struct ReplicaStatus { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub messages: Vec, pub kv_store: KvStoreStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub acceptor: Option, /// Election/lease state (term, election count, step-downs, heartbeat /// age, lease remaining). `None` for replicas without election state /// (e.g. a remote placeholder). @@ -429,6 +431,14 @@ pub struct ReplicaStatus { pub election: Option, } +/// In-memory consensus history retained for repair and journal readers. +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))] +pub struct AcceptorStateView { + pub trim_before_slot: u64, + pub retained_slots: usize, +} + /// KV store state within a `ReplicaStatus`. #[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(utoipa::ToSchema))] diff --git a/lib/crowdb-protocol/src/types/kv_consensus.rs b/lib/crowdb-protocol/src/types/kv_consensus.rs index a805336b4..f504b1331 100644 --- a/lib/crowdb-protocol/src/types/kv_consensus.rs +++ b/lib/crowdb-protocol/src/types/kv_consensus.rs @@ -202,6 +202,8 @@ pub struct HeartbeatRequest { pub committed_safe_slot: u64, pub lease_grant_until_ms_mono: u64, pub t_send_ms_mono: u64, + pub gc_before_slot: u64, + pub gc_membership_epoch: u64, pub request_id: u64, pub request_create_ms: u64, } diff --git a/pixi.toml b/pixi.toml index 9c218005c..8dcee39ff 100644 --- a/pixi.toml +++ b/pixi.toml @@ -201,7 +201,6 @@ tree-ubsan = { cmd = "bash tools/pixi-tasks/tree-ubsan.sh" } coverage = { cmd = "cargo tarpaulin --workspace --out Html --output-dir target/coverage --exclude-files '*/tests/*'", depends-on = ["install-deps"] } profile-write = { cmd = "bash tools/profiling/profile-write.sh", depends-on = ["build"] } -<<<<<<< HEAD ################################################################################ # OCI containers — build once, run isolated clusters, stop or clean owned resources ################################################################################ @@ -225,9 +224,8 @@ test-boto3-copy-error = { cmd = "python tools/pixi-tasks/test.py unit boto3" } [feature.iceberg-ecosystem.tasks] test-iceberg-container-ecosystem = "bash e2e-test/clients/ecosystem/run.sh" -======= + [pypi-dependencies] maturin = ">=1.15, <2" pip = ">=25, <27" auditwheel = ">=6, <7" ->>>>>>> d3e285b2 (Fix native Python packaging and bounded multipart producers) From defc125cb06388affdeb25197d524f729d56634d Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 22:31:27 +0800 Subject: [PATCH 086/258] Reclaim superseded stream metadata behind serving authority --- app/crowdb-chunk-kv-server/src/main.rs | 20 +- app/crowdb-chunk-kv-server/src/server.rs | 54 +- .../src/server/maintenance.rs | 108 +++ app/crowdb-chunk-kv-server/src/storage.rs | 9 +- .../tests/server_test.rs | 2 + .../tests/server_test/maintenance.rs | 93 ++ .../tests/stream_metadata_reclaim_test.rs | 105 +++ .../chunkds/design-crowdb-chunk-kv-server.md | 11 +- doc/design/chunkds/design-crowdb-chunk-kv.md | 16 + doc/working/plan-dataset-native-sdk.md | 69 +- lib/crowdb-chunk-kv/src/partition.rs | 174 +--- .../src/partition/checkpoint.rs | 223 +++++ .../src/partition/maintenance.rs | 58 ++ .../tests/history_maintenance_test.rs | 244 +++++ lib/crowdb-chunk-kv/tests/partition_test.rs | 6 +- lib/crowdb-chunk-stream/tests/stream_test.rs | 3 + .../tests/stream_test/reclamation.rs | 24 + lib/crowdb-kv/src/cluster.rs | 3 - lib/crowdb-kv/src/cluster/group_tests.rs | 3 - lib/crowdb-kv/src/rpc/px_rpc_service.rs | 857 +----------------- .../src/rpc/px_rpc_service/election.rs | 93 -- lib/crowdb-kv/src/rpc/px_rpc_transport.rs | 61 -- .../src/rpc/px_rpc_transport/heartbeat.rs | 9 +- pixi.toml | 4 - 24 files changed, 989 insertions(+), 1260 deletions(-) create mode 100644 app/crowdb-chunk-kv-server/src/server/maintenance.rs create mode 100644 app/crowdb-chunk-kv-server/tests/server_test/maintenance.rs create mode 100644 app/crowdb-chunk-kv-server/tests/stream_metadata_reclaim_test.rs create mode 100644 lib/crowdb-chunk-kv/src/partition/checkpoint.rs create mode 100644 lib/crowdb-chunk-kv/src/partition/maintenance.rs create mode 100644 lib/crowdb-chunk-kv/tests/history_maintenance_test.rs create mode 100644 lib/crowdb-chunk-stream/tests/stream_test/reclamation.rs diff --git a/app/crowdb-chunk-kv-server/src/main.rs b/app/crowdb-chunk-kv-server/src/main.rs index a8f2a3648..7a2eeb751 100644 --- a/app/crowdb-chunk-kv-server/src/main.rs +++ b/app/crowdb-chunk-kv-server/src/main.rs @@ -453,12 +453,26 @@ async fn main() -> std::process::ExitCode { let materialization_interval = std::time::Duration::from_millis(config.catalog_refresh_interval_ms); let materialization_task = tokio::spawn(async move { let mut interval = tokio::time::interval(materialization_interval); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); interval.tick().await; + let mut pending = false; loop { - interval.tick().await; - if let Err(error) = materialization_service.materialize_split_overlays().await { - warn!(%error, "chunk KV split-overlay materialization pass failed"); + tokio::select! { + biased; + _ = interval.tick() => { + if let Err(error) = materialization_service.materialize_split_overlays().await { + warn!(%error, "chunk KV split-overlay materialization pass failed"); + } + } + () = std::future::ready(()), if pending => tokio::task::yield_now().await, } + pending = match materialization_service.maintain_serving_metadata().await { + Ok(pending) => pending, + Err(error) => { + warn!(%error, "chunk KV extent metadata maintenance failed; retaining watermark for next pass"); + false + } + }; } }); let heartbeat_service = Arc::clone(&service); diff --git a/app/crowdb-chunk-kv-server/src/server.rs b/app/crowdb-chunk-kv-server/src/server.rs index b96acd103..957c8c332 100644 --- a/app/crowdb-chunk-kv-server/src/server.rs +++ b/app/crowdb-chunk-kv-server/src/server.rs @@ -35,6 +35,7 @@ mod drain; mod group; use assignments::{local_partition_for_entry, partition_matches_entry}; mod load_sampling; +mod maintenance; mod observation; mod prepared_transfer; mod readiness; @@ -493,59 +494,6 @@ impl ChunkKvService { std::future::ready(Ok(observation)) } - /// Runs one bounded ownership-materialization pass for every local split child. - /// - /// Returns child identities whose own checkpoint no longer depends on the - /// split-parent suffix. - /// - /// # Errors - /// - /// Returns the first materialization or checkpoint error. - pub async fn materialize_split_overlays(&self) -> Result, ChunkKvError> { - let catalog = self.catalog.load_full(); - let partitions = self.partitions.load_full(); - let mut completed = Vec::new(); - for entry in &catalog.entries { - if entry.owner.instance_id != self.instance_id - || entry.state != ChunkKvRangeCatalogPartitionState::Serving - || entry.artifact.tail_overlay.is_none() - || self - .independently_recoverable - .load() - .contains(&(entry.partition_id, entry.owner_epoch)) - { - continue; - } - let Some(partition) = partitions.get(&entry.partition_id) else { - continue; - }; - if partition.snapshot().lifecycle != crowdb_chunk_kv::PartitionLifecycle::Serving { - continue; - } - if !partition - .materialize_split_ownership(entry.owner_epoch) - .await? - .complete - { - continue; - } - let checkpoint = partition.checkpoint(entry.owner_epoch).await?; - let Some(overlay) = entry.artifact.tail_overlay.as_ref() else { - continue; - }; - if checkpoint.applied_seq < overlay.cutover_seq { - return Err(ChunkKvError::ApplyStateUnknown); - } - self.independently_recoverable.rcu(|current| { - let mut next = (**current).clone(); - next.insert((entry.partition_id, entry.owner_epoch)); - Arc::new(next) - }); - completed.push(entry.partition_id); - } - Ok(completed) - } - /// Validates and atomically activates a newer complete catalog snapshot. /// /// # Errors diff --git a/app/crowdb-chunk-kv-server/src/server/maintenance.rs b/app/crowdb-chunk-kv-server/src/server/maintenance.rs new file mode 100644 index 000000000..840b24eea --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/server/maintenance.rs @@ -0,0 +1,108 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Catalog-fenced extent metadata and split-overlay maintenance. + +use super::{ + local_partition_for_entry, Arc, ChunkKvError, ChunkKvRangeCatalogPartitionState, ChunkKvService, Id128, +}; + +impl ChunkKvService { + /// Runs one bounded ownership-materialization pass for every local split child. + /// + /// Returns child identities whose own checkpoint no longer depends on the + /// split-parent suffix. + /// + /// # Errors + /// + /// Returns the first materialization or checkpoint error. + pub async fn materialize_split_overlays(&self) -> Result, ChunkKvError> { + let catalog = self.catalog.load_full(); + let partitions = self.partitions.load_full(); + let mut completed = Vec::new(); + for entry in &catalog.entries { + if entry.owner.instance_id != self.instance_id + || entry.state != ChunkKvRangeCatalogPartitionState::Serving + || entry.artifact.tail_overlay.is_none() + || self + .independently_recoverable + .load() + .contains(&(entry.partition_id, entry.owner_epoch)) + { + continue; + } + let Some(partition) = partitions.get(&entry.partition_id) else { + continue; + }; + if partition.snapshot().lifecycle != crowdb_chunk_kv::PartitionLifecycle::Serving { + continue; + } + if !partition + .materialize_split_ownership(entry.owner_epoch) + .await? + .complete + { + continue; + } + let checkpoint = partition.checkpoint(entry.owner_epoch).await?; + let Some(overlay) = entry.artifact.tail_overlay.as_ref() else { + continue; + }; + if checkpoint.applied_seq < overlay.cutover_seq { + return Err(ChunkKvError::ApplyStateUnknown); + } + self.independently_recoverable.rcu(|current| { + let mut next = (**current).clone(); + next.insert((entry.partition_id, entry.owner_epoch)); + Arc::new(next) + }); + completed.push(entry.partition_id); + } + Ok(completed) + } + + /// Reclaims a bounded extent metadata batch for each independently recoverable owner. + /// + /// Returns true while any owner has a metadata backlog. Catalog transitions, + /// overlays, expired grants and mismatched local handles cannot authorize GC. + /// + /// # Errors + /// Returns a storage maintenance failure. + pub async fn maintain_serving_metadata(&self) -> Result { + let catalog = self.catalog.load_full(); + let partitions = self.partitions.load_full(); + let mut pending = false; + let mut first_error = None; + for entry in &catalog.entries { + if entry.owner.instance_id != self.instance_id + || entry.state != ChunkKvRangeCatalogPartitionState::Serving + || entry.artifact.tail_overlay.is_some() + || entry.transition_id.is_some() + || self + .authority + .authorize( + catalog.generation, + entry.partition_id, + entry.owner_epoch, + self.monotonic_ms(), + ) + .is_err() + { + continue; + } + let Some(partition) = partitions + .get(&entry.partition_id) + .and_then(|partition| local_partition_for_entry(partition, entry)) + else { + continue; + }; + match partition.maintain_serving_metadata(entry.owner_epoch).await { + Ok(backlog) => pending |= backlog, + Err(error) => { + first_error.get_or_insert(error); + } + } + } + first_error.map_or(Ok(pending), Err) + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage.rs b/app/crowdb-chunk-kv-server/src/storage.rs index ccd64353a..239539f99 100644 --- a/app/crowdb-chunk-kv-server/src/storage.rs +++ b/app/crowdb-chunk-kv-server/src/storage.rs @@ -642,17 +642,14 @@ impl ChunkKvStorage { ) .await?; let checkpoint = source - .checkpoint(transition.source_epoch) - .await - .map_err(|error| storage_plan_error(&error.to_string()))?; - source - .retain_generation_pin( + .checkpoint_with_pin( + transition.source_epoch, crowdb_chunk_kv::TransitionId { high: transition.transition_id.high, low: transition.transition_id.low, }, - checkpoint.root_manifest_generation, ) + .await .map_err(|error| storage_plan_error(&error.to_string()))?; let frontier = source .capture_journal_frontier(transition.source_epoch) diff --git a/app/crowdb-chunk-kv-server/tests/server_test.rs b/app/crowdb-chunk-kv-server/tests/server_test.rs index c37982748..19e86b2f7 100644 --- a/app/crowdb-chunk-kv-server/tests/server_test.rs +++ b/app/crowdb-chunk-kv-server/tests/server_test.rs @@ -1358,3 +1358,5 @@ mod serving_fixture; #[path = "server_test/split_scan.rs"] mod split_scan; +#[path = "server_test/maintenance.rs"] +mod maintenance; diff --git a/app/crowdb-chunk-kv-server/tests/server_test/maintenance.rs b/app/crowdb-chunk-kv-server/tests/server_test/maintenance.rs new file mode 100644 index 000000000..089fb9719 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/server_test/maintenance.rs @@ -0,0 +1,93 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_chunk_kv::{MutationOperation, RequestId}; + +fn install_grant(service: &ChunkKvService, generation: u64) { + let mut grant = ServingGrant { + instance_id: INSTANCE_ID, + lease_sequence: generation, + catalog_generation: generation, + issued_at_ms: 1_000, + expires_at_ms: 13_000, + assignments: vec![ServingAssignment { + partition_id: Id128 { high: 1, low: 2 }, + owner_epoch: EPOCH, + }], + assignment_digest: [0; 32], + }; + grant.seal(); + service + .authority() + .install(grant, &policy(), 1_000, service.monotonic_ms()) + .unwrap(); +} + +async fn write(partition: &Partition, sequence: u64) { + partition + .mutate( + EPOCH, + RequestId { + client_high: 1, + client_low: 1, + client_sequence: sequence, + }, + MutationOperation::Put { + key: b"key".to_vec(), + value: vec![1], + }, + ) + .await + .unwrap(); +} + +#[tokio::test] +async fn metadata_maintenance_requires_a_live_independent_assignment() { + let (service, partition) = fixture().await; + write(&partition, 1).await; + write(&partition, 2).await; + service.authority().clear(); + assert!(!service.maintain_serving_metadata().await.unwrap()); + assert_eq!(partition.metrics().snapshot().reclaimed_stream_metadata_pages, 0); + install_grant(&service, 1); + assert!(!service.maintain_serving_metadata().await.unwrap()); + let reclaimed = partition.metrics().snapshot().reclaimed_stream_metadata_pages; + assert!(reclaimed > 0); + + let (mut head, mut page) = catalog( + Id128 { high: 1, low: 2 }, + partition.snapshot().stream_name, + EPOCH, + 2, + Some(1), + ); + page.entries[0].transition_id = Some(Id128 { high: 20, low: 21 }); + page.seal().unwrap(); + head.pages[0].page_checksum = page.checksum; + head.seal().unwrap(); + service + .install_catalog_and_reconcile(&head, &[page], &[]) + .unwrap(); + install_grant(&service, 2); + write(&partition, 3).await; + assert!(!service.maintain_serving_metadata().await.unwrap()); + assert_eq!( + partition.metrics().snapshot().reclaimed_stream_metadata_pages, + reclaimed + ); + + let (head, page) = catalog( + Id128 { high: 1, low: 2 }, + partition.snapshot().stream_name, + EPOCH, + 3, + Some(2), + ); + service + .install_catalog_and_reconcile(&head, &[page], &[]) + .unwrap(); + install_grant(&service, 3); + assert!(!service.maintain_serving_metadata().await.unwrap()); + assert!(partition.metrics().snapshot().reclaimed_stream_metadata_pages > reclaimed); +} diff --git a/app/crowdb-chunk-kv-server/tests/stream_metadata_reclaim_test.rs b/app/crowdb-chunk-kv-server/tests/stream_metadata_reclaim_test.rs new file mode 100644 index 000000000..692612a7c --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/stream_metadata_reclaim_test.rs @@ -0,0 +1,105 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, KvStreamMetadataStore, StreamBinding, StreamBindingState, + StreamConfig, StreamMetadataStore, StreamName, StreamPurpose, +}; +use crowdb_kv_client::{CrowdbKvClient, ReadMode}; +use crowdb_protocol::key::{BinaryKey, StreamExtentPageKey}; + +#[path = "common/metadata.rs"] +mod metadata; + +async fn pages(kv: &CrowdbKvClient, name: StreamName) -> Vec { + kv.scan( + 0, + 1, + &StreamExtentPageKey::stream_prefix(name), + &[], + &[], + 0, + ReadMode::Linearizable, + None, + true, + None, + ) + .await + .unwrap() + .items + .into_iter() + .map(|(key, _)| StreamExtentPageKey::from_bytes(&key).unwrap()) + .collect() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn production_page_reclaim_batches_only_keys_below_the_retained_generation() { + let fixture = metadata::TestMetadata::start().await; + let chunks = Arc::new(MemoryStreamStore::new(512)); + let metadata = Arc::new(KvStreamMetadataStore::new(fixture.client.clone(), 0, 1).unwrap()); + let name = StreamName { high: 81, low: 82 }; + let stream = ChunkStream::create( + StreamBinding { + purpose: StreamPurpose::Wal, + stream_name: name, + metadata_group_id: 1, + binding_generation: 1, + state: StreamBindingState::Active, + owner_kind: None, + }, + 7, + StreamConfig { + extent_page_entries: 4, + ..StreamConfig::default() + }, + chunks.clone(), + metadata.clone(), + chunks, + ) + .await + .unwrap(); + for byte in 0..16 { + stream.append(&[Bytes::from(vec![byte])]).await.unwrap(); + } + let retained = metadata.load_current(name).await.unwrap().unwrap(); + for byte in 16..24 { + stream.append(&[Bytes::from(vec![byte])]).await.unwrap(); + } + let before = pages(&fixture.client, name).await.len(); + assert_eq!( + stream + .reclaim_metadata_before(retained.generation, 3) + .await + .unwrap(), + 3 + ); + assert_eq!(pages(&fixture.client, name).await.len(), before - 3); + for _ in 0..64 { + if stream + .reclaim_metadata_before(retained.generation, 3) + .await + .unwrap() + == 0 + { + break; + } + } + assert!(pages(&fixture.client, name) + .await + .iter() + .all(|key| key.generation >= retained.generation)); + for fence in retained.extent_pages { + assert!(metadata + .load_extent_page(name, retained.writer_epoch, retained.generation, fence.page_index) + .await + .unwrap() + .is_some()); + } + assert_eq!( + stream.read_at(0, 24).await.unwrap().as_ref(), + &(0..24).collect::>() + ); +} diff --git a/doc/design/chunkds/design-crowdb-chunk-kv-server.md b/doc/design/chunkds/design-crowdb-chunk-kv-server.md index a182772fe..b95c78a7e 100644 --- a/doc/design/chunkds/design-crowdb-chunk-kv-server.md +++ b/doc/design/chunkds/design-crowdb-chunk-kv-server.md @@ -344,8 +344,15 @@ match the current Serving catalog entry. Transfer overlays retain their separate committed-transfer validation. Missing or conflicting evidence leaves the writer Prepared; repeated matching grant refresh is idempotent. -The local maintenance loop performs bounded ownership materialization and a -serving checkpoint. A heartbeat then proves independent recovery, group 0 +For independent serving assignments, the local maintenance loop reclaims +superseded immutable WAL extent directories in bounded batches. It requires a +live serving grant, an exact local assignment, and no catalog transition or tail +overlay. Backlogged batches yield across hosted partitions while periodic split +materialization keeps its cadence. WAL bytes and tree-root retention do not +advance as part of extent metadata cleanup. + +For split overlays, the maintenance loop performs bounded ownership +materialization and a serving checkpoint. A heartbeat then proves independent recovery, group 0 publishes a generation that clears both the overlay and the completed split `transition_id`, and only that catalog state is eligible for another split or owner balance. Releasing both markers atomically prevents a later transfer from diff --git a/doc/design/chunkds/design-crowdb-chunk-kv.md b/doc/design/chunkds/design-crowdb-chunk-kv.md index 622889de0..8333b53e1 100644 --- a/doc/design/chunkds/design-crowdb-chunk-kv.md +++ b/doc/design/chunkds/design-crowdb-chunk-kv.md @@ -103,6 +103,22 @@ reapplying them, then applies only the suffix above the checkpoint sequence. WAL trim cannot cross the checkpoint's replay offset. Requests below the declared retained floor return `RequestExpired` and are not executed anew. +Serving maintenance reclaims only superseded immutable WAL extent directories. +The current stream manifest maps the complete retained WAL, including the +suffix needed by older tree checkpoints. A pass deletes at most the configured +metadata page count, then yields to other hosted partitions; a finite backlog +continues without waiting for the next periodic tick. This does not publish a +new tree root or trim WAL bytes. Split preparation and exact-root transition +pins suppress metadata reclamation under the existing maintenance exclusion. +Source checkpoint publication and its durable generation pin share that +exclusion, so GC cannot pass between them. + +Physical WAL trim at a nonzero replay offset requires durable expired-request +floors as well as retained results. That floor state is not yet checkpointed, +so the explicit trim API rejects such reclamation with `MaintenanceDegraded`. +The WAL remains retained until the recovery format carries this state; keeping +only the recent result window would let expired requests execute after restart. + The frontiers satisfy `checkpoint_seq <= applied_seq <= journal_durable_seq`. The stream resolves uncertain cursor and manifest outcomes against durable state before completing diff --git a/doc/working/plan-dataset-native-sdk.md b/doc/working/plan-dataset-native-sdk.md index 3ed87f8b3..49448d6e8 100644 --- a/doc/working/plan-dataset-native-sdk.md +++ b/doc/working/plan-dataset-native-sdk.md @@ -59,13 +59,22 @@ bounded owner-backed delivery and independently packaged Python bindings. replay, TTL, corruption/truncation, cancellation, dropped pulls, held memory, stale routing, partial transport failure and S3/runtime coexistence. Files: native and access-dataset `tests/`, access-server parity tests. -- [~] **Bound shared KV retention**: retain acceptor nodes through concurrent +- [x] **Bound shared KV retention**: retain acceptor nodes through concurrent lookup/reclamation, reclaim only below an all-learner and durable-snapshot certificate, and propagate that certificate to followers with membership and term fencing. Verify RPC, maintenance, replay and concurrent access; - prove the unchanged full workload no longer OOMs before closing this task. + full-workload memory acceptance + also requires the separate immutable metadata cleanup below. Files: `lib/crowdb-kv/src/{paxos,cluster,rpc}/`, related tests, protocol schema. -- [ ] **Run real services and compare**: use the same corpus/snapshot for HTTP +- [x] **Reclaim serving metadata history**: run bounded immutable extent-page + reclamation for independent serving assignments; keep WAL bytes and tree + checkpoint retention unchanged. Serialize source checkpoint pins with metadata + GC using existing maintenance exclusion, batch deletion, and verify old-root + recovery, retained retry results and Split/transfer pins. Fix complete-chunk + trim eligibility before any existing explicit trim can delete a partial chunk. + Files: chunk-kv `partition/checkpoint.rs`, `partition/maintenance.rs`, server + maintenance/storage, chunk-stream `kv.rs`, focused integration tests. +- [~] **Run real services and compare**: use the same corpus/snapshot for HTTP and native Rust/Python, restart storage, validate explicit resume and held buffers, TCP and local shared-HCA policies. Run the same pinned workload and report measured trade-offs, not an assumed throughput win. Physical multi-node @@ -254,10 +263,10 @@ This `--semantics-only` run is functional evidence, not performance acceptance. ## Handoff 2026-10-23 -### Scope and pause +### Scope and resume -- User requested a handoff, commit of completed work, and deferred long-running - acceptance. Stop after the commit; do not start another image build or workload. +- The handoff was committed in `0f5d94a8`. The user has resumed work and + authorized completing the remaining acceptance and cleanup. - Branch: `task-rdma`. R5 is complete; retain its accepted evidence in [RDMA acceptance record](plan-rdma.md). Physical multi-node testing is only a refined [independent plan](plan-rdma-multi-node.md), not a remaining R5 gate. @@ -338,7 +347,7 @@ This `--semantics-only` run is functional evidence, not performance acceptance. Commands from the repository root (run long commands asynchronously and poll): ```bash -pixi run image-build --privileged +pixi run image-build pixi run image-import target/crowdb.oci.tar --image crowdb-node:dev pixi run -- python tools/sdk/dataset_workload.py pixi run -- python tools/rdma/run-native-dataset.py --address 10.200.1.1 --device rocep35s0f0 @@ -357,3 +366,49 @@ Rust singleton trials completed, then native multi-flight failed with a Chunk-KV deadline; metadata replicas 1 and 2 exited 137 with `OOMKilled=true` at 2 GiB. Those partial rates are not accepted comparative performance. There is no remaining user design decision and no requirement `Blocked` condition. + +### Resumed verification + +- Standard rootless `pixi run image-build` and image import pass from + `0f5d94a8`, source tree `9299ad506bccb2c0080cd60970627a9a6717014e`. + Image config is `sha256:74c2be3126997b3899d803dd64ba224019608d0a39a5ecc345f4ef1353b62da5`. + The old handoff's `--privileged` command was inapplicable to the current + non-root user; no source/build policy was changed to use the standard path. +- Complete unchanged workload failed in + `dataset-workload/6581a65b0b9243be9b048b16b2caca71`. Three metadata containers + remain capped at 2 GiB. Extra read-only topology/cgroup observations are + retained as `retention-observations.jsonl` in that run directory. + +- Run `6581a65b0b9243be9b048b16b2caca71` completed all six HTTP and three native + Rust singleton trials. Native multi-flight failed with `chunk KV operation + deadline elapsed`; all three metadata replicas were OOM-killed (exit 137). + Acceptor reclamation advanced correctly (64,877 slots: trim 64,161, retained + 716), while tree live value bytes grew from 343 MB to 663 MB before failure. + Cgroup observations show anonymous RSS plus substantial swap, not file cache. + These partial rates are not accepted comparative performance. +- Second OOM root-cause fix is in progress: ordinary partitions never call + checkpoint metadata reclamation, while each append publishes fresh copies + of historical extent pages. Do not change the workload or resource limits. + +- Recovery regression exposed a pre-existing WAL trim gap: the checkpoint's + replay offset retains recent results but does not persist expired client + floors. Automatic maintenance therefore reclaims only superseded extent + directories; it does not invoke WAL/tree checkpoint reclamation. Recovery + through an older root must still replay the complete required WAL and reject + an expired request. Persisting retry-floor state is required before enabling + automatic physical WAL trim, outside the metadata-memory fix. + +- Focused regressions pass for bounded obsolete-page batches, old-checkpoint + replay and expired-request rejection, concurrent exact-root pins, and active/ + boundary chunk retention. Nonzero physical WAL trim now fails explicitly until + durable retry-floor metadata exists, instead of deleting recovery evidence. + +- Verification passes for this metadata fix: all targets of `crowdb-chunk-stream`, + `crowdb-chunk-kv`, and `crowdb-chunk-kv-server`; the added physical-WAL guard + and modified trim/serving maintenance paths were rerun after final edits. + Production KV batch deletion preserves the protected manifest generation. + Workspace all-target/all-feature clippy, `rs-lint`, Rust fmt and diff checks + pass. No C++ source, new dependency, lock or unsafe exception was introduced. +- Next: rebuild/import this source, then rerun the unchanged full workload. + Metadata history cleanup is verified at component level; full-service memory + and comparative performance remain unaccepted until that run completes. diff --git a/lib/crowdb-chunk-kv/src/partition.rs b/lib/crowdb-chunk-kv/src/partition.rs index 94b27c4e6..249e2c8c4 100644 --- a/lib/crowdb-chunk-kv/src/partition.rs +++ b/lib/crowdb-chunk-kv/src/partition.rs @@ -6,10 +6,12 @@ //! [`tree`] (ordered storage). mod drain; +mod checkpoint; mod frame; mod frontier; mod ingress; mod journal; +mod maintenance; mod observation; mod scan; mod split; @@ -167,6 +169,7 @@ pub struct Partition { applied_seq: Arc, applied_position: Arc, retry_replay_offset: Arc, + metadata_reclaim: Arc>, checkpoint_pin_generation: Arc, checkpoint_pin_transition_high: Arc, checkpoint_pin_transition_low: Arc, @@ -851,6 +854,7 @@ impl Partition { applied_seq, applied_position, retry_replay_offset, + metadata_reclaim: Arc::new(ArcSwapOption::empty()), checkpoint_pin_generation, checkpoint_pin_transition_high, checkpoint_pin_transition_low, @@ -1142,50 +1146,6 @@ impl Partition { self.tree.tree_id() } - /// Persists an exact root-generation reference before publishing a - /// topology artifact that depends on it. - /// - /// # Errors - /// - /// Returns a storage error when the generation is absent or conflicts - /// with the transition's existing pin. - pub fn retain_generation_pin(&self, transition_id: TransitionId, generation: u64) -> Result<()> { - let current = self.checkpoint_pin_generation.load(Ordering::Acquire); - if current != 0 - && (current != generation - || self.checkpoint_pin_transition_high.load(Ordering::Acquire) != transition_id.high - || self.checkpoint_pin_transition_low.load(Ordering::Acquire) != transition_id.low) - { - return Err(ChunkKvError::InvalidRequest( - "another transition already pins the partition checkpoint lineage".into(), - )); - } - self.tree.pin_generation(transition_id, generation)?; - self.checkpoint_pin_transition_high - .store(transition_id.high, Ordering::Relaxed); - self.checkpoint_pin_transition_low - .store(transition_id.low, Ordering::Relaxed); - self.checkpoint_pin_generation - .store(generation, Ordering::Release); - Ok(()) - } - - /// Releases this partition tree's durable generation pin after the - /// authoritative catalog has removed its overlay dependency. - /// - /// # Errors - /// - /// Returns a storage error when durable pin deletion cannot complete. - pub fn release_generation_pin(&self, transition_id: TransitionId) -> Result<()> { - self.tree.unpin_generation(transition_id)?; - if self.checkpoint_pin_transition_high.load(Ordering::Acquire) == transition_id.high - && self.checkpoint_pin_transition_low.load(Ordering::Acquire) == transition_id.low - { - self.checkpoint_pin_generation.store(0, Ordering::Release); - } - Ok(()) - } - #[must_use] pub fn lifecycle(&self) -> PartitionLifecycle { lifecycle_from_code(self.lifecycle.load(Ordering::Acquire)) @@ -1640,132 +1600,6 @@ impl Partition { Ok(()) } - /// Creates an exact checkpoint while a transfer source is quiesced. - /// - /// # Errors - /// - /// Returns an error unless the partition is drained and fenced, or if the - /// tree checkpoint fails. - pub async fn checkpoint_quiesced(&self, ownership_epoch: u64) -> Result { - self.validate_epoch(ownership_epoch)?; - let _maintenance = self.split_transition.lock().await; - if self.lifecycle() != PartitionLifecycle::TransferQuiesced - || self.queued_requests.load(Ordering::Acquire) != 0 - { - return Err(ChunkKvError::InvalidRequest( - "checkpoint requires a quiesced mutation source".into(), - )); - } - self.create_checkpoint().await - } - - /// Publishes a recoverable checkpoint while the partition continues - /// serving. The replay offset is captured before the tree flush, so a - /// concurrent mutation can only make recovery replay an already included - /// record; it cannot create a WAL gap. - /// - /// # Errors - /// - /// Returns a stale-epoch, lifecycle, tree, or frontier error. - pub async fn checkpoint(&self, ownership_epoch: u64) -> Result { - self.validate_epoch(ownership_epoch)?; - let _maintenance = self.split_transition.lock().await; - if self.checkpoint_pin_generation.load(Ordering::Acquire) != 0 { - return Err(ChunkKvError::SplitRetry( - "checkpoint publication is suppressed by an exact-root transition pin".into(), - )); - } - match self.lifecycle() { - PartitionLifecycle::Serving - | PartitionLifecycle::TransferQuiesced - | PartitionLifecycle::SplitFinalizing => self.create_checkpoint().await, - state => Err(read_state_error(state)), - } - } - - async fn create_checkpoint(&self) -> Result { - let replay_offset = self.retry_replay_offset.load(Ordering::Acquire); - self.journal.checkpoint().await?; - let stream_generation = self.journal.record_generation(); - let (tree_manifest, applied_seq) = self.tree.checkpoint(replay_offset).await?; - let root_manifest_generation = self.tree.root_manifest_generation()?; - if applied_seq > self.journal_durable_seq.load(Ordering::Acquire) - || self.tree.last_applied_seq() < applied_seq - { - return Err(ChunkKvError::ApplyStateUnknown); - } - self.metrics.checkpoint(); - Ok(Checkpoint { - tree_id: self.tree.tree_id(), - tree_manifest, - root_manifest_generation, - applied_seq, - stream_name: self.journal.stream_name(), - stream_generation, - replay_offset, - }) - } - - /// Trims WAL bytes only after the caller supplies the published checkpoint. - /// - /// # Errors - /// - /// Returns an error for stale identity/frontiers or journal GC failure. - pub async fn trim_published_checkpoint( - &self, - ownership_epoch: u64, - checkpoint: &Checkpoint, - ) -> Result { - self.validate_epoch(ownership_epoch)?; - if checkpoint.stream_name != self.journal.stream_name() - || checkpoint.stream_generation == 0 - || checkpoint.stream_generation > self.journal.record_generation() - || checkpoint.applied_seq > self.applied_seq.load(Ordering::Acquire) - || checkpoint.replay_offset > self.journal.tail() - || checkpoint.replay_offset > self.retry_replay_offset.load(Ordering::Acquire) - { - return Err(ChunkKvError::InvalidRequest( - "checkpoint does not belong to this frontier".into(), - )); - } - self.journal.trim_prefix(checkpoint.replay_offset).await - } - - /// Advances both durable retention watermarks after the caller has - /// published this exact checkpoint, then reclaims unreferenced chunk - /// objects left by interrupted page-store work. - /// - /// # Errors - /// - /// Returns an error for stale authority, a mismatched checkpoint, or a - /// journal/tree maintenance failure. - pub async fn reclaim_published_checkpoint( - &self, - ownership_epoch: u64, - checkpoint: &Checkpoint, - ) -> Result { - let journal_bytes = self - .trim_published_checkpoint(ownership_epoch, checkpoint) - .await?; - let metadata_records = self - .journal - .reclaim_metadata_before( - checkpoint.stream_generation, - self.config.metadata_reclaim_records_per_pass, - ) - .await?; - let tree_bytes = self.tree.reclaim_before(checkpoint.root_manifest_generation)?; - let orphan_bytes = self.tree.reclaim_orphans()?; - self.metrics - .reclaim(journal_bytes, metadata_records, tree_bytes, orphan_bytes); - Ok(CheckpointReclaim { - journal_bytes, - metadata_records, - tree_bytes, - orphan_bytes, - }) - } - /// Returns native chunk page-store counters when this partition uses the /// R140 backend. /// diff --git a/lib/crowdb-chunk-kv/src/partition/checkpoint.rs b/lib/crowdb-chunk-kv/src/partition/checkpoint.rs new file mode 100644 index 000000000..fcc477f0c --- /dev/null +++ b/lib/crowdb-chunk-kv/src/partition/checkpoint.rs @@ -0,0 +1,223 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Checkpoint frontiers and transition retention. + +use std::sync::atomic::Ordering; + +use super::{ + read_state_error, Checkpoint, CheckpointReclaim, ChunkKvError, Partition, PartitionLifecycle, Result, + TransitionId, +}; + +impl Partition { + /// Creates an exact checkpoint while a transfer source is quiesced. + /// + /// # Errors + /// + /// Returns an error unless the partition is drained and fenced, or if the + /// tree checkpoint fails. + pub async fn checkpoint_quiesced(&self, ownership_epoch: u64) -> Result { + let _maintenance = self.split_transition.lock().await; + self.validate_epoch(ownership_epoch)?; + if self.lifecycle() != PartitionLifecycle::TransferQuiesced + || self.queued_requests.load(Ordering::Acquire) != 0 + { + return Err(ChunkKvError::InvalidRequest( + "checkpoint requires a quiesced mutation source".into(), + )); + } + self.create_checkpoint().await + } + + /// Publishes a recoverable checkpoint while the partition continues + /// serving. The replay offset is captured before the tree flush, so a + /// concurrent mutation can only make recovery replay an already included + /// record; it cannot create a WAL gap. + /// + /// # Errors + /// + /// Returns a stale-epoch, lifecycle, tree, or frontier error. + pub async fn checkpoint(&self, ownership_epoch: u64) -> Result { + let _maintenance = self.split_transition.lock().await; + self.validate_epoch(ownership_epoch)?; + if self.checkpoint_pin_generation.load(Ordering::Acquire) != 0 { + return Err(ChunkKvError::SplitRetry( + "checkpoint publication is suppressed by an exact-root transition pin".into(), + )); + } + match self.lifecycle() { + PartitionLifecycle::Serving + | PartitionLifecycle::TransferQuiesced + | PartitionLifecycle::SplitFinalizing => self.create_checkpoint().await, + state => Err(read_state_error(state)), + } + } + + pub(super) async fn create_checkpoint(&self) -> Result { + let replay_offset = self.retry_replay_offset.load(Ordering::Acquire); + self.journal.checkpoint().await?; + let stream_generation = self.journal.record_generation(); + let (tree_manifest, applied_seq) = self.tree.checkpoint(replay_offset).await?; + let root_manifest_generation = self.tree.root_record_generation()?; + if applied_seq > self.journal_durable_seq.load(Ordering::Acquire) + || self.tree.last_applied_seq() < applied_seq + { + return Err(ChunkKvError::ApplyStateUnknown); + } + self.metrics.checkpoint(); + Ok(Checkpoint { + tree_id: self.tree.tree_id(), + tree_manifest, + root_manifest_generation, + applied_seq, + stream_name: self.journal.stream_name(), + stream_generation, + replay_offset, + }) + } + + /// Trims WAL bytes only after the caller supplies the published checkpoint. + /// + /// # Errors + /// + /// Returns an error for stale identity/frontiers, a nonzero replay offset + /// without durable expired-request floors, or journal GC failure. + pub async fn trim_published_checkpoint( + &self, + ownership_epoch: u64, + checkpoint: &Checkpoint, + ) -> Result { + let _maintenance = self.split_transition.lock().await; + self.ensure_reclaim_allowed(ownership_epoch)?; + self.trim_checkpoint(checkpoint).await + } + + async fn trim_checkpoint(&self, checkpoint: &Checkpoint) -> Result { + if checkpoint.tree_id != self.tree.tree_id() + || checkpoint.stream_name != self.journal.stream_name() + || checkpoint.stream_generation == 0 + || checkpoint.stream_generation > self.journal.record_generation() + || checkpoint.applied_seq > self.applied_seq.load(Ordering::Acquire) + || checkpoint.replay_offset > self.journal.tail() + || checkpoint.replay_offset > self.retry_replay_offset.load(Ordering::Acquire) + { + return Err(ChunkKvError::InvalidRequest( + "checkpoint does not belong to this frontier".into(), + )); + } + self.journal.trim_prefix(checkpoint.replay_offset).await + } + + /// Advances both durable retention watermarks after the caller has + /// published this exact checkpoint, then reclaims unreferenced chunk + /// objects left by interrupted page-store work. + /// + /// # Errors + /// + /// Returns an error for stale authority, a mismatched checkpoint, or a + /// journal/tree maintenance failure. + pub async fn reclaim_published_checkpoint( + &self, + ownership_epoch: u64, + checkpoint: &Checkpoint, + ) -> Result { + let _maintenance = self.split_transition.lock().await; + self.ensure_reclaim_allowed(ownership_epoch)?; + self.reclaim_checkpoint(checkpoint).await + } + + pub(super) async fn reclaim_checkpoint(&self, checkpoint: &Checkpoint) -> Result { + let journal_bytes = self.trim_checkpoint(checkpoint).await?; + let metadata_records = self + .journal + .reclaim_metadata_before( + checkpoint.stream_generation, + self.config.metadata_reclaim_records_per_pass, + ) + .await?; + let tree_bytes = self.tree.reclaim_before(checkpoint.root_manifest_generation)?; + let orphan_bytes = self.tree.reclaim_orphans()?; + self.metrics + .reclaim(journal_bytes, metadata_records, tree_bytes, orphan_bytes); + Ok(CheckpointReclaim { + journal_bytes, + metadata_records, + tree_bytes, + orphan_bytes, + }) + } + + /// Publishes and pins one exact source checkpoint in the same maintenance turn. + /// + /// # Errors + /// Returns a stale authority, lifecycle, checkpoint or durable pin error. + pub async fn checkpoint_with_pin( + &self, + ownership_epoch: u64, + transition_id: TransitionId, + ) -> Result { + let _maintenance = self.split_transition.lock().await; + self.ensure_reclaim_allowed(ownership_epoch)?; + let checkpoint = self.create_checkpoint().await?; + self.retain_generation_pin(transition_id, checkpoint.root_manifest_generation)?; + self.metadata_reclaim.store(None); + Ok(checkpoint) + } + + pub(super) fn ensure_reclaim_allowed(&self, ownership_epoch: u64) -> Result<()> { + self.validate_epoch(ownership_epoch)?; + if self.checkpoint_pin_generation.load(Ordering::Acquire) != 0 { + return Err(ChunkKvError::SplitRetry( + "history is retained by an exact-root transition pin".into(), + )); + } + match self.lifecycle() { + PartitionLifecycle::Serving | PartitionLifecycle::TransferQuiesced => Ok(()), + state => Err(read_state_error(state)), + } + } + /// Persists an exact root-generation reference before publishing a + /// topology artifact that depends on it. + /// + /// # Errors + /// + /// Returns a storage error when the generation is absent or conflicts + /// with the transition's existing pin. + pub(super) fn retain_generation_pin(&self, transition_id: TransitionId, generation: u64) -> Result<()> { + let current = self.checkpoint_pin_generation.load(Ordering::Acquire); + if current != 0 + && (current != generation + || self.checkpoint_pin_transition_high.load(Ordering::Acquire) != transition_id.high + || self.checkpoint_pin_transition_low.load(Ordering::Acquire) != transition_id.low) + { + return Err(ChunkKvError::InvalidRequest( + "another transition already pins the partition checkpoint lineage".into(), + )); + } + self.tree.pin_generation(transition_id, generation)?; + self.checkpoint_pin_transition_high + .store(transition_id.high, Ordering::Relaxed); + self.checkpoint_pin_transition_low + .store(transition_id.low, Ordering::Relaxed); + self.checkpoint_pin_generation + .store(generation, Ordering::Release); + Ok(()) + } + + /// Releases this partition tree's durable generation pin after the + /// authoritative catalog has removed its overlay dependency. + /// + /// # Errors + /// + /// Returns a storage error when durable pin deletion cannot complete. + pub fn release_generation_pin(&self, transition_id: TransitionId) -> Result<()> { + self.tree.unpin_generation(transition_id)?; + if self.checkpoint_pin_transition_high.load(Ordering::Acquire) == transition_id.high + && self.checkpoint_pin_transition_low.load(Ordering::Acquire) == transition_id.low + { + self.checkpoint_pin_generation.store(0, Ordering::Release); + } + Ok(()) + } +} diff --git a/lib/crowdb-chunk-kv/src/partition/maintenance.rs b/lib/crowdb-chunk-kv/src/partition/maintenance.rs new file mode 100644 index 000000000..ad51a183f --- /dev/null +++ b/lib/crowdb-chunk-kv/src/partition/maintenance.rs @@ -0,0 +1,58 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Reclamation of superseded extent directories for independently serving WALs. + +use std::sync::{atomic::Ordering, Arc}; + +use super::{Partition, PartitionLifecycle, Result}; + +pub(super) struct MetadataReclaim { + generation: u64, + pending: bool, +} + +impl Partition { + /// Reclaims one bounded batch of superseded WAL extent pages. + /// + /// The current manifest still maps the complete retained WAL, so existing + /// tree checkpoints and retry history remain recoverable. This does not + /// trim WAL bytes or advance tree retention. Transition pins retain their + /// exact manifest generations and suppress maintenance. + /// + /// Returns true when another batch may remain. Callers revisit other + /// hosted partitions before continuing backlog work. + /// + /// # Errors + /// Returns a stale epoch or metadata-store error. Failed deletion retains + /// its watermark for the next pass. + pub async fn maintain_serving_metadata(&self, ownership_epoch: u64) -> Result { + let _maintenance = self.split_transition.lock().await; + self.validate_epoch(ownership_epoch)?; + if self.lifecycle() != PartitionLifecycle::Serving + || self.checkpoint_pin_generation.load(Ordering::Acquire) != 0 + { + self.metadata_reclaim.store(None); + return Ok(false); + } + let current = self.journal.record_generation(); + let generation = match self.metadata_reclaim.load_full() { + Some(retained) if retained.pending => retained.generation, + Some(retained) if retained.generation == current => return Ok(false), + _ => current, + }; + self.metadata_reclaim.store(Some(Arc::new(MetadataReclaim { + generation, + pending: true, + }))); + let reclaimed = self + .journal + .reclaim_metadata_before(generation, self.config.metadata_reclaim_records_per_pass) + .await?; + self.metrics.reclaim(0, reclaimed, 0, 0); + let pending = reclaimed >= self.config.metadata_reclaim_records_per_pass as u64; + self.metadata_reclaim + .store(Some(Arc::new(MetadataReclaim { generation, pending }))); + Ok(pending) + } +} diff --git a/lib/crowdb-chunk-kv/tests/history_maintenance_test.rs b/lib/crowdb-chunk-kv/tests/history_maintenance_test.rs new file mode 100644 index 000000000..f6a60fa60 --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/history_maintenance_test.rs @@ -0,0 +1,244 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_chunk_kv::memory::MemoryPartitionTree; +use crowdb_chunk_kv::{ + ChunkKvError, MutationOperation, Partition, PartitionConfig, PartitionId, PartitionJournal, + PartitionRange, RequestId, StreamPartitionJournal, TransitionId, +}; +use crowdb_chunk_stream::memory::MemoryStreamStore; +use crowdb_chunk_stream::{ + ChunkStream, StreamBinding, StreamBindingState, StreamConfig, StreamMetadataStore, StreamName, + StreamPurpose, +}; + +const EPOCH: u64 = 7; +const NAME: StreamName = StreamName { high: 9, low: 11 }; +const ID: PartitionId = PartitionId { high: 9, low: 12 }; + +struct TestPartition { + partition: Partition, + store: Arc, + journal: Arc, + config: PartitionConfig, +} + +impl TestPartition { + async fn open() -> Self { + let store = Arc::new(MemoryStreamStore::new(512)); + let stream = ChunkStream::create( + StreamBinding { + purpose: StreamPurpose::Wal, + stream_name: NAME, + metadata_group_id: 1, + binding_generation: 1, + state: StreamBindingState::Active, + owner_kind: Some("chunk-kv-partition".into()), + }, + EPOCH, + StreamConfig { + extent_page_entries: 4, + ..StreamConfig::default() + }, + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + let journal: Arc = Arc::new(StreamPartitionJournal::new(stream, NAME).unwrap()); + let tree = Arc::new(MemoryPartitionTree::default()); + let config = PartitionConfig { + retained_results: 2, + metadata_reclaim_pages_per_pass: 2, + ..PartitionConfig::default() + }; + let partition = + Partition::open(ID, range(), EPOCH, config.clone(), tree.clone(), journal.clone()).unwrap(); + Self { + partition, + store, + journal, + config, + } + } + + async fn write(&self, sequence: u64) -> crowdb_chunk_kv::MutationResponse { + self.partition + .mutate(EPOCH, request(sequence), operation(sequence)) + .await + .unwrap() + } + + async fn drain(&self) { + for _ in 0..256 { + if !self.partition.maintain_serving_metadata(EPOCH).await.unwrap() { + return; + } + } + panic!("bounded maintenance did not drain its finite metadata backlog"); + } +} + +fn range() -> PartitionRange { + PartitionRange { + start: None, + end: None, + } +} + +fn request(sequence: u64) -> RequestId { + RequestId { + client_high: 1, + client_low: 2, + client_sequence: sequence, + } +} + +fn operation(sequence: u64) -> MutationOperation { + MutationOperation::Put { + key: b"key".to_vec(), + value: sequence.to_be_bytes().to_vec(), + } +} + +#[tokio::test] +async fn superseded_directories_are_bounded_and_old_checkpoints_keep_retry_history() { + let fixture = TestPartition::open().await; + let checkpoint = fixture.partition.checkpoint(EPOCH).await.unwrap(); + for round in 0..20 { + for sequence in (round * 8 + 1)..=(round * 8 + 8) { + fixture.write(sequence).await; + } + let checkpoints = fixture.partition.metrics().snapshot().checkpoints; + fixture.drain().await; + assert_eq!(fixture.partition.metrics().snapshot().checkpoints, checkpoints); + let current = fixture.store.load_current(NAME).await.unwrap().unwrap(); + assert_eq!( + fixture.store.extent_page_count().await, + current.extent_pages.len() + ); + fixture.drain().await; + assert_eq!(fixture.partition.metrics().snapshot().checkpoints, checkpoints); + } + let expected = fixture.write(161).await; + fixture.drain().await; + assert_eq!(checkpoint.replay_offset, 0); + fixture.partition.suspend_for_transfer(EPOCH).await.unwrap(); + let recovered = Partition::recover( + ID, + range(), + EPOCH, + checkpoint, + fixture.config, + Arc::new(MemoryPartitionTree::default()), + fixture.journal, + ) + .await + .unwrap(); + assert_eq!( + recovered.get(EPOCH, b"key", None).await.unwrap().unwrap().value, + 161_u64.to_be_bytes() + ); + assert_eq!( + recovered + .mutate(EPOCH, request(161), operation(161)) + .await + .unwrap(), + expected + ); + assert_eq!( + recovered.mutate(EPOCH, request(1), operation(1)).await, + Err(ChunkKvError::RequestExpired) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn concurrent_transition_pin_keeps_source_wal_and_extent_pages() { + let fixture = TestPartition::open().await; + for sequence in 1..=16 { + fixture.write(sequence).await; + } + let transition = TransitionId { high: 90, low: 91 }; + let (_, checkpoint) = tokio::join!( + fixture.partition.maintain_serving_metadata(EPOCH), + fixture.partition.checkpoint_with_pin(EPOCH, transition), + ); + let checkpoint = checkpoint.unwrap(); + let retained = fixture.store.load_current(NAME).await.unwrap().unwrap(); + for sequence in 17..=32 { + fixture.write(sequence).await; + } + let before = fixture.store.extent_page_count().await; + assert!(!fixture.partition.maintain_serving_metadata(EPOCH).await.unwrap()); + assert_eq!(fixture.store.extent_page_count().await, before); + assert!(matches!( + fixture + .partition + .reclaim_published_checkpoint(EPOCH, &checkpoint) + .await, + Err(ChunkKvError::SplitRetry(_)) + )); + for fence in retained.extent_pages { + assert!(fixture + .store + .load_extent_page(NAME, retained.writer_epoch, retained.generation, fence.page_index) + .await + .unwrap() + .is_some()); + } + assert!(!fixture + .journal + .read_window(checkpoint.replay_offset, 4096) + .await + .unwrap() + .is_empty()); + fixture.partition.release_generation_pin(transition).unwrap(); + fixture.drain().await; + assert!(fixture.store.extent_page_count().await < before); + assert_eq!( + fixture.partition.maintain_serving_metadata(EPOCH + 1).await, + Err(ChunkKvError::StaleEpoch) + ); +} + +#[tokio::test] +async fn physical_wal_trim_rejects_a_checkpoint_without_durable_retry_floors() { + let fixture = TestPartition::open().await; + let initial = fixture.partition.checkpoint(EPOCH).await.unwrap(); + for sequence in 1..=8 { + fixture.write(sequence).await; + } + let checkpoint = fixture.partition.checkpoint(EPOCH).await.unwrap(); + assert!(checkpoint.replay_offset > 0); + assert!(matches!( + fixture + .partition + .reclaim_published_checkpoint(EPOCH, &checkpoint) + .await, + Err(ChunkKvError::MaintenanceDegraded(_)) + )); + assert!(!fixture + .journal + .read_window(initial.replay_offset, 4096) + .await + .unwrap() + .is_empty()); + let recovered = Partition::recover( + ID, + range(), + EPOCH, + initial, + fixture.config, + Arc::new(MemoryPartitionTree::default()), + fixture.journal, + ) + .await + .unwrap(); + assert_eq!( + recovered.mutate(EPOCH, request(1), operation(1)).await, + Err(ChunkKvError::RequestExpired) + ); +} diff --git a/lib/crowdb-chunk-kv/tests/partition_test.rs b/lib/crowdb-chunk-kv/tests/partition_test.rs index e5ba5cff5..d3c0c37c7 100644 --- a/lib/crowdb-chunk-kv/tests/partition_test.rs +++ b/lib/crowdb-chunk-kv/tests/partition_test.rs @@ -1359,12 +1359,8 @@ async fn transition_generation_pin_suppresses_source_checkpoint_branching() { ) .await .unwrap(); - let checkpoint = partition.checkpoint(10).await.unwrap(); let transition = TransitionId { high: 70, low: 71 }; - - partition - .retain_generation_pin(transition, checkpoint.root_manifest_generation) - .unwrap(); + partition.checkpoint_with_pin(10, transition).await.unwrap(); assert!(matches!( partition.checkpoint(10).await, Err(ChunkKvError::SplitRetry(_)) diff --git a/lib/crowdb-chunk-stream/tests/stream_test.rs b/lib/crowdb-chunk-stream/tests/stream_test.rs index 7d3c09fd0..644151634 100644 --- a/lib/crowdb-chunk-stream/tests/stream_test.rs +++ b/lib/crowdb-chunk-stream/tests/stream_test.rs @@ -712,3 +712,6 @@ async fn reopen_recovers_tail_record_and_reads_before_another_append() { assert_eq!(observed.chunk.logical_end, 9); assert!(observed.chunk.mapping_count <= 3); } + +#[path = "stream_test/reclamation.rs"] +mod reclamation; diff --git a/lib/crowdb-chunk-stream/tests/stream_test/reclamation.rs b/lib/crowdb-chunk-stream/tests/stream_test/reclamation.rs new file mode 100644 index 000000000..8fd98ab6e --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/stream_test/reclamation.rs @@ -0,0 +1,24 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test] +async fn trim_retains_frames_in_the_active_and_boundary_chunk() { + let store = Arc::new(MemoryStreamStore::new(128)); + let stream = create_stream(&store, 128, StreamConfig::default()).await; + let first = stream.append(&[Bytes::from_static(b"abcd")]).await.unwrap(); + let second = stream.append(&[Bytes::from_static(b"efgh")]).await.unwrap(); + assert_eq!(first.chunk_id, second.chunk_id); + assert_eq!(stream.trim_prefix(4).await.unwrap(), 0); + assert!(!store.is_released(first.chunk_id.unwrap()).await); + assert_eq!(stream.read_at(4, 4).await.unwrap(), Bytes::from_static(b"efgh")); + stream.append(&[Bytes::from_static(b"ijkl")]).await.unwrap(); + let successor = stream.append(&[Bytes::from(vec![b'x'; 64])]).await.unwrap(); + assert_ne!(first.chunk_id, successor.chunk_id); + assert_eq!(stream.trim_prefix(8).await.unwrap(), 0); + assert_eq!(stream.read_at(8, 4).await.unwrap(), Bytes::from_static(b"ijkl")); + assert!(stream.trim_prefix(12).await.unwrap() > 0); + assert!(store.is_released(first.chunk_id.unwrap()).await); + assert_eq!(stream.trim_prefix(12).await.unwrap(), 0); +} diff --git a/lib/crowdb-kv/src/cluster.rs b/lib/crowdb-kv/src/cluster.rs index fe5e4a33e..bc96c6afe 100644 --- a/lib/crowdb-kv/src/cluster.rs +++ b/lib/crowdb-kv/src/cluster.rs @@ -19,11 +19,8 @@ pub mod group_operations; pub(crate) mod group_owner_fence; pub mod group_prepare; pub mod group_propose; -<<<<<<< HEAD mod group_routes; -======= mod group_retention; ->>>>>>> 0f5d94a8 (Reclaim snapshot-backed Paxos history with retained readers) #[cfg(feature = "test-util")] mod group_tests; pub mod kv_server; diff --git a/lib/crowdb-kv/src/cluster/group_tests.rs b/lib/crowdb-kv/src/cluster/group_tests.rs index 31f00c39c..f57e9e1f8 100644 --- a/lib/crowdb-kv/src/cluster/group_tests.rs +++ b/lib/crowdb-kv/src/cluster/group_tests.rs @@ -14,14 +14,12 @@ use crate::paxos::PxNodeId; /// repair step, and peer-applied injection — to integration tests under /// `tests/` without permanently widening the production public API. impl PxGroup { -<<<<<<< HEAD /// Whether a conditional mutation currently owns one key's admission. #[must_use] pub fn conditional_write_pending_for_tests(&self, key: &[u8]) -> bool { self.cas_transient_map.get(key).is_some() } -======= pub fn reclamation_certificate_for_tests(&self) -> u64 { self.reclamation_certificate() } @@ -33,7 +31,6 @@ impl PxGroup { pub fn record_reclamation_heartbeat_for_tests(&self, request: &super::replica::HeartbeatRequestPayload) { self.record_reclamation_heartbeat(request); } ->>>>>>> 0f5d94a8 (Reclaim snapshot-backed Paxos history with retained readers) /// Run one heartbeat round without depending on the election timer. pub async fn heartbeat_round_for_tests(self: &std::sync::Arc) -> bool { matches!( diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service.rs b/lib/crowdb-kv/src/rpc/px_rpc_service.rs index ba69313bf..c26169ce4 100644 --- a/lib/crowdb-kv/src/rpc/px_rpc_service.rs +++ b/lib/crowdb-kv/src/rpc/px_rpc_service.rs @@ -23,8 +23,6 @@ //! `Arc` so it can submit responses from either the dispatch //! thread (sync error path) or the spawned task (async success path). -mod heartbeat; - use std::future::Future; use std::sync::Arc; @@ -32,11 +30,11 @@ use crowdb_protocol::fb::FBMsgType; use crowdb_protocol::kv_consensus_fb::{ FBAcceptRequest, FBAcceptedResponse, FBAcceptedResponseArgs, FBAcceptedValue, FBAcceptedValueArgs, FBBatchChosenNotification, FBChosenNotification, FBFetchGapRequest, FBFetchGapResponse, - FBFetchGapResponseArgs, FBHeartbeatResponse, FBHeartbeatResponseArgs, FBKvRetCode, FBPreVoteRequest, - FBPreVoteResponse, FBPreVoteResponseArgs, FBPrepareRequest, FBPromiseResponse, FBPromiseResponseArgs, - FBRequestVoteRequest, FBRequestVoteResponse, FBRequestVoteResponseArgs, FBSnapshotAbortRequest, - FBSnapshotAbortResponse, FBSnapshotAbortResponseArgs, FBSnapshotBeginRequest, FBSnapshotBeginResponse, - FBSnapshotBeginResponseArgs, FBSnapshotFinishRequest, FBSnapshotFinishResponse, + FBFetchGapResponseArgs, FBHeartbeatRequest, FBHeartbeatResponse, FBHeartbeatResponseArgs, FBKvRetCode, + FBPreVoteRequest, FBPreVoteResponse, FBPreVoteResponseArgs, FBPrepareRequest, FBPromiseResponse, + FBPromiseResponseArgs, FBRequestVoteRequest, FBRequestVoteResponse, FBRequestVoteResponseArgs, + FBSnapshotAbortRequest, FBSnapshotAbortResponse, FBSnapshotAbortResponseArgs, FBSnapshotBeginRequest, + FBSnapshotBeginResponse, FBSnapshotBeginResponseArgs, FBSnapshotFinishRequest, FBSnapshotFinishResponse, FBSnapshotFinishResponseArgs, FBSnapshotReadRequest, FBSnapshotReadResponse, FBSnapshotReadResponseArgs, FBStepDownRequest, FBStepDownResponse, FBStepDownResponseArgs, }; @@ -47,7 +45,9 @@ use tracing::{debug, field, info_span, warn, Instrument, Span}; use crate::cluster::local_replica::PxLocalReplica; use crate::cluster::px_kv_store::PxKvStore; -use crate::cluster::replica::{PxReplicaError, ReplicaHandler, StepDownRequestPayload, VoteRequestPayload}; +use crate::cluster::replica::{ + HeartbeatRequestPayload, PxReplicaError, ReplicaHandler, StepDownRequestPayload, VoteRequestPayload, +}; use crate::paxos::roles::{PxAcceptReply, PxBallot, PxLogEntry, PxPrepareReply}; use crate::rpc::snapshot_registry::{ SnapshotRegistration, SnapshotRegistry, SnapshotSessionError, SnapshotSessionId, SnapshotSourceMetrics, @@ -203,7 +203,6 @@ impl PxRpcService { } } -<<<<<<< HEAD fn target_matches(&self, req: &ServerRequest) -> bool { let consensus = [ FBMsgType::EPrepareRequest, @@ -221,845 +220,6 @@ impl PxRpcService { .any(|kind| u16::try_from(kind.0).ok() == Some(req.msg_type)) { return true; -======= - // ── Prepare ────────────────────────────────────────────────── - - #[allow(clippy::too_many_lines)] - fn handle_prepare(&self, req: ServerRequest, server: &Arc) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle_usize = req.conn_handle as usize; - let msg_type = FBMsgType::EPromiseResponse.0 as u16; - let store = Arc::clone(&self.store); - let server = Arc::clone(server); - spawn_request(&self.rt, "prepare", store.store_id, async move { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::InvalidArgument, - "invalid request flatbuffer", - ); - return; - }; - let group_id = fb_req.group_id(); - record_group_context(&store, group_id); - let slot = fb_req.slot(); - let round = fb_req.round(); - let leader_id = fb_req.leader_id(); - let term = fb_req.term(); - let membership_epoch = fb_req.membership_epoch(); - - let Some(group) = store.get_group(group_id) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::NotFound, - "px group not found", - ); - return; - }; - let responder_epoch = group.membership_epoch(); - - // Membership-epoch fence (same as px_service.rs L131-147). - if membership_epoch != responder_epoch { - let converged_epoch = group.adopt_membership_epoch(membership_epoch); - warn!( - group_id, - slot, - round, - leader_id, - proposer_epoch = membership_epoch, - responder_epoch, - converged_epoch, - "prepare rejected by membership-epoch fence; adopting higher epoch from proposer" - ); - let term = group.local_replica().current_term_snapshot(); - let ctrl = build_promise_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - slot, - round, - leader_id, - None, - false, - 0, - 0, - term, - false, - responder_epoch, - true, - ); - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - return; - } - - let ballot = PxBallot { round, leader_id }; - let replica = group.local_replica(); - let reply = - ::on_prepare(replica, slot, ballot, term, group_id).await; - let ctrl = match reply { - Ok(PxPrepareReply::Promised { slot, accepted }) => { - let mut builder = FlatBufferBuilder::new(); - let prev_accepted_off = accepted.as_ref().map(|entry| { - let payload = builder.create_vector(&entry.payload); - FBAcceptedValue::create( - &mut builder, - &FBAcceptedValueArgs { - slot: entry.slot, - round: entry.ballot.round, - leader_id: entry.ballot.leader_id, - term: entry.term, - payload: Some(payload), - }, - ) - }); - let term = replica.current_term_snapshot(); - finish_promise_response( - builder, - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - slot, - round, - leader_id, - prev_accepted_off, - false, - 0, - 0, - term, - false, - responder_epoch, - false, - ) - } - Ok(PxPrepareReply::TermStale { slot, new_term }) => build_promise_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - slot, - round, - leader_id, - None, - false, - 0, - 0, - new_term, - true, - responder_epoch, - false, - ), - Ok(PxPrepareReply::Rejected { - slot, - current_promised, - }) => { - warn!( - group_id, - slot, - current_round = current_promised.round, - current_leader_id = current_promised.leader_id, - "prepare rejected; next step: proposer should retry with a higher ballot" - ); - let term = replica.current_term_snapshot(); - build_promise_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - slot, - round, - leader_id, - None, - true, - current_promised.round, - current_promised.leader_id, - term, - false, - responder_epoch, - false, - ) - } - Ok(PxPrepareReply::EpochMismatch { .. }) => { - unreachable!("on_prepare does not produce EpochMismatch") - } - Err(e) => { - let (code, msg) = px_error_to_ret_code(&e); - build_promise_response( - req_id, - create_nano, - code, - Some(&msg), - 1, - slot, - round, - leader_id, - None, - false, - 0, - 0, - replica.current_term_snapshot(), - false, - responder_epoch, - false, - ) - } - }; - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - // req dropped here, frame released - }); - } - - // ── Accept ─────────────────────────────────────────────────── - - #[allow(clippy::too_many_lines)] - fn handle_accept(&self, req: ServerRequest, server: &Arc) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle_usize = req.conn_handle as usize; - let msg_type = FBMsgType::EAcceptedResponse.0 as u16; - let store = Arc::clone(&self.store); - let server = Arc::clone(server); - spawn_request(&self.rt, "accept", store.store_id, async move { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::InvalidArgument, - "invalid request flatbuffer", - ); - return; - }; - let group_id = fb_req.group_id(); - record_group_context(&store, group_id); - let slot = fb_req.slot(); - let round = fb_req.round(); - let leader_id = fb_req.leader_id(); - let term = fb_req.term(); - let membership_epoch = fb_req.membership_epoch(); - - let Some(fb_value) = fb_req.value() else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::InvalidArgument, - "missing value", - ); - return; - }; - let payload_bytes: Vec = fb_value.payload().map(|v| v.bytes().to_vec()).unwrap_or_default(); - let entry = PxLogEntry { - slot, - ballot: PxBallot { round, leader_id }, - term, - payload: payload_bytes.into(), - }; - - let Some(group) = store.get_group(group_id) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::NotFound, - "px group not found", - ); - return; - }; - let responder_epoch = group.membership_epoch(); - - if membership_epoch != responder_epoch { - let converged_epoch = group.adopt_membership_epoch(membership_epoch); - warn!( - group_id, - slot, - round, - leader_id, - proposer_epoch = membership_epoch, - responder_epoch, - converged_epoch, - "accept rejected by membership-epoch fence; adopting higher epoch from proposer" - ); - let term = group.local_replica().current_term_snapshot(); - let ctrl = build_accepted_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - slot, - round, - leader_id, - false, - 0, - 0, - term, - false, - responder_epoch, - true, - ); - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - return; - } - - let replica = group.local_replica(); - let reply = ::on_accept(replica, &entry, group_id).await; - let (rejected, rejected_round, rejected_leader_id, term_stale, reply_term) = match reply { - Ok(PxAcceptReply::Accepted { .. }) => (false, 0, 0, false, replica.current_term_snapshot()), - Ok(PxAcceptReply::Rejected { current_promised, .. }) => { - warn!( - group_id, - slot, - current_round = current_promised.round, - current_leader_id = current_promised.leader_id, - "accept rejected; next step: proposer should run prepare with a higher ballot" - ); - ( - true, - current_promised.round, - current_promised.leader_id, - false, - replica.current_term_snapshot(), - ) - } - Ok(PxAcceptReply::TermStale { new_term, .. }) => { - warn!( - group_id, - slot, new_term, "accept rejected by term fence; proposer should step down" - ); - (false, 0, 0, true, new_term) - } - Ok(PxAcceptReply::EpochMismatch { .. }) => { - unreachable!("on_accept does not produce EpochMismatch") - } - Err(e) => { - let (code, msg) = px_error_to_ret_code(&e); - let ctrl = build_accepted_response( - req_id, - create_nano, - code, - Some(&msg), - 1, - slot, - round, - leader_id, - false, - 0, - 0, - replica.current_term_snapshot(), - false, - responder_epoch, - false, - ); - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - return; - } - }; - let ctrl = build_accepted_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - slot, - round, - leader_id, - rejected, - rejected_round, - rejected_leader_id, - reply_term, - term_stale, - responder_epoch, - false, - ); - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - // req dropped here, frame released - }); - } - - // ── PreVote ────────────────────────────────────────────────── - - fn handle_pre_vote(&self, req: ServerRequest, server: &Arc) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle_usize = req.conn_handle as usize; - let msg_type = FBMsgType::EPreVoteResponse.0 as u16; - let store = Arc::clone(&self.store); - let server = Arc::clone(server); - spawn_request(&self.rt, "pre_vote", store.store_id, async move { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::InvalidArgument, - "invalid request flatbuffer", - ); - return; - }; - let group_id = fb_req.group_id(); - record_group_context(&store, group_id); - let Some(group) = store.get_group(group_id) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::NotFound, - "px group not found", - ); - return; - }; - let payload = VoteRequestPayload { - term: fb_req.term(), - candidate_id: fb_req.candidate_id(), - accepted_log_tip_slot: fb_req.accepted_log_tip_slot(), - accepted_log_tip_term: fb_req.accepted_log_tip_term(), - }; - let replica = group.local_replica(); - let reply = ::on_pre_vote(replica, payload, group_id).await; - let ctrl = match reply { - Ok(r) => build_pre_vote_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - group_id, - r.term, - r.granted, - r.contiguous_chosen, - r.last_chosen_term, - r.highest_seen_slot, - ), - Err(e) => { - let (code, msg) = px_error_to_ret_code(&e); - build_pre_vote_response( - req_id, - create_nano, - code, - Some(&msg), - 1, - group_id, - 0, - false, - 0, - 0, - 0, - ) - } - }; - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - // req dropped here, frame released - }); - } - - // ── RequestVote ────────────────────────────────────────────── - - fn handle_request_vote(&self, req: ServerRequest, server: &Arc) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle_usize = req.conn_handle as usize; - let msg_type = FBMsgType::ERequestVoteResponse.0 as u16; - let store = Arc::clone(&self.store); - let server = Arc::clone(server); - spawn_request(&self.rt, "request_vote", store.store_id, async move { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::InvalidArgument, - "invalid request flatbuffer", - ); - return; - }; - let group_id = fb_req.group_id(); - record_group_context(&store, group_id); - let Some(group) = store.get_group(group_id) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::NotFound, - "px group not found", - ); - return; - }; - let payload = VoteRequestPayload { - term: fb_req.term(), - candidate_id: fb_req.candidate_id(), - accepted_log_tip_slot: fb_req.accepted_log_tip_slot(), - accepted_log_tip_term: fb_req.accepted_log_tip_term(), - }; - let replica = group.local_replica(); - let reply = ::on_request_vote(replica, payload, group_id).await; - let ctrl = match reply { - Ok(r) => build_request_vote_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - group_id, - r.term, - r.granted, - r.contiguous_chosen, - r.last_chosen_term, - r.highest_seen_slot, - ), - Err(e) => { - let (code, msg) = px_error_to_ret_code(&e); - build_request_vote_response( - req_id, - create_nano, - code, - Some(&msg), - 1, - group_id, - 0, - false, - 0, - 0, - 0, - ) - } - }; - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - // req dropped here, frame released - }); - } - - // ── StepDown ───────────────────────────────────────────────── - - fn handle_step_down(&self, req: ServerRequest, server: &Arc) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle_usize = req.conn_handle as usize; - let msg_type = FBMsgType::EStepDownResponse.0 as u16; - let store = Arc::clone(&self.store); - let server = Arc::clone(server); - spawn_request(&self.rt, "step_down", store.store_id, async move { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::InvalidArgument, - "invalid request flatbuffer", - ); - return; - }; - let group_id = fb_req.group_id(); - record_group_context(&store, group_id); - let Some(group) = store.get_group(group_id) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::NotFound, - "px group not found", - ); - return; - }; - let reason = fb_req.reason().unwrap_or("").to_string(); - let payload = StepDownRequestPayload { - term: fb_req.term(), - target_leader_id: fb_req.target_leader_id(), - reason, - }; - let replica = group.local_replica(); - let reply = ::on_step_down(replica, &payload, group_id).await; - let ctrl = match reply { - Ok(r) => build_step_down_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - group_id, - r.accepted, - r.current_term, - r.current_leader_id, - ), - Err(e) => { - let (code, msg) = px_error_to_ret_code(&e); - build_step_down_response(req_id, create_nano, code, Some(&msg), 1, group_id, false, 0, 0) - } - }; - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - // req dropped here, frame released - }); - } - - // ── ChosenNotification (fire-and-forget) ───────────────────── - - #[allow(clippy::needless_pass_by_value, reason = "make_handler uniform signature")] - fn handle_chosen_notice(&self, req: ServerRequest, _server: &Arc) { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - debug!("chosen notice: invalid flatbuffer"); - return; - }; - let group_id = fb_req.group_id(); - let slot = fb_req.slot(); - let term = fb_req.term(); - let leader_id = fb_req.leader_id(); - let ballot_round = fb_req.ballot_round(); - - let Some(group) = self.store.get_group(group_id) else { - debug!(slot, term, "chosen notice dropped (group not found)"); - return; - }; - let chosen_ballot = PxBallot { - round: ballot_round, - leader_id, - }; - spawn_group_request( - &self.rt, - "chosen_notice", - self.store.store_id, - group_id, - group.local_replica().id, - async move { - let replica = group.local_replica(); - let accepted = replica.accepted_at(slot).await; - let ballot_matches = accepted.as_ref().is_some_and(|e| e.ballot == chosen_ballot); - if ballot_matches { - replica.learner.update_chosen_frontier(slot, term); - replica.wake_apply_loop(); - debug!( - slot, - term, leader_id, ballot_round, "chosen notification applied (ballot match)" - ); - } else { - replica.note_chosen(slot, term); - replica.record_gap(slot); - if let Some(ref entry) = accepted { - replica.incr_chosen_notice_stale(); - debug!( - slot, - term, - leader_id, - chosen_ballot_round = ballot_round, - accepted_ballot_round = entry.ballot.round, - accepted_ballot_leader = entry.ballot.leader_id, - "chosen notification stale ballot (gap recorded)" - ); - } else { - replica.incr_chosen_notice_missing(); - debug!( - slot, - term, leader_id, "chosen notification missing value (gap recorded)" - ); - } - } - }, - ); - } - - // ── BatchChosenNotification (fire-and-forget) ──────────────── - - #[allow(clippy::needless_pass_by_value, reason = "make_handler uniform signature")] - fn handle_batch_chosen(&self, req: ServerRequest, _server: &Arc) { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - debug!("batch chosen: invalid flatbuffer"); - return; - }; - let group_id = fb_req.group_id(); - let start_slot = fb_req.start_slot(); - let end_slot = fb_req.end_slot(); - let term = fb_req.term(); - let leader_id = fb_req.leader_id(); - let ballot_round = fb_req.ballot_round(); - - let Some(group) = self.store.get_group(group_id) else { - debug!("batch chosen dropped (group not found)"); - return; - }; - let chosen_ballot = PxBallot { - round: ballot_round, - leader_id, - }; - spawn_group_request( - &self.rt, - "batch_chosen", - self.store.store_id, - group_id, - group.local_replica().id, - async move { - let replica = group.local_replica(); - let mut advanced_count = 0u64; - for slot in start_slot..=end_slot { - let accepted = replica.accepted_at(slot).await; - if accepted.as_ref().is_some_and(|e| e.ballot == chosen_ballot) { - replica.learner.update_chosen_frontier(slot, term); - advanced_count += 1; - } else { - replica.note_chosen(slot, term); - replica.record_gap(slot); - } - } - replica.wake_apply_loop(); - debug!( - group_id, - start_slot, - end_slot, - term, - leader_id, - ballot_round, - advanced_count, - "batch chosen notification applied" - ); - }, - ); - } - - // ── FetchGap ───────────────────────────────────────────────── - - #[allow(clippy::needless_pass_by_value, reason = "make_handler uniform signature")] - fn handle_fetch_gap(&self, req: ServerRequest, server: &Arc) { - let span = request_span("fetch_gap", self.store.store_id); - let _entered = span.enter(); - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle = req.conn_handle; - let msg_type = FBMsgType::EFetchGapResponse.0 as u16; - - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - submit_error( - server, - conn_handle, - req_id, - create_nano, - msg_type, - FBKvRetCode::InvalidArgument, - "invalid request flatbuffer", - ); - return; - }; - let group_id = fb_req.group_id(); - record_group_context(&self.store, group_id); - let slot = fb_req.slot(); - - let Some(group) = self.store.get_group(group_id) else { - submit_error( - server, - conn_handle, - req_id, - create_nano, - msg_type, - FBKvRetCode::NotFound, - "px group not found", - ); - return; - }; - if let Some(resp) = group.handle_fetch_gap(slot) { - let payload_vec = resp.payload.clone(); - let ctrl = build_fetch_gap_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - group_id, - resp.slot, - resp.term, - resp.ballot_round, - resp.leader_id, - &payload_vec, - ); - submit_fb_response(server, conn_handle, ctrl, msg_type, req_id); - } else { - debug!(slot, "fetch_gap no value (not yet chosen)"); - submit_error( - server, - conn_handle, - req_id, - create_nano, - msg_type, - FBKvRetCode::NotFound, - "slot not yet chosen", - ); ->>>>>>> 0f5d94a8 (Reclaim snapshot-backed Paxos history with retained readers) } let Ok(target) = flatbuffers::root::(req.control()) @@ -1086,6 +246,7 @@ impl Drop for PxRpcService { mod accept; mod election; +mod heartbeat; mod notifications; #[allow(clippy::too_many_arguments)] mod prepare; diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service/election.rs b/lib/crowdb-kv/src/rpc/px_rpc_service/election.rs index 2f22a699f..f37c79a5f 100644 --- a/lib/crowdb-kv/src/rpc/px_rpc_service/election.rs +++ b/lib/crowdb-kv/src/rpc/px_rpc_service/election.rs @@ -184,99 +184,6 @@ impl PxRpcService { // ── Heartbeat ──────────────────────────────────────────────── - pub(super) fn handle_heartbeat(&self, req: ServerRequest, server: &Arc) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle_usize = req.conn_handle as usize; - let msg_type = FBMsgType::EHeartbeatResponse.0 as u16; - let store = Arc::clone(&self.store); - let server = Arc::clone(server); - spawn_request(&self.rt, "heartbeat", store.store_id, async move { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::InvalidArgument, - "invalid request flatbuffer", - ); - return; - }; - let group_id = fb_req.group_id(); - record_group_context(&store, group_id); - let Some(group) = store.get_group(group_id) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::NotFound, - "px group not found", - ); - return; - }; - let payload = HeartbeatRequestPayload { - term: fb_req.term(), - leader_id: fb_req.leader_id(), - prev_log_slot: fb_req.prev_log_slot(), - prev_log_term: fb_req.prev_log_term(), - committed_safe_slot: fb_req.committed_safe_slot(), - lease_grant_until_ms_mono: fb_req.lease_grant_until_ms_mono(), - t_send_ms_mono: fb_req.t_send_ms_mono(), - }; - let replica = group.local_replica(); - let reply = ::on_heartbeat(replica, payload, group_id).await; - let ctrl = match reply { - Ok(r) => build_heartbeat_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - group_id, - r.term, - r.success, - r.contiguous_chosen, - r.last_chosen_term, - r.contiguous_applied, - r.highest_seen_slot, - r.durable_snapshot_slot, - ), - Err(e) => { - let (code, msg) = px_error_to_ret_code(&e); - build_heartbeat_response( - req_id, - create_nano, - code, - Some(&msg), - 1, - group_id, - 0, - false, - 0, - 0, - 0, - 0, - 0, - ) - } - }; - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - // req dropped here, frame released - }); - } - - // ── StepDown ───────────────────────────────────────────────── - pub(super) fn handle_step_down(&self, req: ServerRequest, server: &Arc) { let req_id = req.request_id; let create_nano = req.rpc_create_nano; diff --git a/lib/crowdb-kv/src/rpc/px_rpc_transport.rs b/lib/crowdb-kv/src/rpc/px_rpc_transport.rs index d0b7b473e..40cc0f9bf 100644 --- a/lib/crowdb-kv/src/rpc/px_rpc_transport.rs +++ b/lib/crowdb-kv/src/rpc/px_rpc_transport.rs @@ -406,67 +406,6 @@ impl PxRpcTransport { }) } -<<<<<<< HEAD - /// Send a `Heartbeat` request via crowdb-rpc. - pub async fn send_heartbeat( - &self, - destination: ReplicaDestination<'_>, - req: HeartbeatRequestPayload, - group_id: u64, - ) -> Result { - let rpc_endpoint = destination.endpoint; - let target_replica_id = destination.replica_id; - let req_id = self.next_id(); - let conn = self.conn_for(rpc_endpoint)?; - let mut builder = FlatBufferBuilder::new(); - let target_cluster_id = self.cluster_id.as_deref().map(|id| builder.create_string(id)); - let args = FBHeartbeatRequestArgs { - id: req_id, - rpc_create_nano: 0, - version: 1, - group_id, - term: req.term, - leader_id: req.leader_id, - prev_log_slot: req.prev_log_slot, - prev_log_term: req.prev_log_term, - committed_safe_slot: req.committed_safe_slot, - lease_grant_until_ms_mono: req.lease_grant_until_ms_mono, - t_send_ms_mono: req.t_send_ms_mono, - target_replica_id, - target_cluster_id, - }; - let fb_req = FBHeartbeatRequest::create(&mut builder, &args); - builder.finish(fb_req, None); - let control = Buffer::from_bytes(builder.finished_data()); - let msg_type = FBMsgType::EHeartbeatRequest.0 as u16; - let fut = self - .rpc - .call(&self.server, &conn, req_id, control, None, msg_type) - .map_err(|e| self.map_rpc_err(e, rpc_endpoint, conn.generation()))?; - let resp = fut - .await - .map_err(|e| self.map_rpc_err(e, rpc_endpoint, conn.generation()))?; - let ctrl = resp - .control - .ok_or_else(|| PxReplicaError::Internal("heartbeat response missing control buffer".into()))?; - let r = FBHeartbeatResponseRef::new(ctrl.bytes()); - if !r.valid() { - return Err(PxReplicaError::Internal("heartbeat response malformed".into())); - } - check_ret_code(r.ret_code(), r.error_msg())?; - Ok(HeartbeatReply { - term: r.term(), - success: r.success(), - contiguous_chosen: r.contiguous_chosen(), - last_chosen_term: r.last_chosen_term(), - contiguous_applied: r.contiguous_applied(), - highest_seen_slot: r.highest_seen_slot(), - durable_snapshot_slot: r.durable_snapshot_slot(), - }) - } - -======= ->>>>>>> 0f5d94a8 (Reclaim snapshot-backed Paxos history with retained readers) /// Send a `StepDown` request via crowdb-rpc. pub async fn send_step_down( &self, diff --git a/lib/crowdb-kv/src/rpc/px_rpc_transport/heartbeat.rs b/lib/crowdb-kv/src/rpc/px_rpc_transport/heartbeat.rs index b99835d5d..f13f59a5a 100644 --- a/lib/crowdb-kv/src/rpc/px_rpc_transport/heartbeat.rs +++ b/lib/crowdb-kv/src/rpc/px_rpc_transport/heartbeat.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{check_ret_code, PxRpcTransport}; +use super::{check_ret_code, PxRpcTransport, ReplicaDestination}; use crate::cluster::replica::{HeartbeatReply, HeartbeatRequestPayload, PxReplicaError}; use crowdb_protocol::{ fb::FBMsgType, @@ -15,14 +15,19 @@ impl PxRpcTransport { /// Send a `Heartbeat` request via crowdb-rpc. pub async fn send_heartbeat( &self, - rpc_endpoint: &str, + destination: ReplicaDestination<'_>, req: HeartbeatRequestPayload, group_id: u64, ) -> Result { + let rpc_endpoint = destination.endpoint; + let target_replica_id = destination.replica_id; let req_id = self.next_id(); let conn = self.conn_for(rpc_endpoint)?; let mut builder = FlatBufferBuilder::new(); + let target_cluster_id = self.cluster_id.as_deref().map(|id| builder.create_string(id)); let args = FBHeartbeatRequestArgs { + target_replica_id, + target_cluster_id, id: req_id, rpc_create_nano: 0, version: 1, diff --git a/pixi.toml b/pixi.toml index 8dcee39ff..d6bc1ade8 100644 --- a/pixi.toml +++ b/pixi.toml @@ -225,7 +225,3 @@ test-boto3-copy-error = { cmd = "python tools/pixi-tasks/test.py unit boto3" } [feature.iceberg-ecosystem.tasks] test-iceberg-container-ecosystem = "bash e2e-test/clients/ecosystem/run.sh" -[pypi-dependencies] -maturin = ">=1.15, <2" -pip = ">=25, <27" -auditwheel = ">=6, <7" From c7eec29443c3b537bc138b197009b4a357c48112 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 23:11:28 +0800 Subject: [PATCH 087/258] Release reclaimed Paxos payloads and completed quorum drains --- doc/working/plan-dataset-native-sdk.md | 38 +++++++++++++ lib/crowdb-kv/src/cluster/group_accept.rs | 14 ++--- lib/crowdb-kv/src/cluster/group_prepare.rs | 7 +-- lib/crowdb-kv/src/paxos/acceptor.rs | 43 +++++++++++---- lib/crowdb-kv/tests/common/payload_owner.rs | 22 ++++++++ lib/crowdb-kv/tests/group_test.rs | 3 ++ .../tests/group_test/quorum_drain_test.rs | 54 +++++++++++++++++++ lib/crowdb-kv/tests/paxos_test.rs | 3 ++ .../tests/paxos_test/acceptor_test.rs | 36 +++++++++++++ 9 files changed, 201 insertions(+), 19 deletions(-) create mode 100644 lib/crowdb-kv/tests/common/payload_owner.rs create mode 100644 lib/crowdb-kv/tests/group_test/quorum_drain_test.rs diff --git a/doc/working/plan-dataset-native-sdk.md b/doc/working/plan-dataset-native-sdk.md index 49448d6e8..6ff5479d6 100644 --- a/doc/working/plan-dataset-native-sdk.md +++ b/doc/working/plan-dataset-native-sdk.md @@ -412,3 +412,41 @@ remaining user design decision and no requirement `Blocked` condition. - Next: rebuild/import this source, then rerun the unchanged full workload. Metadata history cleanup is verified at component level; full-service memory and comparative performance remain unaccepted until that run completes. + +- Metadata reclamation is committed as `1c9cc06e`. Standard rootless image build + and import pass. Image config: + `sha256:2991a6e21f6cb96f544aee0e6d8a3cd295878a7295dbe5aca2e1433a9f8e6a25`; + source tree `0f7d29d911eb74e73f07acc116de305ddd0299ec`. +- Full unchanged workload failed in + `dataset-workload/968559deb2cf49158eefc01a19939ca1`, with service error capture + enabled. Supplemental read-only observations include cgroup RAM, swap and OOM + counters plus retained slots and tree live-value summaries. +- This run completed HTTP six trials and native Rust singleton three trials; + native multi-flight returned `publication lost its head race`. Metadata RSS + still approached the 2-GiB limit, although extent cleanup kept live tree data + near 1–2 MB and acceptor trim/retained counts advanced. Partial rates remain + unaccepted comparative performance. +- Low-overhead sampled heap evidence is in diagnostic run + `dataset-workload/f5bf584463564df8971885734f38fb51`. Live allocations grew from + 255 MB to 1.76 GB and were dominated by `PxKvStore::encode_kv_payload`. This + diagnostic was stopped after locating the leak; it is not a performance run. + An earlier full heap-profile run (`212aa76eed8c463d88e406d7faf67a4b`) distorted + follower progress and must not be used to attribute normal RPC queue growth. +- Root cause: crossbeam-skiplist 0.1.3 `base::RefRange::{next,next_back}` replace + cursor entries without releasing their prior explicit references. Logical + removal succeeds while every scanned payload remains allocated. Acceptor + reclamation/log-tip now use the full iterator, and journal range scans walk + owned entries from `lower_bound`; both paths release cursor references. + The owner-destructor regression previously released 0 of 8192 payloads with + only one visible slot; it now passes while the acceptor remains live. +- Prepare and both accept quorum drains also retained completed tasks: matching + `Some(reply)` disabled the exhausted stream branch while cancellation stayed + pending. Explicit `None => return` releases completed drains immediately. + The real two-voter RPC regression reproduced two leaked group references in + both durable and early-ack modes before the fix and passes after it. +- All KV targets pass, including 53 Paxos and 127 group tests, election, + concurrent readers/removers, RPC, store restart and WAL recovery. Workspace + all-target/all-feature clippy, `rs-lint`, Rust fmt and diff checks pass. + Rebuilt full-workload acceptance remains pending. No C++ source changes, new + locks, unsafe exceptions, dependencies, resource limits, request retries or + deadline changes. diff --git a/lib/crowdb-kv/src/cluster/group_accept.rs b/lib/crowdb-kv/src/cluster/group_accept.rs index e2f24ac21..5b640b9d1 100644 --- a/lib/crowdb-kv/src/cluster/group_accept.rs +++ b/lib/crowdb-kv/src/cluster/group_accept.rs @@ -131,9 +131,10 @@ impl PxGroup { tokio::select! { biased; () = cancel.cancelled() => return, - Some(tagged) = futs.next() => - accept_drain_side_effect(&group_drain, slot, tagged), - else => return, + tagged = futs.next() => match tagged { + Some(tagged) => accept_drain_side_effect(&group_drain, slot, tagged), + None => return, + }, } } } @@ -215,9 +216,10 @@ impl PxGroup { tokio::select! { biased; () = cancel.cancelled() => return, - Some(tagged) = futs.next() => - accept_drain_side_effect(&group_drain, slot, tagged), - else => return, + tagged = futs.next() => match tagged { + Some(tagged) => accept_drain_side_effect(&group_drain, slot, tagged), + None => return, + }, } } } diff --git a/lib/crowdb-kv/src/cluster/group_prepare.rs b/lib/crowdb-kv/src/cluster/group_prepare.rs index 2bb225cc2..879e20d91 100644 --- a/lib/crowdb-kv/src/cluster/group_prepare.rs +++ b/lib/crowdb-kv/src/cluster/group_prepare.rs @@ -211,9 +211,10 @@ impl PxGroup { tokio::select! { biased; () = cancel.cancelled() => return, - Some(tagged) = futs.next() => - prepare_drain_side_effect(&group_drain, slot, tagged), - else => return, + tagged = futs.next() => match tagged { + Some(tagged) => prepare_drain_side_effect(&group_drain, slot, tagged), + None => return, + }, } } } diff --git a/lib/crowdb-kv/src/paxos/acceptor.rs b/lib/crowdb-kv/src/paxos/acceptor.rs index 5b52811ba..96c3f9616 100644 --- a/lib/crowdb-kv/src/paxos/acceptor.rs +++ b/lib/crowdb-kv/src/paxos/acceptor.rs @@ -15,6 +15,7 @@ use crate::paxos::roles::{Acceptor, PxAcceptReply, PxBallot, PxLogEntry, PxPrepareReply, SlotIndex}; use crate::paxos::slot_node::PxSlotNode; use crossbeam_skiplist::{map::Entry, SkipMap}; +use std::ops::Bound; use std::sync::atomic::{AtomicU64, Ordering}; #[derive(Debug, PartialEq, Eq)] @@ -48,11 +49,16 @@ impl PxAcceptor { #[must_use] pub fn accepted_log_tip(&self) -> Option<(SlotIndex, u64)> { - self.slots.range(self.trim_slot()..).rev().find_map(|node| { - node.value() - .accepted_cloned() - .map(|entry| (entry.slot, entry.term)) - }) + let before = self.trim_slot(); + self.slots + .iter() + .rev() + .take_while(|node| *node.key() >= before) + .find_map(|node| { + node.value() + .accepted_cloned() + .map(|entry| (entry.slot, entry.term)) + }) } /// Slot-ordered iteration over accepted entries in @@ -75,10 +81,20 @@ impl PxAcceptor { if start >= end_slot_exclusive { return Vec::new(); } - self.slots - .range(start..end_slot_exclusive) - .filter_map(|node| node.value().accepted_cloned().map(|entry| (*node.key(), entry))) - .collect() + // crossbeam-skiplist 0.1.3's range iterator retains previously visited + // nodes. Walk RAII entries so scans cannot pin reclaimed payloads. + let mut cursor = self.slots.lower_bound(Bound::Included(&start)); + let mut entries = Vec::new(); + while let Some(node) = cursor { + if *node.key() >= end_slot_exclusive { + break; + } + if let Some(entry) = node.value().accepted_cloned() { + entries.push((*node.key(), entry)); + } + cursor = node.next(); + } + entries } /// Current trim slot — slots below this have been GC'd and are no @@ -222,7 +238,14 @@ impl Acceptor for PxAcceptor { self.slots.get(&slot)?.value().promised_cloned() } fn reclaim(&self) -> usize { - self.slots.range(..self.trim_slot()).filter(Entry::remove).count() + let before = self.trim_slot(); + // The full iterator releases its cursor references as it advances; + // the dependency's range iterator does not. + self.slots + .iter() + .take_while(|node| *node.key() < before) + .filter(Entry::remove) + .count() } fn trim(&self, before_slot: SlotIndex) { self.trim_slot.fetch_max(before_slot, Ordering::AcqRel); diff --git a/lib/crowdb-kv/tests/common/payload_owner.rs b/lib/crowdb-kv/tests/common/payload_owner.rs new file mode 100644 index 000000000..893d4ea7b --- /dev/null +++ b/lib/crowdb-kv/tests/common/payload_owner.rs @@ -0,0 +1,22 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +pub struct TestPayload { + pub bytes: Vec, + pub dropped: Arc, +} + +impl AsRef<[u8]> for TestPayload { + fn as_ref(&self) -> &[u8] { + &self.bytes + } +} + +impl Drop for TestPayload { + fn drop(&mut self) { + self.dropped.fetch_add(1, Ordering::Relaxed); + } +} diff --git a/lib/crowdb-kv/tests/group_test.rs b/lib/crowdb-kv/tests/group_test.rs index 7518d2d51..274c7090b 100644 --- a/lib/crowdb-kv/tests/group_test.rs +++ b/lib/crowdb-kv/tests/group_test.rs @@ -34,6 +34,9 @@ mod group_config; #[path = "group_test/group_propose_test.rs"] mod group_propose; +#[path = "group_test/quorum_drain_test.rs"] +mod quorum_drain; + #[path = "group_test/cas_test.rs"] mod cas; diff --git a/lib/crowdb-kv/tests/group_test/quorum_drain_test.rs b/lib/crowdb-kv/tests/group_test/quorum_drain_test.rs new file mode 100644 index 000000000..5c26760fb --- /dev/null +++ b/lib/crowdb-kv/tests/group_test/quorum_drain_test.rs @@ -0,0 +1,54 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; +use std::time::Duration; + +use crowdb_kv::cluster::group::{ProposeResult, PxGroup}; +use crowdb_kv::cluster::{KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; +use crowdb_kv::rpc::PxRpcTransport; + +#[tokio::test] +async fn completed_quorum_drains_release_groups_without_tenure_cancellation() { + let _network = crate::common::net_lock::lock().await; + let follower = Arc::new(PxKvStore::new(2, "127.0.0.1:0".parse().unwrap())); + follower.add_group_without_election(PxGroup::new( + 1, + PxLocalReplica::new(2, PxLocalReplicaRole::Follower), + )); + follower.start().await.unwrap(); + let mut retained = Vec::new(); + for early_ack in [false, true] { + let remote = PxRemoteReplica::new(2, follower.listen_addr().unwrap().to_string()); + remote.set_rpc_transport(Arc::new(PxRpcTransport::new())); + let mut group = PxGroup::new(1, PxLocalReplica::new(1, PxLocalReplicaRole::Leader)); + group.set_remote_replicas(vec![remote]); + group.set_force_classic(true); + group.set_wal_early_ack(early_ack); + let group = Arc::new(group); + group.set_self_weak(); + // Both voters must answer before quorum: the detached prepare and + // accept drains are already empty, without any tenure cancellation. + assert!(matches!( + group.propose_in_tenure(b"payload".to_vec(), None, None, 0).await, + ProposeResult::Chosen { .. } + )); + let weak = Arc::downgrade(&group); + drop(group); + let released = tokio::time::timeout(Duration::from_secs(1), async { + while weak.strong_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await; + if released.is_err() { + retained.push((early_ack, weak.strong_count())); + } + } + follower.stop(); + follower.join().await; + assert!( + retained.is_empty(), + "completed drains retain groups: {retained:?}" + ); +} diff --git a/lib/crowdb-kv/tests/paxos_test.rs b/lib/crowdb-kv/tests/paxos_test.rs index c2fae4e30..eb5327c45 100644 --- a/lib/crowdb-kv/tests/paxos_test.rs +++ b/lib/crowdb-kv/tests/paxos_test.rs @@ -9,6 +9,9 @@ //! `group`, and `store` test binaries; the slot list has its own `slot` //! subsystem binary. +#[path = "common/payload_owner.rs"] +mod payload_owner; + #[path = "paxos_test/acceptor_test.rs"] mod acceptor; diff --git a/lib/crowdb-kv/tests/paxos_test/acceptor_test.rs b/lib/crowdb-kv/tests/paxos_test/acceptor_test.rs index ffe0c3c33..9ad583701 100644 --- a/lib/crowdb-kv/tests/paxos_test/acceptor_test.rs +++ b/lib/crowdb-kv/tests/paxos_test/acceptor_test.rs @@ -249,3 +249,39 @@ async fn concurrent_readers_and_reclaimers_keep_live_slots_and_owned_payloads() assert_eq!(acc.accepted_log_tip(), Some((2049, 1))); assert_eq!(held.payload.as_ref(), b"retained"); } + +#[tokio::test] +async fn reclaimed_slots_release_payload_owners_while_acceptor_stays_live() { + use crate::payload_owner::TestPayload; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + let dropped = Arc::new(AtomicUsize::new(0)); + let acc = PxAcceptor::new(); + for slot in 1..=8192 { + acc.accept(&PxLogEntry { + slot, + ballot: PxBallot::new(1, 1), + term: 1, + payload: bytes::Bytes::from_owner(TestPayload { + bytes: vec![42; 1024], + dropped: dropped.clone(), + }), + }) + .await; + if slot % 128 == 0 { + let scanned = acc.accepted_iter_range(slot - 127, slot + 1); + assert_eq!(scanned.len(), 128); + drop(scanned); + acc.trim(slot); + acc.reclaim(); + } + } + assert_eq!(acc.retained_slots(), 1); + assert!( + dropped.load(Ordering::Relaxed) >= 8192 - 1024, + "reclaimed payload owners remain live: {} released", + dropped.load(Ordering::Relaxed) + ); +} From 8ab7b4cec1ac948d439f1c1975c5637664c10985 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 23 Oct 2026 23:52:58 +0800 Subject: [PATCH 088/258] Keep read cleanup within native request admission --- .../dataset/design-crowdb-access-dataset.md | 5 +- doc/working/plan-dataset-native-sdk.md | 112 +++++++++++++++++- lib/crowdb-sdk-native/README.md | 7 +- lib/crowdb-sdk-native/src/dataset/read.rs | 4 +- .../src/dataset/read/body.rs | 20 +++- .../src/dataset/read/protection.rs | 62 +++++++--- .../src/runtime/resources.rs | 1 + lib/crowdb-sdk-native/tests/common/dataset.rs | 17 ++- .../tests/common/metadata.rs | 50 ++++++++ .../tests/common/read_cleanup.rs | 52 ++++++++ .../tests/dataset_cleanup_test.rs | 80 +++++++++++++ 11 files changed, 381 insertions(+), 29 deletions(-) create mode 100644 lib/crowdb-sdk-native/tests/common/read_cleanup.rs create mode 100644 lib/crowdb-sdk-native/tests/dataset_cleanup_test.rs diff --git a/doc/design/access-server/dataset/design-crowdb-access-dataset.md b/doc/design/access-server/dataset/design-crowdb-access-dataset.md index d7ea873fc..f31ca037b 100644 --- a/doc/design/access-server/dataset/design-crowdb-access-dataset.md +++ b/doc/design/access-server/dataset/design-crowdb-access-dataset.md @@ -139,7 +139,10 @@ may replay the last unconfirmed batch, but may not skip or change the snapshot. Epoch and sampler state remain client state. An active-read lease protects payloads reachable from the selected snapshot while -the read is in progress. Normal close or cancel releases it immediately. If the +the read is in progress. Normal close or cancel starts its release. Native +verified EOF waits for bounded cleanup; cancellation stops production promptly +and retains request admission through cleanup completion or its five-second +bound. Caller-held buffers retain their own byte credits independently. If the connection has no activity for 60 seconds, the lease is invalid and GC may treat the read as disconnected; this TTL is the crash and network-failure fallback. diff --git a/doc/working/plan-dataset-native-sdk.md b/doc/working/plan-dataset-native-sdk.md index 6ff5479d6..98d8e2bd6 100644 --- a/doc/working/plan-dataset-native-sdk.md +++ b/doc/working/plan-dataset-native-sdk.md @@ -263,6 +263,22 @@ This `--semantics-only` run is functional evidence, not performance acceptance. ## Handoff 2026-10-23 +### Current stop point + +- Metadata memory is fixed at `b18efa9a`: unchanged full run + `dataset-workload/c7fd408b216a451e81feb9009e2eeac1` stays below 280 MiB, + without swap/OOM. Native multi-flight still exhausts lease CAS attempts. +- Native EOF/cancel cleanup now retains request admission through its existing + bounded task; focused lifecycle tests pass. This is a resource-accounting fix, + not a claim that the contention failure or R220 acceptance is complete. +- After five contention corrections, the slow durable-CAS regression still + fails. See [Blocked](#blocked) before further implementation. The unaccepted + authority RTT experiment and failing regression are preserved in stash + `a6e19f404d9cb614a3004a09575051196d6d0727`; no experiment is discarded. +- The recorded HCA/Python semantics evidence below predates the final memory + and cleanup changes. Full mixed-size comparison, current-source HCA/restart, + and requirement/doc cleanup remain outstanding. + ### Scope and resume - The handoff was committed in `0f5d94a8`. The user has resumed work and @@ -278,7 +294,8 @@ This `--semantics-only` run is functional evidence, not performance acceptance. - Native Dataset implementation: `f5fefdfe`; common workload: `982c1809`. Shared read-lease CAS fixes: `93aefa43`, `4127a966`, `a8e1571e` with focused competing-renewal regressions. HTTP contention no longer returns 409 in the - recorded full HTTP runs; the subsequent failure is metadata process OOM. + recorded full HTTP runs. Subsequent metadata OOM is fixed; native contention + remains, as recorded at the current stop point above. - Packaging/default namespace/bounded multi-owner producer fixes: `d3e285b2`. Python fills bounded final-frame owners once; multi-part producer chains preserve original pointers through the durable writer. Ordinary Python input @@ -450,3 +467,96 @@ remaining user design decision and no requirement `Blocked` condition. Rebuilt full-workload acceptance remains pending. No C++ source changes, new locks, unsafe exceptions, dependencies, resource limits, request retries or deadline changes. +- The payload/drain fix is committed as `b18efa9a`; rootless build/import pass + with source tree `f47e633e7c2bc4b3fd519f9ef35e5131372a4200` and image config + `sha256:134008ce6622c77355288d34ef8dff7046b46b329ae5484658604d1d392d950f`. +- Full unchanged run `dataset-workload/c7fd408b216a451e81feb9009e2eeac1` again + completes HTTP six trials and native Rust singleton three trials, then fails + native multi-flight with `publication lost its head race`. This time all + metadata replicas remain alive: observed memory stays below 280 MiB through + 70,000 log slots, without swap or OOM. The CAS exhaustion is independent of + the fixed payload leak. Full performance/restart acceptance remains pending. +- Diagnostic run `dataset-workload/0df852363785426086b924ad6c744a4d` retains + per-attempt native lease-conflict revision/latency histories. Its native + library has temporary logging while the frozen server keeps the identical + lease algorithm; it is diagnostic evidence, not a source-matched acceptance. +- The 64 failed native CAS operations took 69–454 ms each (median 126 ms). + The existing 32-ms RTT clamp and 256-ms maximum window understate durable + sequencer latency. The fourth contention attempt uses actual RTT and full-width + random jitter, preserving 64 attempts and all caller deadlines. +- A paused-time diagnostic with three readers and unbounded detached releases + still fails after that change: cleanup can accumulate beyond client request + concurrency. That model omitted the SDK's five-second cleanup timeout and is + not a bounded production lifecycle. Its failure is retained here, not counted + as a passing regression. The authority regression now covers the proposed + bounded acquire/read/release lifecycle; the unchanged service workload remains + the acceptance gate. +- The fifth contention correction makes native verified EOF await the existing + bounded cleanup task. Cancel/drop remains prompt but the task retains request + admission until completion/timeout. Dropped EOF waits reuse the same task; + no new lock, caller retry, TTL/deadline increase, or workload reduction. + Native regressions cover admission during cleanup, cancellation/drop, one + release across interrupted EOF waits, timeout fallback and retained buffers. + All six native lifecycle/read tests pass. The slow authority regression + initially passed alone but failed in the component suite and again alone; + this is not accepted contention evidence. + +## Blocked + +- Failure: `slow_durable_cas_completes_with_bounded_reader_lifetimes` returns + `HeadConflict` at `access_contention_test.rs:154`, after one update exhausts + the unchanged 64-attempt bound. The isolated rerun also fails, excluding + interference from other component tests. Three readers each perform 96 + acquire/read/release cycles over a sequencer with 100-ms durable mutations; + reads see committed state and failed conditions still consume a mutation. +- Exact failing commands: + + ```bash + pixi run -- cargo test -p crowdb-sdk-native -p crowdb-access-dataset -p crowdb-access-server --all-targets + pixi run -- cargo test -p crowdb-access-dataset --test access_contention_test slow_durable_cas_completes_with_bounded_reader_lifetimes -- --exact --nocapture + ``` + +- Attempts and evidence: + 1. `93aefa43`: recompute from the CAS observation; delay the first HTTP + heartbeat. The real workload still exhausts attempts. + 2. `4127a966`: randomized short contention windows. The real workload still + exhausts attempts. + 3. `a8e1571e`: scale the window with durable RTT, capped at 256 ms. HTTP + completes, but native multi-flight still exhausts attempts after memory + fixes; its observed CAS RTT is 69–454 ms. + 4. Remove the RTT clamp and use full-width random jitter. The slow sequencer + with detached releases still exhausts attempts. + 5. Bound native cleanup within the admitted read lifetime. Native lifecycle + regressions pass, but the corresponding bounded slow-sequencer model still + exhausts attempts in both the suite and the isolated rerun. +- Diagnosis: independent read/modify/CAS loops on the same snapshot lease + record have no fairness guarantee. Queueing latency and freshly arriving + updates can repeatedly invalidate a slower contender. Random windows and + bounded producer lifetime reduce pressure but do not provide ordered progress. + The fifth change has not been rebuilt into a full-service acceptance image: + its failing component model prevents claiming end-to-end resolution. +- Preserve the fourth experiment and slow-store regression with + `git stash apply a6e19f404d9cb614a3004a09575051196d6d0727` when resuming. + This stash contains only the four shared-authority experimental files, + including the untracked sequencer fixture; it does not contain the native + cleanup fix. Applying it preserves the stash for recovery. +- Recommended next direction: bounded per-snapshot scheduling and merging of + lease changes, retaining the existing persisted record and remote CAS fences. + Specify queue capacity, per-handle completion/cancellation, cleanup ownership, + and fairness across processes before implementation. A local queue alone is + not a proof of distributed fairness; keep the failing model and real workload. +- Alternative: independently stored per-reader leases. This reduces contention + on one record but changes reclamation fencing, bounded scans, compatibility + and crash recovery; it requires a broader protocol/design change. +- Human choice required by the requirement workflow after five root-cause + attempts. Do not raise deadlines/retry limits, suppress the failing test, + reduce the workload, or mark R220 complete. The verified native cleanup fix + can be committed separately with this blocked record; the failed experiment + remains recoverable and explicitly unaccepted. +- The native cleanup commit passes `clean-env` followed by all targets of + `crowdb-sdk-native`, `crowdb-access-dataset`, and `crowdb-access-server`; + workspace all-target/all-feature clippy, `rs-lint`, Rust fmt and diff checks + also pass. These checks exclude the explicitly preserved, failing authority + experiment. Existing environment-gated hardware/installed-Python cases are + not claimed as passing acceptance. No new production lock, unsafe exception, + C++ change, request retry or deadline increase was introduced. diff --git a/lib/crowdb-sdk-native/README.md b/lib/crowdb-sdk-native/README.md index 6c40c6a30..57641afdb 100644 --- a/lib/crowdb-sdk-native/README.md +++ b/lib/crowdb-sdk-native/README.md @@ -108,8 +108,11 @@ async fn read_object(seeds: Vec) -> Result<(), Box, ) -> Result { - let protection = protection::ReadProtection::acquire(self, &request, batch.snapshot, session).await?; + let request = Arc::new(request); + let protection = + protection::ReadProtection::acquire(self, Arc::clone(&request), batch.snapshot, session).await?; let samples = if batch.sample_ids.is_empty() { Vec::new() } else { diff --git a/lib/crowdb-sdk-native/src/dataset/read/body.rs b/lib/crowdb-sdk-native/src/dataset/read/body.rs index f1776cc68..460f39db2 100644 --- a/lib/crowdb-sdk-native/src/dataset/read/body.rs +++ b/lib/crowdb-sdk-native/src/dataset/read/body.rs @@ -19,11 +19,12 @@ pub struct BatchBody { locators: VecDeque<(usize, FieldLocator)>, field: Option, pending: Option, - request: Option, + request: Option>, resources: Arc, uploads: Arc, protection: Option, sequence: u64, + finishing: bool, complete: bool, failure: Option, } @@ -32,7 +33,7 @@ impl BatchBody { pub(super) fn new( header: Arc, locators: Vec<(usize, FieldLocator)>, - request: RequestLease, + request: Arc, resources: Arc, uploads: Arc, protection: ReadProtection, @@ -47,6 +48,7 @@ impl BatchBody { uploads, protection: Some(protection), sequence: 0, + finishing: false, complete: false, failure: None, } @@ -62,6 +64,7 @@ impl BatchBody { } /// Successful EOF certifies all field and part digests, never processing ack. + /// It waits for bounded read-protection cleanup before returning admission. /// # Errors /// Returns sticky integrity/transport/cancellation failures or expired admission. pub async fn next(&mut self) -> Result> { @@ -102,13 +105,17 @@ impl BatchBody { } async fn pull(&mut self) -> Result> { + if self.finishing { + return self.finish().await; + } let request = self.request.as_ref().ok_or(Error::Cancelled)?; let protection = self.protection.as_ref().ok_or(Error::Cancelled)?; request.run(protection.check()).await?; loop { if self.field.is_none() { let Some((index, locator)) = self.locators.pop_front() else { - return Ok(None); + self.finishing = true; + return self.finish().await; }; self.field = Some(FieldReader::new(index, locator)?); } @@ -137,4 +144,11 @@ impl BatchBody { })); } } + + async fn finish(&mut self) -> Result> { + let request = self.request.as_ref().ok_or(Error::Cancelled)?; + let protection = self.protection.as_mut().ok_or(Error::Cancelled)?; + request.run(protection.finish()).await?; + Ok(None) + } } diff --git a/lib/crowdb-sdk-native/src/dataset/read/protection.rs b/lib/crowdb-sdk-native/src/dataset/read/protection.rs index 52c3d2082..d15892658 100644 --- a/lib/crowdb-sdk-native/src/dataset/read/protection.rs +++ b/lib/crowdb-sdk-native/src/dataset/read/protection.rs @@ -23,14 +23,16 @@ struct Protection { pub(super) struct ReadProtection { state: Arc, + request: Option>, stop: Cancellation, worker: tokio::task::JoinHandle<()>, + cleanup: Option>, } impl ReadProtection { pub(super) async fn acquire( client: &DatasetClient, - request: &RequestLease, + request: Arc, snapshot: SnapshotId, session: Option, ) -> Result { @@ -64,7 +66,13 @@ impl ReadProtection { worker_state.failed.store(true, Ordering::Release); } }); - Ok(Self { state, stop, worker }) + Ok(Self { + state, + request: Some(request), + stop, + worker, + cleanup: None, + }) } pub(super) async fn check(&self) -> Result<()> { @@ -78,29 +86,28 @@ impl ReadProtection { } Ok(()) } -} -impl Protection { - async fn renew(&self) -> Result<()> { - if let Some(session) = self.session { - DatasetSessions::new(Arc::clone(&self.authority)) - .resume(&self.identity, session, now()) - .await?; + pub(super) async fn finish(&mut self) -> Result<()> { + // Keep the task across dropped EOF waits; cancellation detaches the same + // bounded cleanup while its request credit still covers the metadata I/O. + self.start_cleanup(); + if let Some(cleanup) = &mut self.cleanup { + cleanup + .await + .map_err(|error| Error::Storage(format!("read cleanup task failed: {error}")))?; } - self.authority - .touch_read_handle(&self.identity, self.snapshot, self.handle, now(), 60) - .await?; Ok(()) } -} -impl Drop for ReadProtection { - fn drop(&mut self) { + fn start_cleanup(&mut self) { + let Some(request) = self.request.take() else { + return; + }; self.stop.cancel(); self.worker.abort(); let state = Arc::clone(&self.state); if let Ok(runtime) = tokio::runtime::Handle::try_current() { - runtime.spawn(async move { + self.cleanup = Some(runtime.spawn(async move { let release = state .authority @@ -113,7 +120,28 @@ impl Drop for ReadProtection { "Dataset read release failed; inactivity expiry retains cleanup responsibility" ); } - }); + drop(request); + })); + } + } +} + +impl Protection { + async fn renew(&self) -> Result<()> { + if let Some(session) = self.session { + DatasetSessions::new(Arc::clone(&self.authority)) + .resume(&self.identity, session, now()) + .await?; } + self.authority + .touch_read_handle(&self.identity, self.snapshot, self.handle, now(), 60) + .await?; + Ok(()) + } +} + +impl Drop for ReadProtection { + fn drop(&mut self) { + self.start_cleanup(); } } diff --git a/lib/crowdb-sdk-native/src/runtime/resources.rs b/lib/crowdb-sdk-native/src/runtime/resources.rs index c3e8113b2..d79c6820d 100644 --- a/lib/crowdb-sdk-native/src/runtime/resources.rs +++ b/lib/crowdb-sdk-native/src/runtime/resources.rs @@ -118,6 +118,7 @@ impl RuntimeResources { /// Request admission lasts through a stream's terminal result or cancellation. /// Its release is independent of any buffer leases delivered to the caller. +/// Dataset cancellation retains admission until its bounded protection cleanup ends. pub struct RequestLease { resources: Arc, context: OperationContext, diff --git a/lib/crowdb-sdk-native/tests/common/dataset.rs b/lib/crowdb-sdk-native/tests/common/dataset.rs index 05afb0f97..ac51e3022 100644 --- a/lib/crowdb-sdk-native/tests/common/dataset.rs +++ b/lib/crowdb-sdk-native/tests/common/dataset.rs @@ -27,16 +27,25 @@ pub fn context(id: u64) -> OperationContext { } impl TestFixture { + #[allow(dead_code)] // Cleanup tests provide a gated metadata client instead. pub async fn new(held: usize) -> Self { let metadata = super::metadata::client().await; - let authority = DatasetAuthority::new(Arc::new(ChunkKvDatasetStore::new(metadata.clone()))); - let runtime = super::storage::runtime_with_limits( - metadata, + Self::with_metadata( RuntimeLimits { held_bytes: held, ..RuntimeLimits::default() }, - ); + metadata, + ) + .await + } + + pub async fn with_metadata( + limits: RuntimeLimits, + metadata: Arc, + ) -> Self { + let authority = DatasetAuthority::new(Arc::new(ChunkKvDatasetStore::new(metadata.clone()))); + let runtime = super::storage::runtime_with_limits(metadata, limits); let identity = DatasetIdentity::new(NamespacePath::root(), "native").unwrap(); let client = DatasetClient::new(&runtime, identity.clone()).unwrap(); client.create(context(1)).await.unwrap(); diff --git a/lib/crowdb-sdk-native/tests/common/metadata.rs b/lib/crowdb-sdk-native/tests/common/metadata.rs index 278e07a17..1c745cb37 100644 --- a/lib/crowdb-sdk-native/tests/common/metadata.rs +++ b/lib/crowdb-sdk-native/tests/common/metadata.rs @@ -32,6 +32,25 @@ pub struct TestMetadataTransport { lose_mutation_reply: Arc, blackout: Option>, route: Option>, + gate: Option>, +} + +pub struct TestMutationGate { + pub hold: AtomicBool, + pub entered: tokio::sync::Semaphore, + pub permits: tokio::sync::Semaphore, + pub calls: std::sync::atomic::AtomicUsize, +} + +impl Default for TestMutationGate { + fn default() -> Self { + Self { + hold: AtomicBool::new(false), + entered: tokio::sync::Semaphore::new(0), + permits: tokio::sync::Semaphore::new(0), + calls: std::sync::atomic::AtomicUsize::new(0), + } + } } #[derive(Default)] @@ -78,6 +97,17 @@ impl ChunkKvTransport for TestMetadataTransport { )); } let (sender, receiver) = oneshot::channel(); + if !matches!(request.operation, PointOperation::Get { .. }) { + if let Some(gate) = self + .gate + .as_ref() + .filter(|gate| gate.hold.load(Ordering::Acquire)) + { + gate.calls.fetch_add(1, Ordering::Relaxed); + gate.entered.add_permits(1); + gate.permits.acquire().await.unwrap().forget(); + } + } self.sender .send(TestCommand::Point(request.operation.clone(), sender)) .await @@ -275,6 +305,7 @@ pub async fn client_with_source(source: Arc) -> A ClientConfig::default(), None, None, + None, ) .await } @@ -291,6 +322,7 @@ pub async fn client_with_loss(loss: Arc) -> Arc { }, None, None, + None, ) .await } @@ -307,6 +339,7 @@ pub async fn client_with_blackout(blackout: Arc) -> Arc) -> Arc { + let (head, pages) = catalog(1, 1); + client_with_options( + Arc::new(TestCatalog(head, pages)), + Arc::new(AtomicBool::new(false)), + ClientConfig::default(), + None, + None, + Some(gate), ) .await } @@ -332,6 +380,7 @@ async fn client_with_options( config: ClientConfig, blackout: Option>, route: Option>, + gate: Option>, ) -> Arc { let (sender, mut receiver) = mpsc::channel(64); tokio::spawn(async move { @@ -360,6 +409,7 @@ async fn client_with_options( lose_mutation_reply: loss, blackout, route, + gate, }), ) .unwrap(), diff --git a/lib/crowdb-sdk-native/tests/common/read_cleanup.rs b/lib/crowdb-sdk-native/tests/common/read_cleanup.rs new file mode 100644 index 000000000..0cf63927d --- /dev/null +++ b/lib/crowdb-sdk-native/tests/common/read_cleanup.rs @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{dataset, metadata}; +use crowdb_sdk_native::{ + buffer::BufferLease, + dataset::{BatchBody, BatchRequest}, + RuntimeLimits, +}; +use std::sync::{atomic::Ordering, Arc}; + +pub struct TestCleanupFixture { + pub dataset: dataset::TestFixture, + pub gate: Arc, + pub body: BatchBody, + pub held: BufferLease, +} + +impl TestCleanupFixture { + pub async fn new() -> Self { + let gate = Arc::new(metadata::TestMutationGate::default()); + let metadata = metadata::client_with_gate(Arc::clone(&gate)).await; + let dataset = dataset::TestFixture::with_metadata( + RuntimeLimits { + requests: 1, + ..RuntimeLimits::default() + }, + metadata, + ) + .await; + let mut read = dataset + .client + .read_batch( + dataset::context(2), + BatchRequest { + snapshot: dataset.snapshot, + sample_ids: vec![vec![1]], + fields: vec!["value".into()], + }, + ) + .await + .unwrap(); + let held = read.body.next().await.unwrap().unwrap().buffer; + gate.hold.store(true, Ordering::Release); + Self { + dataset, + gate, + body: read.body, + held, + } + } +} diff --git a/lib/crowdb-sdk-native/tests/dataset_cleanup_test.rs b/lib/crowdb-sdk-native/tests/dataset_cleanup_test.rs new file mode 100644 index 000000000..7ef7f569f --- /dev/null +++ b/lib/crowdb-sdk-native/tests/dataset_cleanup_test.rs @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/dataset.rs"] +mod dataset; +#[path = "common/metadata.rs"] +mod metadata; +#[path = "common/read_cleanup.rs"] +mod read_cleanup; +#[path = "common/storage.rs"] +mod storage; + +use crowdb_sdk_native::{CapacityKind, Error}; +use read_cleanup::TestCleanupFixture; +use std::{sync::atomic::Ordering, time::Duration}; + +#[tokio::test] +async fn eof_keeps_admission_until_cleanup_and_dropped_wait_does_not_repeat_release() { + let mut fixture = TestCleanupFixture::new().await; + assert!( + tokio::time::timeout(Duration::from_millis(10), fixture.body.next()) + .await + .is_err() + ); + fixture.gate.entered.acquire().await.unwrap().forget(); + assert!(!fixture.body.is_complete()); + assert_eq!(fixture.dataset.runtime.resources().status().requests, 1); + assert!(matches!( + fixture.dataset.runtime.resources().admit(dataset::context(3)), + Err(Error::Capacity(CapacityKind::Requests)) + )); + fixture.gate.permits.add_permits(1); + assert!(fixture.body.next().await.unwrap().is_none()); + assert!(fixture.body.next().await.unwrap().is_none()); + assert!(fixture.body.is_complete()); + assert_eq!(fixture.gate.calls.load(Ordering::Relaxed), 1); + assert_eq!(fixture.dataset.runtime.resources().status().requests, 0); + assert_eq!(fixture.held.as_slice(), &[0; 16]); +} + +#[tokio::test] +async fn cancelled_or_dropped_reads_keep_cleanup_charged_and_delivered_bytes_valid() { + for cancel in [false, true] { + let mut fixture = TestCleanupFixture::new().await; + if cancel { + fixture.body.cancel(); + assert!(matches!(fixture.body.next().await, Err(Error::Cancelled))); + } + drop(fixture.body); + fixture.gate.entered.acquire().await.unwrap().forget(); + assert_eq!(fixture.dataset.runtime.resources().status().requests, 1); + assert!(matches!( + fixture.dataset.runtime.resources().admit(dataset::context(3)), + Err(Error::Capacity(CapacityKind::Requests)) + )); + fixture.gate.permits.add_permits(1); + tokio::time::timeout(Duration::from_secs(1), async { + while fixture.dataset.runtime.resources().status().requests != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(fixture.gate.calls.load(Ordering::Relaxed), 1); + assert_eq!(fixture.held.as_slice(), &[0; 16]); + } +} + +#[tokio::test(start_paused = true)] +async fn stalled_cleanup_releases_admission_at_the_existing_bound() { + let fixture = TestCleanupFixture::new().await; + drop(fixture.body); + fixture.gate.entered.acquire().await.unwrap().forget(); + assert_eq!(fixture.dataset.runtime.resources().status().requests, 1); + tokio::time::advance(Duration::from_secs(5)).await; + tokio::task::yield_now().await; + assert_eq!(fixture.dataset.runtime.resources().status().requests, 0); + assert_eq!(fixture.gate.calls.load(Ordering::Relaxed), 1); + assert_eq!(fixture.held.as_slice(), &[0; 16]); +} From ec72fc009d1b4bb73750aee3b5f12c45c108f4fe Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 07:38:50 +0800 Subject: [PATCH 089/258] Replace snapshot read leases with delayed reclamation --- Cargo.lock | 1 + app/crowdb-access-server/Cargo.toml | 1 + app/crowdb-access-server/src/dataset.rs | 37 ++- .../src/dataset/api/batch.rs | 98 +------ .../src/dataset/api/management.rs | 56 +--- .../src/dataset/api/v2/auxiliary.rs | 4 +- .../src/dataset/api/v2/preview.rs | 6 +- .../src/dataset/legacy.rs | 14 +- .../tests/common/dataset.rs | 5 + .../tests/dataset_retirement_contract_test.rs | 6 + doc/backlog/R220-dataset-fat-client-plan.md | 11 +- .../dataset/design-crowdb-access-dataset.md | 44 ++-- .../dataset/design-dataset-control.md | 13 +- .../dataset/openapi-crowdb-dataset-v1.yaml | 6 +- .../dataset/openapi-crowdb-dataset-v2.yaml | 10 + .../dataset/spec-crowdb-dataset.md | 35 ++- doc/working/plan-access-simplification.md | 116 +++++++++ doc/working/plan-dataset-native-sdk.md | 116 ++++----- lib/crowdb-access-dataset/Cargo.toml | 4 + lib/crowdb-access-dataset/src/authority.rs | 15 +- .../src/authority/access.rs | 239 ++++++++---------- .../src/authority/access/cleanup.rs | 1 + .../src/authority/access/lifecycle.rs | 15 ++ .../src/authority/access/observation.rs | 13 +- .../src/authority/access/reclaim.rs | 29 ++- .../src/authority/access/sweep.rs | 60 +++++ .../src/composition/edit.rs | 7 - .../src/composition/publication.rs | 9 - .../src/composition/state.rs | 3 +- .../src/inspection/legacy.rs | 19 +- .../src/inspection/reclaim.rs | 14 +- .../src/read_session/planning.rs | 4 +- .../src/read_session/progress.rs | 9 +- .../src/read_session/split/binding.rs | 11 +- .../src/read_session/split/planning.rs | 14 +- .../src/split/evaluation.rs | 2 +- .../src/split/inheritance.rs | 18 +- .../src/split/initial.rs | 23 +- .../src/split/lifecycle.rs | 8 +- .../src/split/prepare.rs | 2 +- .../src/split/publication.rs | 4 +- .../src/split/snapshot_cleanup.rs | 2 +- lib/crowdb-access-dataset/src/split/state.rs | 16 +- lib/crowdb-access-dataset/src/transport.rs | 64 +---- .../src/upload/reclaim.rs | 1 + .../tests/access_contention_test.rs | 114 ++------- .../tests/common/access_store.rs | 49 ++-- .../tests/common/deletion.rs | 79 ++++++ .../tests/composition_upload_test.rs | 6 +- lib/crowdb-access-dataset/tests/core_test.rs | 18 +- .../tests/deletion_grace_test.rs | 154 +++++++++++ .../tests/inspection_legacy_test.rs | 8 +- .../tests/reclaim_race_test.rs | 2 +- .../tests/snapshot_split_cleanup_test.rs | 11 +- .../tests/split_reclaim_test.rs | 13 +- .../tests/split_session_edges_test.rs | 7 +- lib/crowdb-sdk-native/README.md | 14 +- lib/crowdb-sdk-native/src/dataset/error.rs | 1 + lib/crowdb-sdk-native/src/dataset/read.rs | 8 +- .../src/dataset/read/admission.rs | 43 ++++ .../src/dataset/read/body.rs | 35 +-- .../src/dataset/read/protection.rs | 147 ----------- .../src/runtime/resources.rs | 2 +- lib/crowdb-sdk-native/tests/common/dataset.rs | 2 +- .../tests/common/read_cleanup.rs | 2 +- .../tests/dataset_cleanup_test.rs | 62 ++--- .../tests/dataset_read_test.rs | 50 ++++ .../crowdb-sdk-dataset/src/v2/inspection.rs | 5 + .../tests/v2_contract_test.rs | 17 ++ 69 files changed, 1055 insertions(+), 979 deletions(-) create mode 100644 doc/working/plan-access-simplification.md create mode 100644 lib/crowdb-access-dataset/src/authority/access/sweep.rs create mode 100644 lib/crowdb-access-dataset/tests/common/deletion.rs create mode 100644 lib/crowdb-access-dataset/tests/deletion_grace_test.rs create mode 100644 lib/crowdb-sdk-native/src/dataset/read/admission.rs delete mode 100644 lib/crowdb-sdk-native/src/dataset/read/protection.rs diff --git a/Cargo.lock b/Cargo.lock index 64de298e8..acd0b1e6d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -615,6 +615,7 @@ dependencies = [ "async-trait", "bincode", "bytes", + "crowdb-access-dataset", "crowdb-chunk-client", "crowdb-chunk-kv-client", "crowdb-protocol", diff --git a/app/crowdb-access-server/Cargo.toml b/app/crowdb-access-server/Cargo.toml index 3ef96d1a4..b36a54e3c 100644 --- a/app/crowdb-access-server/Cargo.toml +++ b/app/crowdb-access-server/Cargo.toml @@ -55,6 +55,7 @@ thiserror = { workspace = true } sync_wrapper = { version = "1", features = ["futures"] } [dev-dependencies] +crowdb-access-dataset = { path = "../../lib/crowdb-access-dataset", features = ["test-util"] } crowdb-kv = { path = "../../lib/crowdb-kv" } bytes = "1" tempfile = "3" diff --git a/app/crowdb-access-server/src/dataset.rs b/app/crowdb-access-server/src/dataset.rs index b5b015a67..7be7b3d7c 100644 --- a/app/crowdb-access-server/src/dataset.rs +++ b/app/crowdb-access-server/src/dataset.rs @@ -94,6 +94,7 @@ fn status_for_authority(error: &AuthorityError) -> StatusCode { | AuthorityError::ParentConflict | AuthorityError::HeadConflict | AuthorityError::CursorConflict + | AuthorityError::ReclaimPending { .. } | AuthorityError::SnapshotProtected => StatusCode::CONFLICT, AuthorityError::NotFound | AuthorityError::SnapshotNotFound => StatusCode::NOT_FOUND, AuthorityError::AncestryCycle | AuthorityError::Corrupt | AuthorityError::Store(_) => { @@ -191,7 +192,7 @@ pub async fn run_with_shutdown( std::env::var("CROWDB_DATASET_NAME").unwrap_or_else(|_| "default".into()), )?; let reads = Arc::new(DatasetReadService::with_ttl( - authority, + Arc::clone(&authority), identity, access.dataset.max_in_flight, unix_seconds(), @@ -214,21 +215,39 @@ pub async fn run_with_shutdown( service = service.with_bearer(Arc::::from(token)); } let listener = TcpListener::bind(address).await?; - let expiry = tokio::spawn(async move { + let expiry = start_cleanup(authority, uploads); + let result = serve(listener, Arc::new(service), shutdown).await; + expiry.abort(); + let _ = expiry.await; + chunks.shutdown_small_writes().await?; + result?; + Ok(()) +} + +fn start_cleanup( + authority: Arc, + uploads: Arc, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { let mut tick = tokio::time::interval(Duration::from_secs(15)); + let mut after = None; loop { tick.tick().await; + if let Err(error) = authority + .reclaim_due_page( + &uploads, + &mut after, + &crowdb_access_dataset::ReadCancellation::new(), + ) + .await + { + tracing::error!(%error, "Dataset snapshot cleanup failed; durable deletion marks retained for retry"); + } if let Err(error) = uploads.expire(unix_seconds()).await { tracing::error!(%error, "Dataset upload cleanup failed; retaining cleanup records for retry"); } } - }); - let result = serve(listener, Arc::new(service), shutdown).await; - expiry.abort(); - let _ = expiry.await; - chunks.shutdown_small_writes().await?; - result?; - Ok(()) + }) } async fn connect_payloads( diff --git a/app/crowdb-access-server/src/dataset/api/batch.rs b/app/crowdb-access-server/src/dataset/api/batch.rs index b40b7d6e0..c8e20d8c3 100644 --- a/app/crowdb-access-server/src/dataset/api/batch.rs +++ b/app/crowdb-access-server/src/dataset/api/batch.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::{error::ApiError, read_json, snapshot, ApiResult}; -use crate::dataset::{unix_seconds, DatasetBody, DatasetHttpService}; +use crate::dataset::{DatasetBody, DatasetHttpService}; use base64::{engine::general_purpose::STANDARD, Engine}; use crowdb_access_dataset::{DatasetIdentity, FieldLocator, SampleView, SnapshotId}; use futures::stream; @@ -12,10 +12,7 @@ use hyper::{ Request, Response, }; use serde::{Deserialize, Serialize}; -use std::sync::{ - atomic::{AtomicBool, Ordering}, - Arc, -}; +use std::sync::Arc; #[derive(Deserialize)] pub(super) struct BatchRequest { @@ -86,33 +83,7 @@ impl DatasetHttpService { let id = snapshot(&request.snapshot)?; let ids = request.ids()?; let authority = self.reads.authority(); - let lease = Lease::acquire(authority.clone(), identity, id, session).await?; - let weak = Arc::downgrade(&lease); - tokio::spawn(async move { - let period = std::time::Duration::from_secs(15); - let mut timer = tokio::time::interval_at(tokio::time::Instant::now() + period, period); - loop { - timer.tick().await; - let Some(lease) = weak.upgrade() else { break }; - let result = async { - if let Some(session) = lease.session { - crowdb_access_dataset::DatasetSessions::new(lease.authority.clone()) - .resume(&lease.identity, session, unix_seconds()) - .await?; - } - lease - .authority - .touch_read_handle(&lease.identity, lease.snapshot, lease.handle, unix_seconds(), 60) - .await - } - .await; - if let Err(error) = result { - lease.failed.store(true, Ordering::Release); - tracing::error!(%error,"Dataset stream lease renewal failed; delivery cancelled"); - break; - } - } - }); + let admission = ReadAdmission::acquire(authority.clone(), identity, id, session).await?; let samples = if ids.is_empty() { Vec::new() } else if request.sample_id_limit.is_some() { @@ -155,7 +126,7 @@ impl DatasetHttpService { part: 0, pending: Some(Bytes::from(prefix)), finished: false, - lease, + admission, }; Ok(Response::builder() .header("content-type", "application/vnd.crowdb.dataset-batch.v1") @@ -199,48 +170,26 @@ fn make_header( end, } } -pub(super) struct Lease { +pub(super) struct ReadAdmission { authority: Arc, identity: DatasetIdentity, - snapshot: SnapshotId, - handle: crowdb_access_dataset::OperationId, session: Option, - failed: AtomicBool, } -impl Lease { +impl ReadAdmission { pub(super) async fn acquire( authority: Arc, identity: &DatasetIdentity, snapshot: SnapshotId, session: Option, ) -> Result, ApiError> { - let handle = crowdb_access_dataset::OperationId::random(); - authority - .touch_read_handle(identity, snapshot, handle, unix_seconds(), 60) - .await?; + authority.check_snapshot_readable(identity, snapshot).await?; Ok(Arc::new(Self { authority, identity: identity.clone(), - snapshot, - handle, session, - failed: AtomicBool::new(false), })) } } -impl Drop for Lease { - fn drop(&mut self) { - let authority = self.authority.clone(); - let identity = self.identity.clone(); - let snapshot = self.snapshot; - let handle = self.handle; - if let Ok(runtime) = tokio::runtime::Handle::try_current() { - runtime.spawn(async move { - if let Err(error)=authority.release_read_handle(&identity,snapshot,handle).await {tracing::error!(%error,"Dataset read release failed; lease expiry protects disconnected reads");} - }); - } - } -} struct BatchState { uploads: Option>, request_id: String, @@ -249,7 +198,7 @@ struct BatchState { part: u32, pending: Option, finished: bool, - lease: Arc, + admission: Arc, } fn stream_body(state: BatchState) -> DatasetBody { let frames = stream::unfold(state, |mut state| async move { @@ -265,28 +214,15 @@ fn stream_body(state: BatchState) -> DatasetBody { frame(4, &Bytes::from(record.to_string())).expect("bounded error frame") } }; - let owner = FrameOwner { - bytes, - lease: state.lease.clone(), - }; - Some(( - Ok::<_, crate::dataset::BoxError>(Frame::data(Bytes::from_owner(owner))), - state, - )) + Some((Ok::<_, crate::dataset::BoxError>(Frame::data(bytes)), state)) }); StreamBody::new(frames).boxed_unsync() } impl BatchState { async fn next(&mut self) -> Result { - if self.lease.failed.load(Ordering::Acquire) { - return Err(crowdb_access_dataset::AuthorityError::from( - crowdb_access_dataset::DatasetError::ReadCancelled, - ) - .into()); - } - if let Some(session) = self.lease.session { - crowdb_access_dataset::DatasetSessions::new(self.lease.authority.clone()) - .check_open(&self.lease.identity, session) + if let Some(session) = self.admission.session { + crowdb_access_dataset::DatasetSessions::new(self.admission.authority.clone()) + .check_open(&self.admission.identity, session) .await?; } if let Some(bytes) = self.pending.take() { @@ -345,13 +281,3 @@ fn frame(kind: u8, bytes: &Bytes) -> Result { output.extend_from_slice(bytes); Ok(Bytes::from(output)) } -struct FrameOwner { - bytes: Bytes, - lease: Arc, -} -impl AsRef<[u8]> for FrameOwner { - fn as_ref(&self) -> &[u8] { - let _ = &self.lease; - &self.bytes - } -} diff --git a/app/crowdb-access-server/src/dataset/api/management.rs b/app/crowdb-access-server/src/dataset/api/management.rs index 4b40ba08d..3ce305788 100644 --- a/app/crowdb-access-server/src/dataset/api/management.rs +++ b/app/crowdb-access-server/src/dataset/api/management.rs @@ -3,9 +3,7 @@ use super::{conversion, error::ApiError, json, read_body, snapshot, ApiResult}; use crate::dataset::DatasetHttpService; -use crowdb_access_dataset::{ - DatasetIdentity, DatasetUploads, FieldLocator, ManifestRecord, ReclaimState, UploadState, -}; +use crowdb_access_dataset::{DatasetIdentity, ReclaimState}; use hyper::{body::Incoming, Method, Request}; use std::collections::BTreeSet; @@ -73,45 +71,19 @@ impl DatasetHttpService { (Method::POST, ["snapshots", id, "reclaim"]) => { read_body(request, false).await?; let id = snapshot(id)?; - let record = authority.begin_reclaim(identity, id).await?; - let manifest: ManifestRecord = - bincode::deserialize(&record.publication.manifest).map_err(|_| ApiError::integrity())?; - let locations: BTreeSet<_> = manifest - .samples - .iter() - .flat_map(|s| &s.fields) - .filter_map(|f| { - if let FieldLocator::Chunk { location, .. } = &f.value { - Some(location.clone()) - } else { - None - } - }) - .collect(); - let mut pending = false; - for location in locations { - // External legacy references never transfer physical ownership. - if DatasetUploads::upload_id(identity, &location).is_err() { - continue; - } - let uploads = self.uploads.as_ref().ok_or_else(ApiError::unsupported)?; - uploads.release(&location, record.publication.operation).await?; - let metadata = uploads.metadata(&location).await?; - if metadata.state == UploadState::Claimed { - continue; - } - match uploads.abort(&location).await { - Ok(result) => pending |= !result.cleanup_complete, - Err(crowdb_access_dataset::UploadError::Conflict) => {} // Another durable owner now protects it. - Err(error) => return Err(error.into()), - } - } - if !pending { - authority.reclaim_snapshot_metadata(identity, id).await?; - } - json( - &serde_json::json!({"snapshot":id.to_string(),"state":if pending{"pending"}else{"completed"},"chunks_reclaimed":0}), - ) + let uploads = self.uploads.as_ref().ok_or_else(ApiError::unsupported)?; + let progress = authority + .reclaim_snapshot_step( + identity, + id, + uploads, + &crowdb_access_dataset::ReadCancellation::new(), + ) + .await?; + json(&serde_json::json!({"snapshot":id.to_string(), + "state":if progress.complete {"completed"}else{"pending"}, + "reclaim_after":authority.snapshot_reclaim_after(identity, id).await?, + "chunks_reclaimed":0})) } _ => Err(ApiError::not_found()), } diff --git a/app/crowdb-access-server/src/dataset/api/v2/auxiliary.rs b/app/crowdb-access-server/src/dataset/api/v2/auxiliary.rs index 5c53a95d1..f69482ab0 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/auxiliary.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/auxiliary.rs @@ -54,7 +54,7 @@ impl DatasetHttpService { .await?; return json(&serde_json::json!({"snapshot": value.snapshot.to_string(), "operation": value.operation.map(|id| id.to_string()), "latest": value.latest, - "stable": value.stable, "retained": value.retained, "active_read_lease": value.active_read_lease, + "stable": value.stable, "retained": value.retained, "active_read_lease": value.active_read_lease, "reclaim_after":value.reclaim_after, "state": if value.completed { "completed" } else if value.retiring { "pending" } else { "not_started" }, "physical_cleanup_deferred": value.retiring || value.completed, "parent_protection": "not_evaluated"})); @@ -66,7 +66,7 @@ impl DatasetHttpService { .reclaim_snapshot_step(identity, snapshot(id)?, uploads, &ReadCancellation::new()) .await?; return json( - &serde_json::json!({"snapshot":progress.snapshot.to_string(),"phase":progress.phase,"complete":progress.complete,"physical_cleanup_deferred":progress.physical_cleanup_deferred}), + &serde_json::json!({"snapshot":progress.snapshot.to_string(),"phase":progress.phase,"complete":progress.complete,"physical_cleanup_deferred":progress.physical_cleanup_deferred,"reclaim_after":authority.snapshot_reclaim_after(identity, progress.snapshot).await?}), ); } (Method::GET, ["commits", id]) => { diff --git a/app/crowdb-access-server/src/dataset/api/v2/preview.rs b/app/crowdb-access-server/src/dataset/api/v2/preview.rs index 27cacc5b1..8dc1cfd02 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/preview.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/preview.rs @@ -1,7 +1,9 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::super::{batch::Lease, json, read_body, read_json, ApiError, ApiResult, DatasetHttpService}; +use super::super::{ + batch::ReadAdmission, json, read_body, read_json, ApiError, ApiResult, DatasetHttpService, +}; use super::{ inspection_models::Sample, query::{bytes, Query}, @@ -73,7 +75,7 @@ impl DatasetHttpService { } read_body(request, false).await?; let authority = self.reads.authority(); - let _lease = Lease::acquire(authority.clone(), identity, snapshot, None).await?; + let _admission = ReadAdmission::acquire(authority.clone(), identity, snapshot, None).await?; let mut rows = authority .resolve_inspection_batch_locators( identity, diff --git a/app/crowdb-access-server/src/dataset/legacy.rs b/app/crowdb-access-server/src/dataset/legacy.rs index 3fab49717..d1804a799 100644 --- a/app/crowdb-access-server/src/dataset/legacy.rs +++ b/app/crowdb-access-server/src/dataset/legacy.rs @@ -8,9 +8,10 @@ use super::{ DatasetHttpService, MAX_REQUEST_BYTES, }; use crowdb_access_dataset::{ - DatasetManifestResponse, DatasetPublishRequest, DatasetPublishResponse, DatasetReadRequest, - DatasetReclaimResponse, DatasetRetentionResponse, DatasetScanRequest, DatasetSnapshotRequest, - DatasetSnapshotResponse, DatasetSnapshotsResponse, OperationId, ReadTransportError, + AuthorityError, DatasetManifestResponse, DatasetPublishRequest, DatasetPublishResponse, + DatasetReadRequest, DatasetReclaimResponse, DatasetRetentionResponse, DatasetScanRequest, + DatasetSnapshotRequest, DatasetSnapshotResponse, DatasetSnapshotsResponse, OperationId, + ReadTransportError, }; use http_body_util::{BodyExt, Full}; use hyper::{ @@ -254,6 +255,13 @@ impl DatasetHttpService { reclaimed: true, }, ), + Err(AuthorityError::ReclaimPending { .. }) => encode( + StatusCode::OK, + &DatasetReclaimResponse { + snapshot: request.snapshot, + reclaimed: false, + }, + ), Err(error) => response(status_for_authority(&error), error.to_string().into_bytes()), } } diff --git a/app/crowdb-access-server/tests/common/dataset.rs b/app/crowdb-access-server/tests/common/dataset.rs index d4541afdd..7e8217640 100644 --- a/app/crowdb-access-server/tests/common/dataset.rs +++ b/app/crowdb-access-server/tests/common/dataset.rs @@ -16,6 +16,8 @@ pub use upload::TestStore; pub struct TestDatasetHttp { pub origin: String, + #[allow(dead_code)] + pub clock: Arc, stop: Option>, task: JoinHandle>, } @@ -42,6 +44,8 @@ impl TestDatasetHttp { } else { DatasetAuthority::new(store) }; + let clock = Arc::new(std::sync::atomic::AtomicU64::new(1000)); + let authority = authority.with_reclaim_clock_for_tests(clock.clone()); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let origin = format!("http://{}", listener.local_addr().unwrap()); let reads = DatasetReadService::new( @@ -65,6 +69,7 @@ impl TestDatasetHttp { })); Self { origin, + clock, stop: Some(stop), task, } diff --git a/app/crowdb-access-server/tests/dataset_retirement_contract_test.rs b/app/crowdb-access-server/tests/dataset_retirement_contract_test.rs index e9f5cedf9..c2ea57173 100644 --- a/app/crowdb-access-server/tests/dataset_retirement_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_retirement_contract_test.rs @@ -59,6 +59,12 @@ async fn retirement_respects_retention_and_finishes_split_and_composition_cleanu response(client.post(format!("{snapshot}/retain")), 200).await; response(client.post(format!("{snapshot}/reclaim")), 409).await; response(client.post(format!("{snapshot}/release")), 200).await; + let marked = response(client.post(format!("{snapshot}/reclaim")), 200).await; + assert_eq!(marked["phase"], "grace_period"); + assert_eq!(marked["complete"], false); + assert_eq!(marked["reclaim_after"], 1300); + response(client.get(format!("{snapshot}/summary")), 404).await; + test.clock.store(1300, std::sync::atomic::Ordering::Release); for step in 0..100 { let progress = response(client.post(format!("{snapshot}/reclaim")), 200).await; assert_eq!(progress["physical_cleanup_deferred"], true); diff --git a/doc/backlog/R220-dataset-fat-client-plan.md b/doc/backlog/R220-dataset-fat-client-plan.md index 717b3aaf5..a2642e339 100644 --- a/doc/backlog/R220-dataset-fat-client-plan.md +++ b/doc/backlog/R220-dataset-fat-client-plan.md @@ -26,7 +26,7 @@ and [SDK API guide](../../lib/crowdb-sdk-native/README.md). user-secret database, nor external-user capability issuance. 2. Reuse `crowdb-access-dataset` authority, sessions, composition, Split, and upload logic. Preserve Snapshot/Read Plan binding, selection, worker - partitioning, integrity, cleanup, and active-read protection. Topology is + partitioning, integrity, cleanup, and deletion admission. Topology is a supported scheduling surface, not placement authority. 3. Preserve explicit acknowledgement and resume: fetching data, releasing a buffer, and acknowledging processing are separate events. Native delivery @@ -34,7 +34,9 @@ and [SDK API guide](../../lib/crowdb-sdk-native/README.md). stops producers; caller-held leases remain valid and charged until release. 4. Compare native Chunk-KV and chunk-range reads with the HTTP baseline, including topology refresh, stale owners, failures, digest costs, and the - existing 60-second active-read inactivity TTL. Keep placement, fencing, + snapshot deletion contract: reject new reads after a durable deletion mark, + preserve a five-minute cleanup grace period, and allow overdue reads to fail. + Ordinary reads must not register, renew or release persistent reader leases. Keep placement, fencing, durability, and publication rules in their existing components. 5. Expose native Dataset through an opt-in main-workspace Python extension (`crowdb_dataset_native`) to the same Rust data plane. The existing @@ -73,6 +75,11 @@ and [SDK API guide](../../lib/crowdb-sdk-native/README.md). executes, it refreshes, retries where eligible, or fails without changing selection/snapshot or publishing placement. Invariant: non-authoritative topology. Integration test. +- Given concurrent reads of one snapshot, when requests start, stream and end, + no snapshot activity mutations occur. Given a deletion mark, new requests fail + while admitted streams may finish; cleanup waits for the persisted grace + deadline across retries/restarts. Invariant: read independence and delayed GC. + Integration test. - Given caller-held buffers and a slow consumer, when prefetch or cancellation occurs, memory stays bounded and held data stays valid until final release. Invariant: bounded owner-backed delivery. Integration test. diff --git a/doc/design/access-server/dataset/design-crowdb-access-dataset.md b/doc/design/access-server/dataset/design-crowdb-access-dataset.md index f31ca037b..78ad31957 100644 --- a/doc/design/access-server/dataset/design-crowdb-access-dataset.md +++ b/doc/design/access-server/dataset/design-crowdb-access-dataset.md @@ -138,18 +138,21 @@ within the group. The client advances a cursor after batch confirmation. A crash may replay the last unconfirmed batch, but may not skip or change the snapshot. Epoch and sampler state remain client state. -An active-read lease protects payloads reachable from the selected snapshot while -the read is in progress. Normal close or cancel starts its release. Native -verified EOF waits for bounded cleanup; cancellation stops production promptly -and retains request admission through cleanup completion or its five-second -bound. Caller-held buffers retain their own byte credits independently. If the -connection has no activity for 60 seconds, the lease is invalid and GC may treat -the read as disconnected; this TTL is the crash and network-failure fallback. +Read admission checks the selected snapshot's authoritative deletion state without +registering a reader. Deletion durably fences new reads and records a five-minute +`reclaim_after` deadline; retries and restarts preserve that deadline. Existing +streams can finish using resolved metadata and payloads during the grace period. +Unusually long reads may fail after reclamation; partial delivery is never complete +success. Access Servers and native clients do not acknowledge deletion. Session +progress and local request/buffer admission have independent lifetimes. EOF, +cancel and drop release local request admission without remote cleanup writes; +caller-held buffers remain valid and charged until their final release. Dataset owns the lifetime of its metadata and payload references. Reclamation -requires that no retained snapshot, tombstone needed by a retained history, or -active read lease can reach the value or chunk. External S3 or Iceberg references -are provenance unless an explicit cross-model retention contract says otherwise. +preserves explicit retention, live snapshot ancestry and shared payload owners. +External S3 or Iceberg references are provenance unless an explicit cross-model +retention contract says otherwise. Physical I/O identity checks and buffer ownership +remain necessary when storage is reclaimed or reused. ## 8. Interactions @@ -256,10 +259,10 @@ Snapshot remains available. The public binary response streams a bounded JSON header, integrity-checked payload frames, field terminators and a batch terminator. Success also requires -HTTP EOF with no trailing bytes. Each active transfer owns a unique scoped read -lease and admission credits through transport delivery/drop. Heartbeats renew -leases during active streaming. Cancellation stops the producer and returns -credits; retained SDK-owned buffers keep their byte credits until released. +HTTP EOF with no trailing bytes. Each active transfer owns local admission +credits through delivery/drop. Snapshot reads perform no persistent activity +updates or heartbeat writes. Cancellation stops the producer and returns credits; +retained SDK-owned buffers keep their byte credits until released. The internal legacy scan adapter retains its compatibility behavior; public fetch never implies application acknowledgement. @@ -270,8 +273,10 @@ head nonce, and rechecks latest, stable, explicit retention and all live committ child branches. Temporary child admission fences do not prove retirement. A reclaimer owns a CAS token; only that owner can reopen admission or enter the irreversible retiring phase. Concurrent stale reclaimers cannot undo retirement. -An Aborted Snapshot tombstone preserves the original Manifest while payload -cleanup is pending and prevents publication retry resurrection. Upload ownership +After the terminal deletion mark, cleanup waits until the persisted grace +deadline. The Access Server scans bounded pages of deletion records and resumes +cleanup independently of reader processes. An Aborted Snapshot tombstone preserves +the original Manifest while payload cleanup is pending and prevents publication retry resurrection. Upload ownership is released for that operation only; other Snapshot owners preserve shared bytes. Deferred physical cleanup remains pending. Shared range retirement does not count as a reclaimed whole chunk. @@ -354,9 +359,10 @@ conflicts are not blanket retried. File downloads use a private temporary file and rename only after successful integrity verification. Head envelopes decode legacy versions and use version 3 for commit-attempt -markers. Access envelopes decode versions 1 and 2 and write version 3 with scoped -leases, reclaim ownership and terminal retirement. Upload sessions decode version -2; permanent owner retirement upgrades a session to version 3 so an older writer +markers. Access envelopes decode versions 1 through 4 and write version 4 with +reclaim ownership, terminal deletion, grace deadline, cleanup target and completion. +Legacy reader activity is ignored; all authority writers must use this contract. +Upload sessions decode version 2; permanent owner retirement upgrades a session to version 3 so an older writer cannot ignore revocation. This metadata upgrade is one-way: do not downgrade an upgraded authority or run old and new authority writers against the same Dataset store. Immutable Dataset/Snapshot/Manifest encodings diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index 9a0dd5fdc..f16b0ea6a 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -70,7 +70,7 @@ The data head transition makes data and initial Splits visible together. ## 3. Split evaluation and publication Standalone Split drafts pin owner Snapshot and expected current revision at begin. -Source leases protect effective values while active. Whole-set edits invalidate +Preparation checks source availability; deletion may interrupt unfinished work. Whole-set edits invalidate evaluation; prepare requires complete membership and valid overlap/coverage policy. Defaults are disjoint and allow-unassigned. Frozen inherited membership is explicit; reevaluation is a deliberate definition change. @@ -105,14 +105,14 @@ reject oversized external values before materializing their contents. Old snapshots without projections require explicit resumable inspection preparation. Unsupported or oversized legacy state is reported, never represented as an empty -Snapshot. Field preview is explicit and capped at 64 KiB under an owned lease; it +Snapshot. Field preview is explicit and capped at 64 KiB after read-only admission; it does not create a read plan or acknowledge application consumption. V2 read planning pins Snapshot and optional Split revision once, intersects Selection with Split membership, then orders/shuffles and partitions workers before fetching projected payloads. Planning advances twenty-Sample windows with fixed total ID/byte limits. Fetch/preview/status never advance the acknowledged cursor; only explicit -application acknowledgement does so. Cancel releases owned source protection. +application acknowledgement does so. Cancel stops local work without a persistent reader release. ## 5. Ownership and cleanup @@ -129,14 +129,15 @@ payload writes so failed admissions can also be retired. Small receipt/fence rec remain after bulk cleanup to preserve replay and non-resurrection guarantees. Snapshot reclaim checks latest/stable, retained Snapshots, live children, candidate -ownership and scoped leases under a terminal owner fence. It retires upload owners, +ownership under a terminal owner fence. A durable five-minute grace deadline +separates deletion admission from cleanup. After that deadline it retires upload owners, Snapshot-scoped Split drafts, possible index writes and ordinary metadata in phases. Completion reports logical retirement; physical upload deletion remains deferred to its registry cleanup. There is no independent committed Split revision pruning. `GET snapshots/{snapshot}/reclaim` is bounded, read-only observation. It returns -publication operation identity where available, latest/stable/retained/read-lease -facts, and not_started/pending/completed retirement state. Parent reachability is +publication operation identity where available, latest/stable/retained flags, +`reclaim_after` and the compatibility `active_read_lease=false` field, and not_started/pending/completed retirement state. Parent reachability is explicitly not evaluated in this small read; observed flags are never permission to reclaim. Rust/Python `control().lifecycle(snapshot)` share this contract. Metadata HTTP reads remain capped at five seconds, while bounded mutation steps diff --git a/doc/design/access-server/dataset/openapi-crowdb-dataset-v1.yaml b/doc/design/access-server/dataset/openapi-crowdb-dataset-v1.yaml index b80d110df..a652a4889 100644 --- a/doc/design/access-server/dataset/openapi-crowdb-dataset-v1.yaml +++ b/doc/design/access-server/dataset/openapi-crowdb-dataset-v1.yaml @@ -904,7 +904,7 @@ "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/snapshots/{snapshot}/reclaim": { "post": { "operationId": "ReclaimSnapshot", - "description": "Completed only after eligible Dataset metadata and owned payload reclamation are complete. Latest/stable/retained/parent/active-read snapshots conflict; deferred physical cleanup stays pending.", + "description": "Deletion marks the snapshot unavailable to new requests and persists a five-minute grace deadline before cleanup. Latest/stable/retained/parent snapshots conflict. Active readers do not block deletion; overdue reads may fail. Deferred physical cleanup stays pending.", "parameters": [ { "in": "path", @@ -1737,7 +1737,7 @@ }, "delete": { "operationId": "CloseReadPlan", - "description": "Idempotent scoped close/cancel; other sessions continue. No activity for 60 seconds expires the scoped lease. Application processing is at least once around acknowledgement.", + "description": "Idempotent scoped close/cancel; other sessions continue. No activity for 60 seconds expires the read session; it is not a snapshot GC lease. Application processing is at least once around acknowledgement.", "parameters": [ { "in": "path", @@ -2898,6 +2898,6 @@ "batch": "Content-Type application/vnd.crowdb.dataset-batch.v1. Magic is 8 ASCII bytes CDBDS001. Each frame: type u8, body length u32 big endian, then body. Types: 1 JSON BatchHeader (at most 8 MiB); 2 data (16-byte MD5 followed by 1..4 MiB bytes); 3 field end (zero length); 0 batch end (zero length); 4 terminal JSON Error (at most 64 KiB, only after the header). Header declares every sample/field in requested order. Value fields are delivered sequentially with one or more type-2 frames and one type-3; empty values have just type-3. Missing/tombstone have no data frames. Validate each frame MD5, exact whole-field length/MD5, field ends, batch end and HTTP EOF with no trailing bytes. A batch is successful only after completion. A streaming destination may contain verified partial data on error and must be discarded; download-to-path publishes only a verified temporary file. A type-4 frame reports a failed operation, may follow verified partial data, and never permits successful completion. It carries the same structured error model as control responses. The SDK must report the error and discard partial output.", "retry": "SDK uses one overall deadline and stable logical tokens; bounded attempts/backoff only on documented idempotent operations with replayable bodies. Do not retry authorization, integrity or semantic conflicts. Ambiguous mutation errors retain operation/upload identity for status recovery. Never invent new tokens on retry.", "integrity": "Seekable file convenience upload makes a bounded digest pass then a bounded transfer pass. Upload complete independently verifies durable parts and whole MD5. These are deliberate integrity passes, not whole-file buffering. Nonseekable uploads require known length/digest or later explicit streaming-session design.", - "lifecycle": "Remote durability is guaranteed after successful snapshot publication; local staging/checkpoint flush semantics are future SDK work. Read leases and byte credits survive until streaming completes/drops, with periodic renewal during slow active transfers. Keep-alive uses bounded idle/header timeouts and shutdown drain. Upload ownership and reclaim are durable authority state, never client authority. Whole-file integrity completion has a 30-minute server deadline; clients use their configurable transfer deadline. Candidate reclamation retains an aborted operation tombstone; publication retry cannot resurrect it. chunks_reclaimed counts whole chunks, not retired shared ranges." + "lifecycle": "Remote durability is guaranteed after successful snapshot publication; local staging/checkpoint flush semantics are future SDK work. Reads perform no persistent activity registration or renewal. Snapshot deletion rejects new requests and defers cleanup for five minutes; admitted reads may complete or fail if overdue. Local byte credits survive until streaming completes/drops. Keep-alive uses bounded idle/header timeouts and shutdown drain. Upload ownership and reclaim are durable authority state, never client authority. Whole-file integrity completion has a 30-minute server deadline; clients use their configurable transfer deadline. Candidate reclamation retains an aborted operation tombstone; publication retry cannot resurrect it. chunks_reclaimed counts whole chunks, not retired shared ranges." } } diff --git a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml index 9799bf303..6e692a9da 100644 --- a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml +++ b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml @@ -2093,6 +2093,12 @@ components: type: boolean active_read_lease: type: boolean + const: false + description: Compatibility field; readers do not register persistent activity. + reclaim_after: + type: [integer, 'null'] + minimum: 0 + description: Durable Unix-seconds deadline for destructive cleanup. state: type: string enum: [not_started, pending, completed] @@ -2113,12 +2119,16 @@ components: snapshot: *composition001 phase: enum: + - grace_period - upload_owners - split_drafts - index_fences - metadata - complete complete: *composition005 + reclaim_after: + type: integer + minimum: 0 physical_cleanup_deferred: const: true CommitStatus: diff --git a/doc/design/access-server/dataset/spec-crowdb-dataset.md b/doc/design/access-server/dataset/spec-crowdb-dataset.md index eecdc3ffc..89892b2db 100644 --- a/doc/design/access-server/dataset/spec-crowdb-dataset.md +++ b/doc/design/access-server/dataset/spec-crowdb-dataset.md @@ -54,8 +54,8 @@ The following terms have one meaning throughout this specification: shuffle to one immutable Snapshot. - **Cursor** records progress for one Read Plan and is acknowledged only after the corresponding batch is accepted by the caller. -- **Active-read lease** protects data reachable from one Dataset/Snapshot while - a read is admitted or executing. +- **Deletion grace period** separates a durable snapshot deletion mark from + destructive cleanup; it does not guarantee completion of every admitted read. Dataset does not infer semantics from a locator, a Snapshot ID's numeric ordering, a sample ID's encoding, or an external source URI. @@ -67,7 +67,7 @@ the configured `dataset-ns` root namespace for every Dataset. Names are unique within that namespace and are compared as opaque UTF-8 names after validation. `SnapshotId` is immutable and opaque. `latest` and `stable` are mutable names -whose values are Snapshot IDs. A Read Plan, Cursor, Manifest, or lease stores +whose values are Snapshot IDs. A Read Plan, Cursor, or Manifest stores the resolved Snapshot ID and never stores a mutable name in its place. A Snapshot has zero or one declared parent. Parentage is explicit; ID ordering @@ -122,12 +122,12 @@ specified below and in its versioned OpenAPI contract. | `GetSample` / `GetBatch` | Snapshot ID, sample IDs, projection | Returns ordered samples and verified field values. | | `Scan` | Read Plan, optional Cursor | Returns one bounded batch and the next Cursor. | | `SaveProgress` / `Resume` | Read Plan, Cursor | Persists or loads plan-bound progress. | -| `Cancel` | Bound read or service | Stops admission and releases the active-read lease. | +| `Cancel` | Bound read or service | Stops local admission and production without a remote reader release. | `PrepareSnapshot` and `PublishSnapshot` share the same operation token and validation rules. A complete publication may be implemented as one call that performs preparation and completion internally; it MUST have the same durable -state transitions. Read operations are side-effect free except for lease and +state transitions. Ordinary data reads do not persist activity. Explicit sessions retain cursor state required by their documented lifecycle. ## 5. Manifest and field values @@ -228,12 +228,11 @@ Requested sample and projection order is preserved unless the plan explicitly requests another ordering. Unknown samples, invalid projections, checksum mismatches, and malformed cursors are errors. -Each admitted transfer owns a unique scoped active-read lease for its Dataset -and Snapshot. Normal completion, cancellation, and explicit close release that -lease without releasing another reader's lease. Streaming renews its lease -while bytes are in flight. Transport-owned buffers retain protection and -admission through delivery or drop. A bounded inactivity TTL expires a disconnected lease; the expiry is a -GC safety policy, not a publication or cursor transition. +Each request checks authoritative snapshot availability at admission and MUST NOT +register, renew or release persistent reader activity. A durable deletion mark +rejects new requests. Previously admitted transfers may continue; neither EOF nor +cancellation waits for a remote cleanup operation. Local byte/request credits and +caller-owned buffers retain their existing lifetime guarantees. HTTP and native reads use the same authority semantics. Transport framing, serialization, and topology discovery may differ, but the resolved Snapshot, @@ -244,7 +243,14 @@ equivalent. `retain` and `release` apply to an immutable Snapshot ID. The latest Snapshot, stable Snapshot, retained Snapshots, parents of live committed Snapshots on any -branch, and Snapshots with active-read leases are protected from reclamation. +branch are protected from reclamation. Active reads do not prevent deletion. + +Deletion first persists a terminal mark and a five-minute `reclaim_after` deadline. +Cleanup MUST NOT remove snapshot-owned metadata or payload references before +that deadline. Restart and retry MUST preserve it. No Access Server or client +acknowledgements are required. Reads that exceed the grace period may return +an error after partial delivery; they MUST NOT report successful completion or +switch snapshots. Deletion completion does not revoke bytes already delivered. Reclamation is idempotent. It removes only unreachable Dataset-owned metadata and payload references. A payload shared by two retained or readable @@ -358,8 +364,9 @@ extensions. - **DATA-S6 — Plan binding:** a Cursor is valid only for its exact Read Plan. - **DATA-S7 — Bounded delivery:** configured memory, batch, and in-flight limits are enforced before payload admission. -- **DATA-S8 — Lease protection:** active reads protect all reachable metadata - and payloads until release or expiry. +- **DATA-S8 — Delayed reclamation:** deletion rejects new reads and defers + cleanup to its durable grace deadline; overdue reads may fail without + successful truncation or snapshot substitution. - **DATA-S9 — Ownership-safe GC:** reclamation deletes only unowned Dataset data and never external source objects. - **DATA-S10 — Surface parity:** HTTP and native surfaces preserve the same diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md new file mode 100644 index 000000000..77b50be02 --- /dev/null +++ b/doc/working/plan-access-simplification.md @@ -0,0 +1,116 @@ + + + +# Access Server Simplification Plan + +Upstream: [Dataset SDK](../backlog/R220-dataset-fat-client-plan.md), +[Access design](../design/access-server/design-crowdb-access-server.md). + +Goal: remove coordination from independent requests where the business contract +allows interruption or retry, and retain only justified durable authority. +This is a persistent cleanup inventory requested by the user; remove completed +items after recording their verified outcome in the owning design. + +## Agreed Dataset deletion contract + +- Mark deletion durably, reject new requests, then reclaim after a persisted + grace deadline. No Access Server/client acknowledgements or reader registry. +- Requests admitted before the mark may continue during the grace period. + Reclamation may interrupt unusually long reads; return an error, never a + successful truncated response or a different snapshot. +- Default grace: five minutes. It is a delay, not a promise that a running job + can issue new requests forever. Retries/restarts must not reset the deadline. +- Retention, live snapshot ownership, publication fences, physical allocation + identity, in-flight I/O and caller-owned buffers remain distinct protections. +- A read checks authoritative availability at admission. A stale cache is not + sufficient to admit a new read after deletion. Read paths perform no snapshot + activity writes, heartbeats or asynchronous release writes. + +## Execution + +- [ ] **Dataset durable read sessions**: `DatasetSessions::{start,next,resume, + acknowledge,close}` persists bindings, selected IDs, issued tokens, activity + deadlines and cursor changes. `next` writes even when replaying an already + issued batch; streaming checks session state repeatedly. Determine which + callers need server-persisted restart/ack semantics and whether the ordinary + path can use a snapshot-bound client cursor. Keep explicit acknowledged + progress separate from fetching. Files: `lib/crowdb-access-dataset/src/read_session/`, + `app/crowdb-access-server/src/dataset/api/{session.rs,batch.rs}`. +- [ ] **Dataset preparation and scratch journals**: Composition/Split evaluation + persists work pages, input revisions, candidate links, cleanup cursors and + source activity. Separate unpublished scratch that may expire from committed + visibility and shared payload ownership. Measure KV mutations per page and + evaluate coarser checkpoints/expiry before removing recovery facts. Files: + `lib/crowdb-access-dataset/src/{composition,split,inspection}/`. +- [ ] **Dataset upload shared session updates**: inspect part progress and owner + vectors, registry page updates, registration generations and permanent retired + owner lists for contention between independent parts/owners. Preserve digest, + durable payload publication and shared-owner safety; evaluate independent part + records and delayed orphan cleanup. Files: + `lib/crowdb-access-dataset/src/upload/{ownership.rs,retirement.rs,registry.rs}` + and upload part/completion implementations. +- [ ] **Iceberg catalog-wide multipart credits**: every reservation/release + changes catalog-wide `sessions`, `reserved_bytes`, and `pending` state through + `MultipartAdmission`. Independent uploads share a durable coordination point. + Evaluate local bounded admission plus coarse distributed quotas, with explicit + allowance for quota overshoot; do not weaken memory limits. Files: + `lib/crowdb-access-iceberg/src/file/multipart_credits.rs`, + `src/file/multipart_credits/settle.rs`, `src/file/multipart_admission.rs`. +- [ ] **Iceberg parent namespace admission**: namespace mutations and table + lifecycle operations CAS the parent's `pending_operation` and + `mutation_revision`, persist before/after payloads and journal phases, then + release the marker. Independent child creations can help/wait on each other. + Evaluate immutable namespace identities plus deletion marks/grace periods; + retain same-name uniqueness and reject publication into retired identities. + Files: `lib/crowdb-access-iceberg/src/namespace/admission.rs`, + `src/table/lifecycle/admission.rs`, their journals/recovery modules. +- [ ] **Iceberg publication and GC fencing**: distinguish expected table-head + CAS conflicts from extra persisted operation phases, payload copies, admission + markers and GC `operation_fence` writes. Audit ordinary commit versus create, + rename, delete and recovery separately. Simplify only after counting RPCs and + identifying the exact guarantee being removed. Files: + `lib/crowdb-access-iceberg/src/commit/`, `src/gc/{admission.rs,fence.rs,worker/}`. +- [ ] **S3 and Iceberg multipart per-upload contention**: part publication and + completion use session generations and CAS. Check whether independent parts + rewrite shared session state and whether completion/recovery duplicates + persistent work. Keep immutable selected-part identity, integrity and + lost-reply recovery. Files: + `lib/crowdb-access-s3/src/metadata/multipart_repository/`, + `lib/crowdb-access-iceberg/src/file/multipart_repository/`. +- [ ] **Retry hygiene across authority loops**: classify CAS failure as stale + base, business precondition failure or uncertain transport outcome. Where + retry is valid, use returned observed state and recompute; remove redundant + GETs and no-op writes. A transport retry must retain mutation identity while + a new logical mutation must get a new identity. Files: Dataset read-session, + publication/upload loops, Iceberg journals and S3 multipart repositories. + +## Inspection coverage and exclusions + +- Inspected route ownership and coordination sites in + `app/crowdb-access-server/src/` and production sources of + `lib/crowdb-access-{dataset,s3,iceberg,multipart}/src/`, including shared credentials, + admission, streaming, Dataset lifecycle, S3 metadata and Iceberg catalogs. + This inventory identifies candidates; it is not a measured proof that every + listed write is unnecessary. +- No direct `Mutex`/`RwLock` usages found in those five production source trees. + Dependencies and FFI may contain locks; audit them only when profiling points + there. Atomic resource counters are not automatically removal candidates. +- S3 ordinary object PUT/DELETE already use unconditional point mutations; + ordinary GET has no persistent reader registration. Bucket-name deletion CAS, + explicitly conditional writes and multipart completion have different scopes. +- Iceberg `FileReader` verifies physical directory ownership and block digests + without registering each reader. Preserve stale/reused-location rejection. +- Credentials, immutable content publication, explicit retention, shared payload + owners, authorization and physical allocation identity are not expendable + merely because they persist state or use CAS. + +## Verification for each cleanup + +- Record the old guarantee, accepted replacement semantics, conflict scope, + persistent GET/write counts and reader/request lifetime before implementation. +- Unit/integration: independent concurrent operations, deletion races, retries, + restart, partial results, retained buffers and shared references. +- E2E: existing HTTP/native Dataset parity and mixed-size workload; use matched + source/builds. For other domains run the affected existing protocol suites. +- Run affected tests and Rust fmt/clippy through `pixi run`. Do not increase + retry bounds, weaken integrity assertions or call workload reduction a fix. diff --git a/doc/working/plan-dataset-native-sdk.md b/doc/working/plan-dataset-native-sdk.md index 98d8e2bd6..8e8f03757 100644 --- a/doc/working/plan-dataset-native-sdk.md +++ b/doc/working/plan-dataset-native-sdk.md @@ -38,10 +38,10 @@ bounded owner-backed delivery and independently packaged Python bindings. - [x] **Deliver bounded batches**: separate immutable field metadata, payload leases, verified EOF and acknowledgement. Pull at most a bounded part ahead; charge retained allocations, preserve pending data if a pull future is - dropped, and make terminal errors sticky. Renew the existing 60-second - active-read lease while producing; close/cancel stops production and releases - protection without acknowledging or invalidating caller-held buffers. - Files: native `dataset/read/`, shared session/lease code as needed. + dropped, and make terminal errors sticky. Check deletion admission once; + close/cancel returns local request credit without persistent reader cleanup, + acknowledgement or invalidation of caller-held buffers. + Files: native `dataset/read/`, shared authority admission. - [x] **Share topology and lifetime**: expose existing runtime observations with Dataset locations as hints only. S3 and Dataset use independent facades and writer policies over shared admission/transports; closing a facade or @@ -263,7 +263,7 @@ This `--semantics-only` run is functional evidence, not performance acceptance. ## Handoff 2026-10-23 -### Current stop point +### Historical stop point (superseded by the deletion redesign) - Metadata memory is fixed at `b18efa9a`: unchanged full run `dataset-workload/c7fd408b216a451e81feb9009e2eeac1` stays below 280 MiB, @@ -272,7 +272,7 @@ This `--semantics-only` run is functional evidence, not performance acceptance. bounded task; focused lifecycle tests pass. This is a resource-accounting fix, not a claim that the contention failure or R220 acceptance is complete. - After five contention corrections, the slow durable-CAS regression still - fails. See [Blocked](#blocked) before further implementation. The unaccepted + fails. The approved deletion redesign below replaces this model. The unaccepted authority RTT experiment and failing regression are preserved in stash `a6e19f404d9cb614a3004a09575051196d6d0727`; no experiment is discarded. - The recorded HCA/Python semantics evidence below predates the final memory @@ -501,62 +501,48 @@ remaining user design decision and no requirement `Blocked` condition. initially passed alone but failed in the component suite and again alone; this is not accepted contention evidence. -## Blocked - -- Failure: `slow_durable_cas_completes_with_bounded_reader_lifetimes` returns - `HeadConflict` at `access_contention_test.rs:154`, after one update exhausts - the unchanged 64-attempt bound. The isolated rerun also fails, excluding - interference from other component tests. Three readers each perform 96 - acquire/read/release cycles over a sequencer with 100-ms durable mutations; - reads see committed state and failed conditions still consume a mutation. -- Exact failing commands: - - ```bash - pixi run -- cargo test -p crowdb-sdk-native -p crowdb-access-dataset -p crowdb-access-server --all-targets - pixi run -- cargo test -p crowdb-access-dataset --test access_contention_test slow_durable_cas_completes_with_bounded_reader_lifetimes -- --exact --nocapture - ``` - -- Attempts and evidence: - 1. `93aefa43`: recompute from the CAS observation; delay the first HTTP - heartbeat. The real workload still exhausts attempts. - 2. `4127a966`: randomized short contention windows. The real workload still - exhausts attempts. - 3. `a8e1571e`: scale the window with durable RTT, capped at 256 ms. HTTP - completes, but native multi-flight still exhausts attempts after memory - fixes; its observed CAS RTT is 69–454 ms. - 4. Remove the RTT clamp and use full-width random jitter. The slow sequencer - with detached releases still exhausts attempts. - 5. Bound native cleanup within the admitted read lifetime. Native lifecycle - regressions pass, but the corresponding bounded slow-sequencer model still - exhausts attempts in both the suite and the isolated rerun. -- Diagnosis: independent read/modify/CAS loops on the same snapshot lease - record have no fairness guarantee. Queueing latency and freshly arriving - updates can repeatedly invalidate a slower contender. Random windows and - bounded producer lifetime reduce pressure but do not provide ordered progress. - The fifth change has not been rebuilt into a full-service acceptance image: - its failing component model prevents claiming end-to-end resolution. -- Preserve the fourth experiment and slow-store regression with - `git stash apply a6e19f404d9cb614a3004a09575051196d6d0727` when resuming. - This stash contains only the four shared-authority experimental files, - including the untracked sequencer fixture; it does not contain the native - cleanup fix. Applying it preserves the stash for recovery. -- Recommended next direction: bounded per-snapshot scheduling and merging of - lease changes, retaining the existing persisted record and remote CAS fences. - Specify queue capacity, per-handle completion/cancellation, cleanup ownership, - and fairness across processes before implementation. A local queue alone is - not a proof of distributed fairness; keep the failing model and real workload. -- Alternative: independently stored per-reader leases. This reduces contention - on one record but changes reclamation fencing, bounded scans, compatibility - and crash recovery; it requires a broader protocol/design change. -- Human choice required by the requirement workflow after five root-cause - attempts. Do not raise deadlines/retry limits, suppress the failing test, - reduce the workload, or mark R220 complete. The verified native cleanup fix - can be committed separately with this blocked record; the failed experiment - remains recoverable and explicitly unaccepted. -- The native cleanup commit passes `clean-env` followed by all targets of - `crowdb-sdk-native`, `crowdb-access-dataset`, and `crowdb-access-server`; - workspace all-target/all-feature clippy, `rs-lint`, Rust fmt and diff checks - also pass. These checks exclude the explicitly preserved, failing authority - experiment. Existing environment-gated hardware/installed-Python cases are - not claimed as passing acceptance. No new production lock, unsafe exception, - C++ change, request retry or deadline increase was introduced. +## Approved deletion redesign + +- The user replaced active-read GC protection with durable deletion admission and + a five-minute grace deadline. Remove per-reader activity CAS, renewal tasks and + asynchronous release writes; reads may fail if reclamation interrupts them. +- Implementation and Access Server coordination cleanup inventory are tracked in + [Access simplification](plan-access-simplification.md). Historical contention + experiments below are not acceptance requirements for the removed lease model. +- Preserve existing mixed-size workload, HTTP/native parity, memory, publication, + shared payload ownership and local buffer/admission tests. Same-source workload + and final HCA acceptance are still required before closing the SDK requirement. + +## Prior contention investigation (superseded) + +- The old model updated one snapshot-wide reader record on admission, renewal + and release. Returned CAS observations avoid extra GETs but do not prevent + independent readers from repeatedly invalidating each other's base. +- The 96-reader test exhausted the 64-attempt bound with + `reader 32, release=false: publication lost its head race`. The slow durable + model also failed `slow_durable_cas_completes_with_bounded_reader_lifetimes`. + Jitter/RTT backoff and retaining local request credit through cleanup did not + establish distributed fairness. The slow model is diagnostic, not a faithful + production sequencer or accepted end-to-end result. +- Preserve experiment stash `a6e19f404d9cb614a3004a09575051196d6d0727` as historical + evidence. Do not apply it to the replacement design: its lease model and tests + are obsolete. No queue or independent-reader lease replacement was selected. +- The replacement concurrency test runs 96 readers with 64 admission checks each + and verifies zero persistent mutations. Full workload throughput remains an + independent acceptance gate; do not infer TPS from this component test. + +## Deletion redesign verification (2026-10-24) + +- Implemented persistent deletion marks and five-minute grace, read-only + admission, restartable background cleanup, and immediate local EOF/cancel + credit return. New reads reject marked snapshots; already admitted bodies + can finish, and unusually long reads may fail after reclamation. +- Shared Dataset, native SDK and Access Server all-target tests pass. Focused + regressions cover duplicate marks/restart, legacy record decoding, cleanup + boundaries, held buffers, corrupt-mark isolation and owner/index cleanup after + metadata completion. +- Workspace all-target/all-feature clippy, standard Rust lint, independent SDK + tests/lint and dependency-boundary checks pass. Environment-gated hardware or + installed-Python cases are not current-source hardware acceptance. +- Next: rebuild/import the same source and rerun unchanged mixed-size workload, + then final real-service restart/local-HCA acceptance. R220 stays open. diff --git a/lib/crowdb-access-dataset/Cargo.toml b/lib/crowdb-access-dataset/Cargo.toml index 401bbabb1..b85e47c73 100644 --- a/lib/crowdb-access-dataset/Cargo.toml +++ b/lib/crowdb-access-dataset/Cargo.toml @@ -10,6 +10,9 @@ description = "Native Dataset authority and bounded sample access." [lints] workspace = true +[features] +test-util = [] + [dependencies] tokio = { workspace = true, features = ["rt", "macros", "time"] } async-trait = "0.1" @@ -29,5 +32,6 @@ thiserror = { workspace = true } uuid = { version = "1", features = ["v4"] } [dev-dependencies] +crowdb-access-dataset = { path = ".", features = ["test-util"] } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } arc-swap = "1.9" diff --git a/lib/crowdb-access-dataset/src/authority.rs b/lib/crowdb-access-dataset/src/authority.rs index 5b28ece0b..48dd389b9 100644 --- a/lib/crowdb-access-dataset/src/authority.rs +++ b/lib/crowdb-access-dataset/src/authority.rs @@ -62,13 +62,17 @@ pub enum AuthorityError { AncestryCycle, #[error("read cursor lost its compare-and-exchange race")] CursorConflict, - #[error("snapshot is protected by a head, retention marker, parent chain, or active read")] + #[error("snapshot is protected by a head, retention marker, or parent chain")] SnapshotProtected, + #[error("snapshot deletion grace period ends at {reclaim_after}")] + ReclaimPending { reclaim_after: u64 }, } pub struct DatasetAuthority { pub(crate) store: Arc, chunk_reader: Arc, + #[cfg(feature = "test-util")] + reclaim_clock: Option>, } #[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] @@ -86,6 +90,8 @@ impl DatasetAuthority { Self { store, chunk_reader: unavailable_chunk_reader(), + #[cfg(feature = "test-util")] + reclaim_clock: None, } } @@ -93,7 +99,12 @@ impl DatasetAuthority { /// # Panics /// The built-in non-zero lease duration is always valid. pub fn with_chunk_reader(store: Arc, chunk_reader: Arc) -> Self { - Self { store, chunk_reader } + Self { + store, + chunk_reader, + #[cfg(feature = "test-util")] + reclaim_clock: None, + } } /// # Errors diff --git a/lib/crowdb-access-dataset/src/authority/access.rs b/lib/crowdb-access-dataset/src/authority/access.rs index 02f3aa054..27941e60d 100644 --- a/lib/crowdb-access-dataset/src/authority/access.rs +++ b/lib/crowdb-access-dataset/src/authority/access.rs @@ -8,56 +8,79 @@ mod observation; pub use lifecycle::{SnapshotReclaimError, SnapshotReclaimPhase, SnapshotReclaimStatus}; pub use observation::SnapshotLifecycle; mod reclaim; -use crate::{key, AuthorityError, CasOutcome, DatasetError, DatasetIdentity, OperationId, SnapshotId}; +mod sweep; +use crate::{key, AuthorityError, CasOutcome, DatasetIdentity, OperationId, SnapshotId}; pub use cleanup::MetadataCleanupProgress; use serde::{Deserialize, Serialize}; +/// Default delay between logical deletion and destructive cleanup. +const RECLAIM_GRACE_SECONDS: u64 = 300; + #[derive(Default, Serialize, Deserialize)] struct Access { reclaiming: bool, - legacy: u64, - deadline: u64, - leases: Vec<(OperationId, u64)>, reclaim_owner: Option, retiring: bool, + reclaim_after: u64, + target: Option<(DatasetIdentity, SnapshotId)>, + cleanup_complete: bool, } #[derive(Deserialize)] struct AccessV2 { reclaiming: bool, - legacy: u64, - deadline: u64, - leases: Vec<(OperationId, u64)>, + _legacy: u64, + _deadline: u64, + _leases: Vec<(OperationId, u64)>, +} +#[derive(Deserialize)] +struct AccessV3 { + reclaiming: bool, + _legacy: u64, + _deadline: u64, + _leases: Vec<(OperationId, u64)>, + reclaim_owner: Option, + retiring: bool, } fn decode_access(bytes: &[u8]) -> Result { - if bytes.get(..2) == Some(1u16.to_le_bytes().as_slice()) { - let old: crate::ActiveReadLease = decode(bytes)?; - return Ok(Access { - legacy: old.active, - deadline: old.deadline, - ..Access::default() - }); - } - if bytes.get(..2) == Some(2u16.to_le_bytes().as_slice()) { - let record: VersionedRecord = - bincode::deserialize(bytes).map_err(|_| AuthorityError::Corrupt)?; - let old = record.payload; - return Ok(Access { - reclaiming: old.reclaiming, - legacy: old.legacy, - deadline: old.deadline, - leases: old.leases, - ..Access::default() - }); - } - let record: VersionedRecord = bincode::deserialize(bytes).map_err(|_| AuthorityError::Corrupt)?; - if record.version != 3 || (record.payload.retiring && !record.payload.reclaiming) { + let version = bytes.get(..2).ok_or(AuthorityError::Corrupt)?; + let access = match u16::from_le_bytes([version[0], version[1]]) { + 1 => { + let _: crate::ActiveReadLease = decode(bytes)?; + Access::default() + } + 2 => { + let old: VersionedRecord = + bincode::deserialize(bytes).map_err(|_| AuthorityError::Corrupt)?; + Access { + reclaiming: old.payload.reclaiming, + ..Access::default() + } + } + 3 => { + let old: VersionedRecord = + bincode::deserialize(bytes).map_err(|_| AuthorityError::Corrupt)?; + Access { + reclaiming: old.payload.reclaiming, + reclaim_owner: old.payload.reclaim_owner, + retiring: old.payload.retiring, + ..Access::default() + } + } + 4 => { + let record: VersionedRecord = + bincode::deserialize(bytes).map_err(|_| AuthorityError::Corrupt)?; + record.payload + } + _ => return Err(AuthorityError::Corrupt), + }; + if access.retiring && !access.reclaiming { return Err(AuthorityError::Corrupt); } - Ok(record.payload) + Ok(access) } fn encode_access(record: Access) -> Result, AuthorityError> { bincode::serialize(&VersionedRecord { - version: 3, + version: 4, payload: record, }) .map_err(|_| AuthorityError::Corrupt) @@ -77,15 +100,18 @@ impl DatasetAuthority { ) -> Result<(), AuthorityError> { let key = key::read_lease(identity, snapshot)?; let mut old = self.store.get(&key).await?; - for attempt in 0..64 { + for _ in 0..64 { let mut access = old .as_ref() .map(|v| decode_access(&v.bytes)) .transpose()? .unwrap_or_default(); change(&mut access)?; + access.target = Some((identity.clone(), snapshot)); let bytes = encode_access(access)?; - let submitted = std::time::Instant::now(); + if old.as_ref().is_some_and(|v| v.bytes == bytes) { + return Ok(()); + } match self .store .compare_exchange(&key, old.as_ref().map(|v| v.bytes.as_slice()), &bytes) @@ -93,23 +119,8 @@ impl DatasetAuthority { { CasOutcome::Applied(_) => return Ok(()), // Recompute against the sequencer's observation without opening another - // point-read race against concurrent acquisitions and releases. - CasOutcome::Conflict(observed) => { - old = observed; - if attempt % 2 == 1 && attempt < 63 { - // Concurrent acquisitions and asynchronous releases must not - // repeatedly rejoin the same CAS wave and starve each other. - // Include the durable store round trip: a fixed sub-RTT - // window keeps slow writers in the same contention wave. - let round_trip = submitted.elapsed().as_millis().clamp(1, 32); - let window = round_trip * (1_u128 << (attempt / 2).min(3)); - let delay = 1 + u128::from(uuid::Uuid::new_v4().as_bytes()[0]) % window; - let delay = u64::try_from(delay).expect("contention window is at most 256 ms"); - tokio::time::sleep(std::time::Duration::from_millis(delay)).await; - // The conflict observation predates the contention window. - old = self.store.get(&key).await?; - } - } + // point-read race against concurrent lifecycle changes. + CasOutcome::Conflict(observed) => old = observed, } } Err(AuthorityError::HeadConflict) @@ -126,107 +137,71 @@ impl DatasetAuthority { } Ok(()) } - /// Acquires legacy aggregate activity, preserving older persisted records. + /// Checks authoritative admission once without registering or renewing a reader. + /// Requests admitted before deletion may continue until storage is reclaimed. + /// The caller resolves snapshot existence and publication separately. /// # Errors - /// Returns expired/fenced snapshots, invalid TTL or storage failures. - pub async fn acquire_snapshot_read( + /// Returns `SnapshotNotFound` for deletion fences, or storage/decoding failures. + pub async fn check_snapshot_readable( &self, identity: &DatasetIdentity, snapshot: SnapshotId, - time: u64, - ttl: u64, ) -> Result<(), AuthorityError> { - if ttl == 0 { - return Err(DatasetError::InvalidManifest.into()); - } - self.update_access(identity, snapshot, |a| { - if a.reclaiming { - return Err(AuthorityError::SnapshotProtected); - } - if time >= a.deadline { - a.legacy = 0; + if let Some(value) = self.store.get(&key::read_lease(identity, snapshot)?).await? { + if decode_access(&value.bytes)?.reclaiming { + return Err(AuthorityError::SnapshotNotFound); } - a.legacy = a.legacy.checked_add(1).ok_or(AuthorityError::Corrupt)?; - a.deadline = time.saturating_add(ttl); - Ok(()) - }) - .await + } + Ok(()) } + + /// Returns the durable deadline after a successful deletion mark. /// # Errors - /// Returns malformed or unavailable storage. - pub async fn release_snapshot_read( + /// Rejects snapshots without a terminal deletion mark or malformed storage. + pub async fn snapshot_reclaim_after( &self, identity: &DatasetIdentity, snapshot: SnapshotId, - ) -> Result<(), AuthorityError> { - self.update_access(identity, snapshot, |a| { - a.legacy = a.legacy.saturating_sub(1); - Ok(()) - }) - .await + ) -> Result { + let value = self + .store + .get(&key::read_lease(identity, snapshot)?) + .await? + .ok_or(AuthorityError::SnapshotProtected)?; + let access = decode_access(&value.bytes)?; + if !access.retiring { + return Err(AuthorityError::SnapshotProtected); + } + Ok(access.reclaim_after) } - /// # Errors - /// Returns expired activity or unavailable storage. - pub async fn renew_snapshot_read( + + pub(crate) async fn require_reclaim_ready( &self, identity: &DatasetIdentity, snapshot: SnapshotId, - time: u64, - ttl: u64, ) -> Result<(), AuthorityError> { - self.update_access(identity, snapshot, |a| { - if a.reclaiming || a.legacy == 0 || time >= a.deadline { - return Err(DatasetError::ReadCancelled.into()); - } - a.deadline = time.saturating_add(ttl); - Ok(()) - }) - .await + let reclaim_after = self.snapshot_reclaim_after(identity, snapshot).await?; + if self.reclaim_time() < reclaim_after { + return Err(AuthorityError::ReclaimPending { reclaim_after }); + } + Ok(()) } - /// Acquires/renews one independently owned lease; another handle cannot release it. - /// # Errors - /// Returns fenced snapshots, exhausted capacity or unavailable storage. - pub async fn touch_read_handle( - &self, - identity: &DatasetIdentity, - snapshot: SnapshotId, - id: OperationId, - time: u64, - ttl: u64, - ) -> Result<(), AuthorityError> { - if ttl == 0 { - return Err(DatasetError::InvalidManifest.into()); + + fn reclaim_time(&self) -> u64 { + #[cfg(feature = "test-util")] + if let Some(clock) = &self.reclaim_clock { + return clock.load(std::sync::atomic::Ordering::Acquire); } - self.update_access(identity, snapshot, |a| { - if a.reclaiming { - return Err(AuthorityError::SnapshotProtected); - } - a.leases.retain(|(_, deadline)| *deadline > time); - if let Some((_, deadline)) = a.leases.iter_mut().find(|(lease, _)| *lease == id) { - *deadline = time.saturating_add(ttl); - } else { - if a.leases.len() >= 4096 { - return Err(DatasetError::ReadLimitExceeded.into()); - } - a.leases.push((id, time.saturating_add(ttl))); - } - Ok(()) - }) - .await + now() } - /// # Errors - /// Returns malformed or unavailable storage. - pub async fn release_read_handle( - &self, - identity: &DatasetIdentity, - snapshot: SnapshotId, - id: OperationId, - ) -> Result<(), AuthorityError> { - self.update_access(identity, snapshot, |a| { - a.leases.retain(|(lease, _)| *lease != id); - Ok(()) - }) - .await + #[cfg(feature = "test-util")] + #[must_use] + pub fn with_reclaim_clock_for_tests( + mut self, + clock: std::sync::Arc, + ) -> Self { + self.reclaim_clock = Some(clock); + self } } diff --git a/lib/crowdb-access-dataset/src/authority/access/cleanup.rs b/lib/crowdb-access-dataset/src/authority/access/cleanup.rs index 9808a325c..d5e49358f 100644 --- a/lib/crowdb-access-dataset/src/authority/access/cleanup.rs +++ b/lib/crowdb-access-dataset/src/authority/access/cleanup.rs @@ -32,6 +32,7 @@ impl DatasetAuthority { return Err(DatasetError::ReadCancelled.into()); } let record = self.begin_reclaim(identity, snapshot).await?; + self.require_reclaim_ready(identity, snapshot).await?; let scopes = cleanup_scopes(identity, snapshot, record.publication.parent)?; let key = scoped(b"DS\0GMC\0", identity, snapshot)?; let budget = MetadataScanBudget { diff --git a/lib/crowdb-access-dataset/src/authority/access/lifecycle.rs b/lib/crowdb-access-dataset/src/authority/access/lifecycle.rs index a25a02aa2..394864cb4 100644 --- a/lib/crowdb-access-dataset/src/authority/access/lifecycle.rs +++ b/lib/crowdb-access-dataset/src/authority/access/lifecycle.rs @@ -26,6 +26,7 @@ pub enum SnapshotReclaimError { #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum SnapshotReclaimPhase { + GracePeriod, UploadOwners, SplitDrafts, IndexFences, @@ -74,6 +75,15 @@ impl DatasetAuthority { complete: false, physical_cleanup_deferred: true, }; + self.begin_reclaim(identity, snapshot).await?; + match self.require_reclaim_ready(identity, snapshot).await { + Ok(()) => {} + Err(AuthorityError::ReclaimPending { .. }) => { + status.phase = SnapshotReclaimPhase::GracePeriod; + return Ok(status); + } + Err(error) => return Err(error.into()), + } if !self .reclaim_snapshot_uploads_step(identity, snapshot, uploads) .await? @@ -118,6 +128,11 @@ impl DatasetAuthority { { status.phase = SnapshotReclaimPhase::Complete; status.complete = true; + self.update_access(identity, snapshot, |access| { + access.cleanup_complete = true; + Ok(()) + }) + .await?; } Ok(status) } diff --git a/lib/crowdb-access-dataset/src/authority/access/observation.rs b/lib/crowdb-access-dataset/src/authority/access/observation.rs index 161b8b0f9..c85fc431b 100644 --- a/lib/crowdb-access-dataset/src/authority/access/observation.rs +++ b/lib/crowdb-access-dataset/src/authority/access/observation.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{decode_access, now}; +use super::decode_access; use crate::{ authority::{decode, decode_head}, inspection::{work::InspectionWork, InspectionError}, @@ -21,6 +21,7 @@ pub struct SnapshotLifecycle { pub stable: bool, pub retained: bool, pub active_read_lease: bool, + pub reclaim_after: Option, pub retiring: bool, pub completed: bool, } @@ -68,6 +69,11 @@ impl DatasetAuthority { .map(|value| decode_access(&value.bytes)) .transpose()? .unwrap_or_default(); + let completed = if access.target.is_some() { + access.cleanup_complete + } else { + completed + }; if !completed && !access.retiring && !self @@ -90,15 +96,14 @@ impl DatasetAuthority { let operation = self .inspection_snapshot_operation(identity, snapshot, &mut work) .await?; - let time = now(); Ok(SnapshotLifecycle { snapshot, operation, latest: head.as_ref().is_some_and(|head| head.latest == Some(snapshot)), stable: head.as_ref().is_some_and(|head| head.stable == Some(snapshot)), retained, - active_read_lease: (access.legacy > 0 && access.deadline > time) - || access.leases.iter().any(|(_, deadline)| *deadline > time), + active_read_lease: false, + reclaim_after: access.retiring.then_some(access.reclaim_after), retiring: access.reclaiming, completed, }) diff --git a/lib/crowdb-access-dataset/src/authority/access/reclaim.rs b/lib/crowdb-access-dataset/src/authority/access/reclaim.rs index 8d8c19f81..2b643647a 100644 --- a/lib/crowdb-access-dataset/src/authority/access/reclaim.rs +++ b/lib/crowdb-access-dataset/src/authority/access/reclaim.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{now, DatasetAuthority}; +use super::DatasetAuthority; use crate::authority::{decode, decode_head, encode_head}; use crate::{key, AuthorityError, CasOutcome, DatasetIdentity, HeadRecord, OperationId, SnapshotId}; @@ -18,10 +18,6 @@ impl DatasetAuthority { if retiring { return Ok(()); } - let time = now(); - if (a.legacy > 0 && a.deadline > time) || a.leases.iter().any(|(_, d)| *d > time) { - return Err(AuthorityError::SnapshotProtected); - } a.reclaiming = true; a.reclaim_owner = Some(owner); Ok(()) @@ -80,12 +76,20 @@ impl DatasetAuthority { /// Fences publication/read admission and retains an aborted operation tombstone. /// The original manifest remains in that record for crash-safe payload cleanup. /// # Errors - /// Rejects head/ancestry/retention/read protection or storage failures. + /// Rejects head/ancestry/retention protection or storage failures. pub async fn begin_reclaim( &self, identity: &DatasetIdentity, snapshot: SnapshotId, ) -> Result { + // Avoid fencing an obviously protected head. The checks after fencing + // remain authoritative against concurrent publication and retention. + if let Some(value) = self.store.get(&key::head(identity)?).await? { + let head = decode_head(&value.bytes)?; + if head.latest == Some(snapshot) || head.stable == Some(snapshot) { + return Err(AuthorityError::SnapshotProtected); + } + } if let Some(owner) = self.fence_reclaim(identity, snapshot).await? { let protected = async { let head = self.store.get(&key::head(identity)?).await?; @@ -114,7 +118,13 @@ impl DatasetAuthority { return Err(error); } } - self.begin_terminal_retirement(identity, snapshot, owner).await?; + self.begin_terminal_retirement( + identity, + snapshot, + owner, + self.reclaim_time().saturating_add(super::RECLAIM_GRACE_SECONDS), + ) + .await?; } self.retire_snapshot_record(identity, snapshot).await } @@ -123,12 +133,14 @@ impl DatasetAuthority { identity: &DatasetIdentity, snapshot: SnapshotId, owner: OperationId, + reclaim_after: u64, ) -> Result<(), AuthorityError> { self.update_access(identity, snapshot, |a| { if !a.reclaiming || a.retiring || a.reclaim_owner != Some(owner) { return Err(AuthorityError::HeadConflict); } a.retiring = true; + a.reclaim_after = reclaim_after; Ok(()) }) .await @@ -195,7 +207,8 @@ impl DatasetAuthority { self.unfence_reclaim(identity, snapshot, owner).await?; return Ok(Some(receipt)); } - self.begin_terminal_retirement(identity, snapshot, owner).await?; + self.begin_terminal_retirement(identity, snapshot, owner, 0) + .await?; } else if let Some(receipt) = self.snapshot_commit(identity, snapshot).await? { return Ok(Some(receipt)); } diff --git a/lib/crowdb-access-dataset/src/authority/access/sweep.rs b/lib/crowdb-access-dataset/src/authority/access/sweep.rs new file mode 100644 index 000000000..f9a85742f --- /dev/null +++ b/lib/crowdb-access-dataset/src/authority/access/sweep.rs @@ -0,0 +1,60 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::decode_access; +use crate::{ + key, AuthorityError, DatasetAuthority, DatasetUploads, MetadataScan, MetadataScanBudget, + ReadCancellation, SnapshotReclaimError, +}; +use std::time::Duration; + +impl DatasetAuthority { + /// Advances one bounded page of durable deletion marks. The cursor is only + /// a scan optimization; restarting from the beginning loses no cleanup work. + /// On an item failure the cursor advances past it, so unrelated deletions + /// can progress; the failed durable mark is retried on the next scan cycle. + /// # Errors + /// Returns storage, corrupt marker, cancellation or cleanup failures. + pub async fn reclaim_due_page( + &self, + uploads: &DatasetUploads, + after: &mut Option>, + cancel: &ReadCancellation, + ) -> Result<(), SnapshotReclaimError> { + let page = self + .store + .scan_page( + &MetadataScan { + prefix: key::LEASE_PREFIX.to_vec(), + after: after.clone(), + budget: MetadataScanBudget { + items: 20, + bytes: 512 * 1024, + timeout: Duration::from_secs(2), + }, + }, + cancel, + ) + .await?; + for (key, value) in &page.entries { + if cancel.is_cancelled() { + return Err(AuthorityError::from(crate::DatasetError::ReadCancelled).into()); + } + *after = Some(key.clone()); + let access = decode_access(&value.bytes)?; + if !access.retiring || access.cleanup_complete || access.reclaim_after > self.reclaim_time() { + continue; + } + let Some((identity, snapshot)) = access.target else { + continue; + }; + if crate::key::read_lease(&identity, snapshot).map_err(AuthorityError::from)? != *key { + return Err(AuthorityError::Corrupt.into()); + } + self.reclaim_snapshot_step(&identity, snapshot, uploads, cancel) + .await?; + } + *after = page.after; + Ok(()) + } +} diff --git a/lib/crowdb-access-dataset/src/composition/edit.rs b/lib/crowdb-access-dataset/src/composition/edit.rs index ea1ab8f76..cc1c2de09 100644 --- a/lib/crowdb-access-dataset/src/composition/edit.rs +++ b/lib/crowdb-access-dataset/src/composition/edit.rs @@ -123,10 +123,6 @@ impl DatasetAuthority { next.edit = None; self.replace_composition(identity, &old, &state, &next).await?; if let Some(preparation) = &next.draft.preparation { - if let Some(split_operation) = preparation.split_operation { - self.release_initial_split_leases(identity, split_operation) - .await?; - } if let Some(receipt) = self .abort_uncommitted_candidate(identity, preparation.candidate) .await? @@ -137,9 +133,6 @@ impl DatasetAuthority { return Err(CompositionError::Frozen); } } - if let Some(parent) = next.draft.record.parent { - self.release_read_handle(identity, parent, operation).await?; - } Ok(next.draft) } } diff --git a/lib/crowdb-access-dataset/src/composition/publication.rs b/lib/crowdb-access-dataset/src/composition/publication.rs index 983afb611..0a632531b 100644 --- a/lib/crowdb-access-dataset/src/composition/publication.rs +++ b/lib/crowdb-access-dataset/src/composition/publication.rs @@ -117,15 +117,6 @@ impl DatasetAuthority { attempt: receipt.attempt.id, }; self.replace_composition(identity, old, state, &next).await?; - if let Some(parent) = next.draft.record.parent { - self.release_read_handle(identity, parent, next.draft.record.operation) - .await?; - } - if let Some(preparation) = &next.draft.preparation { - if let Some(operation) = preparation.split_operation { - self.release_initial_split_leases(identity, operation).await?; - } - } Ok(()) } } diff --git a/lib/crowdb-access-dataset/src/composition/state.rs b/lib/crowdb-access-dataset/src/composition/state.rs index e827739a9..93dfad0df 100644 --- a/lib/crowdb-access-dataset/src/composition/state.rs +++ b/lib/crowdb-access-dataset/src/composition/state.rs @@ -171,8 +171,7 @@ impl DatasetAuthority { ) -> Result<(), CompositionError> { if let Some(parent) = draft.record.parent { self.require_committed_snapshot(identity, parent).await?; - self.touch_read_handle(identity, parent, draft.record.operation, now(), 3600) - .await?; + self.check_snapshot_readable(identity, parent).await?; if let CompositionSource::Split { source, .. } = &draft.record.source { let revision = self .get_split_revision( diff --git a/lib/crowdb-access-dataset/src/inspection/legacy.rs b/lib/crowdb-access-dataset/src/inspection/legacy.rs index 4a38092fb..b98e82638 100644 --- a/lib/crowdb-access-dataset/src/inspection/legacy.rs +++ b/lib/crowdb-access-dataset/src/inspection/legacy.rs @@ -8,13 +8,13 @@ use super::{ InspectionError, InspectionPreparation, InspectionState, PreparationLimits, PreparationState, }; use crate::{ - AuthorityError, CasOutcome, DatasetAuthority, DatasetIdentity, ManifestRecord, OperationId, - PublicationState, ReadCancellation, SnapshotId, SnapshotPublication, SnapshotRecord, + AuthorityError, CasOutcome, DatasetAuthority, DatasetIdentity, ManifestRecord, PublicationState, + ReadCancellation, SnapshotId, SnapshotPublication, SnapshotRecord, }; use bincode::Options; use serde::Deserialize; use sha2::{Digest, Sha256}; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::Duration; #[derive(Deserialize)] struct LegacyRecord { @@ -64,14 +64,9 @@ impl DatasetAuthority { let source = self .load_legacy_inspection(identity, snapshot, limits, cancel) .await?; - let lease = OperationId::random(); - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); tokio::time::timeout( Duration::from_secs(2), - self.touch_read_handle(identity, snapshot, lease, now, 60), + self.check_snapshot_readable(identity, snapshot), ) .await .map_err(|_| InspectionError::Limit)??; @@ -80,13 +75,7 @@ impl DatasetAuthority { self.prepare_inspection_inner(identity, snapshot, limits, cancel, source), ) .await; - let released = tokio::time::timeout( - Duration::from_secs(2), - self.release_read_handle(identity, snapshot, lease), - ) - .await; let progress = result.map_err(|_| InspectionError::Limit)??; - released.map_err(|_| InspectionError::Limit)??; Ok(progress) } diff --git a/lib/crowdb-access-dataset/src/inspection/reclaim.rs b/lib/crowdb-access-dataset/src/inspection/reclaim.rs index 0ad76ad43..7ad258cf0 100644 --- a/lib/crowdb-access-dataset/src/inspection/reclaim.rs +++ b/lib/crowdb-access-dataset/src/inspection/reclaim.rs @@ -101,11 +101,15 @@ impl DatasetAuthority { identity: &DatasetIdentity, snapshot: SnapshotId, ) -> Result<(), AuthorityError> { - match self - .store - .compare_exchange(&retired_key(identity, snapshot)?, None, &[1]) - .await? - { + let key = retired_key(identity, snapshot)?; + if let Some(value) = self.store.get(&key).await? { + return if value.bytes == [1] { + Ok(()) + } else { + Err(AuthorityError::Corrupt) + }; + } + match self.store.compare_exchange(&key, None, &[1]).await? { CasOutcome::Applied(_) => Ok(()), CasOutcome::Conflict(Some(value)) if value.bytes == [1] => Ok(()), CasOutcome::Conflict(_) => Err(AuthorityError::Corrupt), diff --git a/lib/crowdb-access-dataset/src/read_session/planning.rs b/lib/crowdb-access-dataset/src/read_session/planning.rs index 11a8a7571..cf8d396e0 100644 --- a/lib/crowdb-access-dataset/src/read_session/planning.rs +++ b/lib/crowdb-access-dataset/src/read_session/planning.rs @@ -58,9 +58,7 @@ impl DatasetSessions { deadline: now.saturating_add(TTL), closed: false, }; - self.authority - .touch_read_handle(identity, snapshot, spec.id, now, TTL) - .await?; + self.authority.check_snapshot_readable(identity, snapshot).await?; match self .authority .store diff --git a/lib/crowdb-access-dataset/src/read_session/progress.rs b/lib/crowdb-access-dataset/src/read_session/progress.rs index c264ba780..238cf7321 100644 --- a/lib/crowdb-access-dataset/src/read_session/progress.rs +++ b/lib/crowdb-access-dataset/src/read_session/progress.rs @@ -25,7 +25,7 @@ impl DatasetSessions { .require_committed_snapshot(identity, record.snapshot) .await?; self.authority - .touch_read_handle(identity, record.snapshot, id, now, TTL) + .check_snapshot_readable(identity, record.snapshot) .await?; record.deadline = now.saturating_add(TTL); if matches!( @@ -62,7 +62,7 @@ impl DatasetSessions { .clone(); r.deadline = now.saturating_add(TTL); self.authority - .touch_read_handle(identity, r.snapshot, id, now, TTL) + .check_snapshot_readable(identity, r.snapshot) .await?; if matches!( self.authority @@ -108,7 +108,7 @@ impl DatasetSessions { r.cursor = next.into(); r.deadline = now.saturating_add(TTL); self.authority - .touch_read_handle(identity, r.snapshot, id, now, TTL) + .check_snapshot_readable(identity, r.snapshot) .await?; if matches!( self.authority @@ -153,9 +153,6 @@ impl DatasetSessions { .await?, CasOutcome::Applied(_) ) { - self.authority - .release_read_handle(identity, r.snapshot, id) - .await?; return Ok(r.public()); } } diff --git a/lib/crowdb-access-dataset/src/read_session/split/binding.rs b/lib/crowdb-access-dataset/src/read_session/split/binding.rs index 6ba5af9cb..41a5b6418 100644 --- a/lib/crowdb-access-dataset/src/read_session/split/binding.rs +++ b/lib/crowdb-access-dataset/src/read_session/split/binding.rs @@ -14,7 +14,7 @@ impl DatasetSessions { &self, identity: &DatasetIdentity, spec: &SessionSpecV2, - now: u64, + _now: u64, cancel: &ReadCancellation, ) -> Result { self.reserve_session_version(identity, spec.read.id, 2).await?; @@ -62,9 +62,7 @@ impl DatasetSessions { }; self.validate_split_read_schema(identity, snapshot, spec, cancel) .await?; - self.authority - .touch_read_handle(identity, snapshot, spec.read.id, now, super::super::TTL) - .await?; + self.authority.check_snapshot_readable(identity, snapshot).await?; let bound = BoundSplitSpec { version: 2, spec: spec.clone(), @@ -79,11 +77,6 @@ impl DatasetSessions { Ok(()) => Ok(bound), Err(AuthorityError::CursorConflict) => { let winner = self.bound_split_read_spec(identity, spec.read.id, cancel).await?; - if winner.snapshot != snapshot { - self.authority - .release_read_handle(identity, snapshot, spec.read.id) - .await?; - } if winner.spec != *spec { return Err(AuthorityError::CursorConflict); } diff --git a/lib/crowdb-access-dataset/src/read_session/split/planning.rs b/lib/crowdb-access-dataset/src/read_session/split/planning.rs index c64ecc913..97b80802a 100644 --- a/lib/crowdb-access-dataset/src/read_session/split/planning.rs +++ b/lib/crowdb-access-dataset/src/read_session/split/planning.rs @@ -53,18 +53,6 @@ impl DatasetSessions { .prepare_split_read_step(identity, spec, expected_page, now, cancel) .await; if cancel.is_cancelled() { - match self - .bound_split_read_spec(identity, spec.read.id, &ReadCancellation::new()) - .await - { - Ok(bound) => { - self.authority - .release_read_handle(identity, bound.snapshot, spec.read.id) - .await?; - } - Err(AuthorityError::NotFound) => {} - Err(error) => return Err(error), - } return Err(DatasetError::ReadCancelled.into()); } result @@ -91,7 +79,7 @@ impl DatasetSessions { .require_committed_snapshot(identity, bound.snapshot) .await?; self.authority - .touch_read_handle(identity, bound.snapshot, spec.read.id, now, super::super::TTL) + .check_snapshot_readable(identity, bound.snapshot) .await?; match self.record(identity, spec.read.id).await { Ok(_) => return self.prepared_split_read(identity, &bound, &progress, now).await, diff --git a/lib/crowdb-access-dataset/src/split/evaluation.rs b/lib/crowdb-access-dataset/src/split/evaluation.rs index a9e140748..858732281 100644 --- a/lib/crowdb-access-dataset/src/split/evaluation.rs +++ b/lib/crowdb-access-dataset/src/split/evaluation.rs @@ -120,7 +120,7 @@ impl DatasetAuthority { ) { return Err(SplitLifecycleError::Frozen); } - self.renew_split_draft_lease(identity, &state).await?; + self.check_split_draft_sources(identity, &state).await?; let mut progress = state .evaluation .clone() diff --git a/lib/crowdb-access-dataset/src/split/inheritance.rs b/lib/crowdb-access-dataset/src/split/inheritance.rs index faac1f587..90b7026fb 100644 --- a/lib/crowdb-access-dataset/src/split/inheritance.rs +++ b/lib/crowdb-access-dataset/src/split/inheritance.rs @@ -2,8 +2,8 @@ // Licensed under the Apache License, Version 2.0. use super::{ - key, state::DraftState, MembershipOrigin, MembershipPage, SplitDraftRecord, SplitLifecycleError, - SplitName, SplitRevisionRecord, + key, MembershipOrigin, MembershipPage, SplitDraftRecord, SplitLifecycleError, SplitName, + SplitRevisionRecord, }; use crate::{ inspection::work::InspectionWork, DatasetAuthority, DatasetIdentity, MetadataScanBudget, @@ -64,19 +64,7 @@ impl DatasetAuthority { } Ok(()) } - pub(super) async fn release_split_draft_leases( - &self, - identity: &DatasetIdentity, - state: &DraftState, - ) -> Result<(), SplitLifecycleError> { - let mut snapshots: BTreeSet<_> = state.source_leases.iter().copied().collect(); - snapshots.insert(state.record.snapshot); - for snapshot in snapshots { - self.release_read_handle(identity, snapshot, state.record.operation) - .await?; - } - Ok(()) - } + pub(crate) async fn frozen_split_contains( &self, identity: &DatasetIdentity, diff --git a/lib/crowdb-access-dataset/src/split/initial.rs b/lib/crowdb-access-dataset/src/split/initial.rs index e49280494..1974ff7ed 100644 --- a/lib/crowdb-access-dataset/src/split/initial.rs +++ b/lib/crowdb-access-dataset/src/split/initial.rs @@ -9,7 +9,7 @@ use crate::{ composition::DraftProgress, AuthorityError, DatasetAuthority, DatasetIdentity, OperationId, SnapshotId, SplitRevisionId, }; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::Duration; impl DatasetAuthority { pub(crate) async fn fence_initial_split_reservation( @@ -21,7 +21,6 @@ impl DatasetAuthority { let stored = self.split_state_get(&key).await?; if let Some(stored) = &stored { if stored.bytes != super::cleanup::SEALED { - self.release_initial_split_leases(identity, operation).await?; return Ok(true); } } else { @@ -51,17 +50,6 @@ impl DatasetAuthority { crate::CasOutcome::Conflict(_) => Err(SplitLifecycleError::Conflict), } } - pub(crate) async fn release_initial_split_leases( - &self, - identity: &DatasetIdentity, - operation: OperationId, - ) -> Result<(), SplitLifecycleError> { - let (_, state) = self.load_split_draft(identity, operation).await?; - if !state.initial { - return Err(SplitLifecycleError::Conflict); - } - self.release_split_draft_leases(identity, &state).await - } pub(crate) async fn begin_initial_split( &self, @@ -132,7 +120,7 @@ impl DatasetAuthority { if !current.initial { return Err(SplitLifecycleError::Conflict); } - self.renew_split_draft_lease(identity, ¤t).await?; + self.check_split_draft_sources(identity, ¤t).await?; Ok(current.record) } pub(crate) async fn require_initial_split_owner( @@ -151,12 +139,7 @@ impl DatasetAuthority { if let Some(parent) = candidate.parent { self.require_committed_snapshot(identity, parent).await?; } - let time = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - self.touch_read_handle(identity, snapshot, operation, time, 3600) - .await + self.check_snapshot_readable(identity, snapshot).await } pub(crate) async fn validate_initial_split( &self, diff --git a/lib/crowdb-access-dataset/src/split/lifecycle.rs b/lib/crowdb-access-dataset/src/split/lifecycle.rs index f5d8b0e82..9d050aaed 100644 --- a/lib/crowdb-access-dataset/src/split/lifecycle.rs +++ b/lib/crowdb-access-dataset/src/split/lifecycle.rs @@ -96,7 +96,7 @@ impl DatasetAuthority { current.record.progress, DraftProgress::Abandoned | DraftProgress::Published { .. } ) { - self.renew_split_draft_lease(identity, ¤t).await?; + self.check_split_draft_sources(identity, ¤t).await?; } Ok(current.record) } @@ -141,7 +141,7 @@ impl DatasetAuthority { if state.record.revision != edit.expected_revision { return Err(SplitLifecycleError::Conflict); } - self.renew_split_draft_lease(identity, &state).await?; + self.check_split_draft_sources(identity, &state).await?; let mut next = state.clone(); next.record .progress @@ -156,7 +156,7 @@ impl DatasetAuthority { sources.extend(super::inheritance::source_owners(&next.record)); next.source_leases = sources.into_iter().collect(); self.replace_split_draft(identity, &old, &state, &next).await?; - self.renew_split_draft_lease(identity, &next).await?; + self.check_split_draft_sources(identity, &next).await?; Ok(next.record) } @@ -209,7 +209,7 @@ impl DatasetAuthority { next.record.progress = DraftProgress::Abandoned; next.edit = None; self.replace_split_draft(identity, &old, &state, &next).await?; - self.release_split_draft_leases(identity, &next).await?; + Ok(next.record) } } diff --git a/lib/crowdb-access-dataset/src/split/prepare.rs b/lib/crowdb-access-dataset/src/split/prepare.rs index 22f2fab2f..eab0171ae 100644 --- a/lib/crowdb-access-dataset/src/split/prepare.rs +++ b/lib/crowdb-access-dataset/src/split/prepare.rs @@ -19,7 +19,7 @@ impl DatasetAuthority { if state.record.revision != expected_revision { return Err(SplitLifecycleError::Conflict); } - self.renew_split_draft_lease(identity, &state).await?; + self.check_split_draft_sources(identity, &state).await?; if let DraftProgress::Prepared { target: CommitTarget::SplitRevision { snapshot, revision }, } = state.record.progress diff --git a/lib/crowdb-access-dataset/src/split/publication.rs b/lib/crowdb-access-dataset/src/split/publication.rs index 5173c754a..38c29806b 100644 --- a/lib/crowdb-access-dataset/src/split/publication.rs +++ b/lib/crowdb-access-dataset/src/split/publication.rs @@ -63,7 +63,7 @@ impl DatasetAuthority { .await?; return Ok(receipt); } - self.renew_split_draft_lease(identity, &state).await?; + self.check_split_draft_sources(identity, &state).await?; match state.record.progress { DraftProgress::Prepared { .. } => { let mut next = state.clone(); @@ -170,7 +170,7 @@ impl DatasetAuthority { attempt: receipt.attempt.id, }; self.replace_split_draft(identity, old, state, &next).await?; - self.release_split_draft_leases(identity, state).await?; + Ok(()) } } diff --git a/lib/crowdb-access-dataset/src/split/snapshot_cleanup.rs b/lib/crowdb-access-dataset/src/split/snapshot_cleanup.rs index a8cd6f8af..1a92cb124 100644 --- a/lib/crowdb-access-dataset/src/split/snapshot_cleanup.rs +++ b/lib/crowdb-access-dataset/src/split/snapshot_cleanup.rs @@ -57,6 +57,7 @@ impl DatasetAuthority { { return Err(SplitLifecycleError::Frozen); } + self.require_reclaim_ready(identity, snapshot).await?; let mut prefix = b"DS\0SD\0".to_vec(); prefix.extend(identity.key().map_err(AuthorityError::from)?); let mut cursor_key = b"DS\0GSD\0".to_vec(); @@ -109,7 +110,6 @@ impl DatasetAuthority { { return Ok(progress); } - self.release_split_draft_leases(identity, &state).await?; } } progress.after = Some(key.clone()); diff --git a/lib/crowdb-access-dataset/src/split/state.rs b/lib/crowdb-access-dataset/src/split/state.rs index 2aa39b214..573631493 100644 --- a/lib/crowdb-access-dataset/src/split/state.rs +++ b/lib/crowdb-access-dataset/src/split/state.rs @@ -7,7 +7,7 @@ use crate::{ StoredValue, }; use serde::{Deserialize, Serialize}; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::Duration; #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -164,7 +164,7 @@ impl DatasetAuthority { CasOutcome::Conflict(_) => Err(SplitLifecycleError::Conflict), } } - pub(super) async fn renew_split_draft_lease( + pub(super) async fn check_split_draft_sources( &self, identity: &DatasetIdentity, state: &DraftState, @@ -178,11 +178,9 @@ impl DatasetAuthority { self.validate_split_sources(identity, record).await?; } for source in super::inheritance::source_owners(record) { - self.touch_read_handle(identity, source, record.operation, now(), 3600) - .await?; + self.check_snapshot_readable(identity, source).await?; } - self.touch_read_handle(identity, record.snapshot, record.operation, now(), 3600) - .await?; + self.check_snapshot_readable(identity, record.snapshot).await?; Ok(()) } } @@ -200,9 +198,3 @@ pub(super) fn state_key( } Ok(key) } -fn now() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() -} diff --git a/lib/crowdb-access-dataset/src/transport.rs b/lib/crowdb-access-dataset/src/transport.rs index b86d2b1e8..b77722a2e 100644 --- a/lib/crowdb-access-dataset/src/transport.rs +++ b/lib/crowdb-access-dataset/src/transport.rs @@ -103,23 +103,9 @@ impl DatasetReadService { return Err(ReadTransportError::WindowFull); } let permit = WindowPermit(Arc::clone(&self.window)); - let handle = crate::OperationId::random(); self.authority - .touch_read_handle( - &self.identity, - request.snapshot, - handle, - now_seconds, - crate::DEFAULT_READ_LEASE_SECONDS, - ) + .check_snapshot_readable(&self.identity, request.snapshot) .await?; - let mut read = ReadGuard { - authority: self.authority.clone(), - identity: self.identity.clone(), - snapshot: request.snapshot, - handle, - released: false, - }; self.lease.touch(now_seconds); let fields: Vec<&str> = request.fields.iter().map(String::as_str).collect(); let samples = self @@ -132,10 +118,6 @@ impl DatasetReadService { &fields, ) .await; - self.authority - .release_read_handle(&self.identity, request.snapshot, handle) - .await?; - read.released = true; drop(permit); self.lease.touch(now_seconds); Ok(DatasetReadResponse::from_batch(request.snapshot, samples?)) @@ -171,23 +153,9 @@ impl DatasetReadService { return Err(ReadTransportError::WindowFull); } let permit = WindowPermit(Arc::clone(&self.window)); - let handle = crate::OperationId::random(); self.authority - .touch_read_handle( - &self.identity, - request.plan.snapshot, - handle, - now_seconds, - crate::DEFAULT_READ_LEASE_SECONDS, - ) + .check_snapshot_readable(&self.identity, request.plan.snapshot) .await?; - let mut read = ReadGuard { - authority: self.authority.clone(), - identity: self.identity.clone(), - snapshot: request.plan.snapshot, - handle, - released: false, - }; let windows = match request.shuffle.as_ref() { Some(spec) => { let manifest = self @@ -239,10 +207,6 @@ impl DatasetReadService { .read_batch_surface(surface, &self.identity, request.plan.snapshot, &ids, &fields) .await? }; - self.authority - .release_read_handle(&self.identity, request.plan.snapshot, handle) - .await?; - read.released = true; let next_group = group .checked_add(usize::from(!ids.is_empty())) .and_then(|value| u32::try_from(value).ok()) @@ -301,27 +265,3 @@ impl Drop for WindowPermit { self.0.release(); } } - -struct ReadGuard { - authority: Arc, - identity: DatasetIdentity, - snapshot: crate::SnapshotId, - handle: crate::OperationId, - released: bool, -} -impl Drop for ReadGuard { - fn drop(&mut self) { - if self.released { - return; - } - let authority = self.authority.clone(); - let identity = self.identity.clone(); - let snapshot = self.snapshot; - let handle = self.handle; - if let Ok(runtime) = tokio::runtime::Handle::try_current() { - runtime.spawn(async move { - let _ = authority.release_read_handle(&identity, snapshot, handle).await; - }); - } - } -} diff --git a/lib/crowdb-access-dataset/src/upload/reclaim.rs b/lib/crowdb-access-dataset/src/upload/reclaim.rs index fa83e224e..6c0e63905 100644 --- a/lib/crowdb-access-dataset/src/upload/reclaim.rs +++ b/lib/crowdb-access-dataset/src/upload/reclaim.rs @@ -75,6 +75,7 @@ impl DatasetAuthority { .await? .ok_or(AuthorityError::SnapshotNotFound)?; let record = self.begin_reclaim(identity, snapshot).await?; + self.require_reclaim_ready(identity, snapshot).await?; if record.publication.manifest.len() > 8 * 1024 * 1024 { return Err(UploadReclaimError::Limit); } diff --git a/lib/crowdb-access-dataset/tests/access_contention_test.rs b/lib/crowdb-access-dataset/tests/access_contention_test.rs index e40b68ea5..72dcdcf9e 100644 --- a/lib/crowdb-access-dataset/tests/access_contention_test.rs +++ b/lib/crowdb-access-dataset/tests/access_contention_test.rs @@ -6,112 +6,28 @@ mod access_store; #[path = "common/publication.rs"] mod publication; use crowdb_access_dataset::*; -use std::sync::{atomic::Ordering, Arc}; +use std::sync::atomic::Ordering; #[tokio::test] -async fn competing_renewal_does_not_starve_acquisition_or_release_another_reader() { - let identity = DatasetIdentity::new(NamespacePath::root(), "lease-contention").unwrap(); - let inner = access_store::store(); - let setup = DatasetAuthority::new(inner.clone()); - setup.create_dataset(identity.clone()).await.unwrap(); - let snapshot = setup - .publish_snapshot(&identity, None, publication::publication_manifest(1)) - .await - .unwrap(); - let store = Arc::new(access_store::TestAccessStore { - inner, - race: true.into(), - interleave: false, - renewals: 0.into(), - identity: identity.clone(), - snapshot, - other: OperationId::random(), - }); +async fn concurrent_readers_admit_without_any_persistent_mutation() { + let store = access_store::TestAccessStore::shared(); let authority = DatasetAuthority::new(store.clone()); - let reader = OperationId::random(); - authority - .touch_read_handle(&identity, snapshot, reader, u64::MAX / 2, 60) - .await - .unwrap(); - authority - .release_read_handle(&identity, snapshot, reader) - .await - .unwrap(); - assert_eq!(store.renewals.load(Ordering::Relaxed), 2); - store.race.store(false, Ordering::Relaxed); - // Remove head protection so only the competing reader prevents reclamation. - let next = setup - .prepare_snapshot_candidate( - &identity, - None, - publication::publication_manifest(2), - OperationId::random(), - ) - .await - .unwrap(); - setup - .publish_snapshot_candidate( - &identity, - &CommitAttempt { - version: 1, - id: OperationId::random(), - target: CommitTarget::Snapshot { snapshot: next }, - expected_head: setup.inspect_head(&identity).await.unwrap().generation, - }, - ) - .await - .unwrap(); - assert!(matches!( - authority.begin_reclaim(&identity, snapshot).await, - Err(AuthorityError::SnapshotProtected) - )); - authority - .release_read_handle(&identity, snapshot, store.other) - .await - .unwrap(); - authority.begin_reclaim(&identity, snapshot).await.unwrap(); - assert!(matches!( - authority - .touch_read_handle(&identity, snapshot, reader, u64::MAX / 2, 60) - .await, - Err(AuthorityError::SnapshotProtected) - )); -} - -#[tokio::test] -async fn concurrent_readers_acquire_and_release_without_exhausting_cas_budget() { let identity = DatasetIdentity::new(NamespacePath::root(), "concurrent-leases").unwrap(); - let inner = access_store::store(); - let setup = DatasetAuthority::new(inner.clone()); - setup.create_dataset(identity.clone()).await.unwrap(); - let snapshot = setup + authority.create_dataset(identity.clone()).await.unwrap(); + let snapshot = authority .publish_snapshot(&identity, None, publication::publication_manifest(1)) .await .unwrap(); - let store = Arc::new(access_store::TestAccessStore { - inner, - race: false.into(), - interleave: true, - renewals: 0.into(), - identity: identity.clone(), - snapshot, - other: OperationId::random(), - }); - let authority = DatasetAuthority::new(store); - let handles: Vec<_> = (0..96).map(|_| OperationId::random()).collect(); - for release in [false, true] { - let results = futures::future::join_all(handles.iter().map(|handle| async { - if release { - authority.release_read_handle(&identity, snapshot, *handle).await - } else { - authority - .touch_read_handle(&identity, snapshot, *handle, u64::MAX / 2, 60) - .await - } - })) - .await; - for (index, result) in results.into_iter().enumerate() { - result.unwrap_or_else(|error| panic!("reader {index}, release={release}: {error}")); + let writes = store.writes.load(Ordering::Relaxed); + let results = futures::future::join_all((0..96).map(|_| async { + for _ in 0..64 { + authority.check_snapshot_readable(&identity, snapshot).await?; } + Ok::<_, AuthorityError>(()) + })) + .await; + for result in results { + result.unwrap(); } + assert_eq!(store.writes.load(Ordering::Relaxed), writes); } diff --git a/lib/crowdb-access-dataset/tests/common/access_store.rs b/lib/crowdb-access-dataset/tests/common/access_store.rs index 690666522..8c3981074 100644 --- a/lib/crowdb-access-dataset/tests/common/access_store.rs +++ b/lib/crowdb-access-dataset/tests/common/access_store.rs @@ -6,26 +6,28 @@ mod dataset_store; use crowdb_access_dataset::*; use std::sync::{ - atomic::{AtomicBool, AtomicU64, Ordering}, + atomic::{AtomicU64, Ordering}, Arc, }; +#[derive(Default)] pub struct TestAccessStore { - pub inner: Arc, - pub race: AtomicBool, - pub interleave: bool, - pub renewals: AtomicU64, - pub identity: DatasetIdentity, - pub snapshot: SnapshotId, - pub other: OperationId, + inner: dataset_store::TestDatasetStore, + pub writes: AtomicU64, + pub reads: AtomicU64, } - -pub fn store() -> Arc { - Arc::new(dataset_store::TestDatasetStore::default()) +impl TestAccessStore { + pub fn shared() -> Arc { + Arc::new(Self::default()) + } } - #[async_trait::async_trait] impl DatasetStore for TestAccessStore { + async fn get(&self, key: &[u8]) -> Result, StoreError> { + self.reads.fetch_add(1, Ordering::Relaxed); + tokio::task::yield_now().await; + self.inner.get(key).await + } async fn get_bounded( &self, key: &[u8], @@ -34,7 +36,6 @@ impl DatasetStore for TestAccessStore { ) -> Result, StoreError> { self.inner.get_bounded(key, budget, cancel).await } - async fn scan_page( &self, request: &MetadataScan, @@ -42,29 +43,17 @@ impl DatasetStore for TestAccessStore { ) -> Result { self.inner.scan_page(request, cancel).await } - - async fn get(&self, key: &[u8]) -> Result, StoreError> { - let value = self.inner.get(key).await?; - if self.race.load(Ordering::Relaxed) { - // A different reader renews between our point read and conditional write. - let time = u64::MAX / 2 + self.renewals.fetch_add(1, Ordering::Relaxed); - DatasetAuthority::new(self.inner.clone()) - .touch_read_handle(&self.identity, self.snapshot, self.other, time, 3600) - .await - .unwrap(); - } - Ok(value) - } - async fn compare_exchange( &self, key: &[u8], expected: Option<&[u8]>, value: &[u8], ) -> Result { - if self.interleave { - tokio::task::yield_now().await; - } + self.writes.fetch_add(1, Ordering::Relaxed); self.inner.compare_exchange(key, expected, value).await } + async fn delete(&self, key: &[u8]) -> Result<(), StoreError> { + self.writes.fetch_add(1, Ordering::Relaxed); + self.inner.delete(key).await + } } diff --git a/lib/crowdb-access-dataset/tests/common/deletion.rs b/lib/crowdb-access-dataset/tests/common/deletion.rs new file mode 100644 index 000000000..ecf46568c --- /dev/null +++ b/lib/crowdb-access-dataset/tests/common/deletion.rs @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{access_store, publication}; +use crowdb_access_dataset::*; +use std::sync::{atomic::AtomicU64, Arc}; + +pub struct TestDeletionFixture { + pub store: Arc, + pub clock: Arc, + pub authority: DatasetAuthority, + pub identity: DatasetIdentity, + pub snapshot: SnapshotId, + pub next: SnapshotId, + pub held: ManifestRecord, +} +impl TestDeletionFixture { + pub async fn new() -> Self { + let store = access_store::TestAccessStore::shared(); + let clock = Arc::new(AtomicU64::new(1000)); + let authority = DatasetAuthority::new(store.clone()).with_reclaim_clock_for_tests(clock.clone()); + let identity = DatasetIdentity::new(NamespacePath::root(), "grace").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + let snapshot = authority + .publish_snapshot(&identity, None, publication::publication_manifest(1)) + .await + .unwrap(); + // A persisted old reader, even with a distant expiry, cannot hold deletion. + let mut old_key = b"DS\0L\0".to_vec(); + old_key.extend(identity.key().unwrap()); + old_key.extend(snapshot.as_bytes()); + let old_access = bincode::serialize(&( + 3u16, + false, + 0u64, + 0u64, + vec![(OperationId::random(), u64::MAX)], + None::, + false, + )) + .unwrap(); + store.compare_exchange(&old_key, None, &old_access).await.unwrap(); + authority + .check_snapshot_readable(&identity, snapshot) + .await + .unwrap(); + let held = authority.get_manifest(&identity, snapshot).await.unwrap(); + let next = authority + .prepare_snapshot_candidate( + &identity, + None, + publication::publication_manifest(2), + OperationId::random(), + ) + .await + .unwrap(); + authority + .publish_snapshot_candidate( + &identity, + &CommitAttempt { + version: 1, + id: OperationId::random(), + target: CommitTarget::Snapshot { snapshot: next }, + expected_head: authority.inspect_head(&identity).await.unwrap().generation, + }, + ) + .await + .unwrap(); + Self { + store, + clock, + authority, + identity, + snapshot, + next, + held, + } + } +} diff --git a/lib/crowdb-access-dataset/tests/composition_upload_test.rs b/lib/crowdb-access-dataset/tests/composition_upload_test.rs index be3df4a02..f5ef40b6c 100644 --- a/lib/crowdb-access-dataset/tests/composition_upload_test.rs +++ b/lib/crowdb-access-dataset/tests/composition_upload_test.rs @@ -247,7 +247,9 @@ async fn child_candidate_claims_inherited_upload_and_cancellation_preserves_pare #[tokio::test] async fn reclaimed_snapshot_retires_its_upload_owner_after_index_cleanup() { - let f = TestComposition::new().await; + let mut f = TestComposition::new().await; + let clock = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(1000)); + f.authority = f.authority.with_reclaim_clock_for_tests(clock.clone()); let snapshot = f.prepare().await; f.own().await; let attempt = common::attempt(&f.authority, &f.identity, snapshot).await; @@ -282,6 +284,8 @@ async fn reclaimed_snapshot_retires_its_upload_owner_after_index_cleanup() { ) .await .unwrap(); + f.authority.begin_reclaim(&f.identity, snapshot).await.unwrap(); + clock.store(1300, std::sync::atomic::Ordering::Release); f.authority .reclaim_snapshot_metadata(&f.identity, snapshot) .await diff --git a/lib/crowdb-access-dataset/tests/core_test.rs b/lib/crowdb-access-dataset/tests/core_test.rs index 7dd8f9169..cdc4fa4e1 100644 --- a/lib/crowdb-access-dataset/tests/core_test.rs +++ b/lib/crowdb-access-dataset/tests/core_test.rs @@ -500,23 +500,7 @@ async fn authority_persists_snapshot_bound_manifest_partitions_idempotently() { crowdb_access_dataset::ReclaimState::Pending ); authority - .acquire_snapshot_read( - &identity, - snapshot, - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs(), - 60, - ) - .await - .unwrap(); - assert!(matches!( - authority.reclaim_snapshot_metadata(&identity, snapshot).await, - Err(AuthorityError::SnapshotProtected) - )); - authority - .release_snapshot_read(&identity, snapshot) + .check_snapshot_readable(&identity, snapshot) .await .unwrap(); let manifest = ManifestRecord { diff --git a/lib/crowdb-access-dataset/tests/deletion_grace_test.rs b/lib/crowdb-access-dataset/tests/deletion_grace_test.rs new file mode 100644 index 000000000..2a6cebceb --- /dev/null +++ b/lib/crowdb-access-dataset/tests/deletion_grace_test.rs @@ -0,0 +1,154 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/access_store.rs"] +mod access_store; +#[path = "common/deletion.rs"] +mod deletion; +#[path = "common/publication.rs"] +mod publication; +#[path = "common/upload.rs"] +#[allow(dead_code)] +mod upload; +use crowdb_access_dataset::*; +use std::sync::{atomic::Ordering, Arc}; + +#[tokio::test] +async fn deletion_rejects_new_reads_and_keeps_the_grace_deadline_across_restarts() { + let deletion::TestDeletionFixture { + store, + clock, + authority, + identity, + snapshot, + next, + held, + } = deletion::TestDeletionFixture::new().await; + authority.begin_reclaim(&identity, snapshot).await.unwrap(); + assert_eq!( + authority + .snapshot_reclaim_after(&identity, snapshot) + .await + .unwrap(), + 1300 + ); + assert!(matches!( + authority.check_snapshot_readable(&identity, snapshot).await, + Err(AuthorityError::SnapshotNotFound) + )); + assert!(!held.samples.is_empty()); + clock.store(1299, Ordering::Release); + let restarted = DatasetAuthority::new(store.clone()).with_reclaim_clock_for_tests(clock.clone()); + restarted.begin_reclaim(&identity, snapshot).await.unwrap(); + assert_eq!( + restarted + .snapshot_reclaim_after(&identity, snapshot) + .await + .unwrap(), + 1300 + ); + let writes = store.writes.load(Ordering::Relaxed); + let uploads = DatasetUploads::new(store.clone(), Arc::new(upload::TestPayload::default())); + restarted + .reclaim_due_page(&uploads, &mut None, &ReadCancellation::new()) + .await + .unwrap(); + assert!(matches!( + restarted + .reclaim_snapshot_metadata_step(&identity, snapshot, &ReadCancellation::new()) + .await, + Err(AuthorityError::ReclaimPending { reclaim_after: 1300 }) + )); + assert_eq!(store.writes.load(Ordering::Relaxed), writes); + clock.store(1300, Ordering::Release); + loop { + if restarted + .reclaim_snapshot_metadata_step(&identity, snapshot, &ReadCancellation::new()) + .await + .unwrap() + .complete + { + break; + } + } + assert_eq!( + restarted + .reclaim_progress(&identity, snapshot) + .await + .unwrap() + .state, + ReclaimState::Completed + ); + assert!(!held.samples.is_empty()); + assert!(matches!( + restarted.get_snapshot(&identity, snapshot).await, + Err(AuthorityError::SnapshotNotFound) + )); + assert!(restarted.get_snapshot(&identity, next).await.is_ok()); + // Metadata completion alone must not make the background worker skip owner/index cleanup. + for _ in 0..32 { + restarted + .reclaim_due_page(&uploads, &mut None, &ReadCancellation::new()) + .await + .unwrap(); + if restarted + .inspect_snapshot_lifecycle(&identity, snapshot, &ReadCancellation::new()) + .await + .unwrap() + .completed + { + break; + } + } + assert!( + restarted + .inspect_snapshot_lifecycle(&identity, snapshot, &ReadCancellation::new()) + .await + .unwrap() + .completed + ); + let writes = store.writes.load(Ordering::Relaxed); + restarted + .reclaim_due_page(&uploads, &mut None, &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(store.writes.load(Ordering::Relaxed), writes); +} + +#[tokio::test] +async fn corrupt_deletion_marks_do_not_starve_other_snapshots() { + let fixture = deletion::TestDeletionFixture::new().await; + let authority = &fixture.authority; + authority + .begin_reclaim(&fixture.identity, fixture.snapshot) + .await + .unwrap(); + fixture.clock.store(1300, Ordering::Release); + let corrupt_key = b"DS\0L\0".to_vec(); + fixture + .store + .compare_exchange(&corrupt_key, None, &[255]) + .await + .unwrap(); + let uploads = DatasetUploads::new(fixture.store.clone(), Arc::new(upload::TestPayload::default())); + let cancel = ReadCancellation::new(); + let mut after = None; + assert!(authority + .reclaim_due_page(&uploads, &mut after, &cancel) + .await + .is_err()); + assert_eq!(after, Some(corrupt_key)); + for _ in 0..32 { + // A corrupt record is reported again each cycle without starving later records. + let _ = authority.reclaim_due_page(&uploads, &mut after, &cancel).await; + if authority + .inspect_snapshot_lifecycle(&fixture.identity, fixture.snapshot, &cancel) + .await + .unwrap() + .completed + { + return; + } + } + panic!("unrelated corrupt deletion mark prevented cleanup"); +} diff --git a/lib/crowdb-access-dataset/tests/inspection_legacy_test.rs b/lib/crowdb-access-dataset/tests/inspection_legacy_test.rs index 8ae8747f4..932115c8a 100644 --- a/lib/crowdb-access-dataset/tests/inspection_legacy_test.rs +++ b/lib/crowdb-access-dataset/tests/inspection_legacy_test.rs @@ -204,7 +204,7 @@ async fn legacy_limits_cancellation_and_opaque_data_are_explicit() { ); } #[tokio::test] -async fn legacy_work_releases_its_lease_after_limit_failure() { +async fn legacy_work_never_persists_reader_activity_on_limit_failure() { let (store, identity, snapshot, _) = fixture(false).await; let authority = DatasetAuthority::new(store.clone()); let _ = authority @@ -218,7 +218,7 @@ async fn legacy_work_releases_its_lease_after_limit_failure() { &ReadCancellation::new(), ) .await; - // The existing reclaim operation tests the same durable lease authority. + // Preparing inspection work must not register snapshot reader activity. let mut key = b"DS\0L\0".to_vec(); key.extend_from_slice(&identity.key().unwrap()); key.extend_from_slice(snapshot.as_bytes()); @@ -233,10 +233,8 @@ async fn legacy_work_releases_its_lease_after_limit_failure() { &ReadCancellation::new(), ) .await - .unwrap() .unwrap(); - let access: TestEnvelope = bincode::deserialize(&stored.bytes).unwrap(); - assert!(access.payload.leases.is_empty()); + assert!(stored.is_none()); } #[derive(serde::Serialize, serde::Deserialize)] diff --git a/lib/crowdb-access-dataset/tests/reclaim_race_test.rs b/lib/crowdb-access-dataset/tests/reclaim_race_test.rs index 7fa99cdfb..06b66dc46 100644 --- a/lib/crowdb-access-dataset/tests/reclaim_race_test.rs +++ b/lib/crowdb-access-dataset/tests/reclaim_race_test.rs @@ -25,7 +25,7 @@ async fn old_reclaimer_cannot_unfence_a_new_reclaimers_terminal_decision() { store.resume.notify_one(); assert!(matches!(task.await.unwrap(), Err(AuthorityError::HeadConflict))); assert!(authority - .touch_read_handle(&identity, snapshot, OperationId::random(), 1, 3600) + .check_snapshot_readable(&identity, snapshot) .await .is_err()); assert_eq!( diff --git a/lib/crowdb-access-dataset/tests/snapshot_split_cleanup_test.rs b/lib/crowdb-access-dataset/tests/snapshot_split_cleanup_test.rs index 2adc9bcc1..d9babb575 100644 --- a/lib/crowdb-access-dataset/tests/snapshot_split_cleanup_test.rs +++ b/lib/crowdb-access-dataset/tests/snapshot_split_cleanup_test.rs @@ -12,6 +12,8 @@ use crowdb_access_dataset::{split::*, *}; #[tokio::test] async fn retired_owner_recovers_all_draft_work_without_touching_other_snapshots() { let (store, authority, identity, parent) = split_store::fixture().await; + let clock = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(1000)); + let authority = authority.with_reclaim_clock_for_tests(clock.clone()); let snapshot = split_store::publish_binary_samples(&authority, &identity, parent).await; let record = split_store::prepared(&authority, &identity, snapshot, None).await; assert!(matches!( @@ -24,11 +26,7 @@ async fn retired_owner_recovers_all_draft_work_without_touching_other_snapshots( let key = key::membership_page(&identity, snapshot, record.revision, &split.name, 0).unwrap(); let delayed = store.get(&key).await.unwrap().unwrap(); store.delete(&key).await.unwrap(); - // A terminated draft worker leaves only its expiring target lease. - authority - .release_read_handle(&identity, snapshot, record.operation) - .await - .unwrap(); + // A terminated draft worker does not block snapshot deletion. let root = authority .prepare_snapshot_candidate( &identity, @@ -45,7 +43,8 @@ async fn retired_owner_recovers_all_draft_work_without_touching_other_snapshots( let foreign = split_store::prepared(&authority, &identity, root, None).await; split_store::publish_revision(&authority, &identity, &foreign).await; authority.begin_reclaim(&identity, snapshot).await.unwrap(); - let restarted = DatasetAuthority::new(store.clone()); + clock.store(1300, std::sync::atomic::Ordering::Release); + let restarted = DatasetAuthority::new(store.clone()).with_reclaim_clock_for_tests(clock); let mut complete = false; for _ in 0..100 { let progress = restarted diff --git a/lib/crowdb-access-dataset/tests/split_reclaim_test.rs b/lib/crowdb-access-dataset/tests/split_reclaim_test.rs index 9e249ef29..33f3e8fbb 100644 --- a/lib/crowdb-access-dataset/tests/split_reclaim_test.rs +++ b/lib/crowdb-access-dataset/tests/split_reclaim_test.rs @@ -12,6 +12,8 @@ use crowdb_access_dataset::*; #[tokio::test] async fn retired_snapshot_cleans_all_revision_pages_without_touching_other_owners() { let (store, authority, identity, snapshot) = split_store::fixture().await; + let clock = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(1000)); + let authority = authority.with_reclaim_clock_for_tests(clock.clone()); let first = split_store::prepared(&authority, &identity, snapshot, None).await; let first_attempt = split_store::attempt(&authority, &identity, &first).await; authority @@ -42,13 +44,16 @@ async fn retired_snapshot_cleans_all_revision_pages_without_touching_other_owner ) .await .unwrap(); + authority.begin_reclaim(&identity, snapshot).await.unwrap(); + clock.store(1300, std::sync::atomic::Ordering::Release); let cancel = ReadCancellation::new(); let partial = authority .reclaim_snapshot_metadata_step(&identity, snapshot, &cancel) .await .unwrap(); assert!(!partial.complete); - let restarted = DatasetAuthority::new(store.clone()); + clock.store(1300, std::sync::atomic::Ordering::Release); + let restarted = DatasetAuthority::new(store.clone()).with_reclaim_clock_for_tests(clock); let mut completed = false; for _ in 0..100 { if restarted @@ -110,6 +115,8 @@ async fn retired_snapshot_cleans_all_revision_pages_without_touching_other_owner #[tokio::test] async fn cleanup_completion_is_helped_after_final_status_write_failure() { let (store, authority, identity, snapshot) = split_store::fixture().await; + let clock = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(1000)); + let authority = authority.with_reclaim_clock_for_tests(clock.clone()); let root = authority .prepare_snapshot_candidate( &identity, @@ -123,6 +130,8 @@ async fn cleanup_completion_is_helped_after_final_status_write_failure() { .publish_snapshot_candidate(&identity, &common::attempt(&authority, &identity, root).await) .await .unwrap(); + authority.begin_reclaim(&identity, snapshot).await.unwrap(); + clock.store(1300, std::sync::atomic::Ordering::Release); store .reject_cleanup_finish .store(true, std::sync::atomic::Ordering::Relaxed); @@ -146,7 +155,7 @@ async fn cleanup_completion_is_helped_after_final_status_write_failure() { .state, ReclaimState::Pending ); - let restarted = DatasetAuthority::new(store); + let restarted = DatasetAuthority::new(store).with_reclaim_clock_for_tests(clock); assert!( restarted .reclaim_snapshot_metadata_step(&identity, snapshot, &ReadCancellation::new()) diff --git a/lib/crowdb-access-dataset/tests/split_session_edges_test.rs b/lib/crowdb-access-dataset/tests/split_session_edges_test.rs index 57f3f9c68..15f7a5077 100644 --- a/lib/crowdb-access-dataset/tests/split_session_edges_test.rs +++ b/lib/crowdb-access-dataset/tests/split_session_edges_test.rs @@ -66,7 +66,7 @@ async fn v2_session_accepts_empty_inline_equality_without_changing_legacy_plans( } #[tokio::test] -async fn interrupted_preparation_releases_its_lease_without_losing_durable_progress() { +async fn interrupted_preparation_preserves_progress_without_blocking_deletion() { let (store, authority, identity, parent) = split_store::fixture().await; let snapshot = split_store::publish_binary_samples(&authority, &identity, parent).await; let authority = Arc::new(authority); @@ -115,10 +115,7 @@ async fn interrupted_preparation_releases_its_lease_without_losing_durable_progr .publish_snapshot_candidate(&identity, &attempt) .await .unwrap(); - assert!(matches!( - authority.begin_reclaim(&identity, snapshot).await, - Err(AuthorityError::SnapshotProtected) - )); + authority.begin_reclaim(&identity, snapshot).await.unwrap(); cancel.cancel(); assert!(sessions .prepare_split_read(&identity, &spec, resumed.next_page, now, &cancel) diff --git a/lib/crowdb-sdk-native/README.md b/lib/crowdb-sdk-native/README.md index 57641afdb..5b6d61535 100644 --- a/lib/crowdb-sdk-native/README.md +++ b/lib/crowdb-sdk-native/README.md @@ -106,13 +106,13 @@ async fn read_object(seeds: Vec) -> Result<(), Box for Error { AuthorityError::NotFound | AuthorityError::SnapshotNotFound => FailureKind::NotFound, AuthorityError::AncestryCycle | AuthorityError::Corrupt => FailureKind::Integrity, AuthorityError::Store(_) => FailureKind::Storage, + AuthorityError::ReclaimPending { .. } => FailureKind::Pending, _ => FailureKind::Conflict, }; error(kind, value) diff --git a/lib/crowdb-sdk-native/src/dataset/read.rs b/lib/crowdb-sdk-native/src/dataset/read.rs index 3fd18ca53..8f698c5d8 100644 --- a/lib/crowdb-sdk-native/src/dataset/read.rs +++ b/lib/crowdb-sdk-native/src/dataset/read.rs @@ -1,9 +1,9 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +mod admission; mod body; mod field; -mod protection; pub use body::{BatchBody, FieldBuffer}; use super::{now, DatasetClient}; @@ -90,9 +90,7 @@ impl DatasetClient { end: bool, session: Option, ) -> Result { - let request = Arc::new(request); - let protection = - protection::ReadProtection::acquire(self, Arc::clone(&request), batch.snapshot, session).await?; + let admission = admission::ReadAdmission::admit(self, &request, batch.snapshot, session).await?; let samples = if batch.sample_ids.is_empty() { Vec::new() } else { @@ -125,7 +123,7 @@ impl DatasetClient { request, Arc::clone(&self.runtime.resources), Arc::clone(&self.uploads), - protection, + admission, ); Ok(ReadBatch { header, body }) } diff --git a/lib/crowdb-sdk-native/src/dataset/read/admission.rs b/lib/crowdb-sdk-native/src/dataset/read/admission.rs new file mode 100644 index 000000000..e41356f76 --- /dev/null +++ b/lib/crowdb-sdk-native/src/dataset/read/admission.rs @@ -0,0 +1,43 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::super::DatasetClient; +use crate::{runtime::RequestLease, Result}; +use crowdb_access_dataset::{DatasetAuthority, DatasetIdentity, DatasetSessions, OperationId, SnapshotId}; +use std::sync::Arc; + +pub(super) struct ReadAdmission { + authority: Arc, + identity: DatasetIdentity, + session: Option, +} +impl ReadAdmission { + pub(super) async fn admit( + client: &DatasetClient, + request: &RequestLease, + snapshot: SnapshotId, + session: Option, + ) -> Result { + request + .run(async { + Ok(client + .authority + .check_snapshot_readable(&client.identity, snapshot) + .await?) + }) + .await?; + Ok(Self { + authority: Arc::clone(&client.authority), + identity: client.identity.clone(), + session, + }) + } + pub(super) async fn check(&self) -> Result<()> { + if let Some(session) = self.session { + DatasetSessions::new(Arc::clone(&self.authority)) + .check_open(&self.identity, session) + .await?; + } + Ok(()) + } +} diff --git a/lib/crowdb-sdk-native/src/dataset/read/body.rs b/lib/crowdb-sdk-native/src/dataset/read/body.rs index 460f39db2..b8648648f 100644 --- a/lib/crowdb-sdk-native/src/dataset/read/body.rs +++ b/lib/crowdb-sdk-native/src/dataset/read/body.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{field::FieldReader, protection::ReadProtection, BatchHeader}; +use super::{admission::ReadAdmission, field::FieldReader, BatchHeader}; use crate::{buffer::BufferLease, runtime::RequestLease, Error, Result, RuntimeResources}; use crowdb_access_dataset::{DatasetUploads, FieldLocator}; use crowdb_rpc_ffi::OwnedPayload; @@ -19,12 +19,11 @@ pub struct BatchBody { locators: VecDeque<(usize, FieldLocator)>, field: Option, pending: Option, - request: Option>, + request: Option, resources: Arc, uploads: Arc, - protection: Option, + admission: Option, sequence: u64, - finishing: bool, complete: bool, failure: Option, } @@ -33,10 +32,10 @@ impl BatchBody { pub(super) fn new( header: Arc, locators: Vec<(usize, FieldLocator)>, - request: Arc, + request: RequestLease, resources: Arc, uploads: Arc, - protection: ReadProtection, + admission: ReadAdmission, ) -> Self { Self { header, @@ -46,9 +45,8 @@ impl BatchBody { request: Some(request), resources, uploads, - protection: Some(protection), + admission: Some(admission), sequence: 0, - finishing: false, complete: false, failure: None, } @@ -64,7 +62,7 @@ impl BatchBody { } /// Successful EOF certifies all field and part digests, never processing ack. - /// It waits for bounded read-protection cleanup before returning admission. + /// EOF releases local request admission without metadata writes. /// # Errors /// Returns sticky integrity/transport/cancellation failures or expired admission. pub async fn next(&mut self) -> Result> { @@ -100,22 +98,18 @@ impl BatchBody { self.field = None; self.pending = None; self.locators.clear(); - self.protection = None; + self.admission = None; self.request = None; } async fn pull(&mut self) -> Result> { - if self.finishing { - return self.finish().await; - } let request = self.request.as_ref().ok_or(Error::Cancelled)?; - let protection = self.protection.as_ref().ok_or(Error::Cancelled)?; - request.run(protection.check()).await?; + let admission = self.admission.as_ref().ok_or(Error::Cancelled)?; + request.run(admission.check()).await?; loop { if self.field.is_none() { let Some((index, locator)) = self.locators.pop_front() else { - self.finishing = true; - return self.finish().await; + return Ok(None); }; self.field = Some(FieldReader::new(index, locator)?); } @@ -144,11 +138,4 @@ impl BatchBody { })); } } - - async fn finish(&mut self) -> Result> { - let request = self.request.as_ref().ok_or(Error::Cancelled)?; - let protection = self.protection.as_mut().ok_or(Error::Cancelled)?; - request.run(protection.finish()).await?; - Ok(None) - } } diff --git a/lib/crowdb-sdk-native/src/dataset/read/protection.rs b/lib/crowdb-sdk-native/src/dataset/read/protection.rs deleted file mode 100644 index d15892658..000000000 --- a/lib/crowdb-sdk-native/src/dataset/read/protection.rs +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::super::{now, DatasetClient}; -use crate::{runtime::RequestLease, Cancellation, Error, Result}; -use crowdb_access_dataset::{DatasetAuthority, DatasetIdentity, DatasetSessions, OperationId, SnapshotId}; -use std::{ - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, - time::Duration, -}; - -struct Protection { - authority: Arc, - identity: DatasetIdentity, - snapshot: SnapshotId, - handle: OperationId, - session: Option, - failed: AtomicBool, -} - -pub(super) struct ReadProtection { - state: Arc, - request: Option>, - stop: Cancellation, - worker: tokio::task::JoinHandle<()>, - cleanup: Option>, -} - -impl ReadProtection { - pub(super) async fn acquire( - client: &DatasetClient, - request: Arc, - snapshot: SnapshotId, - session: Option, - ) -> Result { - let state = Arc::new(Protection { - authority: Arc::clone(&client.authority), - identity: client.identity.clone(), - snapshot, - handle: OperationId::random(), - session, - failed: AtomicBool::new(false), - }); - request.run(state.renew()).await?; - let stop = Cancellation::default(); - let closing = client.runtime.resources.closing.clone(); - let worker_stop = stop.clone(); - let worker_state = Arc::clone(&state); - let context = request.context().clone(); - let worker = tokio::spawn(async move { - let renewal = async { - loop { - tokio::time::sleep(Duration::from_secs(15)).await; - worker_state.renew().await?; - } - }; - let result = tokio::select! { - () = worker_stop.cancelled() => Ok(()), - () = closing.cancelled() => Err(Error::Shutdown), - result = context.run(renewal) => result, - }; - if result.is_err() { - worker_state.failed.store(true, Ordering::Release); - } - }); - Ok(Self { - state, - request: Some(request), - stop, - worker, - cleanup: None, - }) - } - - pub(super) async fn check(&self) -> Result<()> { - if self.state.failed.load(Ordering::Acquire) { - return Err(Error::Cancelled); - } - if let Some(session) = self.state.session { - DatasetSessions::new(Arc::clone(&self.state.authority)) - .check_open(&self.state.identity, session) - .await?; - } - Ok(()) - } - - pub(super) async fn finish(&mut self) -> Result<()> { - // Keep the task across dropped EOF waits; cancellation detaches the same - // bounded cleanup while its request credit still covers the metadata I/O. - self.start_cleanup(); - if let Some(cleanup) = &mut self.cleanup { - cleanup - .await - .map_err(|error| Error::Storage(format!("read cleanup task failed: {error}")))?; - } - Ok(()) - } - - fn start_cleanup(&mut self) { - let Some(request) = self.request.take() else { - return; - }; - self.stop.cancel(); - self.worker.abort(); - let state = Arc::clone(&self.state); - if let Ok(runtime) = tokio::runtime::Handle::try_current() { - self.cleanup = Some(runtime.spawn(async move { - let release = - state - .authority - .release_read_handle(&state.identity, state.snapshot, state.handle); - if !matches!( - tokio::time::timeout(Duration::from_secs(5), release).await, - Ok(Ok(())) - ) { - tracing::error!( - "Dataset read release failed; inactivity expiry retains cleanup responsibility" - ); - } - drop(request); - })); - } - } -} - -impl Protection { - async fn renew(&self) -> Result<()> { - if let Some(session) = self.session { - DatasetSessions::new(Arc::clone(&self.authority)) - .resume(&self.identity, session, now()) - .await?; - } - self.authority - .touch_read_handle(&self.identity, self.snapshot, self.handle, now(), 60) - .await?; - Ok(()) - } -} - -impl Drop for ReadProtection { - fn drop(&mut self) { - self.start_cleanup(); - } -} diff --git a/lib/crowdb-sdk-native/src/runtime/resources.rs b/lib/crowdb-sdk-native/src/runtime/resources.rs index d79c6820d..ce3995435 100644 --- a/lib/crowdb-sdk-native/src/runtime/resources.rs +++ b/lib/crowdb-sdk-native/src/runtime/resources.rs @@ -118,7 +118,7 @@ impl RuntimeResources { /// Request admission lasts through a stream's terminal result or cancellation. /// Its release is independent of any buffer leases delivered to the caller. -/// Dataset cancellation retains admission until its bounded protection cleanup ends. +/// Dataset EOF/cancellation releases this credit without a remote reader cleanup. pub struct RequestLease { resources: Arc, context: OperationContext, diff --git a/lib/crowdb-sdk-native/tests/common/dataset.rs b/lib/crowdb-sdk-native/tests/common/dataset.rs index ac51e3022..617c45664 100644 --- a/lib/crowdb-sdk-native/tests/common/dataset.rs +++ b/lib/crowdb-sdk-native/tests/common/dataset.rs @@ -61,7 +61,7 @@ impl TestFixture { } } -fn manifest() -> Vec { +pub fn manifest() -> Vec { let manifest = ManifestRecord { version: 1, snapshot: SnapshotId::random(), diff --git a/lib/crowdb-sdk-native/tests/common/read_cleanup.rs b/lib/crowdb-sdk-native/tests/common/read_cleanup.rs index 0cf63927d..6edb47fb8 100644 --- a/lib/crowdb-sdk-native/tests/common/read_cleanup.rs +++ b/lib/crowdb-sdk-native/tests/common/read_cleanup.rs @@ -28,6 +28,7 @@ impl TestCleanupFixture { metadata, ) .await; + gate.hold.store(true, Ordering::Release); let mut read = dataset .client .read_batch( @@ -41,7 +42,6 @@ impl TestCleanupFixture { .await .unwrap(); let held = read.body.next().await.unwrap().unwrap().buffer; - gate.hold.store(true, Ordering::Release); Self { dataset, gate, diff --git a/lib/crowdb-sdk-native/tests/dataset_cleanup_test.rs b/lib/crowdb-sdk-native/tests/dataset_cleanup_test.rs index 7ef7f569f..e0fff73b6 100644 --- a/lib/crowdb-sdk-native/tests/dataset_cleanup_test.rs +++ b/lib/crowdb-sdk-native/tests/dataset_cleanup_test.rs @@ -10,71 +10,51 @@ mod read_cleanup; #[path = "common/storage.rs"] mod storage; -use crowdb_sdk_native::{CapacityKind, Error}; +use crowdb_sdk_native::Error; use read_cleanup::TestCleanupFixture; use std::{sync::atomic::Ordering, time::Duration}; #[tokio::test] -async fn eof_keeps_admission_until_cleanup_and_dropped_wait_does_not_repeat_release() { - let mut fixture = TestCleanupFixture::new().await; - assert!( - tokio::time::timeout(Duration::from_millis(10), fixture.body.next()) - .await - .is_err() - ); - fixture.gate.entered.acquire().await.unwrap().forget(); - assert!(!fixture.body.is_complete()); +async fn reading_and_eof_need_no_metadata_mutations_or_cleanup_wait() { + let mut fixture = tokio::time::timeout(Duration::from_secs(2), TestCleanupFixture::new()) + .await + .unwrap(); assert_eq!(fixture.dataset.runtime.resources().status().requests, 1); - assert!(matches!( - fixture.dataset.runtime.resources().admit(dataset::context(3)), - Err(Error::Capacity(CapacityKind::Requests)) - )); - fixture.gate.permits.add_permits(1); - assert!(fixture.body.next().await.unwrap().is_none()); - assert!(fixture.body.next().await.unwrap().is_none()); + assert!(tokio::time::timeout(Duration::from_secs(1), fixture.body.next()) + .await + .unwrap() + .unwrap() + .is_none()); assert!(fixture.body.is_complete()); - assert_eq!(fixture.gate.calls.load(Ordering::Relaxed), 1); + assert_eq!(fixture.gate.calls.load(Ordering::Relaxed), 0); assert_eq!(fixture.dataset.runtime.resources().status().requests, 0); assert_eq!(fixture.held.as_slice(), &[0; 16]); } #[tokio::test] -async fn cancelled_or_dropped_reads_keep_cleanup_charged_and_delivered_bytes_valid() { +async fn cancelled_or_dropped_reads_release_admission_without_writes_and_keep_owned_bytes() { for cancel in [false, true] { - let mut fixture = TestCleanupFixture::new().await; + let mut fixture = tokio::time::timeout(Duration::from_secs(2), TestCleanupFixture::new()) + .await + .unwrap(); if cancel { fixture.body.cancel(); assert!(matches!(fixture.body.next().await, Err(Error::Cancelled))); } drop(fixture.body); - fixture.gate.entered.acquire().await.unwrap().forget(); - assert_eq!(fixture.dataset.runtime.resources().status().requests, 1); - assert!(matches!( - fixture.dataset.runtime.resources().admit(dataset::context(3)), - Err(Error::Capacity(CapacityKind::Requests)) - )); - fixture.gate.permits.add_permits(1); - tokio::time::timeout(Duration::from_secs(1), async { - while fixture.dataset.runtime.resources().status().requests != 0 { - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - assert_eq!(fixture.gate.calls.load(Ordering::Relaxed), 1); + assert_eq!(fixture.dataset.runtime.resources().status().requests, 0); + assert_eq!(fixture.gate.calls.load(Ordering::Relaxed), 0); assert_eq!(fixture.held.as_slice(), &[0; 16]); } } #[tokio::test(start_paused = true)] -async fn stalled_cleanup_releases_admission_at_the_existing_bound() { +async fn idle_read_has_no_background_lease_renewals() { let fixture = TestCleanupFixture::new().await; - drop(fixture.body); - fixture.gate.entered.acquire().await.unwrap().forget(); - assert_eq!(fixture.dataset.runtime.resources().status().requests, 1); - tokio::time::advance(Duration::from_secs(5)).await; + tokio::time::advance(Duration::from_secs(61)).await; tokio::task::yield_now().await; + assert_eq!(fixture.gate.calls.load(Ordering::Relaxed), 0); + drop(fixture.body); assert_eq!(fixture.dataset.runtime.resources().status().requests, 0); - assert_eq!(fixture.gate.calls.load(Ordering::Relaxed), 1); assert_eq!(fixture.held.as_slice(), &[0; 16]); } diff --git a/lib/crowdb-sdk-native/tests/dataset_read_test.rs b/lib/crowdb-sdk-native/tests/dataset_read_test.rs index 7fdc8913e..3fb90ac00 100644 --- a/lib/crowdb-sdk-native/tests/dataset_read_test.rs +++ b/lib/crowdb-sdk-native/tests/dataset_read_test.rs @@ -36,6 +36,56 @@ fn spec() -> SessionSpecV2 { } } +#[tokio::test] +async fn deletion_rejects_new_requests_without_interrupting_an_admitted_body() { + use crowdb_access_dataset::{CommitAttempt, CommitTarget}; + let fixture = TestFixture::new(1024).await; + let batch = || BatchRequest { + snapshot: fixture.snapshot, + sample_ids: vec![vec![1]], + fields: vec!["value".into()], + }; + let mut old = fixture.client.read_batch(context(2), batch()).await.unwrap(); + fixture + .client + .control(context(3), |scope| async move { + let next = scope + .authority + .prepare_snapshot_candidate(&scope.identity, None, dataset::manifest(), OperationId::random()) + .await?; + scope + .authority + .publish_snapshot_candidate( + &scope.identity, + &CommitAttempt { + version: 1, + id: OperationId::random(), + target: CommitTarget::Snapshot { snapshot: next }, + expected_head: scope.authority.inspect_head(&scope.identity).await?.generation, + }, + ) + .await?; + scope + .authority + .begin_reclaim(&scope.identity, fixture.snapshot) + .await?; + Ok(()) + }) + .await + .unwrap(); + assert!(matches!( + fixture.client.read_batch(context(4), batch()).await, + Err(Error::Dataset { + kind: crowdb_sdk_native::dataset::FailureKind::NotFound, + .. + }) + )); + let held = old.body.next().await.unwrap().unwrap().buffer; + assert_eq!(held.as_slice(), &[0; 16]); + assert!(old.body.next().await.unwrap().is_none()); + assert_eq!(held.as_slice(), &[0; 16]); +} + #[tokio::test] async fn fetching_and_releasing_owners_never_acknowledge_progress() { let fixture = TestFixture::new(1024).await; diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2/inspection.rs b/sdk/crates/crowdb-sdk-dataset/src/v2/inspection.rs index af3e90a68..8f73b2197 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/v2/inspection.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/v2/inspection.rs @@ -96,6 +96,7 @@ pub struct InspectionPreparation { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ReclaimPhase { + GracePeriod, UploadOwners, SplitDrafts, IndexFences, @@ -108,6 +109,8 @@ pub struct SnapshotReclaim { pub phase: ReclaimPhase, pub complete: bool, pub physical_cleanup_deferred: bool, + #[serde(default)] + pub reclaim_after: Option, } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct RetainedSnapshot { @@ -127,6 +130,8 @@ pub struct SnapshotLifecycle { pub stable: bool, pub retained: bool, pub active_read_lease: bool, + #[serde(default)] + pub reclaim_after: Option, pub state: LifecycleState, pub physical_cleanup_deferred: bool, pub parent_protection: String, diff --git a/sdk/crates/crowdb-sdk-dataset/tests/v2_contract_test.rs b/sdk/crates/crowdb-sdk-dataset/tests/v2_contract_test.rs index 9c32b71c3..d8d96f41f 100644 --- a/sdk/crates/crowdb-sdk-dataset/tests/v2_contract_test.rs +++ b/sdk/crates/crowdb-sdk-dataset/tests/v2_contract_test.rs @@ -9,6 +9,23 @@ use serde_json::json; #[path = "../../crowdb-sdk-core/tests/common/peer.rs"] mod peer; +#[test] +fn reclaim_accepts_grace_deadlines_and_older_responses() { + use crowdb_sdk_dataset::v2::{ReclaimPhase, SnapshotReclaim}; + let mut value = json!({ + "snapshot": "01".repeat(16), "phase": "grace_period", "complete": false, + "physical_cleanup_deferred": true, "reclaim_after": 1300 + }); + let status: SnapshotReclaim = serde_json::from_value(value.clone()).unwrap(); + assert!(matches!(status.phase, ReclaimPhase::GracePeriod)); + assert_eq!(status.reclaim_after, Some(1300)); + value["phase"] = json!("upload_owners"); + value.as_object_mut().unwrap().remove("reclaim_after"); + let status: SnapshotReclaim = serde_json::from_value(value).unwrap(); + assert!(matches!(status.phase, ReclaimPhase::UploadOwners)); + assert_eq!(status.reclaim_after, None); +} + #[test] fn v2_stream_accepts_long_binary_ids_without_changing_v1_bounds() { use bytes::BytesMut; From f830039c2ceeb5fbaccb910511ff2bf6a55bae6f Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 07:56:01 +0800 Subject: [PATCH 090/258] Remove redundant access metadata coordination --- app/crowdb-access-server/src/dataset.rs | 3 +- .../src/dataset/api/publication.rs | 5 - .../dataset/design-crowdb-access-dataset.md | 39 ++- .../dataset/design-dataset-control.md | 8 +- .../dataset/openapi-crowdb-dataset-v2.yaml | 4 +- .../dataset/spec-crowdb-dataset.md | 5 +- .../design-crowdb-access-server.md | 19 ++ .../iceberge/design-crowdb-iceberg.md | 7 +- .../s3/design-crowdb-access-s3.md | 7 + doc/working/plan-access-simplification.md | 110 ++++---- .../src/composition/begin.rs | 8 +- .../src/composition/edit.rs | 6 +- .../src/composition/evaluation.rs | 2 +- .../src/composition/prepare.rs | 2 +- .../src/composition/publication.rs | 2 +- .../src/composition/record.rs | 1 + .../src/composition/state.rs | 16 +- lib/crowdb-access-dataset/src/read_session.rs | 10 +- .../src/read_session/planning.rs | 3 +- .../src/read_session/progress.rs | 109 +++++--- .../src/read_session/split/finish.rs | 6 +- .../src/read_session/split/state.rs | 15 ++ lib/crowdb-access-dataset/src/split/state.rs | 14 +- lib/crowdb-access-dataset/src/upload.rs | 20 +- .../src/upload/expiry.rs | 81 ++++++ .../src/upload/ownership.rs | 48 ++-- lib/crowdb-access-dataset/src/upload/parts.rs | 3 - .../src/upload/registry.rs | 255 ------------------ .../src/upload/retirement.rs | 26 +- .../tests/common/access_store.rs | 16 ++ .../tests/draft_cost_test.rs | 66 +++++ .../tests/session_cost_test.rs | 64 +++++ .../tests/upload_registry_test.rs | 157 ++++++----- .../tests/upload_test.rs | 9 +- .../src/metadata/multipart_repository.rs | 10 +- .../metadata/multipart_repository/parts.rs | 27 +- lib/crowdb-access-s3/src/metadata/store.rs | 16 +- 37 files changed, 655 insertions(+), 544 deletions(-) create mode 100644 lib/crowdb-access-dataset/src/upload/expiry.rs delete mode 100644 lib/crowdb-access-dataset/src/upload/registry.rs create mode 100644 lib/crowdb-access-dataset/tests/draft_cost_test.rs create mode 100644 lib/crowdb-access-dataset/tests/session_cost_test.rs diff --git a/app/crowdb-access-server/src/dataset.rs b/app/crowdb-access-server/src/dataset.rs index 7be7b3d7c..7e9a8de45 100644 --- a/app/crowdb-access-server/src/dataset.rs +++ b/app/crowdb-access-server/src/dataset.rs @@ -231,6 +231,7 @@ fn start_cleanup( tokio::spawn(async move { let mut tick = tokio::time::interval(Duration::from_secs(15)); let mut after = None; + let mut upload_after = None; loop { tick.tick().await; if let Err(error) = authority @@ -243,7 +244,7 @@ fn start_cleanup( { tracing::error!(%error, "Dataset snapshot cleanup failed; durable deletion marks retained for retry"); } - if let Err(error) = uploads.expire(unix_seconds()).await { + if let Err(error) = uploads.expire(unix_seconds(), &mut upload_after).await { tracing::error!(%error, "Dataset upload cleanup failed; retaining cleanup records for retry"); } } diff --git a/app/crowdb-access-server/src/dataset/api/publication.rs b/app/crowdb-access-server/src/dataset/api/publication.rs index c13076ea8..12f488bca 100644 --- a/app/crowdb-access-server/src/dataset/api/publication.rs +++ b/app/crowdb-access-server/src/dataset/api/publication.rs @@ -57,11 +57,6 @@ impl DatasetHttpService { .await }; let _snapshot = result?; - if let Some(uploads) = &self.uploads { - for location in &locations { - uploads.untrack(location).await?; - } - } json(&conversion::snapshot( authority.publication_status(identity, operation).await?, authority.latest(identity).await?, diff --git a/doc/design/access-server/dataset/design-crowdb-access-dataset.md b/doc/design/access-server/dataset/design-crowdb-access-dataset.md index 78ad31957..fb2ae1c55 100644 --- a/doc/design/access-server/dataset/design-crowdb-access-dataset.md +++ b/doc/design/access-server/dataset/design-crowdb-access-dataset.md @@ -227,6 +227,17 @@ pass integrity checks. Readable but unconfirmed bytes remain pending. A fenced abandoned allocation is reclaimed before replacing it under the same part ID. Completion verifies every part and the whole declared MD5 with bounded buffers. +Expiry scans bounded pages of upload authorities directly. Each caller retains +its scan cursor locally; restart repeats the scan and loses no cleanup obligation. +Independent upload creation writes only its own authority. No shared registry tail, +registration generation, registry fence or durable scan cursor participates in +creation, part upload or owner release. Corrupt/failed entries are reported and +retried next cycle without starving subsequent records. Legacy registry pages are +ignored. Claimed payloads and terminal owner revocation remain authoritative; +removing an owner cannot release another owner's bytes. Accepted-part index pages +remain bounded to 128 entries to avoid a point read for every part during listing +and completion. + Prepare validates the complete current sample membership, schema, parent and operation token, then persists an invisible candidate and claims completed uploads. Omitted samples disappear; surviving samples inherit omitted optional @@ -250,12 +261,16 @@ partition execute before fetching payloads. Projection walks explicit ancestry only until all requested fields are resolved. Plans admit at most 100000 IDs; metadata is capped at 8 MiB and selection pages contain at most 1024 IDs. -Fetching issues a plan-bound opaque next cursor without advancing acknowledged -progress. Repeated fetches replay that batch. Acknowledgement CAS-advances only -its issued cursor and is idempotent; progress inspection performs no payload -read. Resume restores acknowledged progress. Explicit close affects only that -handle; a disconnected handle expires after 60 seconds and can resume while its -Snapshot remains available. +Creation and acknowledgement persist the next plan-bound opaque token together +with progress. Fetch reads that token and the immutable ID pages without any +mutation or activity renewal. Repeated fetches replay the same batch. Acknowledgement +CAS-advances only the prepared token and is idempotent; progress inspection performs +no payload read. Creation, acknowledgement and explicit resume set a 60-second +admission deadline; an expired handle resumes at acknowledged progress while its +Snapshot remains available. Explicit close affects only that handle. Stream pulls +check its closed state to preserve remote cancellation; they do not update it. +CAS conflicts reuse the returned record, while identical resume/close operations +need no mutation. Legacy sessions without a prepared token acquire one once. The public binary response streams a bounded JSON header, integrity-checked payload frames, field terminators and a batch terminator. Success also requires @@ -362,12 +377,12 @@ Head envelopes decode legacy versions and use version 3 for commit-attempt markers. Access envelopes decode versions 1 through 4 and write version 4 with reclaim ownership, terminal deletion, grace deadline, cleanup target and completion. Legacy reader activity is ignored; all authority writers must use this contract. -Upload sessions decode version 2; permanent owner retirement upgrades a session to version 3 so an older writer -cannot ignore revocation. This metadata upgrade is one-way: -do not downgrade an upgraded authority or run old and new authority writers -against the same Dataset store. Immutable Dataset/Snapshot/Manifest encodings -remain compatible. Upload registry pages also decode the initial array form and -write generation-tagged entries. Private read-session progress is version 2; +Upload sessions decode versions 2 through 4 and write version 4 without registry +fields. Permanent retired-owner identities still reject delayed claims. This +metadata upgrade is one-way: do not downgrade an upgraded authority or mix old +and new authority writers against the same Dataset store. Immutable Dataset/Snapshot/Manifest encodings +remain compatible. Private read-session progress decodes version 2 and writes +version 3 with a prepared next token; public API versions do not expose these private persistence formats. Hot-path synchronization uses atomics, immutable owned views, CAS and bounded diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index f16b0ea6a..22e0fc7ae 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -127,13 +127,19 @@ late write keys, including currently absent keys, rather than deleting a value a allowing a stale CAS(None) writer to recreate it. Edit IDs are registered before payload writes so failed admissions can also be retired. Small receipt/fence records remain after bulk cleanup to preserve replay and non-resurrection guarantees. +Helping an already stored edit receipt only reads and validates it; retries do not +submit a doomed CAS for the same bytes. Evaluation journals and cleanup cursors +are scoped to one draft, not to unrelated readers. They retain exact partial-work +recovery and prevent a delayed writer from reviving cleaned candidate data. +Sources use read-only availability checks. The compatibility +`source_lease_deadline` field is zero for new drafts and edits. Snapshot reclaim checks latest/stable, retained Snapshots, live children, candidate ownership under a terminal owner fence. A durable five-minute grace deadline separates deletion admission from cleanup. After that deadline it retires upload owners, Snapshot-scoped Split drafts, possible index writes and ordinary metadata in phases. Completion reports logical retirement; physical upload deletion remains deferred -to its registry cleanup. There is no independent committed Split revision pruning. +to the bounded upload-authority scan. There is no independent committed Split revision pruning. `GET snapshots/{snapshot}/reclaim` is bounded, read-only observation. It returns publication operation identity where available, latest/stable/retained flags, diff --git a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml index 6e692a9da..b756b87d2 100644 --- a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml +++ b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml @@ -1915,7 +1915,9 @@ components: $ref: '#/components/schemas/SplitPolicy' progress: $ref: '#/components/schemas/DraftProgress' - source_lease_deadline: *composition004 + source_lease_deadline: + <<: *composition004 + description: Compatibility field; new drafts and edits use zero and hold no source lease. preparation: anyOf: - type: object diff --git a/doc/design/access-server/dataset/spec-crowdb-dataset.md b/doc/design/access-server/dataset/spec-crowdb-dataset.md index 89892b2db..4ff7f1562 100644 --- a/doc/design/access-server/dataset/spec-crowdb-dataset.md +++ b/doc/design/access-server/dataset/spec-crowdb-dataset.md @@ -308,7 +308,10 @@ remain protected until every candidate/Snapshot owner is released. Unowned sessions expire; physical cleanup may remain pending and MUST report that state. Public read handles bind the resolved immutable Snapshot, selection, projection, -shuffle and worker partition. Fetch MUST NOT acknowledge consumption. The caller +shuffle and worker partition. Creation and acknowledgement prepare the next token. +Fetch MUST NOT acknowledge consumption or persist activity. Creation, explicit +resume and acknowledgement refresh the 60-second admission deadline; fetching +alone does not extend it. The caller explicitly acknowledges an issued next cursor; repeating that acknowledgement is idempotent. Progress inspection MUST NOT fetch another batch. Resume restores acknowledged progress. Closing one handle MUST NOT cancel another. Worker diff --git a/doc/design/access-server/design-crowdb-access-server.md b/doc/design/access-server/design-crowdb-access-server.md index 1226f8f88..5995d8950 100644 --- a/doc/design/access-server/design-crowdb-access-server.md +++ b/doc/design/access-server/design-crowdb-access-server.md @@ -263,6 +263,25 @@ Frontend query failures do not mutate these backend health observations. completion can each resume an idle transfer without changing protocol publication authority. +Coordination follows the semantic owner, rather than the Access Server process: + +- Dataset read admission is read-only. Explicit session acknowledgements persist + application progress; normal fetching and snapshot activity do not. Draft + journals recover one draft's partial work and fence late writes after cleanup. +- Dataset upload expiry scans upload authorities; creation does not enter a + global registry. Shared payload owner records prevent one snapshot's retirement + from reclaiming another snapshot's bytes. +- Iceberg streamed uploads use independent session bounds and per-part writes. + Catalog-wide credit records are retained only for legacy recovery. Ordinary + table commits use one head CAS; GC fences protect committed file reachability. +- Namespace deletion rejects live children. The associated parent admission and + recovery state preserves that strict contract and is not a reader lease. +- S3 multipart coordinates accounting and exact completion selection within one + upload. Ordinary object reads do not register persistent activity. +- A failed CAS observation can supply the next base for an allowed retry. Unknown + transport outcomes require reconciliation with the same mutation identity; + application preconditions must not silently move to a newer base. + ## 9. Direction and risks The main architectural direction is to keep HTTP broadly compatible while diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg.md b/doc/design/access-server/iceberge/design-crowdb-iceberg.md index e39bc5a66..15aeb8d18 100644 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg.md +++ b/doc/design/access-server/iceberge/design-crowdb-iceberg.md @@ -111,7 +111,8 @@ and after a table has disappeared, without a table-ID reverse index: identity; creation can precede name/head publication and rename spans two live namespaces. Their selected heads carry the frozen storage identity. - Multipart sessions and parts are keyed by upload identity/part number. Catalog - admission accounts for all sessions, including unpublished and expired uploads. + admission records apply only to legacy credited sessions; new streamed sessions + use independent per-session bounds. Every session and part carries and validates the full storage identity. - Write intents/fences track allocated but potentially unpublished native extents, including failed creation. Catalog maintenance scans their bounded @@ -263,6 +264,10 @@ Recursive creation helping shares one bounded phase budget. Root-admission backend identities include the individual creation operation, so a different creator cannot reuse a cached no-op CAS result from before its reservation. +Namespace drop must reject a nonempty namespace. Parent admission and its recovery +markers remain required for that strict lifecycle contract; deletion does not turn +into an unconditional cascading retirement. + Namespace drop persists a Ready-to-Dropping fence before scanning its two child index ranges. Durable cursors advance across bounded pages; reservations are helped, stale namespace mappings are conditionally removed, and corruption blocks diff --git a/doc/design/access-server/s3/design-crowdb-access-s3.md b/doc/design/access-server/s3/design-crowdb-access-s3.md index 08baa002b..a07b694d2 100644 --- a/doc/design/access-server/s3/design-crowdb-access-s3.md +++ b/doc/design/access-server/s3/design-crowdb-access-s3.md @@ -185,6 +185,13 @@ and raw MD5 reuse the retained candidate under the same idempotency rule as a published part; different contents select the next unused generation. Generation numbers therefore increase monotonically and can have gaps. Retained candidates remain immutable, and accounting changes only through the session fence. +Session and part-pointer replay checks consume the CAS conflict observation; +a confirmed conflict does not require another GET to test whether the exact +proposed bytes were installed. An uncertain transport outcome still requires +reconciliation. Part publication carries its latest loaded session into the next +iteration instead of discarding it and reading again. This coordination is scoped +to one upload and preserves exact staged-byte accounting and completion selection; +independent uploads share no reservation record. Completion validates the ordered selected part numbers, raw MD5 values, minimum nonfinal size, and current generations. It freezes the selection under diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 77b50be02..9b5807660 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -28,61 +28,61 @@ items after recording their verified outcome in the owning design. ## Execution -- [ ] **Dataset durable read sessions**: `DatasetSessions::{start,next,resume, - acknowledge,close}` persists bindings, selected IDs, issued tokens, activity - deadlines and cursor changes. `next` writes even when replaying an already - issued batch; streaming checks session state repeatedly. Determine which - callers need server-persisted restart/ack semantics and whether the ordinary - path can use a snapshot-bound client cursor. Keep explicit acknowledged - progress separate from fetching. Files: `lib/crowdb-access-dataset/src/read_session/`, - `app/crowdb-access-server/src/dataset/api/{session.rs,batch.rs}`. -- [ ] **Dataset preparation and scratch journals**: Composition/Split evaluation - persists work pages, input revisions, candidate links, cleanup cursors and - source activity. Separate unpublished scratch that may expire from committed - visibility and shared payload ownership. Measure KV mutations per page and - evaluate coarser checkpoints/expiry before removing recovery facts. Files: - `lib/crowdb-access-dataset/src/{composition,split,inspection}/`. -- [ ] **Dataset upload shared session updates**: inspect part progress and owner - vectors, registry page updates, registration generations and permanent retired - owner lists for contention between independent parts/owners. Preserve digest, - durable payload publication and shared-owner safety; evaluate independent part - records and delayed orphan cleanup. Files: - `lib/crowdb-access-dataset/src/upload/{ownership.rs,retirement.rs,registry.rs}` - and upload part/completion implementations. -- [ ] **Iceberg catalog-wide multipart credits**: every reservation/release - changes catalog-wide `sessions`, `reserved_bytes`, and `pending` state through - `MultipartAdmission`. Independent uploads share a durable coordination point. - Evaluate local bounded admission plus coarse distributed quotas, with explicit - allowance for quota overshoot; do not weaken memory limits. Files: - `lib/crowdb-access-iceberg/src/file/multipart_credits.rs`, - `src/file/multipart_credits/settle.rs`, `src/file/multipart_admission.rs`. -- [ ] **Iceberg parent namespace admission**: namespace mutations and table - lifecycle operations CAS the parent's `pending_operation` and - `mutation_revision`, persist before/after payloads and journal phases, then - release the marker. Independent child creations can help/wait on each other. - Evaluate immutable namespace identities plus deletion marks/grace periods; - retain same-name uniqueness and reject publication into retired identities. - Files: `lib/crowdb-access-iceberg/src/namespace/admission.rs`, - `src/table/lifecycle/admission.rs`, their journals/recovery modules. -- [ ] **Iceberg publication and GC fencing**: distinguish expected table-head - CAS conflicts from extra persisted operation phases, payload copies, admission - markers and GC `operation_fence` writes. Audit ordinary commit versus create, - rename, delete and recovery separately. Simplify only after counting RPCs and - identifying the exact guarantee being removed. Files: - `lib/crowdb-access-iceberg/src/commit/`, `src/gc/{admission.rs,fence.rs,worker/}`. -- [ ] **S3 and Iceberg multipart per-upload contention**: part publication and - completion use session generations and CAS. Check whether independent parts - rewrite shared session state and whether completion/recovery duplicates - persistent work. Keep immutable selected-part identity, integrity and - lost-reply recovery. Files: - `lib/crowdb-access-s3/src/metadata/multipart_repository/`, - `lib/crowdb-access-iceberg/src/file/multipart_repository/`. -- [ ] **Retry hygiene across authority loops**: classify CAS failure as stale - base, business precondition failure or uncertain transport outcome. Where - retry is valid, use returned observed state and recompute; remove redundant - GETs and no-op writes. A transport retry must retain mutation identity while - a new logical mutation must get a new identity. Files: Dataset read-session, - publication/upload loops, Iceberg journals and S3 multipart repositories. +- [~] **Verify and deliver the cleanup**: run affected Dataset/S3/native/HTTP + regressions, format/lint, rebuild the same server source, and run the unchanged + mixed-size workload with restart acceptance. Retain measured limits; do not + infer throughput from mutation counts. Commit the coherent implementation and + record acceptance here before removing this task. + +## Decisions applied to the inventory + +- Dataset sessions retain explicit durable acknowledgement and remote close. + Normal fetch/replay is read-only; start/ack prepares the next token. Only + start/ack/resume extends admission. Legacy unprepared sessions upgrade once. + See [Dataset read flow](../design/access-server/dataset/design-crowdb-access-dataset.md#102-read-flow). +- Composition/Split retain draft-scoped recovery checkpoints and cleanup fences + for exact lost-reply recovery and late-writer exclusion. Source activity leases + are absent; receipt helping avoids redundant CAS. No reader-shared journal was + found. See [control lifecycle](../design/access-server/dataset/design-dataset-control.md). +- Dataset uploads no longer register in shared tail/pages or persist a scan + cursor. Expiry directly scans upload authorities with local continuation. + Permanent owner revocation protects other live snapshots; bounded part indexes + remain to avoid one metadata point read per part during listing/completion. + See [upload flow](../design/access-server/dataset/design-crowdb-access-dataset.md#101-upload-and-publication-flow). +- Iceberg catalog-wide credits are legacy recovery only. New streamed sessions + already use independent session limits; no foreground reserve/release was + found. Keep legacy recovery so old outstanding records can settle. +- **User decision, 2026-10-24**: nonempty namespace deletion must fail. Keep the + parent admission protocol where required by creation/rename/drop recovery; + do not replace it with cascading deletion. Ordinary table creation already + avoids a parent authority mutation. See [Iceberg authority](../design/access-server/iceberge/design-crowdb-iceberg.md#2-authority-model). +- Ordinary Iceberg commits already use one head CAS; the legacy multi-phase + publisher is not the normal HTTP path. Preserve GC/publication fencing because + it prevents deleting referenced files, rather than only interrupting a reader. +- Iceberg streamed parts already publish independently by part number. S3 keeps + upload-local accounting and the completion/part-pointer fence; independent + uploads do not contend on one global record. Confirmed S3 CAS conflicts now + use returned observed bytes, and part retry retains its latest loaded state. + See [S3 data flow](../design/access-server/s3/design-crowdb-access-s3.md). +- Dataset session and upload-owner retry loops use returned observed state. + Confirmed stale-base retries and uncertain transport reconciliation remain + distinct. Explicit revision preconditions still return conflict; retrying an + application mutation against a new base is not universally valid. + +## Files and focused checks + +- Dataset: `read_session/`, `upload/`, `composition/`, `split/state.rs`; Access + Server Dataset cleanup wiring and publication; Dataset design/spec/OpenAPI. +- S3: `metadata/store.rs`, `metadata/multipart_repository{.rs,/parts.rs}`; + owning design. Iceberg design corrections document existing foreground paths. +- Cost regressions: `session_cost_test`, `draft_cost_test`, + `upload_registry_test`; lifecycle regressions retain restart, shared-owner, + lost-response, invalid-content and cancellation assertions. +- Integration: affected Access Server/native/S3 suites; Iceberg namespace drop + and admission tests retain the explicitly selected nonempty-delete contract. +- Gates: Rust fmt and workspace clippy through Pixi. Full workload and restart + use `tools/sdk/dataset_workload.py` with its unchanged defaults and same-source + image; physical multi-host acceptance remains in its independent plan. ## Inspection coverage and exclusions diff --git a/lib/crowdb-access-dataset/src/composition/begin.rs b/lib/crowdb-access-dataset/src/composition/begin.rs index 70548e007..5d01c3624 100644 --- a/lib/crowdb-access-dataset/src/composition/begin.rs +++ b/lib/crowdb-access-dataset/src/composition/begin.rs @@ -3,7 +3,7 @@ use super::{ key, - state::{budget, now, CompositionState}, + state::{budget, CompositionState}, CompositionDraft, CompositionError, CompositionInput, CompositionRecord, CompositionSource, DraftProgress, }; @@ -69,7 +69,7 @@ impl DatasetAuthority { state.draft.record.progress, DraftProgress::Published { .. } | DraftProgress::Abandoned ) { - self.renew_composition_source(identity, &state.draft).await?; + self.check_composition_source(identity, &state.draft).await?; } Ok(state.draft) } @@ -146,7 +146,7 @@ impl DatasetAuthority { source: input.source.clone(), revision: 1, edit_pages: 0, - source_lease_deadline: now().saturating_add(3600), + source_lease_deadline: 0, progress: DraftProgress::Editing, }, schema, @@ -155,7 +155,7 @@ impl DatasetAuthority { preparation: None, }; self.validate_composition_splits(identity, &draft).await?; - self.renew_composition_source(identity, &draft).await?; + self.check_composition_source(identity, &draft).await?; let state = CompositionState { uploads: None, draft, diff --git a/lib/crowdb-access-dataset/src/composition/edit.rs b/lib/crowdb-access-dataset/src/composition/edit.rs index cc1c2de09..eb493d4ac 100644 --- a/lib/crowdb-access-dataset/src/composition/edit.rs +++ b/lib/crowdb-access-dataset/src/composition/edit.rs @@ -62,14 +62,14 @@ impl DatasetAuthority { )?; } self.validate_composition_splits(identity, &next.draft).await?; - self.renew_composition_source(identity, &next.draft).await?; - next.draft.record.source_lease_deadline = super::state::now().saturating_add(3600); + self.check_composition_source(identity, &next.draft).await?; + next.draft.record.source_lease_deadline = 0; next.encode()?; let (old, state) = self .register_composition_edit(identity, &old, &state, edit.id) .await?; next = edited_state(&state, edit, marker)?; - next.draft.record.source_lease_deadline = super::state::now().saturating_add(3600); + next.draft.record.source_lease_deadline = 0; // Edit IDs own immutable pages. Losing a root CAS cannot occupy the winner's ordinal. let mut page_key = key::scope(b"DS\0CE\0", identity, operation)?; page_key.extend_from_slice(edit.id.as_bytes()); diff --git a/lib/crowdb-access-dataset/src/composition/evaluation.rs b/lib/crowdb-access-dataset/src/composition/evaluation.rs index c1b3a5712..355efe87b 100644 --- a/lib/crowdb-access-dataset/src/composition/evaluation.rs +++ b/lib/crowdb-access-dataset/src/composition/evaluation.rs @@ -78,7 +78,7 @@ impl DatasetAuthority { if expected_page < current.preview.next_page || current.preview.complete { return Ok(current.preview); } - self.renew_composition_source(identity, &state.draft).await?; + self.check_composition_source(identity, &state.draft).await?; let step_key = key::page(b"DS\0CW\0", identity, operation, expected_revision, expected_page)?; let step = if let Some(value) = self.composition_get(&step_key).await? { crate::split::codec::decode::(&value.bytes)? diff --git a/lib/crowdb-access-dataset/src/composition/prepare.rs b/lib/crowdb-access-dataset/src/composition/prepare.rs index 2350fb281..831442034 100644 --- a/lib/crowdb-access-dataset/src/composition/prepare.rs +++ b/lib/crowdb-access-dataset/src/composition/prepare.rs @@ -33,7 +33,7 @@ impl DatasetAuthority { if !matches!(state.draft.record.progress, DraftProgress::Evaluated { .. }) { return Err(CompositionError::Frozen); } - self.renew_composition_source(identity, &state.draft).await?; + self.check_composition_source(identity, &state.draft).await?; let mut preparation = if let Some(preparation) = &state.draft.preparation { preparation.clone() } else { diff --git a/lib/crowdb-access-dataset/src/composition/publication.rs b/lib/crowdb-access-dataset/src/composition/publication.rs index 0a632531b..b116a3b9a 100644 --- a/lib/crowdb-access-dataset/src/composition/publication.rs +++ b/lib/crowdb-access-dataset/src/composition/publication.rs @@ -42,7 +42,7 @@ impl DatasetAuthority { self.persist_attempt(identity, attempt).await?; match state.draft.record.progress { DraftProgress::Prepared { .. } => { - self.renew_composition_source(identity, &state.draft).await?; + self.check_composition_source(identity, &state.draft).await?; let mut next = state.clone(); next.edit = None; next.draft.record.progress = DraftProgress::Committing { diff --git a/lib/crowdb-access-dataset/src/composition/record.rs b/lib/crowdb-access-dataset/src/composition/record.rs index e33833426..1be40e2de 100644 --- a/lib/crowdb-access-dataset/src/composition/record.rs +++ b/lib/crowdb-access-dataset/src/composition/record.rs @@ -99,6 +99,7 @@ pub struct CompositionRecord { pub source: CompositionSource, pub revision: u64, pub edit_pages: u64, + /// Compatibility field; new drafts have no source activity lease and use zero. pub source_lease_deadline: u64, pub progress: DraftProgress, } diff --git a/lib/crowdb-access-dataset/src/composition/state.rs b/lib/crowdb-access-dataset/src/composition/state.rs index 93dfad0df..76a611b57 100644 --- a/lib/crowdb-access-dataset/src/composition/state.rs +++ b/lib/crowdb-access-dataset/src/composition/state.rs @@ -125,11 +125,15 @@ impl DatasetAuthority { marker: marker.clone(), draft: state.draft.clone(), }; - self.put_composition( - &key::edit_receipt(identity, state.draft.record.operation, marker.id)?, - &crate::split::codec::encode(&receipt)?, - ) - .await?; + let key = key::edit_receipt(identity, state.draft.record.operation, marker.id)?; + let bytes = crate::split::codec::encode(&receipt)?; + if let Some(existing) = self.composition_get(&key).await? { + if existing.bytes != bytes { + return Err(CompositionError::Conflict); + } + } else { + self.put_composition(&key, &bytes).await?; + } } Ok(()) } @@ -164,7 +168,7 @@ impl DatasetAuthority { CasOutcome::Conflict(_) => Err(CompositionError::Conflict), } } - pub(super) async fn renew_composition_source( + pub(super) async fn check_composition_source( &self, identity: &DatasetIdentity, draft: &CompositionDraft, diff --git a/lib/crowdb-access-dataset/src/read_session.rs b/lib/crowdb-access-dataset/src/read_session.rs index 4e434ebce..24e4c6e18 100644 --- a/lib/crowdb-access-dataset/src/read_session.rs +++ b/lib/crowdb-access-dataset/src/read_session.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! Durable scoped reads. Fetch records an issued cursor; only acknowledgement advances it. +//! Durable scoped progress. Creation and acknowledgement prepare the next token; fetch is read-only. mod pages; mod planning; @@ -65,6 +65,12 @@ struct SessionRecord { closed: bool, } impl SessionRecord { + fn prepare_next(&mut self) { + self.version = 3; + let end = self.position.saturating_add(self.batch_size).min(self.count); + self.issued + .get_or_insert_with(|| (OperationId::random().to_string(), end)); + } fn public(&self) -> ReadSession { ReadSession { id: self.id.to_string(), @@ -98,7 +104,7 @@ fn encode(record: &SessionRecord) -> Result, AuthorityError> { } fn decode(bytes: &[u8]) -> Result { let record: SessionRecord = bincode::deserialize(bytes).map_err(|_| AuthorityError::Corrupt)?; - if record.version != 2 { + if !matches!(record.version, 2 | 3) || (record.version == 3 && record.issued.is_none()) { return Err(AuthorityError::Corrupt); } Ok(record) diff --git a/lib/crowdb-access-dataset/src/read_session/planning.rs b/lib/crowdb-access-dataset/src/read_session/planning.rs index cf8d396e0..110f119de 100644 --- a/lib/crowdb-access-dataset/src/read_session/planning.rs +++ b/lib/crowdb-access-dataset/src/read_session/planning.rs @@ -44,7 +44,7 @@ impl DatasetSessions { bincode::serialize(&(identity.key()?, snapshot, &spec)).map_err(|_| AuthorityError::Corrupt)?, ) .into(); - let record = SessionRecord { + let mut record = SessionRecord { version: 2, id: spec.id, fields: spec.fields.clone(), @@ -58,6 +58,7 @@ impl DatasetSessions { deadline: now.saturating_add(TTL), closed: false, }; + record.prepare_next(); self.authority.check_snapshot_readable(identity, snapshot).await?; match self .authority diff --git a/lib/crowdb-access-dataset/src/read_session/progress.rs b/lib/crowdb-access-dataset/src/read_session/progress.rs index 238cf7321..be8a3cabb 100644 --- a/lib/crowdb-access-dataset/src/read_session/progress.rs +++ b/lib/crowdb-access-dataset/src/read_session/progress.rs @@ -2,12 +2,12 @@ // Licensed under the Apache License, Version 2.0. use super::{ - encode, session_key, AuthorityError, CasOutcome, DatasetError, DatasetIdentity, DatasetSessions, + decode, encode, session_key, AuthorityError, CasOutcome, DatasetError, DatasetIdentity, DatasetSessions, OperationId, ReadSession, SessionBatch, TTL, }; impl DatasetSessions { - /// Resumes acknowledged progress, including after a disconnected lease expired. + /// Resumes acknowledged progress and refreshes its admission deadline. /// # Errors /// Rejects closed/reclaimed sessions or unavailable storage. pub async fn resume( @@ -16,8 +16,9 @@ impl DatasetSessions { id: OperationId, now: u64, ) -> Result { + let mut current = self.record(identity, id).await?; for _ in 0..64 { - let (old, mut record) = self.record(identity, id).await?; + let (old, mut record) = current; if record.closed { return Err(DatasetError::ReadCancelled.into()); } @@ -27,20 +28,27 @@ impl DatasetSessions { self.authority .check_snapshot_readable(identity, record.snapshot) .await?; - record.deadline = now.saturating_add(TTL); - if matches!( - self.authority - .store - .compare_exchange(&session_key(identity, id)?, Some(&old), &encode(&record)?) - .await?, - CasOutcome::Applied(_) - ) { + record.deadline = record.deadline.max(now.saturating_add(TTL)); + record.prepare_next(); + let bytes = encode(&record)?; + if bytes == old { return Ok(record.public()); } + match self + .authority + .store + .compare_exchange(&session_key(identity, id)?, Some(&old), &bytes) + .await? + { + CasOutcome::Applied(_) => return Ok(record.public()), + CasOutcome::Conflict(Some(value)) => current = (value.bytes.clone(), decode(&value.bytes)?), + CasOutcome::Conflict(None) => return Err(AuthorityError::NotFound), + } } Err(AuthorityError::CursorConflict) } - /// Fetch never advances acknowledged progress. Identical fetches replay the issued batch. + /// Fetch is read-only for prepared sessions and replays the same next token. + /// Only creation, explicit resume and acknowledgement refresh admission. /// # Errors /// Rejects disconnected/closed sessions or unavailable storage. pub async fn next( @@ -49,28 +57,16 @@ impl DatasetSessions { id: OperationId, now: u64, ) -> Result { + let mut current = self.record(identity, id).await?; for _ in 0..64 { - let (old, mut r) = self.record(identity, id).await?; + let (old, mut r) = current; if r.closed || now >= r.deadline { return Err(DatasetError::ReadCancelled.into()); } - let end = r.position.saturating_add(r.batch_size).min(r.count); - let next = r - .issued - .get_or_insert_with(|| (OperationId::random().to_string(), end)) - .0 - .clone(); - r.deadline = now.saturating_add(TTL); self.authority .check_snapshot_readable(identity, r.snapshot) .await?; - if matches!( - self.authority - .store - .compare_exchange(&session_key(identity, id)?, Some(&old), &encode(&r)?) - .await?, - CasOutcome::Applied(_) - ) { + if let Some((next, end)) = r.issued.clone() { return Ok(SessionBatch { snapshot: r.snapshot, ids: self.load_ids(identity, id, r.position, end).await?, @@ -79,6 +75,18 @@ impl DatasetSessions { end: end == r.count, }); } + // Old records may not have a prepared token yet. Upgrade once. + r.prepare_next(); + match self + .authority + .store + .compare_exchange(&session_key(identity, id)?, Some(&old), &encode(&r)?) + .await? + { + CasOutcome::Applied(_) => current = (encode(&r)?, r), + CasOutcome::Conflict(Some(value)) => current = (value.bytes.clone(), decode(&value.bytes)?), + CasOutcome::Conflict(None) => return Err(AuthorityError::NotFound), + } } Err(AuthorityError::CursorConflict) } @@ -92,8 +100,9 @@ impl DatasetSessions { next: &str, now: u64, ) -> Result { + let mut current = self.record(identity, id).await?; for _ in 0..64 { - let (old, mut r) = self.record(identity, id).await?; + let (old, mut r) = current; if r.closed || now >= r.deadline { return Err(DatasetError::ReadCancelled.into()); } @@ -106,19 +115,25 @@ impl DatasetSessions { let (_, position) = r.issued.take().ok_or(AuthorityError::CursorConflict)?; r.position = position; r.cursor = next.into(); - r.deadline = now.saturating_add(TTL); + r.deadline = r.deadline.max(now.saturating_add(TTL)); + r.prepare_next(); self.authority .check_snapshot_readable(identity, r.snapshot) .await?; - if matches!( - self.authority - .store - .compare_exchange(&session_key(identity, id)?, Some(&old), &encode(&r)?) - .await?, - CasOutcome::Applied(_) - ) { + let bytes = encode(&r)?; + if bytes == old { return Ok(r.public()); } + match self + .authority + .store + .compare_exchange(&session_key(identity, id)?, Some(&old), &bytes) + .await? + { + CasOutcome::Applied(_) => return Ok(r.public()), + CasOutcome::Conflict(Some(value)) => current = (value.bytes.clone(), decode(&value.bytes)?), + CasOutcome::Conflict(None) => return Err(AuthorityError::NotFound), + } } Err(AuthorityError::CursorConflict) } @@ -143,18 +158,24 @@ impl DatasetSessions { identity: &DatasetIdentity, id: OperationId, ) -> Result { + let mut current = self.record(identity, id).await?; for _ in 0..64 { - let (old, mut r) = self.record(identity, id).await?; + let (old, mut r) = current; r.closed = true; - if matches!( - self.authority - .store - .compare_exchange(&session_key(identity, id)?, Some(&old), &encode(&r)?) - .await?, - CasOutcome::Applied(_) - ) { + let bytes = encode(&r)?; + if bytes == old { return Ok(r.public()); } + match self + .authority + .store + .compare_exchange(&session_key(identity, id)?, Some(&old), &bytes) + .await? + { + CasOutcome::Applied(_) => return Ok(r.public()), + CasOutcome::Conflict(Some(value)) => current = (value.bytes.clone(), decode(&value.bytes)?), + CasOutcome::Conflict(None) => return Err(AuthorityError::NotFound), + } } Err(AuthorityError::CursorConflict) } diff --git a/lib/crowdb-access-dataset/src/read_session/split/finish.rs b/lib/crowdb-access-dataset/src/read_session/split/finish.rs index afada0fd9..056b1ebb3 100644 --- a/lib/crowdb-access-dataset/src/read_session/split/finish.rs +++ b/lib/crowdb-access-dataset/src/read_session/split/finish.rs @@ -129,7 +129,7 @@ fn split_session_record( .map_err(|_| AuthorityError::Corrupt)?, ) .into(); - Ok(super::super::SessionRecord { + let mut record = super::super::SessionRecord { version: 2, id: bound.spec.read.id, fields: bound.spec.read.fields.clone(), @@ -142,5 +142,7 @@ fn split_session_record( issued: None, deadline: now.saturating_add(super::super::TTL), closed, - }) + }; + record.prepare_next(); + Ok(record) } diff --git a/lib/crowdb-access-dataset/src/read_session/split/state.rs b/lib/crowdb-access-dataset/src/read_session/split/state.rs index bf320c1d1..7dc84dd7b 100644 --- a/lib/crowdb-access-dataset/src/read_session/split/state.rs +++ b/lib/crowdb-access-dataset/src/read_session/split/state.rs @@ -100,6 +100,18 @@ impl DatasetSessions { key: &[u8], bytes: &[u8], ) -> Result<(), AuthorityError> { + if let Some(existing) = self + .authority + .store + .get_bounded(key, budget(), &ReadCancellation::new()) + .await? + { + return if existing.bytes == bytes { + Ok(()) + } else { + Err(AuthorityError::CursorConflict) + }; + } match self.authority.store.compare_exchange(key, None, bytes).await? { CasOutcome::Applied(_) => Ok(()), CasOutcome::Conflict(Some(value)) if value.bytes == bytes => Ok(()), @@ -138,6 +150,9 @@ impl DatasetSessions { next: &PreparationProgress, ) -> Result<(), AuthorityError> { let bytes = encode(next)?; + if old.is_some_and(|old| old.bytes == bytes) { + return Ok(()); + } match self .authority .store diff --git a/lib/crowdb-access-dataset/src/split/state.rs b/lib/crowdb-access-dataset/src/split/state.rs index 573631493..8f3b98413 100644 --- a/lib/crowdb-access-dataset/src/split/state.rs +++ b/lib/crowdb-access-dataset/src/split/state.rs @@ -124,11 +124,15 @@ impl DatasetAuthority { marker: marker.clone(), record: state.record.clone(), }; - self.put_split_state( - &state_key(b"DS\0SX\0", identity, state.record.operation, Some(marker.id))?, - &codec::encode(&receipt)?, - ) - .await?; + let key = state_key(b"DS\0SX\0", identity, state.record.operation, Some(marker.id))?; + let bytes = codec::encode(&receipt)?; + if let Some(existing) = self.split_state_get(&key).await? { + if existing.bytes != bytes { + return Err(SplitLifecycleError::Conflict); + } + } else { + self.put_split_state(&key, &bytes).await?; + } } Ok(()) } diff --git a/lib/crowdb-access-dataset/src/upload.rs b/lib/crowdb-access-dataset/src/upload.rs index 6bf7eb183..285517c52 100644 --- a/lib/crowdb-access-dataset/src/upload.rs +++ b/lib/crowdb-access-dataset/src/upload.rs @@ -3,10 +3,10 @@ //! Durable Dataset uploads with independent immutable part records. +mod expiry; mod index; mod ownership; mod parts; -mod registry; mod retirement; mod stream; pub use stream::VerifiedUploadPart; @@ -34,7 +34,7 @@ pub enum UploadError { Integrity, #[error("upload is awaiting durable storage recovery")] Pending, - #[error("upload registry capacity exhausted")] + #[error("upload ownership capacity exhausted")] Capacity, #[error("upload storage unavailable: {0}")] Storage(String), @@ -72,11 +72,6 @@ pub struct UploadRecord { owners: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] retired_owners: Vec, - registry_page: Option, - #[serde(default)] - registry_generation: Option, - #[serde(default)] - registry_fence: Option, pub cleanup_complete: bool, } @@ -135,7 +130,7 @@ impl DatasetUploads { return Err(UploadError::Invalid); } let record = UploadRecord { - version: 2, + version: 4, length, part_size, md5, @@ -144,9 +139,6 @@ impl DatasetUploads { parts: Vec::new(), owners: Vec::new(), retired_owners: Vec::new(), - registry_page: None, - registry_generation: None, - registry_fence: None, cleanup_complete: false, }; let created = match self @@ -164,7 +156,6 @@ impl DatasetUploads { } CasOutcome::Conflict(None) => return Err(UploadError::Conflict), }; - self.register(location).await?; if created { self.session(location).await } else { @@ -283,8 +274,8 @@ fn decode(bytes: &[u8]) -> Result { serde_json::from_slice(bytes).map_err(|_| UploadError::Integrity) } fn decode_session(bytes: &[u8]) -> Result { - let record: UploadRecord = decode(bytes)?; - if !matches!(record.version, 2 | 3) + let mut record: UploadRecord = decode(bytes)?; + if !matches!(record.version, 2..=4) || (record.version == 2 && !record.retired_owners.is_empty()) || record.retired_owners.len() > 4096 || record.retired_owners.windows(2).any(|pair| pair[0] >= pair[1]) @@ -300,6 +291,7 @@ fn decode_session(bytes: &[u8]) -> Result { { return Err(UploadError::Integrity); } + record.version = 4; Ok(record) } fn decode_part(bytes: &[u8]) -> Result { diff --git a/lib/crowdb-access-dataset/src/upload/expiry.rs b/lib/crowdb-access-dataset/src/upload/expiry.rs new file mode 100644 index 000000000..8b8a80c64 --- /dev/null +++ b/lib/crowdb-access-dataset/src/upload/expiry.rs @@ -0,0 +1,81 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{decode_session, DatasetUploads, UploadError, UploadState}; +use crate::{MetadataScan, MetadataScanBudget, ReadCancellation}; +use std::time::Duration; + +const PREFIX: &[u8] = b"DS\0U\0"; + +impl DatasetUploads { + /// Scans one bounded page of upload records. The caller-owned cursor is an + /// optimization only; restarting the scan cannot lose a cleanup obligation. + /// # Errors + /// Reports corrupt records or cleanup/storage failures. An item failure + /// advances the cursor so unrelated uploads can progress before it is retried. + pub async fn expire(&self, now: u64, after: &mut Option>) -> Result<(), UploadError> { + let page = self + .store + .scan_page( + &MetadataScan { + prefix: PREFIX.to_vec(), + after: after.clone(), + budget: MetadataScanBudget { + items: 20, + bytes: 1024 * 1024, + timeout: Duration::from_secs(2), + }, + }, + &ReadCancellation::new(), + ) + .await?; + for (key, stored) in &page.entries { + *after = Some(key.clone()); + if !is_session_key(key)? { + continue; + } + let session = decode_session(&stored.bytes)?; + if session.cleanup_complete || !session.owners.is_empty() { + continue; + } + if session.state == UploadState::Aborted || now >= session.expires_at { + match self.abort(key).await { + Ok(_) | Err(UploadError::Conflict) => {} + Err(error) => return Err(error), + } + } + } + *after = page.after; + Ok(()) + } +} + +// Qualified identity = length-prefixed namespace segments, 0xff, name; +// the upload ID follows it. Part and index records append six bytes. +fn is_session_key(key: &[u8]) -> Result { + let mut rest = key.strip_prefix(PREFIX).ok_or(UploadError::Integrity)?; + while rest.first() != Some(&0xff) { + rest = skip_name(rest)?; + } + rest = skip_name(&rest[1..])?; + let id = rest.get(..16).ok_or(UploadError::Integrity)?; + crate::OperationId::from_bytes(id).map_err(|_| UploadError::Integrity)?; + match &rest[16..] { + [] => Ok(true), + [0, b'P' | b'I', _, _, _, _] => Ok(false), + _ => Err(UploadError::Integrity), + } +} + +fn skip_name(bytes: &[u8]) -> Result<&[u8], UploadError> { + let length: [u8; 2] = bytes + .get(..2) + .ok_or(UploadError::Integrity)? + .try_into() + .map_err(|_| UploadError::Integrity)?; + let length = usize::from(u16::from_be_bytes(length)); + if length == 0 || length > 255 { + return Err(UploadError::Integrity); + } + bytes.get(2 + length..).ok_or(UploadError::Integrity) +} diff --git a/lib/crowdb-access-dataset/src/upload/ownership.rs b/lib/crowdb-access-dataset/src/upload/ownership.rs index e2b0ea863..1345a98fd 100644 --- a/lib/crowdb-access-dataset/src/upload/ownership.rs +++ b/lib/crowdb-access-dataset/src/upload/ownership.rs @@ -17,8 +17,8 @@ impl DatasetUploads { owner: OperationId, now: u64, ) -> Result { + let mut stored = self.store.get(location).await?.ok_or(UploadError::NotFound)?; for _ in 0..16 { - let stored = self.store.get(location).await?.ok_or(UploadError::NotFound)?; let mut record = decode_session(&stored.bytes)?; if record.retired_owners.binary_search(&owner).is_ok() { return Err(UploadError::Conflict); @@ -37,13 +37,14 @@ impl DatasetUploads { record.state = UploadState::Claimed; record.owners.push(owner); record.owners.sort(); - if matches!( - self.store - .compare_exchange(location, Some(&stored.bytes), &encode(&record)?) - .await?, - CasOutcome::Applied(_) - ) { - return Ok(record); + match self + .store + .compare_exchange(location, Some(&stored.bytes), &encode(&record)?) + .await? + { + CasOutcome::Applied(_) => return Ok(record), + CasOutcome::Conflict(Some(value)) => stored = value, + CasOutcome::Conflict(None) => return Err(UploadError::NotFound), } } Err(UploadError::Pending) @@ -53,30 +54,24 @@ impl DatasetUploads { /// # Errors /// Returns unavailable storage, malformed records or exhausted retries. pub async fn release(&self, location: &[u8], owner: OperationId) -> Result<(), UploadError> { - self.register(location).await?; + let mut stored = self.store.get(location).await?.ok_or(UploadError::NotFound)?; for _ in 0..16 { - let stored = self.store.get(location).await?.ok_or(UploadError::NotFound)?; let mut record = decode_session(&stored.bytes)?; if !record.owners.contains(&owner) { - if record.state == UploadState::Claimed || record.cleanup_complete { - self.untrack(location).await?; - } return Ok(()); } record.owners.retain(|v| *v != owner); if record.owners.is_empty() { record.state = UploadState::Complete; } - if matches!( - self.store - .compare_exchange(location, Some(&stored.bytes), &encode(&record)?) - .await?, - CasOutcome::Applied(_) - ) { - if !record.owners.is_empty() { - self.untrack(location).await?; - } - return Ok(()); + match self + .store + .compare_exchange(location, Some(&stored.bytes), &encode(&record)?) + .await? + { + CasOutcome::Applied(_) => return Ok(()), + CasOutcome::Conflict(Some(value)) => stored = value, + CasOutcome::Conflict(None) => return Err(UploadError::NotFound), } } Err(UploadError::Pending) @@ -108,6 +103,9 @@ impl DatasetUploads { } async fn cleanup(&self, location: &[u8], record: UploadRecord) -> Result { + if record.cleanup_complete { + return self.get(location).await; + } let entries = self.entries(location, &record).await?; let mut complete = true; for entry in entries { @@ -120,6 +118,9 @@ impl DatasetUploads { if current.state != UploadState::Aborted { return Err(UploadError::Conflict); } + if current.cleanup_complete { + break; + } current.cleanup_complete = true; if matches!( self.store @@ -127,7 +128,6 @@ impl DatasetUploads { .await?, CasOutcome::Applied(_) ) { - self.untrack(location).await?; break; } } diff --git a/lib/crowdb-access-dataset/src/upload/parts.rs b/lib/crowdb-access-dataset/src/upload/parts.rs index ff5e39778..67a413be3 100644 --- a/lib/crowdb-access-dataset/src/upload/parts.rs +++ b/lib/crowdb-access-dataset/src/upload/parts.rs @@ -52,9 +52,6 @@ impl DatasetUploads { let session = self.session(location).await?; let length = data.length(); validate(&session, index, length, now)?; - if session.state == UploadState::Open { - self.register(location).await?; - } let key = part_key(location, index); let pending = PartEntry { version: 1, diff --git a/lib/crowdb-access-dataset/src/upload/registry.rs b/lib/crowdb-access-dataset/src/upload/registry.rs deleted file mode 100644 index 560d809b6..000000000 --- a/lib/crowdb-access-dataset/src/upload/registry.rs +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -//! Paged cleanup registry with registration generations and a CAS fence for stale entries. - -use super::{decode, decode_session, encode, DatasetUploads, UploadError, UploadState}; -use crate::{CasOutcome, OperationId}; -use serde::{Deserialize, Serialize}; -const TAIL: &[u8] = b"DS\0UTAIL\0"; -const SCAN: &[u8] = b"DS\0USCAN\0"; -const PAGE_SIZE: usize = 128; -#[derive(Clone, Eq, PartialEq, Serialize, Deserialize)] -struct Entry { - location: Vec, - generation: Option, -} -#[derive(Serialize, Deserialize)] -struct Page { - version: u16, - entries: Vec, -} -fn page(bytes: &[u8]) -> Result { - if let Ok(record) = decode::(bytes) { - if record.version != 2 { - return Err(UploadError::Integrity); - } - return Ok(record); - } - let old: Vec> = decode(bytes)?; - Ok(Page { - version: 2, - entries: old - .into_iter() - .map(|location| Entry { - location, - generation: None, - }) - .collect(), - }) -} -fn page_key(index: u64) -> Vec { - let mut key = b"DS\0UPAGE\0".to_vec(); - key.extend(index.to_be_bytes()); - key -} -impl DatasetUploads { - pub(super) async fn register(&self, location: &[u8]) -> Result<(), UploadError> { - for _ in 0..16 { - let stored = self.store.get(location).await?.ok_or(UploadError::NotFound)?; - let mut record = decode_session(&stored.bytes)?; - if record.cleanup_complete { - return Ok(()); - } - if let Some(index) = record.registry_page { - return self - .add_entry( - index, - &Entry { - location: location.to_vec(), - generation: record.registry_generation, - }, - ) - .await; - } - let entry = Entry { - location: location.to_vec(), - generation: Some(OperationId::random()), - }; - let index = self.append_registry(&entry).await?; - record.registry_page = Some(index); - record.registry_generation = entry.generation; - if matches!( - self.store - .compare_exchange(location, Some(&stored.bytes), &encode(&record)?) - .await?, - CasOutcome::Applied(_) - ) { - return self.add_entry(index, &entry).await; - } - self.remove_stale(index, &entry).await?; - } - Err(UploadError::Pending) - } - async fn append_registry(&self, entry: &Entry) -> Result { - for _ in 0..32 { - let tail = self.store.get(TAIL).await?; - let index = tail - .as_ref() - .map(|v| decode::(&v.bytes)) - .transpose()? - .unwrap_or(0); - match self.add_entry(index, entry).await { - Ok(()) => return Ok(index), - Err(UploadError::Capacity) => { - let next = index.checked_add(1).ok_or(UploadError::Capacity)?; - self.store - .compare_exchange(TAIL, tail.as_ref().map(|v| v.bytes.as_slice()), &encode(&next)?) - .await?; - } - Err(error) => return Err(error), - } - } - Err(UploadError::Pending) - } - async fn add_entry(&self, index: u64, entry: &Entry) -> Result<(), UploadError> { - let key = page_key(index); - for _ in 0..32 { - let stored = self.store.get(&key).await?; - let mut record = stored - .as_ref() - .map(|v| page(&v.bytes)) - .transpose()? - .unwrap_or(Page { - version: 2, - entries: Vec::new(), - }); - if record.entries.contains(entry) { - return Ok(()); - } - if record.entries.len() >= PAGE_SIZE { - return Err(UploadError::Capacity); - } - record.entries.push(entry.clone()); - if matches!( - self.store - .compare_exchange( - &key, - stored.as_ref().map(|v| v.bytes.as_slice()), - &encode(&record)? - ) - .await?, - CasOutcome::Applied(_) - ) { - return Ok(()); - } - } - Err(UploadError::Pending) - } - /// Removes tracking after durable binding or completed abort. A later - /// registration has a distinct generation and cannot be removed here. - /// # Errors - /// Returns storage failures or a session that still needs expiry tracking. - pub async fn untrack(&self, location: &[u8]) -> Result<(), UploadError> { - for _ in 0..16 { - let stored = self.store.get(location).await?.ok_or(UploadError::NotFound)?; - let mut record = decode_session(&stored.bytes)?; - let Some(index) = record.registry_page else { - return Ok(()); - }; - if record.state != UploadState::Claimed && !record.cleanup_complete { - return Err(UploadError::Conflict); - } - let entry = Entry { - location: location.to_vec(), - generation: record.registry_generation, - }; - record.registry_page = None; - if matches!( - self.store - .compare_exchange(location, Some(&stored.bytes), &encode(&record)?) - .await?, - CasOutcome::Applied(_) - ) { - return self.remove_entry(index, &entry).await; - } - } - Err(UploadError::Pending) - } - async fn remove_entry(&self, index: u64, entry: &Entry) -> Result<(), UploadError> { - let key = page_key(index); - for _ in 0..32 { - let Some(stored) = self.store.get(&key).await? else { - return Ok(()); - }; - let mut record = page(&stored.bytes)?; - record.entries.retain(|v| v != entry); - if matches!( - self.store - .compare_exchange(&key, Some(&stored.bytes), &encode(&record)?) - .await?, - CasOutcome::Applied(_) - ) { - return Ok(()); - } - } - Err(UploadError::Pending) - } - async fn remove_stale(&self, index: u64, entry: &Entry) -> Result<(), UploadError> { - // Fence registrars using the observed session bytes before removing an - // unbound entry. Otherwise a delayed registration could publish a link - // to an entry the scanner has just removed. - for _ in 0..32 { - let stored = self - .store - .get(&entry.location) - .await? - .ok_or(UploadError::NotFound)?; - let mut record = decode_session(&stored.bytes)?; - if record.registry_page == Some(index) && record.registry_generation == entry.generation { - return Ok(()); - } - record.registry_fence = Some(OperationId::random()); - if matches!( - self.store - .compare_exchange(&entry.location, Some(&stored.bytes), &encode(&record)?) - .await?, - CasOutcome::Applied(_) - ) { - return self.remove_entry(index, entry).await; - } - } - Err(UploadError::Pending) - } - /// Scans one bounded registry page; restart preserves the scan cursor. - /// # Errors - /// Returns storage/cleanup failures. Deferred reclamation remains tracked. - pub async fn expire(&self, now: u64) -> Result<(), UploadError> { - let tail = self - .store - .get(TAIL) - .await? - .map(|v| decode::(&v.bytes)) - .transpose()? - .unwrap_or(0); - let scan = self.store.get(SCAN).await?; - let index = scan - .as_ref() - .map(|v| decode::(&v.bytes)) - .transpose()? - .unwrap_or(0) - .min(tail); - if let Some(stored) = self.store.get(&page_key(index)).await? { - for entry in page(&stored.bytes)?.entries { - let session = self.session(&entry.location).await?; - if session.registry_page != Some(index) || session.registry_generation != entry.generation { - self.remove_stale(index, &entry).await?; - continue; - } - if session.state == UploadState::Aborted - || (session.owners.is_empty() && now >= session.expires_at) - { - match self.abort(&entry.location).await { - Ok(_) | Err(UploadError::Conflict) => {} - Err(error) => return Err(error), - } - } - } - } - let next = if index >= tail { 0 } else { index + 1 }; - self.store - .compare_exchange(SCAN, scan.as_ref().map(|v| v.bytes.as_slice()), &encode(&next)?) - .await?; - Ok(()) - } -} diff --git a/lib/crowdb-access-dataset/src/upload/retirement.rs b/lib/crowdb-access-dataset/src/upload/retirement.rs index 23f6481c0..a591992c1 100644 --- a/lib/crowdb-access-dataset/src/upload/retirement.rs +++ b/lib/crowdb-access-dataset/src/upload/retirement.rs @@ -11,36 +11,30 @@ impl DatasetUploads { /// Returns unknown uploads, corrupt metadata, bounded retirement capacity or storage failures. pub async fn retire_owner(&self, location: &[u8], owner: OperationId) -> Result<(), UploadError> { OperationId::from_bytes(owner.as_bytes()).map_err(|_| UploadError::Invalid)?; - self.register(location).await?; + let mut stored = self.store.get(location).await?.ok_or(UploadError::NotFound)?; for _ in 0..16 { - let stored = self.store.get(location).await?.ok_or(UploadError::NotFound)?; let mut record = decode_session(&stored.bytes)?; if record.retired_owners.binary_search(&owner).is_ok() { - if !record.owners.is_empty() || record.cleanup_complete { - self.untrack(location).await?; - } return Ok(()); } if record.retired_owners.len() >= 4096 { return Err(UploadError::Capacity); } - record.version = 3; + record.version = 4; record.retired_owners.push(owner); record.retired_owners.sort(); record.owners.retain(|current| *current != owner); if record.owners.is_empty() && record.state == UploadState::Claimed { record.state = UploadState::Complete; } - if matches!( - self.store - .compare_exchange(location, Some(&stored.bytes), &encode(&record)?) - .await?, - CasOutcome::Applied(_) - ) { - if !record.owners.is_empty() || record.cleanup_complete { - self.untrack(location).await?; - } - return Ok(()); + match self + .store + .compare_exchange(location, Some(&stored.bytes), &encode(&record)?) + .await? + { + CasOutcome::Applied(_) => return Ok(()), + CasOutcome::Conflict(Some(value)) => stored = value, + CasOutcome::Conflict(None) => return Err(UploadError::NotFound), } } Err(UploadError::Pending) diff --git a/lib/crowdb-access-dataset/tests/common/access_store.rs b/lib/crowdb-access-dataset/tests/common/access_store.rs index 8c3981074..053ac8db7 100644 --- a/lib/crowdb-access-dataset/tests/common/access_store.rs +++ b/lib/crowdb-access-dataset/tests/common/access_store.rs @@ -15,6 +15,7 @@ pub struct TestAccessStore { inner: dataset_store::TestDatasetStore, pub writes: AtomicU64, pub reads: AtomicU64, + pub upload_conflicts: AtomicU64, } impl TestAccessStore { pub fn shared() -> Arc { @@ -50,6 +51,21 @@ impl DatasetStore for TestAccessStore { value: &[u8], ) -> Result { self.writes.fetch_add(1, Ordering::Relaxed); + if self + .upload_conflicts + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |n| n.checked_sub(1)) + .is_ok() + { + let old = self.inner.get(key).await?.expect("upload exists"); + let mut record: serde_json::Value = serde_json::from_slice(&old.bytes).unwrap(); + record["expires_at"] = (record["expires_at"].as_u64().unwrap() + 1).into(); + assert!(matches!( + self.inner + .compare_exchange(key, Some(&old.bytes), &serde_json::to_vec(&record).unwrap()) + .await?, + CasOutcome::Applied(_) + )); + } self.inner.compare_exchange(key, expected, value).await } async fn delete(&self, key: &[u8]) -> Result<(), StoreError> { diff --git a/lib/crowdb-access-dataset/tests/draft_cost_test.rs b/lib/crowdb-access-dataset/tests/draft_cost_test.rs new file mode 100644 index 000000000..e7eb7249e --- /dev/null +++ b/lib/crowdb-access-dataset/tests/draft_cost_test.rs @@ -0,0 +1,66 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/access_store.rs"] +mod access_store; +#[path = "common/composition.rs"] +#[allow(dead_code)] +mod composition; +#[path = "common/publication.rs"] +mod publication; +use crowdb_access_dataset::{composition::SampleChange, split::*, *}; +use std::sync::atomic::Ordering; + +#[tokio::test] +async fn draft_edit_replays_read_existing_receipts_without_mutations() { + let store = access_store::TestAccessStore::shared(); + let authority = DatasetAuthority::new(store.clone()); + let identity = DatasetIdentity::new(NamespacePath::root(), "draft-cost").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + let parent = authority + .publish_snapshot(&identity, None, publication::publication_manifest(1)) + .await + .unwrap(); + let composition = OperationId::random(); + authority + .begin_composition(&identity, composition, &composition::input(parent)) + .await + .unwrap(); + let edit = composition::edit(1, vec![SampleChange::Upsert(composition::sample(b"new", 2))]); + let expected = authority + .edit_composition(&identity, composition, &edit) + .await + .unwrap(); + assert_eq!(expected.record.source_lease_deadline, 0); + let writes = store.writes.load(Ordering::Relaxed); + for _ in 0..8 { + assert_eq!( + authority + .edit_composition(&identity, composition, &edit) + .await + .unwrap(), + expected + ); + } + assert_eq!(store.writes.load(Ordering::Relaxed), writes); + let split = OperationId::random(); + authority + .begin_split_draft(&identity, split, parent, None, SplitPolicy::default(), vec![]) + .await + .unwrap(); + let edit = SplitDraftEdit { + id: OperationId::random(), + expected_revision: 1, + policy: SplitPolicy::default(), + splits: vec![], + }; + let expected = authority.edit_split_draft(&identity, split, &edit).await.unwrap(); + let writes = store.writes.load(Ordering::Relaxed); + for _ in 0..8 { + assert_eq!( + authority.edit_split_draft(&identity, split, &edit).await.unwrap(), + expected + ); + } + assert_eq!(store.writes.load(Ordering::Relaxed), writes); +} diff --git a/lib/crowdb-access-dataset/tests/session_cost_test.rs b/lib/crowdb-access-dataset/tests/session_cost_test.rs new file mode 100644 index 000000000..410126211 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/session_cost_test.rs @@ -0,0 +1,64 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/access_store.rs"] +mod access_store; +#[path = "common/publication.rs"] +mod publication; +use crowdb_access_dataset::*; +use std::sync::{atomic::Ordering, Arc}; + +#[tokio::test] +async fn concurrent_fetch_and_replay_never_write_or_renew_progress() { + let store = access_store::TestAccessStore::shared(); + let authority = Arc::new(DatasetAuthority::new(store.clone())); + let identity = DatasetIdentity::new(NamespacePath::root(), "read-cost").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + authority + .publish_snapshot(&identity, None, publication::publication_manifest(1)) + .await + .unwrap(); + let sessions = DatasetSessions::new(authority.clone()); + let spec = SessionSpec { + id: OperationId::random(), + snapshot: "latest".into(), + selection: Selection::Prefix(b"sample-".to_vec()), + fields: vec!["marker".into()], + batch_size: 1, + partition: SessionPartition { + worker: 0, + workers: 1, + rank: 0, + world_size: 1, + }, + shuffle: None, + }; + let opened = sessions.start(&identity, spec.clone(), 10).await.unwrap(); + let writes = store.writes.load(Ordering::Relaxed); + let batches = futures::future::join_all((0..96).map(|_| sessions.next(&identity, spec.id, 69))).await; + let token = &batches[0].as_ref().unwrap().next; + for batch in &batches { + assert_eq!(&batch.as_ref().unwrap().next, token); + } + assert_eq!(store.writes.load(Ordering::Relaxed), writes); + assert!(sessions.next(&identity, spec.id, 70).await.is_err()); + let restarted = DatasetSessions::new(authority); + restarted.resume(&identity, spec.id, 70).await.unwrap(); + assert_eq!(&restarted.next(&identity, spec.id, 71).await.unwrap().next, token); + let writes = store.writes.load(Ordering::Relaxed); + restarted + .acknowledge(&identity, spec.id, &opened.acknowledged, token, 71) + .await + .unwrap(); + assert_eq!(store.writes.load(Ordering::Relaxed), writes + 1); + restarted + .acknowledge(&identity, spec.id, &opened.acknowledged, token, 71) + .await + .unwrap(); + restarted.next(&identity, spec.id, 72).await.unwrap(); + assert_eq!(store.writes.load(Ordering::Relaxed), writes + 1); + restarted.close(&identity, spec.id).await.unwrap(); + let writes = store.writes.load(Ordering::Relaxed); + restarted.close(&identity, spec.id).await.unwrap(); + assert_eq!(store.writes.load(Ordering::Relaxed), writes); +} diff --git a/lib/crowdb-access-dataset/tests/upload_registry_test.rs b/lib/crowdb-access-dataset/tests/upload_registry_test.rs index 2223525eb..a9553bb93 100644 --- a/lib/crowdb-access-dataset/tests/upload_registry_test.rs +++ b/lib/crowdb-access-dataset/tests/upload_registry_test.rs @@ -1,65 +1,98 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. + +#[path = "common/access_store.rs"] +mod access_store; #[path = "common/upload.rs"] +#[allow(dead_code)] mod upload; -use async_trait::async_trait; use bytes::Bytes; -use crowdb_access_dataset::{ - CasOutcome, DatasetIdentity, DatasetStore, DatasetUploads, NamespacePath, OperationId, StoreError, - StoredValue, -}; +use crowdb_access_dataset::*; use md5::{Digest, Md5}; -use std::sync::{ - atomic::{AtomicBool, Ordering}, - Arc, -}; -use tokio::sync::Notify; -#[derive(Default)] -struct TestRegistryStore { - inner: upload::TestStore, - pause: AtomicBool, - entered: Notify, - resume: Notify, -} -#[async_trait] -impl DatasetStore for TestRegistryStore { - async fn get(&self, key: &[u8]) -> Result, StoreError> { - self.inner.get(key).await +use std::sync::{atomic::Ordering, Arc}; + +#[tokio::test] +async fn independent_uploads_need_one_create_write_and_no_shared_registry() { + let store = access_store::TestAccessStore::shared(); + let payloads = Arc::new(upload::TestPayload::default()); + let uploads = DatasetUploads::new(store.clone(), payloads.clone()); + let identity = DatasetIdentity::new(NamespacePath::root(), "expiry").unwrap(); + let keys: Vec<_> = (0..64) + .map(|_| DatasetUploads::location(&identity, OperationId::random()).unwrap()) + .collect(); + let results = futures::future::join_all( + keys.iter() + .map(|key| uploads.create(key, 4, 4, Md5::digest(b"data").into(), 10)), + ) + .await; + for result in results { + result.unwrap(); } - async fn compare_exchange( - &self, - key: &[u8], - expected: Option<&[u8]>, - value: &[u8], - ) -> Result { - if key.starts_with(b"DS\0UPAGE\0") { - let count = |bytes: &[u8]| { - serde_json::from_slice::(bytes).unwrap()["entries"] - .as_array() - .unwrap() - .len() - }; - if expected.is_some_and(|old| count(old) > count(value)) - && self.pause.swap(false, Ordering::AcqRel) - { - self.entered.notify_one(); - self.resume.notified().await; - } + assert_eq!(store.writes.load(Ordering::Relaxed), 64); + for key in [b"DS\0UTAIL\0".as_slice(), b"DS\0USCAN\0", b"DS\0UPAGE\0"] { + assert!(store.get(key).await.unwrap().is_none()); + } + uploads + .put_part(&keys[0], 0, Bytes::from_static(b"data"), 1) + .await + .unwrap(); + uploads.complete(&keys[0], 2).await.unwrap(); + let owner = OperationId::random(); + uploads.claim(&keys[0], owner, 3).await.unwrap(); + let restarted = DatasetUploads::new(store, payloads); + let mut after = None; + loop { + restarted.expire(20, &mut after).await.unwrap(); + if after.is_none() { + break; } - self.inner.compare_exchange(key, expected, value).await } - async fn delete(&self, key: &[u8]) -> Result<(), StoreError> { - self.inner.delete(key).await + assert_eq!(restarted.get(&keys[0]).await.unwrap().state, UploadState::Claimed); + for key in &keys[1..] { + assert!(restarted.get(key).await.unwrap().cleanup_complete); + } + restarted.release(&keys[0], owner).await.unwrap(); + loop { + restarted.expire(20, &mut after).await.unwrap(); + if after.is_none() { + break; + } } + assert!(restarted.get(&keys[0]).await.unwrap().cleanup_complete); } + #[tokio::test] -async fn untracking_old_registration_cannot_remove_a_new_cleanup_registration() { - let store = Arc::new(TestRegistryStore::default()); - let uploads = Arc::new(DatasetUploads::new( - store.clone(), - Arc::new(upload::TestPayload::default()), - )); - let identity = DatasetIdentity::new(NamespacePath::root(), "registry").unwrap(); +async fn legacy_registry_fields_are_unnecessary_for_expiry_after_restart() { + let store = access_store::TestAccessStore::shared(); + let payloads = Arc::new(upload::TestPayload::default()); + let uploads = DatasetUploads::new(store.clone(), payloads.clone()); + let identity = DatasetIdentity::new(NamespacePath::root(), "legacy-registry").unwrap(); + let location = DatasetUploads::location(&identity, OperationId::random()).unwrap(); + uploads.create(&location, 4, 4, [0; 16], 10).await.unwrap(); + let stored = store.get(&location).await.unwrap().unwrap(); + let mut old: serde_json::Value = serde_json::from_slice(&stored.bytes).unwrap(); + old["version"] = 2.into(); + old["registry_page"] = 42.into(); + old["registry_generation"] = serde_json::Value::Null; + old["registry_fence"] = serde_json::Value::Null; + store + .compare_exchange(&location, Some(&stored.bytes), &serde_json::to_vec(&old).unwrap()) + .await + .unwrap(); + let restarted = DatasetUploads::new(store.clone(), payloads); + restarted.expire(20, &mut None).await.unwrap(); + assert!(restarted.get(&location).await.unwrap().cleanup_complete); + let writes = store.writes.load(Ordering::Relaxed); + restarted.abort(&location).await.unwrap(); + restarted.expire(20, &mut None).await.unwrap(); + assert_eq!(store.writes.load(Ordering::Relaxed), writes); +} + +#[tokio::test] +async fn owner_retries_rebase_on_observations_without_another_get() { + let store = access_store::TestAccessStore::shared(); + let uploads = DatasetUploads::new(store.clone(), Arc::new(upload::TestPayload::default())); + let identity = DatasetIdentity::new(NamespacePath::root(), "owner-retry").unwrap(); let location = DatasetUploads::location(&identity, OperationId::random()).unwrap(); uploads .create(&location, 4, 4, Md5::digest(b"data").into(), 10) @@ -71,15 +104,17 @@ async fn untracking_old_registration_cannot_remove_a_new_cleanup_registration() .unwrap(); uploads.complete(&location, 2).await.unwrap(); let owner = OperationId::random(); - uploads.claim(&location, owner, 3).await.unwrap(); - store.pause.store(true, Ordering::Release); - let old = uploads.clone(); - let key = location.clone(); - let untrack = tokio::spawn(async move { old.untrack(&key).await }); - store.entered.notified().await; - uploads.release(&location, owner).await.unwrap(); - store.resume.notify_one(); - untrack.await.unwrap().unwrap(); - uploads.expire(20).await.unwrap(); - assert!(uploads.get(&location).await.unwrap().cleanup_complete); + for action in 0..3 { + store.upload_conflicts.store(3, Ordering::Relaxed); + let reads = store.reads.load(Ordering::Relaxed); + match action { + 0 => { + uploads.claim(&location, owner, 3).await.unwrap(); + } + 1 => uploads.release(&location, owner).await.unwrap(), + _ => uploads.retire_owner(&location, owner).await.unwrap(), + } + assert_eq!(store.upload_conflicts.load(Ordering::Relaxed), 0); + assert_eq!(store.reads.load(Ordering::Relaxed), reads + 1); + } } diff --git a/lib/crowdb-access-dataset/tests/upload_test.rs b/lib/crowdb-access-dataset/tests/upload_test.rs index 941942904..230b6bc89 100644 --- a/lib/crowdb-access-dataset/tests/upload_test.rs +++ b/lib/crowdb-access-dataset/tests/upload_test.rs @@ -73,7 +73,6 @@ async fn independent_owners_and_deferred_cleanup_survive_restart() { let second = OperationId::random(); uploads.claim(&key, first, 3).await.unwrap(); uploads.claim(&key, second, 3).await.unwrap(); - uploads.untrack(&key).await.unwrap(); assert!(matches!(uploads.abort(&key).await, Err(UploadError::Conflict))); uploads.release(&key, first).await.unwrap(); assert_eq!(uploads.read_part(&key, 0).await.unwrap(), b"data".as_slice()); @@ -84,7 +83,7 @@ async fn independent_owners_and_deferred_cleanup_survive_restart() { assert!(!aborted.cleanup_complete); let restarted = DatasetUploads::new(store, payloads.clone()); payloads.defer_reclaim.store(false, Ordering::Release); - restarted.expire(20).await.unwrap(); + restarted.expire(20, &mut None).await.unwrap(); assert!(restarted.get(&key).await.unwrap().cleanup_complete); } #[tokio::test] @@ -104,7 +103,7 @@ async fn invalid_length_digest_and_expiry_do_not_publish() { uploads.complete(&key, 2).await, Err(UploadError::Integrity) )); - uploads.expire(11).await.unwrap(); + uploads.expire(11, &mut None).await.unwrap(); assert_eq!(uploads.get(&key).await.unwrap().state, UploadState::Aborted); } @@ -189,7 +188,7 @@ async fn retired_owner_cannot_reclaim_shared_upload_after_restart() { .put_part(&key, 0, Bytes::from_static(b"data"), 1) .await .unwrap(); - assert_eq!(uploads.complete(&key, 2).await.unwrap().version, 2); + assert_eq!(uploads.complete(&key, 2).await.unwrap().version, 4); let first = OperationId::random(); let second = OperationId::random(); uploads.claim(&key, first, 3).await.unwrap(); @@ -202,7 +201,7 @@ async fn retired_owner_cannot_reclaim_shared_upload_after_restart() { Err(UploadError::Conflict) )); let restarted = DatasetUploads::new(store, payloads); - assert_eq!(restarted.metadata(&key).await.unwrap().version, 3); + assert_eq!(restarted.metadata(&key).await.unwrap().version, 4); assert!(matches!( restarted.claim(&key, first, 4).await, Err(UploadError::Conflict) diff --git a/lib/crowdb-access-s3/src/metadata/multipart_repository.rs b/lib/crowdb-access-s3/src/metadata/multipart_repository.rs index da625fa1a..5fd608f41 100644 --- a/lib/crowdb-access-s3/src/metadata/multipart_repository.rs +++ b/lib/crowdb-access-s3/src/metadata/multipart_repository.rs @@ -161,9 +161,13 @@ impl MultipartRepository { let key = self.session_key(previous); let expected = previous.encode()?; let value = next.encode()?; - match self.store.compare_exchange(key, expected, value).await { - Ok(true) => Ok(true), - Ok(false) => Ok(self.load(next).await?.as_ref() == Some(next)), + match self + .store + .compare_exchange_observed(key, expected, value.clone()) + .await + { + Ok((true, _)) => Ok(true), + Ok((false, observed)) => Ok(observed.is_some_and(|observed| observed.value == value)), Err(error) => { if self.load(next).await?.as_ref() == Some(next) { Ok(true) diff --git a/lib/crowdb-access-s3/src/metadata/multipart_repository/parts.rs b/lib/crowdb-access-s3/src/metadata/multipart_repository/parts.rs index 367571382..9366a93a8 100644 --- a/lib/crowdb-access-s3/src/metadata/multipart_repository/parts.rs +++ b/lib/crowdb-access-s3/src/metadata/multipart_repository/parts.rs @@ -33,11 +33,11 @@ impl MultipartRepository { part: &MultipartPartRecord, now_ms: u64, ) -> Result, MultipartRepositoryError> { + let mut current = self + .load(session) + .await? + .ok_or(MultipartRepositoryError::Conflict)?; loop { - let current = self - .load(session) - .await? - .ok_or(MultipartRepositoryError::Conflict)?; if current.pending.is_some() { self.settle_pending_part(¤t).await?; } else if let Some(saved) = self.reserve_stream_part(¤t, part, now_ms).await? { @@ -50,6 +50,7 @@ impl MultipartRepository { if latest.revision == current.revision { return Ok(None); } + current = latest; // Another publication advanced the durable fence. Re-evaluate our // part against that state without uploading its bytes again. } @@ -238,24 +239,22 @@ impl MultipartRepository { let mutation = match current { Some(before) => { self.store - .compare_exchange(key.clone(), before.encode()?, value.clone()) + .compare_exchange_observed(key.clone(), before.encode()?, value.clone()) .await } None => self .store .put_if_absent(key.clone(), value.clone()) .await - .map(|outcome| matches!(outcome, PutIfAbsentOutcome::Inserted { .. })), + .map(|outcome| match outcome { + PutIfAbsentOutcome::Inserted { .. } => (true, None), + PutIfAbsentOutcome::Existing(value) => (false, Some(value)), + }), }; match mutation { - Ok(true) => Ok(()), - Ok(false) => { - if self - .store - .get(key) - .await? - .is_some_and(|entry| entry.value == value) - { + Ok((true, _)) => Ok(()), + Ok((false, observed)) => { + if observed.is_some_and(|observed| observed.value == value) { Ok(()) } else { Err(MultipartRepositoryError::InvalidPart) diff --git a/lib/crowdb-access-s3/src/metadata/store.rs b/lib/crowdb-access-s3/src/metadata/store.rs index df583ac4d..5cb4ca202 100644 --- a/lib/crowdb-access-s3/src/metadata/store.rs +++ b/lib/crowdb-access-s3/src/metadata/store.rs @@ -137,12 +137,24 @@ impl ChunkKvMetadataStore { expected: Vec, value: Vec, ) -> Result { - let (applied, _, _) = mutation_result( + Ok(self.compare_exchange_observed(key, expected, value).await?.0) + } + + /// Preserves the sequencer's conflict observation for retry and replay checks. + /// # Errors + /// Returns a client, server, or response-shape failure. + pub async fn compare_exchange_observed( + &self, + key: Vec, + expected: Vec, + value: Vec, + ) -> Result<(bool, Option), MetadataStoreError> { + let (applied, _, observed) = mutation_result( self.client .compare_exchange(key, RpcCompareCondition::Value(expected), value) .await?, )?; - Ok(applied) + Ok((applied, observed)) } /// Checks whether a bucket currently contains any visible object. From 7d076354f96676f69720bff74b29a99ab7ec6d16 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 08:41:55 +0800 Subject: [PATCH 091/258] Replace persisted dataset read sessions with caller continuations --- Cargo.lock | 1 + app/crowdb-access-server/src/config.rs | 5 - app/crowdb-access-server/src/dataset.rs | 4 +- app/crowdb-access-server/src/dataset/api.rs | 6 +- .../src/dataset/api/batch.rs | 32 +- .../src/dataset/api/{session.rs => query.rs} | 112 ++-- .../src/dataset/api/v2.rs | 6 +- .../src/dataset/api/v2/preview.rs | 6 +- .../src/dataset/api/v2/read_models.rs | 70 +- .../src/dataset/api/v2/reads.rs | 87 +-- .../src/dataset/legacy.rs | 19 +- .../tests/common/dataset.rs | 1 - .../tests/dataset_read_query_contract_test.rs | 101 +++ .../dataset_split_session_contract_test.rs | 120 ---- .../tests/dataset_v2_contract_test.rs | 2 +- .../templates/access.toml | 1 - doc/backlog/R220-dataset-fat-client-plan.md | 19 +- .../dataset/design-crowdb-access-dataset.md | 77 ++- .../dataset/design-dataset-control.md | 5 +- .../dataset/openapi-crowdb-dataset-v1.yaml | 452 ++----------- .../dataset/openapi-crowdb-dataset-v2.yaml | 141 +--- .../dataset/spec-crowdb-dataset.md | 35 +- .../access-server/design-crowdb-native-sdk.md | 14 +- doc/working/plan-access-simplification.md | 607 +++++++++++++++++- doc/working/plan-dataset-native-sdk.md | 23 +- lib/crowdb-access-dataset/Cargo.toml | 1 + lib/crowdb-access-dataset/src/authority.rs | 48 -- lib/crowdb-access-dataset/src/client.rs | 82 +-- lib/crowdb-access-dataset/src/key.rs | 8 - lib/crowdb-access-dataset/src/lease.rs | 42 -- lib/crowdb-access-dataset/src/lib.rs | 13 +- lib/crowdb-access-dataset/src/read_query.rs | 140 ++++ .../src/read_query/binding.rs | 135 ++++ .../src/read_query/page.rs | 169 +++++ lib/crowdb-access-dataset/src/read_session.rs | 142 ---- .../src/read_session/pages.rs | 130 ---- .../src/read_session/planning.rs | 200 ------ .../src/read_session/progress.rs | 182 ------ .../src/read_session/split.rs | 37 -- .../src/read_session/split/binding.rs | 170 ----- .../src/read_session/split/finish.rs | 148 ----- .../src/read_session/split/planning.rs | 214 ------ .../src/read_session/split/state.rs | 171 ----- lib/crowdb-access-dataset/src/retention.rs | 90 +-- lib/crowdb-access-dataset/src/transport.rs | 80 +-- .../tests/common/read_query.rs | 43 ++ .../tests/common/split_session.rs | 63 -- lib/crowdb-access-dataset/tests/core_test.rs | 46 +- .../tests/read_query_cost_test.rs | 90 +++ ...edges_test.rs => read_query_edges_test.rs} | 83 +-- .../tests/read_query_test.rs | 141 ++++ .../tests/read_session_test.rs | 157 ----- .../tests/retention_test.rs | 26 +- .../tests/session_cost_test.rs | 64 -- .../tests/split_read_query_test.rs | 92 +++ .../tests/split_session_test.rs | 141 ---- .../tests/transport_test.rs | 5 +- lib/crowdb-sdk-native-python/README.md | 12 +- lib/crowdb-sdk-native-python/src/client.rs | 7 +- lib/crowdb-sdk-native-python/src/control.rs | 25 +- .../tests/real_consumer.py | 57 +- lib/crowdb-sdk-native/README.md | 12 +- lib/crowdb-sdk-native/src/dataset.rs | 2 +- lib/crowdb-sdk-native/src/dataset/query.rs | 19 + lib/crowdb-sdk-native/src/dataset/read.rs | 40 +- .../src/dataset/read/admission.rs | 43 -- .../src/dataset/read/body.rs | 9 +- lib/crowdb-sdk-native/src/dataset/session.rs | 68 -- .../tests/dataset_read_test.rs | 70 +- sdk/README.md | 45 +- .../crowdb-sdk-dataset/examples/payload.rs | 22 +- .../examples/split_control.rs | 54 +- sdk/crates/crowdb-sdk-dataset/src/lib.rs | 2 +- sdk/crates/crowdb-sdk-dataset/src/models.rs | 17 +- sdk/crates/crowdb-sdk-dataset/src/query.rs | 53 ++ sdk/crates/crowdb-sdk-dataset/src/session.rs | 70 -- sdk/crates/crowdb-sdk-dataset/src/v2/read.rs | 19 +- .../crowdb-sdk-dataset/src/v2/requests.rs | 63 +- sdk/crates/crowdb-sdk-python/src/control.rs | 6 +- sdk/crates/crowdb-sdk-python/src/operation.rs | 11 +- sdk/crates/crowdb-sdk-python/src/stream.rs | 2 +- sdk/crowdb_dataset/client.py | 12 +- sdk/crowdb_dataset/control.py | 12 +- sdk/crowdb_dataset/iterable.py | 34 +- sdk/test_iterable.py | 20 +- sdk/tests/real_consumer.py | 20 +- sdk/tests/split_consumer.py | 18 +- 87 files changed, 2117 insertions(+), 3826 deletions(-) rename app/crowdb-access-server/src/dataset/api/{session.rs => query.rs} (56%) create mode 100644 app/crowdb-access-server/tests/dataset_read_query_contract_test.rs delete mode 100644 app/crowdb-access-server/tests/dataset_split_session_contract_test.rs delete mode 100644 lib/crowdb-access-dataset/src/lease.rs create mode 100644 lib/crowdb-access-dataset/src/read_query.rs create mode 100644 lib/crowdb-access-dataset/src/read_query/binding.rs create mode 100644 lib/crowdb-access-dataset/src/read_query/page.rs delete mode 100644 lib/crowdb-access-dataset/src/read_session.rs delete mode 100644 lib/crowdb-access-dataset/src/read_session/pages.rs delete mode 100644 lib/crowdb-access-dataset/src/read_session/planning.rs delete mode 100644 lib/crowdb-access-dataset/src/read_session/progress.rs delete mode 100644 lib/crowdb-access-dataset/src/read_session/split.rs delete mode 100644 lib/crowdb-access-dataset/src/read_session/split/binding.rs delete mode 100644 lib/crowdb-access-dataset/src/read_session/split/finish.rs delete mode 100644 lib/crowdb-access-dataset/src/read_session/split/planning.rs delete mode 100644 lib/crowdb-access-dataset/src/read_session/split/state.rs create mode 100644 lib/crowdb-access-dataset/tests/common/read_query.rs delete mode 100644 lib/crowdb-access-dataset/tests/common/split_session.rs create mode 100644 lib/crowdb-access-dataset/tests/read_query_cost_test.rs rename lib/crowdb-access-dataset/tests/{split_session_edges_test.rs => read_query_edges_test.rs} (54%) create mode 100644 lib/crowdb-access-dataset/tests/read_query_test.rs delete mode 100644 lib/crowdb-access-dataset/tests/read_session_test.rs delete mode 100644 lib/crowdb-access-dataset/tests/session_cost_test.rs create mode 100644 lib/crowdb-access-dataset/tests/split_read_query_test.rs delete mode 100644 lib/crowdb-access-dataset/tests/split_session_test.rs create mode 100644 lib/crowdb-sdk-native/src/dataset/query.rs delete mode 100644 lib/crowdb-sdk-native/src/dataset/read/admission.rs delete mode 100644 lib/crowdb-sdk-native/src/dataset/session.rs create mode 100644 sdk/crates/crowdb-sdk-dataset/src/query.rs delete mode 100644 sdk/crates/crowdb-sdk-dataset/src/session.rs diff --git a/Cargo.lock b/Cargo.lock index acd0b1e6d..6a6401f36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -613,6 +613,7 @@ version = "0.3.0" dependencies = [ "arc-swap", "async-trait", + "base64", "bincode", "bytes", "crowdb-access-dataset", diff --git a/app/crowdb-access-server/src/config.rs b/app/crowdb-access-server/src/config.rs index c37c7bd35..524b5901e 100644 --- a/app/crowdb-access-server/src/config.rs +++ b/app/crowdb-access-server/src/config.rs @@ -251,7 +251,6 @@ pub struct DatasetConfig { pub max_in_flight: usize, pub max_batch_size: usize, pub request_timeout_seconds: u64, - pub lease_ttl_seconds: u64, } impl Default for DatasetConfig { @@ -261,7 +260,6 @@ impl Default for DatasetConfig { max_in_flight: 8, max_batch_size: 10_000, request_timeout_seconds: 60, - lease_ttl_seconds: 60, } } } @@ -438,9 +436,6 @@ impl AccessConfig { if self.dataset.request_timeout_seconds == 0 || self.dataset.request_timeout_seconds > 300 { return Err("dataset.request_timeout_seconds must be between 1 and 300".into()); } - if self.dataset.lease_ttl_seconds == 0 || self.dataset.lease_ttl_seconds > 3600 { - return Err("dataset.lease_ttl_seconds must be between 1 and 3600".into()); - } Ok(()) } diff --git a/app/crowdb-access-server/src/dataset.rs b/app/crowdb-access-server/src/dataset.rs index 7e9a8de45..78d5f419b 100644 --- a/app/crowdb-access-server/src/dataset.rs +++ b/app/crowdb-access-server/src/dataset.rs @@ -191,12 +191,10 @@ pub async fn run_with_shutdown( crowdb_access_dataset::NamespacePath::root(), std::env::var("CROWDB_DATASET_NAME").unwrap_or_else(|_| "default".into()), )?; - let reads = Arc::new(DatasetReadService::with_ttl( + let reads = Arc::new(DatasetReadService::new( Arc::clone(&authority), identity, access.dataset.max_in_flight, - unix_seconds(), - access.dataset.lease_ttl_seconds, )?); let mut service = DatasetHttpService::new(reads, Duration::from_secs(access.dataset.request_timeout_seconds)) diff --git a/app/crowdb-access-server/src/dataset/api.rs b/app/crowdb-access-server/src/dataset/api.rs index b67932f25..bf9b2cf18 100644 --- a/app/crowdb-access-server/src/dataset/api.rs +++ b/app/crowdb-access-server/src/dataset/api.rs @@ -9,7 +9,7 @@ mod conversion; mod error; mod management; mod publication; -mod session; +mod query; mod upload; mod v2; @@ -110,8 +110,8 @@ impl DatasetHttpService { let tail: Vec<&str> = parts[7..].iter().map(String::as_str).collect(); let method = request.method().clone(); let authority = self.reads.authority(); - if tail.first() == Some(&"read-plans") { - return self.session_route(&identity, &tail, request).await; + if tail.first() == Some(&"reads") && tail.len() > 1 { + return self.query_route(&identity, &tail, request).await; } if tail.first() == Some(&"uploads") { return self.upload_route(&identity, &tail, request).await; diff --git a/app/crowdb-access-server/src/dataset/api/batch.rs b/app/crowdb-access-server/src/dataset/api/batch.rs index c8e20d8c3..95a3f2ea6 100644 --- a/app/crowdb-access-server/src/dataset/api/batch.rs +++ b/app/crowdb-access-server/src/dataset/api/batch.rs @@ -69,7 +69,7 @@ impl DatasetHttpService { .0 .clone(); let request: BatchRequest = read_json(request).await?; - self.batch(identity, request, None, true, None, request_id).await + self.batch(identity, request, None, true, request_id).await } pub(super) async fn batch( &self, @@ -77,13 +77,12 @@ impl DatasetHttpService { request: BatchRequest, next: Option, end: bool, - session: Option, request_id: String, ) -> ApiResult { let id = snapshot(&request.snapshot)?; let ids = request.ids()?; let authority = self.reads.authority(); - let admission = ReadAdmission::acquire(authority.clone(), identity, id, session).await?; + authority.check_snapshot_readable(identity, id).await?; let samples = if ids.is_empty() { Vec::new() } else if request.sample_id_limit.is_some() { @@ -126,7 +125,6 @@ impl DatasetHttpService { part: 0, pending: Some(Bytes::from(prefix)), finished: false, - admission, }; Ok(Response::builder() .header("content-type", "application/vnd.crowdb.dataset-batch.v1") @@ -170,26 +168,6 @@ fn make_header( end, } } -pub(super) struct ReadAdmission { - authority: Arc, - identity: DatasetIdentity, - session: Option, -} -impl ReadAdmission { - pub(super) async fn acquire( - authority: Arc, - identity: &DatasetIdentity, - snapshot: SnapshotId, - session: Option, - ) -> Result, ApiError> { - authority.check_snapshot_readable(identity, snapshot).await?; - Ok(Arc::new(Self { - authority, - identity: identity.clone(), - session, - })) - } -} struct BatchState { uploads: Option>, request_id: String, @@ -198,7 +176,6 @@ struct BatchState { part: u32, pending: Option, finished: bool, - admission: Arc, } fn stream_body(state: BatchState) -> DatasetBody { let frames = stream::unfold(state, |mut state| async move { @@ -220,11 +197,6 @@ fn stream_body(state: BatchState) -> DatasetBody { } impl BatchState { async fn next(&mut self) -> Result { - if let Some(session) = self.admission.session { - crowdb_access_dataset::DatasetSessions::new(self.admission.authority.clone()) - .check_open(&self.admission.identity, session) - .await?; - } if let Some(bytes) = self.pending.take() { return Ok(bytes); } diff --git a/app/crowdb-access-server/src/dataset/api/session.rs b/app/crowdb-access-server/src/dataset/api/query.rs similarity index 56% rename from app/crowdb-access-server/src/dataset/api/session.rs rename to app/crowdb-access-server/src/dataset/api/query.rs index c8df7077f..ed59ea4dc 100644 --- a/app/crowdb-access-server/src/dataset/api/session.rs +++ b/app/crowdb-access-server/src/dataset/api/query.rs @@ -1,28 +1,26 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{batch::BatchRequest, error::ApiError, json, operation, read_body, read_json, ApiResult}; -use crate::dataset::{unix_seconds, DatasetHttpService}; +use super::{batch::BatchRequest, json, read_json, ApiError, ApiResult, DatasetHttpService}; use base64::{engine::general_purpose::STANDARD, Engine}; use crowdb_access_dataset::{ - DatasetIdentity, DatasetSessions, Selection, SessionPartition, SessionShuffle, SessionSpec, + DatasetIdentity, DatasetReads, ReadCancellation, ReadPartition, ReadShuffle, ReadSpec, Selection, }; use hyper::{body::Incoming, Method, Request}; use serde::Deserialize; #[derive(Deserialize)] struct Plan { - id: String, snapshot: String, selection: Predicate, fields: Vec, batch_size: usize, - partition: SessionPartition, + partition: ReadPartition, #[serde(deserialize_with = "required_shuffle")] - shuffle: Option, + shuffle: Option, } -fn required_shuffle<'de, D: serde::Deserializer<'de>>(d: D) -> Result, D::Error> { - Option::::deserialize(d) +fn required_shuffle<'de, D: serde::Deserializer<'de>>(d: D) -> Result, D::Error> { + Option::::deserialize(d) } #[derive(Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] @@ -83,77 +81,69 @@ impl Predicate { } } #[derive(Deserialize)] -struct Acknowledge { - expected: String, - next: String, +#[serde(deny_unknown_fields)] +pub(super) struct NextRead { + pub cursor: String, } impl DatasetHttpService { - pub(super) async fn session_route( + pub(super) async fn query_route( &self, identity: &DatasetIdentity, tail: &[&str], request: Request, ) -> ApiResult { - let request_id = request - .extensions() - .get::() - .ok_or_else(ApiError::integrity)? - .0 - .clone(); - let sessions = DatasetSessions::new(self.reads.authority()); - let now = unix_seconds(); - match (request.method().clone(), tail) { - (Method::POST, ["read-plans"]) => { + if request.method() != Method::POST { + return Err(ApiError::not_found()); + } + match tail { + ["reads", "bind"] => { let p: Plan = read_json(request).await?; - let spec = SessionSpec { - id: operation(&p.id)?, + let spec = ReadSpec { snapshot: p.snapshot, selection: p.selection.native(0, &mut 1024)?, fields: p.fields, batch_size: p.batch_size, partition: p.partition, shuffle: p.shuffle, + split: None, }; - json(&sessions.start(identity, spec, now).await?) - } - (Method::GET, ["read-plans", id]) => json(&sessions.resume(identity, operation(id)?, now).await?), - (Method::DELETE, ["read-plans", id]) => { - read_body(request, false).await?; - json(&sessions.close(identity, operation(id)?).await?) - } - (Method::POST, ["read-plans", id, "next"]) => { - read_body(request, false).await?; - let batch = sessions.next(identity, operation(id)?, now).await?; - self.batch( - identity, - BatchRequest { - snapshot: batch.snapshot.to_string(), - sample_ids: batch.ids.iter().map(|id| STANDARD.encode(id)).collect(), - fields: batch.fields, - sample_id_limit: None, - }, - Some(batch.next), - batch.end, - Some(operation(id)?), - request_id, - ) - .await - } - (Method::PUT, ["read-plans", id, "progress"]) => { - let ack: Acknowledge = read_json(request).await?; - if ack.expected.len() > 4096 || ack.next.len() > 4096 { - return Err(ApiError::too_large()); - } - let r = sessions - .acknowledge(identity, operation(id)?, &ack.expected, &ack.next, now) + let binding = DatasetReads::new(self.reads.authority()) + .bind(identity, spec) .await?; - json(&serde_json::json!({"plan":r.plan,"cursor":r.acknowledged})) - } - (Method::GET, ["read-plans", id, "progress"]) => { - let r = sessions.resume(identity, operation(id)?, now).await?; - json(&serde_json::json!({"plan":r.plan,"cursor":r.acknowledged})) + json(&serde_json::json!({"snapshot":binding.snapshot.to_string(), "cursor":binding.cursor})) } + ["reads", "next"] => self.query_next(identity, request).await, _ => Err(ApiError::not_found()), } } + pub(super) async fn query_next( + &self, + identity: &DatasetIdentity, + request: Request, + ) -> ApiResult { + let request_id = request + .extensions() + .get::() + .ok_or_else(ApiError::integrity)? + .0 + .clone(); + let input: NextRead = read_json(request).await?; + let page = DatasetReads::new(self.reads.authority()) + .next(identity, &input.cursor, &ReadCancellation::new()) + .await?; + let end = page.next.is_none(); + self.batch( + identity, + BatchRequest { + snapshot: page.snapshot.to_string(), + sample_ids: page.ids.iter().map(|id| STANDARD.encode(id)).collect(), + fields: page.fields, + sample_id_limit: Some(16 * 1024), + }, + page.next, + end, + request_id, + ) + .await + } } diff --git a/app/crowdb-access-server/src/dataset/api/v2.rs b/app/crowdb-access-server/src/dataset/api/v2.rs index f5dbdcf6f..9cd31221e 100644 --- a/app/crowdb-access-server/src/dataset/api/v2.rs +++ b/app/crowdb-access-server/src/dataset/api/v2.rs @@ -53,7 +53,7 @@ impl DatasetHttpService { "inspection-preparation", "dataset-management", "split-revisions", - "split-reads", + "stateless-reads", ]; if self.uploads.is_some() { capabilities.extend(["uploads", "composition", "snapshot-reclaim"]); @@ -111,8 +111,8 @@ impl DatasetHttpService { if tail.first() == Some(&"compositions") { return self.v2_compositions(&identity, &tail, request).await; } - if tail.first() == Some(&"read-plans") { - return self.v2_read_session(&identity, &tail, request).await; + if tail.first() == Some(&"reads") { + return self.v2_read_query(&identity, &tail, request).await; } if matches!( tail.as_slice(), diff --git a/app/crowdb-access-server/src/dataset/api/v2/preview.rs b/app/crowdb-access-server/src/dataset/api/v2/preview.rs index 8dc1cfd02..642f6439a 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/preview.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/preview.rs @@ -1,9 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::super::{ - batch::ReadAdmission, json, read_body, read_json, ApiError, ApiResult, DatasetHttpService, -}; +use super::super::{json, read_body, read_json, ApiError, ApiResult, DatasetHttpService}; use super::{ inspection_models::Sample, query::{bytes, Query}, @@ -75,7 +73,7 @@ impl DatasetHttpService { } read_body(request, false).await?; let authority = self.reads.authority(); - let _admission = ReadAdmission::acquire(authority.clone(), identity, snapshot, None).await?; + authority.check_snapshot_readable(identity, snapshot).await?; let mut rows = authority .resolve_inspection_batch_locators( identity, diff --git a/app/crowdb-access-server/src/dataset/api/v2/read_models.rs b/app/crowdb-access-server/src/dataset/api/v2/read_models.rs index 32b7a45aa..54115022d 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/read_models.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/read_models.rs @@ -2,21 +2,17 @@ // Licensed under the Apache License, Version 2.0. use super::{ - super::{operation, snapshot, ApiError}, + super::{snapshot, ApiError}, predicate::Predicate, split_models::{decimal, required_option, revision, Source}, }; -use crowdb_access_dataset::{ - split::SplitName, ReadPreparation, ReadSession, SessionPartition, SessionShuffle, SessionSpec, - SessionSpecV2, SessionSplit, -}; +use crowdb_access_dataset::{split::SplitName, ReadBinding, ReadPartition, ReadShuffle, ReadSpec, ReadSplit}; use serde::Deserialize; use serde_json::{json, Value}; #[derive(Deserialize)] #[serde(deny_unknown_fields)] pub(super) struct Plan { - pub id: String, pub snapshot: String, pub selection: Predicate, pub fields: Vec, @@ -26,7 +22,6 @@ pub(super) struct Plan { pub shuffle: Option, #[serde(deserialize_with = "required_split")] pub split: Option, - pub expected_page: String, } #[derive(Deserialize)] #[serde(deny_unknown_fields)] @@ -56,55 +51,42 @@ fn required_split<'de, D: serde::Deserializer<'de>>(d: D) -> Result Result<(SessionSpecV2, u64), ApiError> { + pub(super) fn native(self) -> Result { if self.snapshot != "latest" && self.snapshot != "stable" { snapshot(&self.snapshot)?; } - let spec = SessionSpecV2 { - read: SessionSpec { - id: operation(&self.id)?, - snapshot: self.snapshot, - selection: self.selection.native()?, - fields: self.fields, - batch_size: self.batch_size, - partition: SessionPartition { - worker: self.partition.worker, - workers: self.partition.workers, - rank: self.partition.rank, - world_size: self.partition.world_size, - }, - shuffle: self - .shuffle - .map(|s| { - Ok::<_, ApiError>(SessionShuffle { - seed: decimal(&s.seed)?, - group_size: s.group_size, - }) - }) - .transpose()?, + Ok(ReadSpec { + snapshot: self.snapshot, + selection: self.selection.native()?, + fields: self.fields, + batch_size: self.batch_size, + partition: ReadPartition { + worker: self.partition.worker, + workers: self.partition.workers, + rank: self.partition.rank, + world_size: self.partition.world_size, }, + shuffle: self + .shuffle + .map(|s| { + Ok::<_, ApiError>(ReadShuffle { + seed: decimal(&s.seed)?, + group_size: s.group_size, + }) + }) + .transpose()?, split: self .split .map(|s| { - Ok::<_, ApiError>(SessionSplit { + Ok::<_, ApiError>(ReadSplit { revision: s.revision.as_deref().map(revision).transpose()?, name: SplitName::new(s.name).map_err(|_| ApiError::invalid())?, }) }) .transpose()?, - }; - Ok((spec, decimal(&self.expected_page)?)) + }) } } -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct Acknowledge { - pub expected: String, - pub next: String, -} -pub(super) fn session(s: &ReadSession) -> Value { - json!({"id":s.id, "snapshot":s.snapshot, "plan":s.plan, "acknowledged":s.acknowledged, "expires_at":s.expires_at.to_string(), "closed":s.closed}) -} -pub(super) fn preparation(p: ReadPreparation) -> Value { - json!({"snapshot":p.snapshot.to_string(), "split":p.split.map(Source::from), "next_page":p.next_page.to_string(), "admitted":p.admitted.to_string(), "complete":p.complete, "session":p.session.as_ref().map(session)}) +pub(super) fn binding(value: ReadBinding) -> Value { + json!({"snapshot":value.snapshot.to_string(), "split":value.split.map(Source::from), "cursor":value.cursor}) } diff --git a/app/crowdb-access-server/src/dataset/api/v2/reads.rs b/app/crowdb-access-server/src/dataset/api/v2/reads.rs index f0df35d9b..b6327edd0 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/reads.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/reads.rs @@ -1,92 +1,31 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::super::{ - batch::BatchRequest, json, operation, read_body, read_json, ApiError, ApiResult, DatasetHttpService, - RequestId, -}; +use super::super::{json, read_json, ApiError, ApiResult, DatasetHttpService}; use super::{query::Query, read_models as models}; -use crate::dataset::unix_seconds; -use base64::{engine::general_purpose::STANDARD, Engine}; -use crowdb_access_dataset::{DatasetIdentity, DatasetSessions, ReadCancellation}; +use crowdb_access_dataset::{DatasetIdentity, DatasetReads}; use hyper::{body::Incoming, Method, Request}; - impl DatasetHttpService { - pub(super) async fn v2_read_session( + pub(super) async fn v2_read_query( &self, identity: &DatasetIdentity, tail: &[&str], request: Request, ) -> ApiResult { Query::parse(request.uri().query(), &[])?; - let sessions = DatasetSessions::new(self.reads.authority()); - let now = unix_seconds(); - let cancel = ReadCancellation::new(); - if request.method() == Method::POST && tail == ["read-plans"] { - let input: models::Plan = read_json(request).await?; - let (spec, page) = input.native()?; - return json(&models::preparation( - sessions - .prepare_split_read(identity, &spec, page, now, &cancel) - .await?, - )); - } - let ["read-plans", id, rest @ ..] = tail else { + if request.method() != Method::POST { return Err(ApiError::not_found()); - }; - let id = operation(id)?; - if request.method() == Method::DELETE && rest.is_empty() { - read_body(request, false).await?; - return json(&models::session(&sessions.cancel_split_read(identity, id).await?)); } - let status = sessions.split_read_status(identity, id, now, &cancel).await?; - match (request.method().clone(), rest) { - (Method::GET, []) => { - read_body(request, false).await?; - json(&models::preparation(status)) - } - (Method::GET, ["progress"]) => { - read_body(request, false).await?; - let session = status.session.ok_or_else(ApiError::conflict)?; - json(&serde_json::json!({"plan":session.plan, "cursor":session.acknowledged})) - } - (Method::PUT, ["progress"]) => { - let ack: models::Acknowledge = read_json(request).await?; - if ack.expected.len() > 4096 || ack.next.len() > 4096 { - return Err(ApiError::too_large()); - } - let session = sessions - .acknowledge(identity, id, &ack.expected, &ack.next, now) - .await?; - json(&serde_json::json!({"plan":session.plan,"cursor":session.acknowledged})) - } - (Method::POST, ["next"]) => { - let request_id = request - .extensions() - .get::() - .ok_or_else(ApiError::integrity)? - .0 - .clone(); - read_body(request, false).await?; - if !status.complete { - return Err(ApiError::conflict()); - } - let batch = sessions.next(identity, id, now).await?; - self.batch( - identity, - BatchRequest { - snapshot: batch.snapshot.to_string(), - sample_ids: batch.ids.iter().map(|id| STANDARD.encode(id)).collect(), - fields: batch.fields, - sample_id_limit: Some(16 * 1024), - }, - Some(batch.next), - batch.end, - Some(id), - request_id, - ) - .await + match tail { + ["reads", "bind"] => { + let input: models::Plan = read_json(request).await?; + json(&models::binding( + DatasetReads::new(self.reads.authority()) + .bind(identity, input.native()?) + .await?, + )) } + ["reads", "next"] => self.query_next(identity, request).await, _ => Err(ApiError::not_found()), } } diff --git a/app/crowdb-access-server/src/dataset/legacy.rs b/app/crowdb-access-server/src/dataset/legacy.rs index d1804a799..e313122e4 100644 --- a/app/crowdb-access-server/src/dataset/legacy.rs +++ b/app/crowdb-access-server/src/dataset/legacy.rs @@ -4,8 +4,8 @@ //! Compatibility routes for the original bincode client. use super::{ - cancelled_status, decode_snapshot_request, encode, response, status_for_authority, unix_seconds, - DatasetHttpService, MAX_REQUEST_BYTES, + cancelled_status, decode_snapshot_request, encode, response, status_for_authority, DatasetHttpService, + MAX_REQUEST_BYTES, }; use crowdb_access_dataset::{ AuthorityError, DatasetManifestResponse, DatasetPublishRequest, DatasetPublishResponse, @@ -30,17 +30,6 @@ impl DatasetHttpService { if request.method() == Method::POST && request.uri().path() == "/v1/dataset/scan" { return self.scan(request).await; } - if request.method() == Method::POST - && matches!( - request.uri().path(), - "/v1/dataset/start-read-plan" - | "/v1/dataset/next-batch" - | "/v1/dataset/save-progress" - | "/v1/dataset/resume" - ) - { - return self.scan(request).await; - } if request.method() == Method::POST && request.uri().path() == "/v1/dataset/create" { return self.create().await; } @@ -303,7 +292,7 @@ impl DatasetHttpService { Ok(request) => request, Err(_) => return response(StatusCode::BAD_REQUEST, b"invalid dataset request".to_vec()), }; - match self.reads.read(request, unix_seconds()).await { + match self.reads.read(request).await { Ok(result) => encode(StatusCode::OK, &result), Err(ReadTransportError::WindowFull) => { response(StatusCode::TOO_MANY_REQUESTS, b"read window full".to_vec()) @@ -332,7 +321,7 @@ impl DatasetHttpService { Ok(request) => request, Err(_) => return response(StatusCode::BAD_REQUEST, b"invalid scan request".to_vec()), }; - match self.reads.scan(request, unix_seconds()).await { + match self.reads.scan(request).await { Ok(result) => encode(StatusCode::OK, &result), Err(ReadTransportError::WindowFull) => { response(StatusCode::TOO_MANY_REQUESTS, b"read window full".to_vec()) diff --git a/app/crowdb-access-server/tests/common/dataset.rs b/app/crowdb-access-server/tests/common/dataset.rs index 7e8217640..95bfc66d9 100644 --- a/app/crowdb-access-server/tests/common/dataset.rs +++ b/app/crowdb-access-server/tests/common/dataset.rs @@ -52,7 +52,6 @@ impl TestDatasetHttp { Arc::new(authority), DatasetIdentity::new(NamespacePath::root(), "legacy").unwrap(), 3, - u64::MAX - 1000, ) .unwrap(); let service = DatasetHttpService::new(Arc::new(reads), Duration::from_secs(30)) diff --git a/app/crowdb-access-server/tests/dataset_read_query_contract_test.rs b/app/crowdb-access-server/tests/dataset_read_query_contract_test.rs new file mode 100644 index 000000000..a6ba3165d --- /dev/null +++ b/app/crowdb-access-server/tests/dataset_read_query_contract_test.rs @@ -0,0 +1,101 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +#[path = "common/dataset_v2.rs"] +mod common; +#[path = "common/dataset.rs"] +mod dataset; +use common::response; +use dataset::{TestDatasetHttp, TestStore}; +use reqwest::Client; +use serde_json::{json, Value}; +use std::sync::Arc; + +#[tokio::test] +async fn split_read_continuation_replays_without_remote_progress_or_cleanup() { + let test = TestDatasetHttp::start(Arc::new(TestStore::default())).await; + let client = Client::new(); + let (root, snapshot) = common::fixture(&client, &test.origin).await; + let revision = published_split(&client, &root, &snapshot).await; + let binding = response(client.post(format!("{root}/reads/bind")).json(&plan()), 200).await; + assert_eq!(binding["snapshot"], snapshot); + assert_eq!(binding["split"]["revision"], revision); + let first = batch(&client, &root, &binding["cursor"]).await; + assert!(!first["sample_ids"].as_array().unwrap().is_empty()); + let independent = Client::new(); + assert_eq!(batch(&independent, &root, &binding["cursor"]).await, first); + let second = batch(&client, &root, &first["next"]).await; + assert_ne!(first["sample_ids"], second["sample_ids"]); + assert_eq!(batch(&client, &root, &binding["cursor"]).await, first); + for suffix in ["", "/progress", "/next"] { + let url = format!("{root}/read-plans/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa{suffix}"); + response(client.get(&url), 404).await; + response(client.post(&url), 404).await; + response(client.put(&url), 404).await; + response(client.delete(&url), 404).await; + } + drop(independent); + drop(client); + test.stop().await; +} +fn plan() -> Value { + json!({"snapshot":"latest","selection":{"kind":"all"},"fields":["value"],"batch_size":3, + "partition":{"worker":0,"workers":1,"rank":0,"world_size":1},"shuffle":null, + "split":{"revision":null,"name":"train/a b"}}) +} +async fn published_split(client: &Client, root: &str, snapshot: &str) -> String { + let op = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + response( + client + .post(format!("{root}/snapshots/{snapshot}/split-drafts")) + .json(&common::begin(op, &Value::Null)), + 200, + ) + .await; + let draft = format!("{root}/split-drafts/{op}"); + let prepared = common::prepare(client, &draft, "1").await; + common::publish( + client, + &draft, + &prepared, + "cccccccccccccccccccccccccccccccc", + Value::Null, + ) + .await; + prepared["revision"].as_str().unwrap().into() +} +async fn batch(client: &Client, url: &str, cursor: &Value) -> Value { + let response = client + .post(format!("{url}/reads/next")) + .json(&json!({"cursor":cursor})) + .bearer_auth("secret") + .send() + .await + .unwrap(); + let status = response.status(); + let bytes = response.bytes().await.unwrap(); + assert_eq!(status, 200, "{}", String::from_utf8_lossy(&bytes)); + assert_eq!(&bytes[..8], b"CDBDS001"); + assert_eq!(bytes[8], 1); + let length = u32::from_be_bytes(bytes[9..13].try_into().unwrap()) as usize; + serde_json::from_slice(&bytes[13..13 + length]).unwrap() +} + +#[tokio::test] +async fn binding_rejects_unknown_split_and_malformed_cursor() { + let test = TestDatasetHttp::start(Arc::new(TestStore::default())).await; + let client = Client::new(); + let (root, _) = common::fixture(&client, &test.origin).await; + response(client.post(format!("{root}/reads/bind")).json(&plan()), 404).await; + response( + client + .post(format!("{root}/reads/next")) + .json(&json!({"cursor":"invalid"})), + 400, + ) + .await; + let mut input = plan(); + input.as_object_mut().unwrap().remove("split"); + response(client.post(format!("{root}/reads/bind")).json(&input), 400).await; + drop(client); + test.stop().await; +} diff --git a/app/crowdb-access-server/tests/dataset_split_session_contract_test.rs b/app/crowdb-access-server/tests/dataset_split_session_contract_test.rs deleted file mode 100644 index 170d8c943..000000000 --- a/app/crowdb-access-server/tests/dataset_split_session_contract_test.rs +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. -#[path = "common/dataset_v2.rs"] -mod common; -#[path = "common/dataset.rs"] -mod dataset; -use common::response; -use dataset::{TestDatasetHttp, TestStore}; -use reqwest::Client; -use serde_json::{json, Value}; -use std::sync::Arc; - -#[tokio::test] -async fn split_plan_pins_revision_and_only_explicit_acknowledgement_advances_delivery() { - let test = TestDatasetHttp::start(Arc::new(TestStore::default())).await; - let client = Client::new(); - let (root, snapshot) = common::fixture(&client, &test.origin).await; - let revision = published_split(&client, &root, &snapshot).await; - let mut input = plan("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - let prepared = prepare(&client, &root, &mut input).await; - assert_eq!(prepared["snapshot"], snapshot); - assert_eq!(prepared["split"]["revision"], revision); - let id = input["id"].as_str().unwrap(); - let url = format!("{root}/read-plans/{id}"); - let initial = prepared["session"]["acknowledged"].clone(); - assert_eq!( - response(client.get(&url), 200).await["session"]["acknowledged"], - initial - ); - let first = batch(&client, &url).await; - assert!(!first["sample_ids"].as_array().unwrap().is_empty()); - assert_eq!(batch(&client, &url).await, first); - assert_eq!( - response(client.get(format!("{url}/progress")), 200).await["cursor"], - initial - ); - let ack = json!({"expected":initial, "next":first["next"]}); - let accepted = response(client.put(format!("{url}/progress")).json(&ack), 200).await; - assert_eq!(accepted["cursor"], first["next"]); - let second = batch(&client, &url).await; - assert_ne!(first["sample_ids"], second["sample_ids"]); - assert_eq!(response(client.delete(&url), 200).await["closed"], true); - response(client.get(&url), 499).await; - response(client.post(format!("{root}/read-plans")).json(&input), 499).await; - drop(client); - test.stop().await; -} -fn plan(id: &str) -> Value { - json!({"id":id,"snapshot":"latest","selection":{"kind":"all"},"fields":["value"],"batch_size":3,"partition":{"worker":0,"workers":1,"rank":0,"world_size":1},"shuffle":null,"split":{"revision":null,"name":"train/a b"},"expected_page":"0"}) -} -async fn prepare(client: &Client, root: &str, input: &mut Value) -> Value { - for _ in 0..10 { - let result = response(client.post(format!("{root}/read-plans")).json(input), 200).await; - if result["complete"] == true { - return result; - } - input["expected_page"] = result["next_page"].clone(); - } - panic!("read preparation did not finish"); -} -async fn published_split(client: &Client, root: &str, snapshot: &str) -> String { - let op = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; - response( - client - .post(format!("{root}/snapshots/{snapshot}/split-drafts")) - .json(&common::begin(op, &Value::Null)), - 200, - ) - .await; - let draft = format!("{root}/split-drafts/{op}"); - let prepared = common::prepare(client, &draft, "1").await; - common::publish( - client, - &draft, - &prepared, - "cccccccccccccccccccccccccccccccc", - Value::Null, - ) - .await; - prepared["revision"].as_str().unwrap().into() -} -async fn batch(client: &Client, url: &str) -> Value { - let response = client - .post(format!("{url}/next")) - .bearer_auth("secret") - .send() - .await - .unwrap(); - let status = response.status(); - let bytes = response.bytes().await.unwrap(); - assert_eq!(status, 200, "{}", String::from_utf8_lossy(&bytes)); - assert_eq!(&bytes[..8], b"CDBDS001"); - assert_eq!(bytes[8], 1); - let length = u32::from_be_bytes(bytes[9..13].try_into().unwrap()) as usize; - serde_json::from_slice(&bytes[13..13 + length]).unwrap() -} - -#[tokio::test] -async fn incomplete_read_can_be_inspected_and_cancelled_without_creating_delivery() { - let test = TestDatasetHttp::start(Arc::new(TestStore::default())).await; - let client = Client::new(); - let (root, _) = common::fixture(&client, &test.origin).await; - let mut input = plan("dddddddddddddddddddddddddddddddd"); - input["split"] = Value::Null; - let first = response(client.post(format!("{root}/read-plans")).json(&input), 200).await; - assert_eq!(first["complete"], false); - assert_eq!(first["session"], Value::Null); - let url = format!("{root}/read-plans/{}", input["id"].as_str().unwrap()); - assert_eq!(response(client.get(&url), 200).await, first); - response(client.post(format!("{url}/next")), 409).await; - response(client.get(format!("{url}/progress")), 409).await; - assert_eq!(response(client.delete(&url), 200).await["closed"], true); - input["expected_page"] = first["next_page"].clone(); - response(client.post(format!("{root}/read-plans")).json(&input), 499).await; - input["id"] = json!("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"); - input.as_object_mut().unwrap().remove("split"); - response(client.post(format!("{root}/read-plans")).json(&input), 400).await; - drop(client); - test.stop().await; -} diff --git a/app/crowdb-access-server/tests/dataset_v2_contract_test.rs b/app/crowdb-access-server/tests/dataset_v2_contract_test.rs index 2fdbad4a9..1f5257d40 100644 --- a/app/crowdb-access-server/tests/dataset_v2_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_v2_contract_test.rs @@ -38,7 +38,7 @@ async fn capabilities_use_existing_auth_and_match_configured_operations() { "inspection-preparation", "dataset-management", "split-revisions", - "split-reads" + "stateless-reads" ]) ); let response = client.post(&endpoint).bearer_auth("secret").send().await.unwrap(); diff --git a/container/single-node-container/templates/access.toml b/container/single-node-container/templates/access.toml index e75b63408..961a58459 100644 --- a/container/single-node-container/templates/access.toml +++ b/container/single-node-container/templates/access.toml @@ -89,5 +89,4 @@ catalogs = [] listen = "0.0.0.0:9093" max_in_flight = 8 request_timeout_seconds = 60 -lease_ttl_seconds = 60 max_batch_size = 10000 diff --git a/doc/backlog/R220-dataset-fat-client-plan.md b/doc/backlog/R220-dataset-fat-client-plan.md index a2642e339..75789f85f 100644 --- a/doc/backlog/R220-dataset-fat-client-plan.md +++ b/doc/backlog/R220-dataset-fat-client-plan.md @@ -24,12 +24,16 @@ and [SDK API guide](../../lib/crowdb-sdk-native/README.md). The fat client shares Access Server's internal trust domain, takes explicit Dataset identity/context, and needs neither SigV4 verification, the S3 user-secret database, nor external-user capability issuance. -2. Reuse `crowdb-access-dataset` authority, sessions, composition, Split, and +2. Reuse `crowdb-access-dataset` authority, stateless reads, composition, Split, and upload logic. Preserve Snapshot/Read Plan binding, selection, worker partitioning, integrity, cleanup, and deletion admission. Topology is a supported scheduling surface, not placement authority. -3. Preserve explicit acknowledgement and resume: fetching data, releasing a - buffer, and acknowledging processing are separate events. Native delivery +3. Keep read progress entirely client-owned. Each request supplies its query + and continuation, bound to Dataset, immutable Snapshot and Split revision. + No read-session record, selected-ID copy, progress CAS, TTL or remote close + is stored. Replaying a continuation replays its batch. The client saves its + checkpoint after successful processing; buffer release does not advance it. + Native delivery uses the shared owner-backed streams and bounded prefetch. Cancellation stops producers; caller-held leases remain valid and charged until release. 4. Compare native Chunk-KV and chunk-range reads with the HTTP baseline, @@ -53,7 +57,7 @@ and [SDK API guide](../../lib/crowdb-sdk-native/README.md). - Existing `ClientRuntime`, topology/locality observations, owner-backed delivery and first native S3 frontend. Dataset uses the common transport - and budget contracts while retaining its own selection and acknowledgement. + and budget contracts while retaining its own selection and client checkpoints. Successful stream EOF, durable publication, caller release and owned runtime shutdown remain distinct events. The HCA acceptance is same-host functional evidence; it does not replace Dataset workload or cross-host measurements. @@ -84,8 +88,9 @@ and [SDK API guide](../../lib/crowdb-sdk-native/README.md). occurs, memory stays bounded and held data stays valid until final release. Invariant: bounded owner-backed delivery. Integration test. - Given a fetched but unacknowledged batch, when its buffers are released and - the read resumes, progress has not advanced; explicit acknowledgement alone - advances it. Invariant: processing acknowledgement. Integration test. + the same continuation is retried on another server/client instance, the same + batch is returned with no read-side writes. Only choosing the returned next + continuation advances delivery. Invariant: client-owned progress. Integration test. - Given a native Dataset facade sharing a runtime with S3, when either facade executes or closes, resource sharing preserves Dataset authority and the other's operations. Invariant: shared runtime, separate semantics. @@ -96,7 +101,7 @@ and [SDK API guide](../../lib/crowdb-sdk-native/README.md). failure boundary. Integration test. - Given the same native read through Rust and Python, when cancellation and close occur with retained data, logical results, buffer lifetime, isolated - cleanup, and explicit acknowledgement agree. Invariant: binding parity. + cleanup, and continuation replay agree. Invariant: binding parity. Integration test. Run: diff --git a/doc/design/access-server/dataset/design-crowdb-access-dataset.md b/doc/design/access-server/dataset/design-crowdb-access-dataset.md index fb2ae1c55..4a72b370f 100644 --- a/doc/design/access-server/dataset/design-crowdb-access-dataset.md +++ b/doc/design/access-server/dataset/design-crowdb-access-dataset.md @@ -143,8 +143,8 @@ registering a reader. Deletion durably fences new reads and records a five-minut `reclaim_after` deadline; retries and restarts preserve that deadline. Existing streams can finish using resolved metadata and payloads during the grace period. Unusually long reads may fail after reclamation; partial delivery is never complete -success. Access Servers and native clients do not acknowledge deletion. Session -progress and local request/buffer admission have independent lifetimes. EOF, +success. Access Servers and native clients do not acknowledge deletion. Client +checkpoints and local request/buffer admission have independent lifetimes. EOF, cancel and drop release local request admission without remote cleanup writes; caller-held buffers remain valid and charged until their final release. @@ -197,8 +197,9 @@ chunk access, local staging and RDMA remain extensions. limits bound memory and registration. - **DATA-I8 — Cursor binding:** a cursor and Read Plan cannot resume against a different snapshot or selection identity. -- **DATA-I9 — Safe reclamation:** retained snapshots and active reads protect all - reachable metadata and payload. +- **DATA-I9 — Safe reclamation:** retention and live ownership protect reachable + metadata and payload. Deletion rejects new reads and delays reclamation for the + grace period; overdue reads may fail without returning truncated success. - **DATA-I10 — Surface equivalence:** HTTP and native access return the same logical selection and integrity result. @@ -207,8 +208,8 @@ chunk access, local staging and RDMA remain extensions. The implementation is split into a `crowdb-access-dataset` library and a thin `crowdb-access-server` Dataset listener, following the S3 and Iceberg boundary. `DatasetAuthority` owns keys, records, publication state, ancestry, retention, -and durable progress through the `DatasetStore` interface. `DatasetReadService` -owns admission, bounded delivery, plan/cursor execution, and lease lifetime. +through the `DatasetStore` interface; read checkpoints stay with the caller. `DatasetReadService` +owns admission, bounded delivery, plan/cursor execution, and request lifetime. `DatasetHttpService` owns HTTP framing, authentication, route/status mapping, shutdown, and health wiring. `ChunkKvDatasetStore` adapts Dataset metadata and payload access to the chunk-backed KV client without exposing physical layout @@ -254,23 +255,28 @@ explicit reclaim; upload expiry does not reclaim claimed candidate payloads. ### 10.2 Read flow -`DatasetSessions` resolves latest/stable once, binds the request to that -immutable Snapshot, and persists immutable pages of selected IDs separately -from a small progress record. Selection, deterministic shuffle and rank/worker -partition execute before fetching payloads. Projection walks explicit ancestry -only until all requested fields are resolved. Plans admit at most 100000 IDs; -metadata is capped at 8 MiB and selection pages contain at most 1024 IDs. - -Creation and acknowledgement persist the next plan-bound opaque token together -with progress. Fetch reads that token and the immutable ID pages without any -mutation or activity renewal. Repeated fetches replay the same batch. Acknowledgement -CAS-advances only the prepared token and is idempotent; progress inspection performs -no payload read. Creation, acknowledgement and explicit resume set a 60-second -admission deadline; an expired handle resumes at acknowledged progress while its -Snapshot remains available. Explicit close affects only that handle. Stream pulls -check its closed state to preserve remote cancellation; they do not update it. -CAS conflicts reuse the returned record, while identical resume/close operations -need no mutation. Legacy sessions without a prepared token acquire one once. +`DatasetReads` resolves latest/stable and optional currentSplit into immutable +versions and returns a self-contained continuation. The server stores no read +record, selected-ID pages, progress, TTL or cancellation state. Every next request +supplies its continuation and passes ordinary authentication and deletion admission. +The token binds Dataset, Snapshot, Split, selection, projection, worker partition, +shuffle seed/group size and position. It is a cursor, not an authorization grant. + +Selection uses bounded inspection windows without fetching a full Manifest. +Split filtering precedes deterministic shuffle and worker partitioning. The cursor +carries a group start and offset, not the Dataset's selected IDs. A partly consumed +shuffle group is reconstructed from immutable metadata; this bounds memory but +adds metadata reads when groups are much larger than delivered batches. Groups +are capped at 100000 selected IDs and 8 MiB of IDs; selection work has a two-second +deadline and at most 5000 scan windows of up to 20 candidates. Serialized cursor +content is capped at 256 KiB before base64 encoding. Invalid or excessive work +returns an explicit error. Stream metadata and headers remain capped at 8 MiB. + +Repeating a continuation replays its batch, including from another Access Server +or native client. After successful application processing, the caller saves the +returned next continuation in its own checkpoint. EOF and buffer release never +save progress. There is no remote ack, resume or close; cancelling a request stops +its local producer. The SDK iterable's acknowledgement is a local operation. The public binary response streams a bounded JSON header, integrity-checked payload frames, field terminators and a batch terminator. Success also requires @@ -278,8 +284,8 @@ HTTP EOF with no trailing bytes. Each active transfer owns local admission credits through delivery/drop. Snapshot reads perform no persistent activity updates or heartbeat writes. Cancellation stops the producer and returns credits; retained SDK-owned buffers keep their byte credits until released. -The internal legacy scan adapter retains its compatibility behavior; public -fetch never implies application acknowledgement. +The internal legacy scan adapter also returns a caller-owned cursor without +loading or persisting progress. Requests have no shared inactivity TTL. ### 10.3 Retention and reclaim flow @@ -337,7 +343,7 @@ cleanup retires the staging owner and obsolete candidate owners, preserving published candidates and independent owners of shared uploads. Snapshot upload reclaim walks at most twenty retained Manifest field positions per step after terminal admission fencing. It revokes only that Snapshot publication's owner; -physical cleanup stays in the durable upload registry. The retained Manifest +physical cleanup stays discoverable from durable upload authorities. The retained Manifest allows owner cleanup even after inspection indexes have been retired. Composition version 3 registers each edit ID in the root before writing its @@ -352,10 +358,12 @@ session CAS that removes its ownership. A delayed claim for that operation then conflicts; other owners remain valid. Sessions retain at most 4096 retired owner IDs and report capacity explicitly. Ordinary `release` remains reversible. -Upload expiry scans one bounded registry page per background sweep. Registration -and removal use generation nonces and session CAS fences so stale cleanup cannot -remove a newer registration. Restart restores sessions, accepted parts, progress -and pending cleanup from the durable store; in-memory state is not authoritative. +Upload expiry scans a bounded page of upload authorities with a local continuation +cursor. Shared registry registration and removal are absent. Restart restores +sessions, accepted parts, progress and pending cleanup from the durable store; +restarting the scan cannot lose a cleanup obligation. Scan throughput and per-upload +cleanup work determine the delay beyond expiry; the grace deadline is not a +maximum reclamation-latency guarantee. ### 10.4 Transport and compatibility @@ -381,9 +389,12 @@ Upload sessions decode versions 2 through 4 and write version 4 without registry fields. Permanent retired-owner identities still reject delayed claims. This metadata upgrade is one-way: do not downgrade an upgraded authority or mix old and new authority writers against the same Dataset store. Immutable Dataset/Snapshot/Manifest encodings -remain compatible. Private read-session progress decodes version 2 and writes -version 3 with a prepared next token; -public API versions do not expose these private persistence formats. +remain compatible. Persisted read-session records and legacy cursor keys are +unused. Session read routes are absent; clients use `POST reads/bind` and +`POST reads/next`. Upgraded clients and servers must deploy together. Control v2 +advertises `stateless-reads` so an SDK cannot treat the retired session capability +as support for this contract. Physical purging of unused keys is an explicit +maintenance operation, not a read-path side effect. Hot-path synchronization uses atomics, immutable owned views, CAS and bounded notification channels. No application-wide request lock serializes independent diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index 22e0fc7ae..3c40f0433 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -111,8 +111,9 @@ does not create a read plan or acknowledge application consumption. V2 read planning pins Snapshot and optional Split revision once, intersects Selection with Split membership, then orders/shuffles and partitions workers before fetching projected payloads. Planning advances twenty-Sample windows with fixed total ID/byte -limits. Fetch/preview/status never advance the acknowledged cursor; only explicit -application acknowledgement does so. Cancel stops local work without a persistent reader release. +limits. Fetch/preview/status never save consumption progress. The client saves the next +continuation after processing; acknowledgement is local. Cancel stops the current +request without a reader registry or remote close. ## 5. Ownership and cleanup diff --git a/doc/design/access-server/dataset/openapi-crowdb-dataset-v1.yaml b/doc/design/access-server/dataset/openapi-crowdb-dataset-v1.yaml index a652a4889..d6109d219 100644 --- a/doc/design/access-server/dataset/openapi-crowdb-dataset-v1.yaml +++ b/doc/design/access-server/dataset/openapi-crowdb-dataset-v1.yaml @@ -1488,10 +1488,10 @@ } } }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/reads": { + "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/reads/next": { "post": { - "operationId": "GetBatch", - "description": "GetSample uses one sample ID. One streaming response for all requested fields in sample/projection order, with field and frame integrity. Unknown samples/projections reject before payload reads. Missing and tombstone fields are explicit.", + "operationId": "NextBatch", + "description": "Replay the caller continuation. Save next only after successful processing; no remote acknowledgement or close.", "parameters": [ { "in": "path", @@ -1565,17 +1565,17 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/BatchRequest" + "$ref": "#/components/schemas/ReadContinuation" } } } } } }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/read-plans": { + "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/reads/bind": { "post": { - "operationId": "StartReadPlan", - "description": "Resolve latest/stable once; persist immutable selection, projection, deterministic shuffle and rank/worker partition before payload access. Same ID+plan retries recover one session; divergent plan conflicts. Metadata predicates filter before payload transfer.", + "operationId": "BindRead", + "description": "Resolve immutable versions into a client-owned continuation. No read record or progress is stored.", "parameters": [ { "in": "path", @@ -1604,7 +1604,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ReadSession" + "$ref": "#/components/schemas/ReadBinding" } } } @@ -1655,170 +1655,10 @@ } } }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/read-plans/{read}": { - "get": { - "operationId": "ResumeReadPlan", - "description": "", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "read", - "required": true, - "schema": { - "$ref": "#/components/schemas/Id" - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ReadSession" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - } - }, - "delete": { - "operationId": "CloseReadPlan", - "description": "Idempotent scoped close/cancel; other sessions continue. No activity for 60 seconds expires the read session; it is not a snapshot GC lease. Application processing is at least once around acknowledgement.", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "read", - "required": true, - "schema": { - "$ref": "#/components/schemas/Id" - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ReadSession" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/read-plans/{read}/next": { + "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/reads": { "post": { - "operationId": "NextBatch", - "description": "Fetches the batch at the acknowledged position. Repeated fetch replays it. Returns an opaque next cursor; fetching/prefetching never acknowledges. Lease is renewed through active delivery.", + "operationId": "GetBatch", + "description": "GetSample uses one sample ID. One streaming response for all requested fields in sample/projection order, with field and frame integrity. Unknown samples/projections reject before payload reads. Missing and tombstone fields are explicit.", "parameters": [ { "in": "path", @@ -1839,14 +1679,6 @@ "minLength": 1, "maxLength": 255 } - }, - { - "in": "path", - "name": "read", - "required": true, - "schema": { - "$ref": "#/components/schemas/Id" - } } ], "responses": { @@ -1894,175 +1726,15 @@ "499": { "$ref": "#/components/responses/Error" } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/read-plans/{read}/progress": { - "put": { - "operationId": "SaveProgress", - "description": "CAS from expected cursor to one previously issued next cursor. Retrying accepted acknowledgement succeeds; stale/mismatched/forged cursor conflicts. Cursor covers entire logical plan including shuffle/partition.", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "read", - "required": true, - "schema": { - "$ref": "#/components/schemas/Id" - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ReadSession" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } }, "requestBody": { "required": true, "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Acknowledge" - } - } - } - } - }, - "get": { - "operationId": "GetProgress", - "description": "", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "read", - "required": true, - "schema": { - "$ref": "#/components/schemas/Id" - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Progress" - } + "$ref": "#/components/schemas/BatchRequest" } } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" } } } @@ -2732,9 +2404,6 @@ "ReadPlan": { "type": "object", "properties": { - "id": { - "$ref": "#/components/schemas/Id" - }, "snapshot": { "type": "string", "maxLength": 32 @@ -2770,7 +2439,6 @@ } }, "required": [ - "id", "snapshot", "selection", "fields", @@ -2779,71 +2447,6 @@ "shuffle" ] }, - "ReadSession": { - "type": "object", - "properties": { - "id": { - "$ref": "#/components/schemas/Id" - }, - "snapshot": { - "$ref": "#/components/schemas/Id" - }, - "plan": { - "$ref": "#/components/schemas/Md5" - }, - "acknowledged": { - "type": "string", - "maxLength": 4096 - }, - "expires_at": { - "type": "integer" - }, - "closed": { - "type": "boolean" - } - }, - "required": [ - "id", - "snapshot", - "plan", - "acknowledged", - "expires_at", - "closed" - ] - }, - "Acknowledge": { - "type": "object", - "properties": { - "expected": { - "type": "string", - "maxLength": 4096 - }, - "next": { - "type": "string", - "maxLength": 4096 - } - }, - "required": [ - "expected", - "next" - ] - }, - "Progress": { - "type": "object", - "properties": { - "plan": { - "$ref": "#/components/schemas/Md5" - }, - "cursor": { - "type": "string", - "maxLength": 4096 - } - }, - "required": [ - "plan", - "cursor" - ] - }, "SnapshotRequest": { "type": "object", "properties": { @@ -2877,6 +2480,35 @@ "state", "chunks_reclaimed" ] + }, + "ReadBinding": { + "type": "object", + "required": [ + "snapshot", + "cursor" + ], + "properties": { + "snapshot": { + "type": "string" + }, + "cursor": { + "type": "string", + "maxLength": 349530 + } + } + }, + "ReadContinuation": { + "type": "object", + "additionalProperties": false, + "required": [ + "cursor" + ], + "properties": { + "cursor": { + "type": "string", + "maxLength": 349530 + } + } } }, "responses": { @@ -2894,7 +2526,7 @@ }, "x-crowdb-wire": { "identity": "IDs are lowercase 32-hex nonzero 16-byte identities. MD5 is lowercase 32 hex; MD5 is an integrity check, not authentication. Dataset names are UTF-8, percent-encoded as one path segment, compared without Unicode normalization. Only dataset-ns exists in v1. Sample IDs are canonical padded base64, never interpreted as paths.", - "bounds": "JSON request/response at most 8 MiB before allocation; inline decoded values at most 64 KiB, schema/projection 256 fields, sample IDs at most 1024 bytes. Resource credits cover active request bodies and actual response delivery. Request/stream deadlines and deployment admission budgets are explicit configuration. V1 reads admit 100000 selected IDs and immutable 1024-ID pages; progress is a separate CAS record. Manifest membership is the complete current sample set; omitted samples are removed, while omitted optional fields on surviving samples inherit by ancestry. Required fields must be present in each sample; explicit tombstones hide parent fields.", + "bounds": "JSON request/response at most 8 MiB before allocation; inline decoded values at most 64 KiB, schema/projection 256 fields, sample IDs at most 1024 bytes. Resource credits cover active request bodies and actual response delivery. Request/stream deadlines and deployment admission budgets are explicit configuration. Stateless read groups admit 100000 selected IDs and 8 MiB of ID bytes; continuation JSON is capped at 256 KiB before encoding. There are no persisted read pages or progress records. Manifest membership is the complete current sample set; omitted samples are removed, while omitted optional fields on surviving samples inherit by ancestry. Required fields must be present in each sample; explicit tombstones hide parent fields.", "batch": "Content-Type application/vnd.crowdb.dataset-batch.v1. Magic is 8 ASCII bytes CDBDS001. Each frame: type u8, body length u32 big endian, then body. Types: 1 JSON BatchHeader (at most 8 MiB); 2 data (16-byte MD5 followed by 1..4 MiB bytes); 3 field end (zero length); 0 batch end (zero length); 4 terminal JSON Error (at most 64 KiB, only after the header). Header declares every sample/field in requested order. Value fields are delivered sequentially with one or more type-2 frames and one type-3; empty values have just type-3. Missing/tombstone have no data frames. Validate each frame MD5, exact whole-field length/MD5, field ends, batch end and HTTP EOF with no trailing bytes. A batch is successful only after completion. A streaming destination may contain verified partial data on error and must be discarded; download-to-path publishes only a verified temporary file. A type-4 frame reports a failed operation, may follow verified partial data, and never permits successful completion. It carries the same structured error model as control responses. The SDK must report the error and discard partial output.", "retry": "SDK uses one overall deadline and stable logical tokens; bounded attempts/backoff only on documented idempotent operations with replayable bodies. Do not retry authorization, integrity or semantic conflicts. Ambiguous mutation errors retain operation/upload identity for status recovery. Never invent new tokens on retry.", "integrity": "Seekable file convenience upload makes a bounded digest pass then a bounded transfer pass. Upload complete independently verifies durable parts and whole MD5. These are deliberate integrity passes, not whole-file buffering. Nonseekable uploads require known length/digest or later explicit streaming-session design.", diff --git a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml index b756b87d2..4f662c4ae 100644 --- a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml +++ b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml @@ -13,7 +13,7 @@ info: bounded-inspection; explicit POST inspection preparation requires inspection-preparation. Dataset creation, stable/retention changes and receipts require dataset-management; Split lifecycle requires split-revisions; versioned - read plans require split-reads. Upload, composition and Snapshot retirement + stateless reads require stateless-reads. Upload, composition and Snapshot retirement require uploads, composition and snapshot-reclaim respectively, advertised only when the upload service is configured. Query parameters are strict and duplicate keys fail. Composition JSON bodies are limited to 1 MiB. @@ -543,9 +543,10 @@ paths: schema: {$ref: '#/components/schemas/CommitReceipt'} default: $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/read-plans: + /namespaces/dataset-ns/datasets/{dataset}/reads/bind: post: - operationId: prepareSplitRead + operationId: bindRead + description: Resolve immutable Snapshot and Split versions without creating read state. parameters: - {$ref: '#/components/parameters/dataset'} requestBody: @@ -555,99 +556,29 @@ paths: schema: {$ref: '#/components/schemas/ReadPlanV2'} responses: '200': - description: Revision-bound read result; only explicit acknowledgement advances consumption. + description: Client-owned continuation; no TTL, remote acknowledgement or close. content: application/json: - schema: {$ref: '#/components/schemas/ReadPreparation'} + schema: {$ref: '#/components/schemas/ReadBinding'} default: $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/read-plans/{operation}: - get: - operationId: splitReadStatus - parameters: - - {$ref: '#/components/parameters/dataset'} - - in: path - name: operation - required: true - schema: {$ref: '#/components/schemas/OpaqueId'} - responses: - '200': - description: Revision-bound read result; only explicit acknowledgement advances consumption. - content: - application/json: - schema: {$ref: '#/components/schemas/ReadPreparation'} - default: - $ref: '#/components/responses/InspectionError' - delete: - operationId: cancelSplitRead - parameters: - - {$ref: '#/components/parameters/dataset'} - - in: path - name: operation - required: true - schema: {$ref: '#/components/schemas/OpaqueId'} - responses: - '200': - description: Revision-bound read result; only explicit acknowledgement advances consumption. - content: - application/json: - schema: {$ref: '#/components/schemas/ReadSessionV2'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/read-plans/{operation}/progress: - get: - operationId: splitReadProgress - parameters: - - {$ref: '#/components/parameters/dataset'} - - in: path - name: operation - required: true - schema: {$ref: '#/components/schemas/OpaqueId'} - responses: - '200': - description: Revision-bound read result; only explicit acknowledgement advances consumption. - content: - application/json: - schema: {$ref: '#/components/schemas/ReadProgress'} - default: - $ref: '#/components/responses/InspectionError' - put: - operationId: acknowledgeSplitRead + /namespaces/dataset-ns/datasets/{dataset}/reads/next: + post: + operationId: nextReadBatch + description: Replay this continuation; save the returned next cursor after processing. parameters: - {$ref: '#/components/parameters/dataset'} - - in: path - name: operation - required: true - schema: {$ref: '#/components/schemas/OpaqueId'} requestBody: required: true content: application/json: - schema: {$ref: '#/components/schemas/ReadAcknowledgement'} + schema: {$ref: '#/components/schemas/ReadContinuation'} responses: '200': - description: Revision-bound read result; only explicit acknowledgement advances consumption. - content: - application/json: - schema: {$ref: '#/components/schemas/ReadProgress'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/read-plans/{operation}/next: - post: - operationId: nextSplitBatch - parameters: - - {$ref: '#/components/parameters/dataset'} - - in: path - name: operation - required: true - schema: {$ref: '#/components/schemas/OpaqueId'} - responses: - '200': - description: Revision-bound read result; only explicit acknowledgement advances consumption. + description: Bounded batch; null next means end of selection. content: application/vnd.crowdb.dataset-batch.v1: schema: {type: string, format: binary} - description: Existing CDBDS001 framing, field integrity and end/error records. default: $ref: '#/components/responses/InspectionError' /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/preview: @@ -1599,19 +1530,16 @@ components: ReadPlanV2: type: object additionalProperties: false - required: [id, snapshot, selection, fields, batch_size, partition, shuffle, split, expected_page] + required: [snapshot, selection, fields, batch_size, partition, shuffle, split] description: >- - Retry preserves all inputs except expected_page. Snapshot latest/stable and - null Split revision resolve once before evaluation. Selection and Split - intersection precede shuffle and worker/rank partitioning. GET status does - not prepare or acknowledge a batch. Cancelled IDs cannot be reopened. + Snapshot latest/stable and null Split revision resolve into the returned + continuation. Selection and Split filtering precede shuffle and worker + partitioning. There is no persisted read plan, preparation or progress. properties: - id: {$ref: '#/components/schemas/OpaqueId'} snapshot: {type: string, description: Concrete ID or latest/stable} selection: {$ref: '#/components/schemas/Predicate'} fields: {type: array, maxItems: 256, uniqueItems: true, items: {type: string}} batch_size: {type: integer, minimum: 1, maximum: 10000} - expected_page: {$ref: '#/components/schemas/Decimal'} partition: type: object additionalProperties: false @@ -1629,7 +1557,7 @@ components: required: [seed, group_size] properties: seed: {$ref: '#/components/schemas/Decimal'} - group_size: {type: integer, minimum: 1, maximum: 1000000} + group_size: {type: integer, minimum: 1, maximum: 100000} split: oneOf: - {type: 'null'} @@ -1639,45 +1567,22 @@ components: properties: revision: {$ref: '#/components/schemas/NullableRevision'} name: {type: string} - ReadSessionV2: + ReadBinding: type: object - required: [id, snapshot, plan, acknowledged, expires_at, closed] - properties: - id: {$ref: '#/components/schemas/OpaqueId'} - snapshot: {$ref: '#/components/schemas/OpaqueId'} - plan: {type: string} - acknowledged: {type: string} - expires_at: {$ref: '#/components/schemas/Decimal'} - closed: {type: boolean} - ReadPreparation: - type: object - required: [snapshot, split, next_page, admitted, complete, session] + required: [snapshot, split, cursor] properties: snapshot: {$ref: '#/components/schemas/OpaqueId'} split: oneOf: - {type: 'null'} - {$ref: '#/components/schemas/SplitSource'} - next_page: {$ref: '#/components/schemas/Decimal'} - admitted: {$ref: '#/components/schemas/Decimal'} - complete: {type: boolean} - session: - oneOf: - - {type: 'null'} - - {$ref: '#/components/schemas/ReadSessionV2'} - ReadAcknowledgement: + cursor: {type: string, maxLength: 349530} + ReadContinuation: type: object additionalProperties: false - required: [expected, next] - properties: - expected: {type: string, maxLength: 4096} - next: {type: string, maxLength: 4096} - ReadProgress: - type: object - required: [plan, cursor] + required: [cursor] properties: - plan: {type: string} - cursor: {type: string} + cursor: {type: string, maxLength: 349530} CompositionSchema: type: object additionalProperties: false diff --git a/doc/design/access-server/dataset/spec-crowdb-dataset.md b/doc/design/access-server/dataset/spec-crowdb-dataset.md index 4ff7f1562..c1d75e5df 100644 --- a/doc/design/access-server/dataset/spec-crowdb-dataset.md +++ b/doc/design/access-server/dataset/spec-crowdb-dataset.md @@ -121,14 +121,14 @@ specified below and in its versioned OpenAPI contract. | `ReclaimSnapshot` / `GetReclaimStatus` | Snapshot ID | Reclaims eligible state or returns durable reclaim progress. | | `GetSample` / `GetBatch` | Snapshot ID, sample IDs, projection | Returns ordered samples and verified field values. | | `Scan` | Read Plan, optional Cursor | Returns one bounded batch and the next Cursor. | -| `SaveProgress` / `Resume` | Read Plan, Cursor | Persists or loads plan-bound progress. | +| `BindRead` / `NextBatch` | Query or continuation | Pins immutable versions or replays a caller-owned position without read-side writes. | | `Cancel` | Bound read or service | Stops local admission and production without a remote reader release. | `PrepareSnapshot` and `PublishSnapshot` share the same operation token and validation rules. A complete publication may be implemented as one call that performs preparation and completion internally; it MUST have the same durable -state transitions. Ordinary data reads do not persist activity. Explicit sessions retain -cursor state required by their documented lifecycle. +state transitions. Data reads MUST NOT persist activity, selected-ID copies, progress, TTL or +reader registration. Checkpoints are owned by the client. ## 5. Manifest and field values @@ -212,7 +212,7 @@ precedes shuffle and worker partitioning, which precede payload reads. A current change MUST NOT alter an admitted plan or composition source. A Cursor is either at the start or contains a plan-bound group and offset. The -caller acknowledges a batch by persisting the next Cursor. A failed or +caller acknowledges a batch by saving the next Cursor in its own checkpoint. A failed or unacknowledged batch may be replayed; a successful acknowledgement may not be skipped or applied to another plan. @@ -292,7 +292,7 @@ unsupported state MUST NOT appear as an empty result. Preview is opt-in, capped The HTTP surface is the portable wire contract. It exposes Dataset creation and opening, latest and Snapshot inspection, publication and preparation, Manifest retrieval, retention, stable selection, reclamation status, bounded reads, read -plans, progress, resume, and cancellation. +queries and client-owned continuations. Cancellation is request-local. The native surface exposes the same operations through typed calls. Every operation identifies the Dataset explicitly or through a bound service and @@ -307,17 +307,20 @@ Preparation rejects incomplete, aborted or foreign references. Claimed payloads remain protected until every candidate/Snapshot owner is released. Unowned sessions expire; physical cleanup may remain pending and MUST report that state. -Public read handles bind the resolved immutable Snapshot, selection, projection, -shuffle and worker partition. Creation and acknowledgement prepare the next token. -Fetch MUST NOT acknowledge consumption or persist activity. Creation, explicit -resume and acknowledgement refresh the 60-second admission deadline; fetching -alone does not extend it. The caller -explicitly acknowledges an issued next cursor; repeating that acknowledgement -is idempotent. Progress inspection MUST NOT fetch another batch. Resume restores -acknowledged progress. Closing one handle MUST NOT cancel another. Worker -partitioning executes before payload fetching. A v1 plan selects at most 100000 -IDs; JSON metadata and binary headers are capped at 8 MiB. Large fields stream -rather than being materialized as a whole batch. +Public reads bind the resolved immutable Snapshot, optional Split revision, +selection, projection, shuffle and worker partition in a caller-owned continuation. +Bind and next requests MUST NOT create persistent or required in-memory sessions. +The same continuation MUST replay its batch across independent server instances. +A successful consumer saves the next continuation; fetching, EOF and releasing +buffers MUST NOT advance shared progress. There is no remote ack, resume or close. +Missing next continuation means end of selection. Cancellation affects only the +current request. Each request authenticates and checks deletion admission. + +Selection windows, shuffle memory and request deadlines MUST remain bounded. +Groups admit at most 100000 selected IDs and 8 MiB of ID bytes; token JSON is capped +at 256 KiB before base64 encoding. Selected IDs are not copied into persistent +read records or the token. JSON metadata and binary headers are capped at 8 MiB. +Large fields stream rather than being materialized as a whole batch. Binary frame integrity is established at each frame, field integrity at field end, and successful batch completion requires the batch terminator followed by diff --git a/doc/design/access-server/design-crowdb-native-sdk.md b/doc/design/access-server/design-crowdb-native-sdk.md index b54cda21e..a9c6d590c 100644 --- a/doc/design/access-server/design-crowdb-native-sdk.md +++ b/doc/design/access-server/design-crowdb-native-sdk.md @@ -39,7 +39,7 @@ Copy captures a source generation. Multipart completion binds an exact selected part set; a repeated completed request returns its recorded outcome. `DatasetClient` attaches `DatasetAuthority`, `DatasetUploads` and -`DatasetSessions` to these same clients. Shared composition helpers enroll +`DatasetReads` to these same clients. Shared composition helpers enroll upload ownership before edits and bind candidate ownership before publication. Snapshot/Read Plan/Split selection, worker partition, CAS, durable intent recovery and cleanup remain in the existing Dataset services. @@ -130,13 +130,14 @@ GPU pointers or RDMA READ/WRITE authority. The independent Dataset HTTP distribution has no dependency on this storage/FFI runtime. A Dataset facade attaches to the common runtime and delivery contracts -while retaining Dataset authority, Snapshot/Split binding and explicit -acknowledgement. Buffer release, download completion and processing -acknowledgement remain separate events. `BatchBody` delivers field metadata and +while retaining Dataset authority, Snapshot/Split binding and caller-owned +checkpoints. Buffer release, download completion and saving a processing +checkpoint remain separate events. `BatchBody` delivers field metadata and lazy payload leases with per-part and whole-field integrity checks. Active reads renew the existing 60-second protection while live; cancellation stops renewal and releases the active handle, with TTL expiry bounding interrupted -cleanup. Durable session close is explicit and never acknowledges processing. +cleanup. There is no persistent read session or remote ack/close. A continuation replays +the same batch on an independent client; only the caller saves the next position. The optional native Python extension owns a shared runtime and exposes retained immutable leases. Slicing shares allocation ownership; conversion to Python @@ -162,4 +163,5 @@ reuse the Rust facade and shared authority schema without another storage path. - **NS-I7 — Bounded transport failure:** accepted RDMA failures never append TCP data or replay an ambiguous mutation. - **NS-I8 — Explicit Dataset progress:** fetching, verified EOF and final buffer - release cannot advance a read cursor; acknowledgement alone records progress. + release cannot save progress; the client checkpoints the next continuation + only after successful processing. diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 9b5807660..0aea6ea6b 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -6,8 +6,8 @@ Upstream: [Dataset SDK](../backlog/R220-dataset-fat-client-plan.md), [Access design](../design/access-server/design-crowdb-access-server.md). -Goal: remove coordination from independent requests where the business contract -allows interruption or retry, and retain only justified durable authority. +Goal: reduce Access request RPCs, copies, CPU work and shared contention; remove +coordination without a necessary business contract and retain justified authority. This is a persistent cleanup inventory requested by the user; remove completed items after recording their verified outcome in the owning design. @@ -26,13 +26,606 @@ items after recording their verified outcome in the owning design. sufficient to admit a new read after deletion. Read paths perform no snapshot activity writes, heartbeats or asynchronous release writes. +## Approved stateless read contract (2026-10-24) + +- The user explicitly removed persisted read sessions. This supersedes the + earlier inventory's proposals to retain durable ack/resume/remote close. +- Requests carry the query and continuation. The continuation binds Dataset, + immutable Snapshot, Split revision, projection, worker partition and shuffle. + Processing checkpoints belong to the caller; fetching/releasing a buffer does + not commit progress anywhere. Reusing a continuation replays its batch. +- Remove session records, persisted selected-ID pages, preparation progress, + version reservations, TTL renewal, ack/close routes and per-frame state reads. + Do not replace them with a required in-memory session or sticky routing. +- Keep bounded work, cancellation, integrity and buffer ownership. New requests + still check snapshot deletion. A continuation is not an authorization grant. +- Review remaining persistent mutations by use case and total RPC cost before + changing their business semantics. Strict nonempty namespace deletion remains + approved; durable publication and shared data ownership need separate review. + ## Execution -- [~] **Verify and deliver the cleanup**: run affected Dataset/S3/native/HTTP - regressions, format/lint, rebuild the same server source, and run the unchanged - mixed-size workload with restart acceptance. Retain measured limits; do not - infer throughput from mutation counts. Commit the coherent implementation and - record acceptance here before removing this task. +- [ ] **Confirm remaining persistence contracts**: review P1–P14 below before + changing resource/recovery semantics. In particular confirm disposable draft + work, upload restart behavior, exact retry responses and global quotas. Strict + nonempty namespace deletion is already decided and remains unchanged. +- [ ] **Reduce remaining request costs**: execute the still-applicable findings + 1–13 below; session-only findings are superseded by the completed removal. Measure total reads/mutations and serial + dependency depth, not only CAS counts. Consume returned CAS observations on + justified retries; preserve explicit expected-version failures. +- [ ] **Measure real services**: rebuild matching sources and run mixed-size/restart + acceptance after the confirmed simplifications. No throughput claim from static RPC counts. + +### Verified session cleanup + +- Replaced persisted Dataset read sessions with `DatasetReads` and caller-owned + continuation tokens. Removed remote ack/resume/close, reader TTL/configuration, + selected-ID/progress pages and legacy cursor storage. HTTP v1/v2 binding uses + `POST reads/bind`; `POST reads/next` replays a continuation. Existing v1 direct + batch reads remain `POST reads`. Native and Python callers use the same shared + query semantics, with client-side processing checkpoints. +- Added zero-write regressions for bind, 96 concurrent fetches, cross-instance + replay and legacy scan continuation. Covered immutable Snapshot/Split binding, + worker coverage, cancellation, deletion admission and bounded token size. +- Passed Dataset core all-target tests and the final focused query regressions; + Access Server and native SDK all-target tests; independent HTTP SDK tests; + affected all-feature clippy, independent SDK clippy and both fmt checks. + Parsed both OpenAPI documents and resolved every local reference. +- The HTTP SDK wheel was rebuilt and installed into an isolated environment. + Its Split/read round trip passed against the real HTTP listener, with the + consumer script copied outside the repository. +- A full Access run initially exhausted disk during linking, before tests; + deleting only rebuildable incremental cache restored space. A subsequent + payload test exposed a conflicting `reads` route; the independent `reads/bind` + route fixed it, and the complete Access suite then passed. +- This validates session removal, not full R220 performance acceptance. The + container mixed-size/restart workload and local-HCA comparison remain pending; + an OCI image from baseline `878ee901` does not contain these changes. + +The review below describes baseline `878ee901`. Its session preservation and +remote-close alternatives are superseded by the approved contract above; those +entries remain evidence of the cost being removed, not current requirements. + +## Remaining persistence review (2026-10-24) + +Scope: Access Server adapters plus Dataset, S3 and Iceberg metadata repositories. +`persistent` here means durable application metadata, including read-triggered +cache fills. A local atomic, resource permit or transport connection is not a +persistent read session. Chunk allocation/WAL/Paxos durability remains storage +infrastructure and is outside this Access semantic cleanup. Counts are logical +metadata operations; nested validation/replication and payload IO are additional. +These items are review proposals, not authorization to remove business guarantees. + +- **P1 — Iceberg GET can persist an optional projection**. `TableLoader::representation` + in `lib/crowdb-access-iceberg/src/table/load.rs` calls `ProjectionStore::put_document` + on a refs-mode cache miss. `metadata_projection/repository.rs` writes each child + page and root with create-CAS; `validated.rs` rereads the root and writes a + validation receipt. For P child pages this fill alone adds `(P + 2)C + 1G`, + on top of reads, parsing and hashing. Concurrent GETs repeat that durable work. + This is derived data, not publication authority. Proposal: remove synchronous + persistent fills from reads; compare a bounded local cache with publication-time + construction. Review decision: whether a durable projection is needed at all. +- **P2 — Dataset Composition workflow**. `composition/{begin,edit,state,evaluation, + evaluate_step,prepare,cleanup}.rs` stores draft revisions, edit receipts, + evaluation windows/rows, candidate pointers and cleanup progress. Use case: + editing a large derived Dataset and recovering across Access restarts without + redoing all preparation. Each window/row/receipt adds separate create-CAS and + progress replacement, with validation GETs; this is not one publication CAS. + Proposal: client-owned editing specification and replayable/disposable + preparation, retaining only immutable candidate data plus publication authority. + Decision: may an unfinished draft/evaluation be lost and restarted? If yes, + remove durable editing/progress first; do not remove committed candidates. +- **P3 — Dataset Split workflow**. `split/{lifecycle,state,work,evaluation,prepare, + recovery,cleanup,initial}.rs` similarly persists edits, evaluation progress, + overlap examples, membership work pages and fences. Published membership and + revision identity are business data; draft progress is auxiliary recovery state. + Proposal: compute from client-owned rules, publish immutable revision once, + restart incomplete evaluation instead of journaling every window. Same decision + as P2; retain frozen historical membership and explicit currentSplit CAS. +- **P4 — Dataset uploads**. `upload.rs`, `upload/{parts,index,ownership,retirement, + reclaim}.rs` retain upload descriptor, part intent/result/index, completion, + owner list, retired-owner IDs and abort/cleanup status. Trigger: multi-request + payload upload, retry after lost storage replies, sharing one payload across + candidates/Snapshots. Create currently costs `1C`; part acceptance adds intent, + part receipt and index mutations plus repeated state reads. Owner mutation is + `1G + C` attempts on one upload; observed conflicts must feed the next attempt. + Permanent retired-owner IDs can grow, and the part index serializes writers. + Proposal: separate immutable part results from upload-wide state; assess replacing + the growing owner list/index with per-owner/per-part records and bounded discovery. + Decisions: is resumable upload across restarts required, and may an incomplete + upload be discarded? Shared published payloads must not be reclaimed accidentally. +- **P5 — S3 multipart**. `metadata/multipart_repository{,/parts,/completion, + /publication,/terminal}.rs` stores upload identity, immutable parts/index, + completion/abort state and publication evidence. These implement S3 multipart + operations, not login/read sessions. Baseline normal new part is `7G + 4C` at + the helper boundary (finding 5); payload and replica IO are extra. First simplify + redundant state reloads and pass observed/applied values through helpers. + Removing upload durability changes resume/list/complete semantics and requires + explicit review; don't silently break S3 clients while removing Dataset reads. +- **P6 — Iceberg multipart and durable admission credits**. `file/multipart_repository/` + and `file/multipart_credits{,/settle}.rs` contain upload state plus a catalog-wide + sessions/byte reservation and pending-credit journal. Such a shared counter + serializes otherwise independent transfers. The current Access file upload path + uses its active upload-head flow; legacy durable-credit code must be traced by + caller before attributing its cost to active requests (finding 7). + Proposal: remove unreachable credit machinery; use bounded local producer + admission for resource protection. Decision for any reachable global budget: + are hard cross-server quotas a required contract, or is per-server admission enough? +- **P7 — Iceberg file writer recovery**. `file/{writer,write_intent,injection}.rs` + and writer checkpoints persist physical ownership/offsets, exact write intents, + completed file records and adoption receipts. Current active part publication + costs `1G + 1C` after admission; intent/checkpoint paths have separate costs. + Purpose: distinguish committed bytes from readable but unconfirmed bytes after + timeout/crash, and avoid adopting recycled storage. Proposal: retain physical + generation/fencing and immutable completion; review whether incomplete writer + checkpoints can be discarded/replayed. Acceptance must inject lost storage replies. +- **P8 — Iceberg HTTP retry ledger and body copies**. `operation/{retry,result,payload}.rs` + and Access `iceberg/{table_write,namespace_write}` persist request binding, + original body pages, result body and terminal status. `RetryLedger::begin` checks + context and locates a binding before create-CAS; finish checks context, locates + the binding, reads/writes result and updates binding again. Large bodies add + page reads/writes; replay may repair a status marker. Boolean CAS loses the + conflict observation and adds a confirmation GET. Proposal: first reuse observed + values; then merge retry evidence with the actual mutation journal or commit + record. Decision: must every retry reproduce the original response, or may an + ambiguous reply return a conflict/unknown outcome for client reconciliation? +- **P9 — Iceberg namespace/table journals and reservations**. `namespace/{journal, + reservation,admission,publication,drop_fence,drop_finish,update_recovery}.rs`, + `table/lifecycle/` and `commit/create/` persist name reservations, parent entries, + namespace/table generations and recovery phases. Multiple records/RPCs implement + multi-key create/drop/rename recovery. **Nonempty namespace deletion must fail**, + already confirmed by the user; do not ask again or replace it with eventual + emptiness. Review phase journals and duplicated context reads independently. + Removing a journal must still preserve strict parent admission and name uniqueness. +- **P10 — Dataset publication and explicit resource controls**. `authority.rs`, + `authority/publication/`, `split/{head,publication,recovery}.rs` persist Dataset + identity, immutable Manifest/Snapshot/Split, latest/stable pointer, retention + marker, commit attempt/receipt and visibility evidence. User mutations need + durable results; head CAS prevents a stale expected generation from publishing. + Extra attempt/receipt/visibility records are separately reviewable overhead. + Proposal: minimize the commit record set while preserving lost-reply outcome + discovery; evaluate whether the separate retention API is actually used. + Dropping persistent resource data is not justified by removing read sessions. +- **P11 — S3 object/bucket publication**. `publication.rs`, `bucket.rs`, + `metadata/{store,namespace}.rs` store object version/location/user metadata, + bucket identity and deletion markers. Unconditional PUT/DELETE should use the + overwrite contract without a preliminary GET/CAS; conditional operations retain + their expected-version check. Bucket tombstone replay still has an avoidable + bool-CAS/put-if-absent sequence (finding 4). These are resource writes; review + unnecessary mutation stages rather than replacing persistence with volatile state. +- **P12 — Dataset inspection indexes**. `inspection/{index,preparation,legacy, + retirement,reclaim}.rs` store schema/member/field projections, summary and + preparation/reclamation progress. They enable bounded reads without decoding a + full Manifest, but build cost is serial and multiplied per row (finding 9). + Explicit legacy preparation remains a mutation. Proposal: keep useful immutable + indexes with publication; compare batched/co-located writes and restarting + interrupted builds, removing auxiliary progress where redo is acceptable. +- **P13 — Cleanup/GC tasks and tombstones**. Dataset `authority/access/`, upload + cleanup and Split/Composition cleanup; Iceberg `gc/{repository,budget,worker}`; + S3 multipart terminal cleanup persist claim/phase/cursors and deletion fences. + Mark + delayed reclamation is already approved for snapshots. Proposal: use + idempotent bounded scans and local cursors where restart only repeats work; + review durable global counters and per-page progress. Keep enough physical + identity/fencing that a delayed delete cannot free a newly reused range. + Decision: accept recomputation after cleanup-worker restart and approximate + progress counters? Distinguish that from a repeated destructive operation. +- **P14 — Catalog lifecycle and credential authority**. Iceberg + `catalog/{repository,storage}.rs` persists active context/root/epoch; Access + `credentials{,/ensure}.rs` persists user credentials in Group-0. These are + administrative identity/revocation boundaries, not per-read sessions. Review + repeated root/context GETs and redundant initialization writes; deleting these + records would change authentication or catalog-reset safety. S3 session-token + credential fields alone are not evidence of server-side session persistence. + +Execution priority after confirmation: P1 read-triggered writes; P2/P3 disposable +workflows; P8 duplicate retry persistence; P4/P5 upload amplification; P6/P13 global +coordination. Every item must name its required invariant, exact normal/conflict/ +unknown-outcome RPC path, and a smaller alternative before implementation. + +### Cost still present in the new stateless read + +- No persistent reader record or selected-ID page is written. Continuations do + not eliminate snapshot/index/locator reads, payload IO or digest work. +- A partially consumed shuffle group is reconstructed; small `batch_size` with + very large `group_size` can therefore increase metadata RPCs. Repeated selection + windows also reload schema/membership validation. This is an explicit remaining + performance review item, not a claimed throughput improvement. Compare bounded + client-side planning/group buffering against server re-evaluation before choosing + a default; do not reintroduce mandatory read-session storage to cache that work. +- Query, payload admission and inspection can repeat deletion/visibility checks. + Consolidate within a request only after preserving the new-request deletion + boundary. Native/HTTP streaming no longer reads a session record per frame. +- Legacy `DatasetReadService::scan_surface` still reconstructs all planned windows + from the Manifest before selecting one cursor window. Removing its persisted + cursor does not remove that CPU/memory amplification; migrate callers to the + bounded inspection path or replace the planner in a separate measured change. +- Old `DS\\0RS\\0` and cursor records are no longer consulted or written. Physical + purge needs a separately scoped maintenance operation and coordinated removal + of old writers; no bulk deletion was executed during this cleanup. + +## Performance review: counting rules and current evidence + +- Static review baseline: `878ee901`. The prior implementation passed Dataset, + Access Server, S3 and native component/integration suites, namespace admission + and drop suites, workspace all-feature clippy and fmt. Its local OCI image + built successfully. Full mixed-size workload/restart was not run on that + revision before the user requested this broader review; do not report it as + accepted performance. +- `G` means one Access metadata point read, `B` one bounded read, `S` one scan, + `C` one conditional mutation, and `W` one unconditional mutation. `C(None)` + is create-if-absent, not a read-modify-write retry. Counts below describe the + named function path, not an entire HTTP request unless explicitly stated. +- These are logical storage calls, not physical wire-packet counts. Dataset `B` + currently becomes a one-key `scan_bounded`; routing, range crossings, topology + discovery, replica reads, Paxos and WAL can add messages and work. Measure + actual client RPC sends and server replication separately. +- A definite CAS conflict returns the value observed when the mutation was + evaluated. It can already be stale by the time it arrives. Use it immediately + to recompute an allowed retry; repeated competition can still fail, so preserve + the existing request deadline and bounded retry budget. Never wait for three + failures before using it. The three-conflict test injects competing writes to + verify this behavior, not a production batching/backoff rule. +- A lost reply is not a definite conflict. It may require exact-operation replay + or a confirmation read. An explicit expected revision/selected generation must + return a conflict when invalidated, rather than silently applying user intent + to a different base. +- Add request-scoped traces/counters in test wrappers and existing metrics, not + per-RPC production logging or a global profiling lock. Count GET/B/scan/CAS/ + PUT/delete, repeated keys, retries, bytes, concurrent operations, and serial + dependency stages. Separately record copied bytes, allocations/retained bytes, + digest/codec CPU, queue wait, active requests, wakeups and cleanup lag. + +## Findings and proposed corrections + +### 1. Repeated snapshot admission within one Dataset batch — confirmed + +- Use case: read the next batch of a persisted session, with deletion rejecting + new requests and admitted reads allowed to finish during grace. +- `DatasetSessions::next` reads session + deletion state + selected ID pages: + `2G + P G`, where P is the number of ID pages. Native `ReadAdmission::admit` + and HTTP `ReadAdmission::acquire` read deletion state again. For a nonempty + batch, `ready_inspection -> inspection_snapshot_committed` reads the same + deletion state a third time through `B`, before separate commit proof reads. +- Proposed: establish one request-local, snapshot-bound admission result and + reuse it across session/locator/body setup. Public standalone entry points + must still admit independently. Do not cache mutable deletion permission + across requests or confuse it with immutable publication proof. +- Verify: count one authoritative deletion check per admitted request; race + deletion at each boundary, direct/session and HTTP/native parity, missing and + unpublished snapshots, retained bodies and failure after physical cleanup. + +### 2. Remote session GET on each body pull — confirmed; contract-sensitive + +- Use case: a separate client closes a session while another client is reading + its current body. Existing code deliberately observes that close remotely. +- Native `BatchBody::pull -> ReadAdmission::check -> check_open` performs `1G` + on each pull. HTTP `BatchState::next` does the same before even returning + already-held `pending` bytes: frame header, payload, field end and stream end + can each incur a session GET. This traffic grows with delivery fragmentation, + not only logical requests or payload size. Ordinary direct batches have no + session ID and skip this cost. +- Proposed: first eliminate duplicate checks within one logical delivery step + if its existing cancellation boundary permits it. Separately evaluate whether + remote close must interrupt an admitted body, or can reject the next request + while local cancel stops the current body. The latter removes per-pull RPCs + but is a distinct contract change, not implied by snapshot deletion approval. +- Verify: current remote-close tests, empty/pending/EOF/capacity-wait cases, + plus RPC counts versus frame count. No new persistent reader registration. + +### 3. Upload metadata repeatedly reloaded while reading — confirmed + +- Use case: read an immutable completed upload referenced by a snapshot field. +- Native `FieldReader` loads upload metadata once, but every + `read_part_stream` loads it again and then loads the part: `1 + 2P` metadata + GETs for P parts, excluding snapshot/session/locator work. +- HTTP `BatchState` loads metadata before each part, and + `read_part_verified` loads it again before the part record. Its normal chunk + branch costs `3P + 1` GETs including the final end-of-field metadata probe, + excluding per-frame session checks and payload reads. +- Proposed: pass a validated, request-owned upload descriptor to part readers; + validate part index/length/digest against it. Reclamation may make later part + I/O fail; it must never yield successful truncation or another upload's bytes. + Keep mutable admission separate from immutable shape and physical identity. +- Also remove upload `create/complete/abort` readbacks that only reconstruct + state already returned by a successful mutation or conflict; assemble parts + for response only when the response needs them. +- Verify: per-field/per-part call budgets, shared owners, corruption, abort and + physical reclamation races; held buffers remain valid after completion. + +### 4. CAS observations still discarded by retry paths — confirmed + +- Dataset `upload/index.rs::index_part` does `GET -> CAS` on every attempt. + Concurrent different part numbers update one 128-part index page; merging + into the returned page is necessary to preserve both, another GET is not. +- Dataset `upload/ownership.rs::{abort,cleanup,cleanup_part}` loops reload the + same upload/part after confirmed CAS conflicts. Retain CAS to prevent abort + racing a claim, a delayed write intent overwriting reclaim, or an owner update + being lost. Recompute state from each returned observation. +- Dataset `authority/access/reclaim.rs::{fence_reclaim,retire_snapshot_record}` + and `split/recovery.rs::invalidate_split_head` do the same. The head nonce + change is a real publisher fence even if latest/stable IDs are unchanged; + it is not removable as a no-op. Reuse observed head and preserve receipt help. +- Dataset `read_session/planning.rs::start` and `read_session/split/finish.rs` + discard a session-create conflict value before entering `resume`; existing + start also reads a record and then resumes through another read. Pass state + through instead. Existing progress and upload-owner retry loops already use + observations; keep them as the expected behavior. +- S3 `bucket.rs::create_bucket` discards a tombstone replacement CAS result + through the bool wrapper, then performs another put-if-absent. Decode and use + the returned active mapping/tombstone immediately. Keep name uniqueness CAS. +- Iceberg `RetryLedger::begin` restarts `ledger_locate` after a failed bool CAS; + use the returned occupant, looking up an overflow key only when the competing + operation actually belongs to a different collision slot. `finish` also GETs + result bytes already supplied by a definite conflict. +- Verify: inject different numbers of actual conflicting writes, assert one + initial read and zero same-key re-GETs solely for each definite conflict; + preserve terminal/missing outcomes, explicit preconditions and lost replies. + +### 5. S3 multipart helpers discard known state — confirmed + +- Use case: publish a new part in an existing open upload; simultaneous parts, + complete and abort share that upload's durable pending/accounting state. +- With no existing part/generation, no pending work, no conflict and no error, + `put_stream_part` currently performs **7G + 4C** before returning: + load index/session (2G), read part (1G), create immutable generation (1C), + reserve session (1C), settle reload index/session (2G), read generation (1G), + read pointer (1G), publish pointer (1C), clear pending (1C). +- `load_identity` reads an index even though the session key is derivable and + `MultipartSessionRecord::decode` checks bucket/object/upload identity. Assess + whether index presence is an independent contract or only listing metadata; + remove it from point operations if the session alone is authoritative. +- Carry the known session, generation and pointer into settling; CAS still + protects against a competing writer. Make `exchange` return typed observed + state through its callers instead of a bool that forces `load` again. +- `publish_completion` has an extra object GET after definite CAS conflict. + `finish_publication`, `abort` and `settle_pending_part` likewise reload the + session after `exchange` discards the observation. Uncertain errors retain + separate confirmation logic. Do not silently overwrite a changed predecessor. +- The four conditional writes have separate current responsibilities. Reducing + them further requires proving quota accounting, same-part replacement and + freeze/abort behavior, not simply replacing them with unconditional PUTs. +- Verify: call counts for new/replaced/replayed parts, concurrent distinct/same + parts, complete-vs-abort, generation orphan cleanup and every lost-write reply. + +### 6. Iceberg phase machines reload their own progress — confirmed + +- Use case: namespace creation/drop/rename/property update and table lifecycle + operations survive Access restart and helpers finishing another request. +- `NamespaceJournal::advance`, table lifecycle/create/commit journals and + `CatalogRepository::cas` reduce CAS to bool. Their driver loops then load + progress from storage after both successful advancement and conflicts. + `NamespaceJournal::load` also checks ActiveRoot, so one reload is at least + `2G`, before payload and authority reads. Preserve applied or observed typed + state within the running request; restart still loads durable progress. +- `NamespaceRepository::load` costs `2 + 2D` GETs for an existing D-component + namespace: two context checks plus mapping and authority for each component. + Nested callers often resolve the same namespace/context again. Identify each + required clear/drop boundary before introducing request-local reuse; a cached + Ready result cannot grant indefinite mutation permission. +- `cleanup_marker` GETs the exact expected value before issuing a conditional + replacement. The CAS itself rejects a changed value. Compare direct-CAS and + existing-marker replay costs; carry known observations when available. +- Preserve the user's strict nonempty-delete guarantee. Parent admission, + name reservation and publication proof cannot be removed merely because the + recovery path has too many RPCs. Simplify state propagation first. +- Verify: count per phase; concurrent create/drop/rename, nonempty rejection, + recreation, epoch change, helper races and lost replies at every durable step. + +### 7. Active Iceberg paths versus legacy work — confirmed distinction + +- `publish_direct_commit` already conditionally publishes one table head and + examines the observed head on conflict. Keep optimistic table requirements; + do not add its legacy multi-phase publisher to normal commit cost estimates. +- Active streamed `put_stream_part` uses one part GET plus one part CAS after + admission, and returns a conflict instead of silently rebasing replacement. + Inspect its callers for duplicate session/context lookup; do not introduce + catalog-wide quota writes already absent from this path. +- Legacy multipart `part` checks `load` before and after reading a part; each + `load` itself reads context + session, giving `5G` for this helper. Completion + can nest further load/exchange checks. `exchange` checks context before and + after CAS. Track these as legacy/recovery costs, with a migration/removal + option if there are no live callers; never attribute them to streamed traffic. +- Native injection selects an immutable receipt, then publishes file authority; + both have request/content identity and deletion replay constraints. Preserve + winner selection while passing known descriptors and admission through. +- Verify: active route tracing, file replacement/deletion, retry identity, + direct-file no-KV reads, old-session recovery and catalog clear/rotation. + +### 8. Fewer writes can still mean more RPCs — confirmed trade-off + +- Previous cleanup added preflight reads to Dataset immutable session metadata + and Composition/Split receipt helping. A new value now costs `G/B + C` + instead of just `C`; an existing identical value costs a read instead of a + failed conditional mutation. Zero-write replay tests alone miss this trade-off. +- `reserve_session_version(version=2)` can read VERSION, check legacy SPEC, then + call `immutable_session_metadata`, which reads VERSION again. Carry that first + observation and avoid redundant preflights. +- Iceberg `PayloadStore::put_page` similarly performs GET then create-if-absent; + a new P-page payload costs `P G + P C`, serially. Journal validation rereads + payloads which the same request may just have produced. +- Proposed: distinguish definitely-new writes from replay/help paths using + existing request-owned evidence, not a new durable flag. Compare direct + create-if-absent, known-value reuse and read-before-write with total round trips + and server work. Immutable CAS preserves identity collisions and cleanup seals; + plain PUT is allowed only if divergent overwrite/resurrection is impossible. +- Verify: first request versus repeated helpers, collisions, late writers after + cleanup, bounds and total GET+CAS counts as well as persistent mutation counts. + +### 9. Serial fine-grained metadata I/O — confirmed shape, cost to measure + +- `resolve_inspection_batch_locators` serially walks samples and fields: one + membership read per sample, a field metadata lookup per ancestor searched, + then a separate locator/inline-value lookup for each present value. Costs grow + with samples × fields × ancestry depth. Schema and snapshot proofs add work. +- These bounded point reads use scans in `chunk_store/metadata.rs`, not a native + bounded point-get RPC. Review protocol support before substituting an unbounded + GET; byte/deadline bounds must remain enforced before materialization. +- `persist_inspection_rows` writes schema, membership, field metadata and value + rows one at a time. For F schema fields, N samples and E present sample-field + entries it emits `F + N + 2E` conditional row writes, then 3 summary/index + writes, excluding manifest/publication records. Session ID pages and Iceberg + payload pages are also written serially. +- Proposed: batch compatible point operations or use bounded concurrent reads/ + writes with a shared byte/work budget; reuse immutable request-local schemas + and proofs. Consider co-locating metadata/locators only after measuring byte + amplification and defining migration. Preserve output order and missing-field + semantics; do not turn a sparse projection into an unbounded Manifest read. +- Verify: samples/fields/ancestry/page-count sweeps, RPC count and dependency + depth, missing/tombstone fields, byte budgets and cancellation under concurrency. + +### 10. Payload copies and full-part materialization — confirmed + +- HTTP Dataset upload collects an entire part before writing. Byte-mode + `ChunkPayloadStorage::read` concatenates receive chunks into `BytesMut` before + HTTP delivery. Inline HTTP delivery uses `Bytes::copy_from_slice` even though + the request already owns the value. Keep the native owner-backed path intact. +- Compatibility `UploadedChunkReader::read` adds another concatenation and then + `output.to_vec()`, copying the assembled result again. Trace its active callers + before prioritizing; it is not the normal native streaming reader. +- Generic S3 coalescing copies frames into a fresh Vec per receive window; + native direct-receive is a separate path. HTTP read-ahead/native fallback and + mixed-size partial frames need separate measurements, not a zero-copy label + applied to all traffic. +- Proposed: move existing owners into outputs, stream retained payload owners + where framing allows, avoid concatenation just to change container type, and + keep bounded aggregation only where measurements justify it. A header-copy + and payload-copy have different costs; count bytes, not only copy calls. +- Verify: address/owner lifetime, partial frames and unaligned sizes, caller-held + buffers, cancel/drop/shutdown, peak physical allocation and explicit fallback. + +### 11. Digest, serialization and allocation CPU — confirmed work, profile first + +- `DatasetUploads::complete` serially rereads every accepted part. Each + `verified_part` verifies its MD5, then completion hashes the bytes again for + whole-upload MD5. Part MD5s cannot simply be combined into whole-upload MD5; + removing this read requires durable trusted whole-content evidence, not + trusting a caller checksum or weakening integrity. +- Native `VerifiedUploadPart::next` and `FieldReader::delivered` both hash + delivered bytes for different scopes. When a field is exactly one verified + part, investigate sharing the certificate; multipart fields still require + their whole-field validation. Measure hash CPU separately from transport. +- Upload/session/draft CAS serializes records repeatedly; owner revocation lists + grow to 4096 entries, part-index pages to 128 receipts. Conflicts repeat + decoding/encoding of whole records. Record encoded bytes and CPU by size and + contention; evaluate narrower records only if lifecycle atomicity survives. +- Inspection stores JSON field/locator values and clones sample/field metadata + for response headers. Profile parsing, allocations and inline byte encoding, + then pre-size or move owned values where justified. An Arc/Bytes clone does + not imply a payload copy; do not optimize it based on syntax alone. +- Verify: CPU samples, allocations/request and bytes processed by each digest + or codec; singleton/multipart, small/large and high-owner-count cases. + +### 12. Upload expiry throughput and coupled cleanup — confirmed + +- Current `start_cleanup` wakes every 15 seconds, runs snapshot cleanup first, + then one upload page. `expire` scans 20 keys from `DS\0U\0`, including part + and index records; it also revisits claimed and cleanup-complete authorities. +- With R stable scanned records, the nominal full-pass scheduling budget is + `15 * ceil(R / 20)` seconds in the no-overrun case. At 100,000 records that is + about 20.8 hours, before I/O delay. Five-minute grace is a minimum delay, not + an upper cleanup bound. Removing the registry eliminated foreground writes + but did not establish acceptable large-scale cleanup latency. +- One expired upload can enumerate up to 10,000 parts and clean them serially; + the two-second scan timeout does not bound the entire cleanup step. A slow + upload delays the next snapshot sweep on the same task. Multiple Access + instances can also duplicate scans and cleanup work. +- Proposed: separate bounded progress budgets for discovery and reclamation; + consume multiple pages while within time/bytes/I/O budget, skip child ranges + where supported, retain incremental cleanup progress and yield fairly. + Compare per-object expiry indexing with full scans only after accounting for + any extra foreground write; do not restore one global registry contention key. +- Verify: sparse expired uploads among live/cleaned records, maximum-part uploads, + repeated restarts, corrupt entries, multiple cleaners, cleanup lag and effect + on foreground p99/CPU/metadata RPCs. Restart cursors must not lose obligations. + +### 13. Admission, scheduling and lower-layer costs — measurement tasks + +- Review count-based versus byte-based admission for mixed sizes, queue residence + and capacity retained by slow consumers; differentiate overload rejection from + insufficient in-flight work. Do not increase capacity just to raise benchmark + throughput or reintroduce unbounded prefetched buffers/tasks. +- Native Dataset pulls one field stream at a time. Measure whether bounded + prefetch hides storage RTT under real mixed sizes, and whether it increases + retained-memory pressure or harms small-request p99. One thread can have + multiple pending operations; the load generator's request/response discipline + must be recorded. +- S3 native allocation credit has a 1-ms retry timer when other producers/CQ + completions can release arena credit; Dataset metadata cancellation polls at + 10 ms. Measure wakeup overhead, stalls and cancellation latency before changing + notification plumbing. Preserve missed-wakeup safety and avoid a new lock. +- Profile allocation arenas, serialization/hash time, scheduler occupancy, + shared writer queues, batching dwell time, transport in-flight WR/byte depth, + CQ processing, disk durability and metadata replication separately. Attribute + a lower-layer bottleneck with traces before expanding this Access change into + a storage or RDMA redesign. +- Include authentication/signing, error/logging paths and metrics overhead in + endpoint CPU profiles. Retain authorization/integrity checks; consolidate + duplicate computation or reuse already-validated request-local evidence. +- No direct production Mutex/RwLock was found in the audited Access libraries; + that does not establish lock-free behavior in dependencies, FFI or the kernel. + +## CAS use cases and disposition + +- **Ordinary reads, fetch replay, payload delivery**: no persistent reader CAS. + Keep only request admission, selected immutable identity and physical integrity; + repeated GETs remain subject to findings 1–3 and 9. +- **Ordinary S3 object PUT/DELETE**: final object publication/deletion already + uses one unconditional mutation; HEAD uses one object GET after bucket + resolution. Do not add preflight CAS. Bucket names still need uniqueness and + generation protection across delete/recreate. +- **Snapshot/table/Split head publication**: competing publishers select one + current generation, respecting an explicit expected base. Keep one conditional + publication point and exact lost-reply proof; do not silently rebase user edits. +- **Upload owners, claim versus reclaim**: two snapshots may reference one + upload. Keep an atomic owner/state update so removing one owner cannot delete + another's live data; permanent owner revocation rejects delayed claims. +- **Part intent/accept/index/complete**: prevent same-part writers, delayed I/O + and abort/complete from publishing inconsistent data. Keep the necessary + atomic state transitions; returned observations and known state eliminate + redundant reads. Index-page sharing is limited to one upload and merits a + separate contention measurement for parallel part uploads. +- **Session ack/resume/close**: same-session cursor/closed/deadline changes must + not overwrite each other. CAS is scoped to that handle; independent sessions + must not share mutable reader state. Normal fetch needs none. +- **Draft edits/checkpoints/cleanup seals**: explicit revision checks protect + concurrent edits; seals reject a late writer after cleanup. Reuse known + immutable receipts without inventing another mutable coordination record. +- **Iceberg namespace parent/name/lifecycle**: preserve strict nonempty delete, + generation isolation and recovery. Ordinary table creation already avoids a + parent mutation. Reduce supporting RPCs without removing the required boundary. +- **Idempotency bindings/immutable records**: create-if-absent selects one payload + per operation/name and detects divergent identity reuse. Plain PUT is not a + generic substitute; prove sole ownership, identical bytes and no cleanup seal + before removing the condition. A failed create-if-absent still costs an RPC. +- **GC progress and conditional deletion**: prevent stale workers from overwriting + newer progress or deleting a reused identity. Background workers need bounded + work and fair progress, not reader leases. Legacy catalog credit accounting + remains only where old records require settlement. +- Remote CAS inventory covers Dataset authority/publication/access, uploads, + sessions, Composition/Split/inspection; S3 bucket and multipart repositories; + Iceberg catalog/namespace/commit/table lifecycle, files/multipart/injection, + retry/payload/projection and GC. Shared `crowdb-access-multipart` helpers do + arithmetic/validation rather than issuing storage RPCs. Local delivery/arena + atomics and metrics are excluded from remote CAS counts. Dynamic counts for + every route and failure branch are still an execution task, not a static proof. + +## Performance verification matrix + +- Run direct read, session start/next/ack/resume/close; upload create/new-part/ + replay/complete/abort; Composition/Split edit/evaluate/publish/recover/cleanup; + S3 ordinary objects and multipart; Iceberg active file/table/namespace paths, + legacy recovery and GC. Include auth/admission entry costs in endpoint totals. +- Vary payload sizes, samples and fields, ancestry, part count, owner count, + concurrency and retained buffers. Separate one shared key from independent + objects. Include no contention, definite conflicts, response loss and restart. +- For each change record before/after logical storage calls, actual routed RPCs, + serial stages, bytes, CPU, allocations, p50/p99 and backlog. Keep byte/digest + correctness and resource limits fixed. Batch operations may lower RPC count + without lowering backend I/O; report both. +- The unchanged full HTTP/native Rust/native Python workload and restart check + remain the final acceptance. Do not compare phases from different source + images or count an interrupted/partial run as a throughput result. Physical + multi-host validation remains in its separate plan. ## Decisions applied to the inventory diff --git a/doc/working/plan-dataset-native-sdk.md b/doc/working/plan-dataset-native-sdk.md index 8e8f03757..69897c39b 100644 --- a/doc/working/plan-dataset-native-sdk.md +++ b/doc/working/plan-dataset-native-sdk.md @@ -7,7 +7,7 @@ Upstream: [R220](../backlog/R220-dataset-fat-client-plan.md), [Dataset](../design/access-server/dataset/design-crowdb-access-dataset.md), [native runtime](../design/access-server/design-crowdb-native-sdk.md). -Goal: attach trusted Dataset access to the accepted runtime with explicit ack, +Goal: attach trusted Dataset access to the accepted runtime with client checkpoints, bounded owner-backed delivery and independently packaged Python bindings. ## Baseline and contract @@ -18,14 +18,11 @@ bounded owner-backed delivery and independently packaged Python bindings. repeats, concurrency, completed requests, per-size p50/p99, throughput, process CPU/RSS and separately timed digest work. Preserve exact-byte checks. Files: `sdk/crates/crowdb-sdk-dataset/examples/`, `tools/sdk/`. -- [x] **Freeze API wiring**: use `DatasetSessions` (including Split preparation), - not legacy `DatasetDirectClient::scan` which acknowledges fetched batches. - Attach `ChunkKvDatasetStore`, `DatasetUploads` and `DatasetAuthority` to the - shared runtime's Dataset writer group. Use explicit `DatasetIdentity`; no - bearer/secret lookup or HTTP fallback. Files: `lib/crowdb-sdk-native/src/dataset/`. - -## Shared payload and native facade - +- [x] **Replace persisted read sessions**: user decision 2026-10-24 supersedes + the earlier ack/resume/close contract. Use `DatasetReads`, caller-owned + continuations and local checkpoints; track cleanup and validation in + [Access cleanup](plan-access-simplification.md). Historical test notes below + describe their original revisions, not the current API. - [x] **Retain payload owners**: extend the existing payload/upload boundary with lazy `OwnedPayload` delivery and verified lengths/digests. Keep the byte-returning HTTP path compatible. Accept frozen producer storage without @@ -51,12 +48,12 @@ bounded owner-backed delivery and independently packaged Python bindings. - [x] **Add optional native binding**: main-workspace PyO3 extension `crowdb_dataset_native`, with owned runtime, Dataset control/read operations, - retained immutable payload objects, explicit ack/cancel/close and exception + retained immutable payload objects, local checkpoints/cancel/close and exception mapping. Keep `sdk/` HTTP distribution free of native storage dependencies. Files: `lib/crowdb-sdk-native-python/`, `tools/sdk/`, workspace manifest. - [x] **Verify semantics**: shared authority tests and native facade tests for - projection/missing/tombstone, Split/revision/worker selection, explicit ack - replay, TTL, corruption/truncation, cancellation, dropped pulls, held memory, + projection/missing/tombstone, Split/revision/worker selection, caller-cursor + replay, deletion admission, corruption/truncation, cancellation, dropped pulls, held memory, stale routing, partial transport failure and S3/runtime coexistence. Files: native and access-dataset `tests/`, access-server parity tests. - [x] **Bound shared KV retention**: retain acceptor nodes through concurrent @@ -75,7 +72,7 @@ bounded owner-backed delivery and independently packaged Python bindings. Files: chunk-kv `partition/checkpoint.rs`, `partition/maintenance.rs`, server maintenance/storage, chunk-stream `kv.rs`, focused integration tests. - [~] **Run real services and compare**: use the same corpus/snapshot for HTTP - and native Rust/Python, restart storage, validate explicit resume and held + and native Rust/Python, restart storage, validate caller-checkpoint replay and held buffers, TCP and local shared-HCA policies. Run the same pinned workload and report measured trade-offs, not an assumed throughput win. Physical multi-node execution belongs to its independent plan and is not scheduled here. Files: diff --git a/lib/crowdb-access-dataset/Cargo.toml b/lib/crowdb-access-dataset/Cargo.toml index b85e47c73..9a1352587 100644 --- a/lib/crowdb-access-dataset/Cargo.toml +++ b/lib/crowdb-access-dataset/Cargo.toml @@ -14,6 +14,7 @@ workspace = true test-util = [] [dependencies] +base64 = "0.22" tokio = { workspace = true, features = ["rt", "macros", "time"] } async-trait = "0.1" bincode = "1.3" diff --git a/lib/crowdb-access-dataset/src/authority.rs b/lib/crowdb-access-dataset/src/authority.rs index 48dd389b9..85cde0a3b 100644 --- a/lib/crowdb-access-dataset/src/authority.rs +++ b/lib/crowdb-access-dataset/src/authority.rs @@ -19,7 +19,6 @@ use crate::key; use crate::manifest::{ManifestPartition, ManifestRecord}; use crate::record::{DatasetRecord, HeadRecord, ManifestBinding, SnapshotRecord}; use crate::store::{CasOutcome, DatasetStore, StoreError}; -use crate::ReadCursor; use crate::{DatasetError, DatasetId, DatasetIdentity, PublicationState, SnapshotId, SnapshotPublication}; const RECORD_VERSION: u16 = 1; @@ -84,8 +83,6 @@ pub struct SampleView { impl DatasetAuthority { #[must_use] - /// # Panics - /// The built-in non-zero lease duration is always valid. pub fn new(store: Arc) -> Self { Self { store, @@ -96,8 +93,6 @@ impl DatasetAuthority { } #[must_use] - /// # Panics - /// The built-in non-zero lease duration is always valid. pub fn with_chunk_reader(store: Arc, chunk_reader: Arc) -> Self { Self { store, @@ -830,49 +825,6 @@ impl DatasetAuthority { self.read_batch(identity, snapshot, sample_ids, fields).await } - /// Loads a durable cursor bound to its logical plan identity. - /// - /// # Errors - /// Returns malformed cursor bytes or storage failures. - pub async fn load_cursor( - &self, - identity: &DatasetIdentity, - plan: &crate::ReadPlan, - ) -> Result, AuthorityError> { - let Some(value) = self.store.get(&key::cursor(identity, plan.identity())?).await? else { - return Ok(None); - }; - let cursor: ReadCursor = decode(&value.bytes)?; - cursor.validate_for(plan)?; - Ok(Some(cursor)) - } - - /// Persists a confirmed cursor using CAS so an older worker cannot move a - /// durable replay boundary backwards after a restart. - /// - /// # Errors - /// Rejects a cursor bound to another plan or an unexpected stored value. - pub async fn persist_cursor( - &self, - identity: &DatasetIdentity, - plan: &crate::ReadPlan, - cursor: &ReadCursor, - expected: Option<&ReadCursor>, - ) -> Result<(), AuthorityError> { - cursor.validate_for(plan)?; - let key = key::cursor(identity, plan.identity())?; - let expected_bytes = expected.map(encode).transpose()?; - let bytes = encode(cursor)?; - match self - .store - .compare_exchange(&key, expected_bytes.as_deref(), &bytes) - .await? - { - CasOutcome::Applied(_) => Ok(()), - CasOutcome::Conflict(_) => Err(AuthorityError::CursorConflict), - } - } - async fn manifest_schema( &self, identity: &DatasetIdentity, diff --git a/lib/crowdb-access-dataset/src/client.rs b/lib/crowdb-access-dataset/src/client.rs index 9ce11cb23..7bd66fa20 100644 --- a/lib/crowdb-access-dataset/src/client.rs +++ b/lib/crowdb-access-dataset/src/client.rs @@ -165,34 +165,6 @@ impl DatasetHttpClient { pub async fn scan(&self, request: DatasetScanRequest) -> Result { self.call("/v1/dataset/scan", &request).await } - - pub async fn start_read_plan( - &self, - request: DatasetScanRequest, - ) -> Result { - self.call("/v1/dataset/start-read-plan", &request).await - } - - pub async fn next_batch( - &self, - request: DatasetScanRequest, - ) -> Result { - self.call("/v1/dataset/next-batch", &request).await - } - - pub async fn save_progress( - &self, - request: DatasetScanRequest, - ) -> Result { - self.call("/v1/dataset/save-progress", &request).await - } - - pub async fn resume( - &self, - request: DatasetScanRequest, - ) -> Result { - self.call("/v1/dataset/resume", &request).await - } } /// Fat client facade for direct access to the same Dataset authority and @@ -208,58 +180,12 @@ impl DatasetDirectClient { Self { service } } - pub async fn read( - &self, - request: DatasetReadRequest, - now_seconds: u64, - ) -> Result { - Ok(self - .service - .read_surface(ReadSurface::Native, request, now_seconds) - .await?) - } - - pub async fn scan( - &self, - request: DatasetScanRequest, - now_seconds: u64, - ) -> Result { - Ok(self - .service - .scan_surface(ReadSurface::Native, request, now_seconds) - .await?) - } - - pub async fn start_read_plan( - &self, - request: DatasetScanRequest, - now_seconds: u64, - ) -> Result { - self.scan(request, now_seconds).await - } - - pub async fn next_batch( - &self, - request: DatasetScanRequest, - now_seconds: u64, - ) -> Result { - self.scan(request, now_seconds).await - } - - pub async fn save_progress( - &self, - request: DatasetScanRequest, - now_seconds: u64, - ) -> Result { - self.scan(request, now_seconds).await + pub async fn read(&self, request: DatasetReadRequest) -> Result { + Ok(self.service.read_surface(ReadSurface::Native, request).await?) } - pub async fn resume( - &self, - request: DatasetScanRequest, - now_seconds: u64, - ) -> Result { - self.scan(request, now_seconds).await + pub async fn scan(&self, request: DatasetScanRequest) -> Result { + Ok(self.service.scan_surface(ReadSurface::Native, request).await?) } #[must_use] diff --git a/lib/crowdb-access-dataset/src/key.rs b/lib/crowdb-access-dataset/src/key.rs index ca6d1bebc..392cd4a9e 100644 --- a/lib/crowdb-access-dataset/src/key.rs +++ b/lib/crowdb-access-dataset/src/key.rs @@ -6,7 +6,6 @@ pub const HEAD_PREFIX: &[u8] = b"DS\0H\0"; pub const OPERATION_PREFIX: &[u8] = b"DS\0O\0"; pub const MANIFEST_PREFIX: &[u8] = b"DS\0M\0"; pub const MANIFEST_HEAD_PREFIX: &[u8] = b"DS\0MH\0"; -pub const CURSOR_PREFIX: &[u8] = b"DS\0C\0"; pub const RETENTION_PREFIX: &[u8] = b"DS\0R\0"; pub const LEASE_PREFIX: &[u8] = b"DS\0L\0"; pub const RECLAIM_PREFIX: &[u8] = b"DS\0G\0"; @@ -61,13 +60,6 @@ pub fn manifest_binding(identity: &DatasetIdentity, snapshot: SnapshotId) -> Res Ok(key) } -pub fn cursor(identity: &DatasetIdentity, plan_identity: [u8; 16]) -> Result, DatasetError> { - let mut key = CURSOR_PREFIX.to_vec(); - key.extend_from_slice(&identity.key()?); - key.extend_from_slice(&plan_identity); - Ok(key) -} - pub fn retention(identity: &DatasetIdentity, id: SnapshotId) -> Result, DatasetError> { let mut key = RETENTION_PREFIX.to_vec(); key.extend_from_slice(&identity.key()?); diff --git a/lib/crowdb-access-dataset/src/lease.rs b/lib/crowdb-access-dataset/src/lease.rs deleted file mode 100644 index d9db9bb8b..000000000 --- a/lib/crowdb-access-dataset/src/lease.rs +++ /dev/null @@ -1,42 +0,0 @@ -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; - -pub const DEFAULT_READ_LEASE_SECONDS: u64 = 60; - -/// Lock-free activity lease used by retention and reclamation checks. -pub struct ReadLease { - last_activity: AtomicU64, - released: AtomicBool, - ttl_seconds: u64, -} - -impl ReadLease { - #[must_use] - pub fn new(now_seconds: u64) -> Self { - Self::with_ttl(now_seconds, DEFAULT_READ_LEASE_SECONDS) - } - - #[must_use] - pub fn with_ttl(now_seconds: u64, ttl_seconds: u64) -> Self { - Self { - last_activity: AtomicU64::new(now_seconds), - released: AtomicBool::new(false), - ttl_seconds, - } - } - - pub fn touch(&self, now_seconds: u64) { - if !self.released.load(Ordering::Acquire) { - self.last_activity.store(now_seconds, Ordering::Release); - } - } - - #[must_use] - pub fn expired(&self, now_seconds: u64) -> bool { - self.released.load(Ordering::Acquire) - || now_seconds.saturating_sub(self.last_activity.load(Ordering::Acquire)) >= self.ttl_seconds - } - - pub fn release(&self) { - self.released.store(true, Ordering::Release); - } -} diff --git a/lib/crowdb-access-dataset/src/lib.rs b/lib/crowdb-access-dataset/src/lib.rs index c6f48bc6d..8b8b89ed6 100644 --- a/lib/crowdb-access-dataset/src/lib.rs +++ b/lib/crowdb-access-dataset/src/lib.rs @@ -15,7 +15,6 @@ mod error; mod identity; pub mod inspection; mod key; -mod lease; mod manifest; mod namespace; mod payload; @@ -46,7 +45,6 @@ pub use delivery::DeliveryWindow; pub use error::DatasetError; pub use identity::{DatasetId, OperationId, SnapshotId, SplitRevisionId}; pub use key::manifest_partition; -pub use lease::{ReadLease, DEFAULT_READ_LEASE_SECONDS}; pub use manifest::{ FieldDefinition, FieldLocator, FieldRecord, ManifestPartition, ManifestRecord, SampleRecord, SchemaRecord, }; @@ -79,15 +77,10 @@ pub use record::{ ActiveReadLease, DatasetRecord, HeadRecord, ManifestBinding, ReclaimProgress, ReclaimState, SnapshotRecord, }; -pub use retention::{ - RetentionLease, RetentionLeaseHandle, RetentionLeaseRegistry, RetentionPlan, RetentionPlanner, -}; +pub use retention::{RetentionPlan, RetentionPlanner}; pub use store::{ CasOutcome, DatasetStore, MetadataPage, MetadataScan, MetadataScanBudget, StoreError, StoredValue, }; -mod read_session; -pub use read_session::{ - DatasetSessions, ReadPreparation, ReadSession, SessionBatch, SessionPartition, SessionShuffle, - SessionSpec, SessionSpecV2, SessionSplit, -}; +mod read_query; +pub use read_query::{DatasetReads, ReadBinding, ReadPage, ReadPartition, ReadShuffle, ReadSpec, ReadSplit}; diff --git a/lib/crowdb-access-dataset/src/read_query.rs b/lib/crowdb-access-dataset/src/read_query.rs new file mode 100644 index 000000000..4376ae733 --- /dev/null +++ b/lib/crowdb-access-dataset/src/read_query.rs @@ -0,0 +1,140 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Stateless immutable reads. Continuations are caller-owned, never metadata records. + +mod binding; +mod page; +use crate::{AuthorityError, DatasetAuthority, DatasetError, DatasetIdentity, Selection, SnapshotId}; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReadPartition { + pub worker: u16, + pub workers: u16, + pub rank: u16, + pub world_size: u16, +} +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReadShuffle { + pub seed: u64, + pub group_size: usize, +} +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReadSplit { + pub revision: Option, + pub name: crate::split::SplitName, +} +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReadSpec { + pub snapshot: String, + pub selection: Selection, + pub fields: Vec, + pub batch_size: usize, + pub partition: ReadPartition, + pub shuffle: Option, + pub split: Option, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ReadBinding { + pub snapshot: SnapshotId, + pub split: Option, + pub cursor: String, +} +pub struct ReadPage { + pub snapshot: SnapshotId, + pub ids: Vec>, + pub fields: Vec, + pub next: Option, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Continuation { + version: u8, + identity: Vec, + spec: ReadSpec, + snapshot: SnapshotId, + split: Option, + after: Option, + ordinal: u64, + offset: usize, +} +const MAX_TOKEN_BYTES: usize = 256 * 1024; +impl Continuation { + fn encode(&self) -> Result { + let bytes = serde_json::to_vec(self).map_err(|_| AuthorityError::Corrupt)?; + if bytes.len() > MAX_TOKEN_BYTES { + return Err(DatasetError::ReadLimitExceeded.into()); + } + Ok(URL_SAFE_NO_PAD.encode(bytes)) + } + fn decode(identity: &DatasetIdentity, token: &str) -> Result { + if token.len() > MAX_TOKEN_BYTES * 4 / 3 + 4 { + return Err(DatasetError::ReadLimitExceeded.into()); + } + let bytes = URL_SAFE_NO_PAD + .decode(token) + .map_err(|_| DatasetError::CursorMismatch)?; + let value: Self = serde_json::from_slice(&bytes).map_err(|_| DatasetError::CursorMismatch)?; + if value.version != 1 || value.identity != identity.key()? { + return Err(DatasetError::CursorMismatch.into()); + } + validate_delivery(&value.spec)?; + if value.spec.snapshot != value.snapshot.to_string() + || value.offset > 100_000 + || match (&value.spec.split, &value.split) { + (None, None) => false, + (Some(request), Some(source)) => { + source.snapshot != value.snapshot + || request.revision != Some(source.revision) + || request.name != source.name + } + _ => true, + } + { + return Err(DatasetError::CursorMismatch.into()); + } + Ok(value) + } +} +pub struct DatasetReads { + authority: Arc, +} +impl DatasetReads { + #[must_use] + pub fn new(authority: Arc) -> Self { + Self { authority } + } +} +pub(super) fn validate_delivery(spec: &ReadSpec) -> Result<(), AuthorityError> { + let p = &spec.partition; + if p.workers == 0 + || p.world_size == 0 + || p.worker >= p.workers + || p.rank >= p.world_size + || spec.fields.len() > 256 + { + return Err(DatasetError::InvalidManifest.into()); + } + let mut names = std::collections::BTreeSet::new(); + if spec.batch_size == 0 + || spec.batch_size > crate::MAX_BATCH_SIZE + || spec + .fields + .iter() + .any(|name| name.is_empty() || !names.insert(name)) + || spec + .shuffle + .as_ref() + .is_some_and(|shuffle| shuffle.group_size == 0 || shuffle.group_size > 100_000) + { + return Err(DatasetError::InvalidManifest.into()); + } + Ok(()) +} diff --git a/lib/crowdb-access-dataset/src/read_query/binding.rs b/lib/crowdb-access-dataset/src/read_query/binding.rs new file mode 100644 index 000000000..702bc9868 --- /dev/null +++ b/lib/crowdb-access-dataset/src/read_query/binding.rs @@ -0,0 +1,135 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + validate_delivery, AuthorityError, Continuation, DatasetError, DatasetIdentity, DatasetReads, + ReadBinding, ReadSpec, SnapshotId, +}; +use crate::{split::SplitSource, MetadataScanBudget, ReadCancellation}; +use std::time::Duration; + +pub(super) fn budget() -> MetadataScanBudget { + MetadataScanBudget { + items: 20, + bytes: 1024 * 1024, + timeout: Duration::from_secs(2), + } +} +impl DatasetReads { + /// Resolves aliases into a portable continuation without registering a reader. + /// # Errors + /// Rejects invalid queries, unavailable snapshots/revisions and metadata limits. + pub async fn bind( + &self, + identity: &DatasetIdentity, + mut spec: ReadSpec, + ) -> Result { + validate_delivery(&spec)?; + let snapshot = self.resolve_snapshot(identity, &spec.snapshot).await?; + self.authority + .require_committed_snapshot(identity, snapshot) + .await?; + self.authority.check_snapshot_readable(identity, snapshot).await?; + let split = if let Some(request) = &mut spec.split { + let revision = match request.revision { + Some(revision) => revision, + None => self + .authority + .inspect_split_head(identity, snapshot) + .await + .map_err(split_error)? + .revision + .ok_or(AuthorityError::NotFound)?, + }; + let record = self + .authority + .get_split_revision(identity, snapshot, revision, budget(), &ReadCancellation::new()) + .await + .map_err(split_error)?; + if !record.splits.iter().any(|s| s.name == request.name) { + return Err(AuthorityError::NotFound); + } + request.revision = Some(revision); + Some(SplitSource { + snapshot, + revision, + name: request.name.clone(), + }) + } else { + None + }; + spec.snapshot = snapshot.to_string(); + let cursor = Continuation { + version: 1, + identity: identity.key()?, + spec, + snapshot, + split: split.clone(), + after: None, + ordinal: 0, + offset: 0, + } + .encode()?; + Ok(ReadBinding { + snapshot, + split, + cursor, + }) + } + pub(super) async fn resolve_snapshot( + &self, + identity: &DatasetIdentity, + value: &str, + ) -> Result { + let snapshot = match value { + "latest" => self + .authority + .latest(identity) + .await? + .ok_or(AuthorityError::SnapshotNotFound)?, + "stable" => self + .authority + .get_dataset(identity) + .await? + .stable + .ok_or(AuthorityError::SnapshotNotFound)?, + value => { + let mut id = [0; 16]; + if value.len() != 32 + || !value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + return Err(DatasetError::InvalidManifest.into()); + } + for (i, byte) in id.iter_mut().enumerate() { + *byte = u8::from_str_radix(&value[i * 2..i * 2 + 2], 16) + .map_err(|_| DatasetError::InvalidManifest)?; + } + SnapshotId::from_bytes(&id)? + } + }; + Ok(snapshot) + } +} +pub(super) fn inspection_error(error: crate::inspection::InspectionError) -> AuthorityError { + match error { + crate::inspection::InspectionError::Authority(error) => error, + crate::inspection::InspectionError::Store(error) => error.into(), + crate::inspection::InspectionError::Cancelled => DatasetError::ReadCancelled.into(), + crate::inspection::InspectionError::Limit + | crate::inspection::InspectionError::NotPrepared + | crate::inspection::InspectionError::Unsupported(_) => DatasetError::ReadLimitExceeded.into(), + crate::inspection::InspectionError::Corrupt => AuthorityError::Corrupt, + } +} +pub(super) fn split_error(error: crate::split::SplitLifecycleError) -> AuthorityError { + match error { + crate::split::SplitLifecycleError::Authority(error) => error, + crate::split::SplitLifecycleError::Store(error) => error.into(), + crate::split::SplitLifecycleError::NotFound => AuthorityError::NotFound, + crate::split::SplitLifecycleError::Inspection(error) => inspection_error(error), + crate::split::SplitLifecycleError::Limit => DatasetError::ReadLimitExceeded.into(), + _ => AuthorityError::Corrupt, + } +} diff --git a/lib/crowdb-access-dataset/src/read_query/page.rs b/lib/crowdb-access-dataset/src/read_query/page.rs new file mode 100644 index 000000000..29ce73815 --- /dev/null +++ b/lib/crowdb-access-dataset/src/read_query/page.rs @@ -0,0 +1,169 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + binding::{budget, inspection_error, split_error}, + AuthorityError, Continuation, DatasetError, DatasetIdentity, DatasetReads, ReadPage, +}; +use crate::{inspection::work::InspectionWork, ReadCancellation}; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use md5::{Digest, Md5}; +use std::time::{Duration, Instant}; + +impl DatasetReads { + /// Replays the caller's position. No read record, heartbeat or acknowledgement is written. + /// A partial shuffle group is reconstructed from its immutable start boundary. + /// # Errors + /// Rejects invalid continuations, deletion admission, unavailable metadata and bounded-work limits. + pub async fn next( + &self, + identity: &DatasetIdentity, + token: &str, + cancel: &ReadCancellation, + ) -> Result { + let mut cursor = Continuation::decode(identity, token)?; + self.authority + .check_snapshot_readable(identity, cursor.snapshot) + .await?; + let (mut ids, after) = + tokio::time::timeout(Duration::from_secs(2), self.group(identity, &cursor, cancel)) + .await + .map_err(|_| DatasetError::ReadLimitExceeded)??; + let count = ids.len(); + if let Some(shuffle) = &cursor.spec.shuffle { + ids.sort_by_cached_key(|id| { + let mut digest = Md5::new(); + digest.update(shuffle.seed.to_le_bytes()); + digest.update(cursor.snapshot.as_bytes()); + digest.update(id); + <[u8; 16]>::from(digest.finalize()) + }); + } + let p = &cursor.spec.partition; + let slots = u64::from(p.world_size) * u64::from(p.workers); + let slot = u64::from(p.rank) * u64::from(p.workers) + u64::from(p.worker); + let ids: Vec<_> = ids + .into_iter() + .enumerate() + .filter_map(|(i, id)| ((cursor.ordinal % slots + i as u64 % slots) % slots == slot).then_some(id)) + .collect(); + if cursor.offset > ids.len() { + return Err(DatasetError::CursorMismatch.into()); + } + let end = cursor + .offset + .saturating_add(cursor.spec.batch_size) + .min(ids.len()); + let batch = ids[cursor.offset..end].to_vec(); + let finished = end == ids.len(); + let next = if finished && after.is_none() { + None + } else { + if finished { + cursor.ordinal = cursor + .ordinal + .checked_add(count as u64) + .ok_or(DatasetError::ReadLimitExceeded)?; + cursor.offset = 0; + cursor.after = after.map(|bytes| URL_SAFE_NO_PAD.encode(bytes)); + } else { + cursor.offset = end; + } + Some(cursor.encode()?) + }; + Ok(ReadPage { + snapshot: cursor.snapshot, + ids: batch, + fields: cursor.spec.fields, + next, + }) + } + + async fn group( + &self, + identity: &DatasetIdentity, + cursor: &Continuation, + cancel: &ReadCancellation, + ) -> Result<(Vec>, Option>), AuthorityError> { + let p = &cursor.spec.partition; + let size = cursor.spec.shuffle.as_ref().map_or_else( + || { + cursor + .spec + .batch_size + .saturating_mul(usize::from(p.workers)) + .saturating_mul(usize::from(p.world_size)) + .min(100_000) + }, + |shuffle| shuffle.group_size, + ); + let mut ids = Vec::new(); + let mut after = cursor + .after + .as_ref() + .map(|value| URL_SAFE_NO_PAD.decode(value)) + .transpose() + .map_err(|_| DatasetError::CursorMismatch)?; + let deadline = Instant::now() + Duration::from_secs(2); + let mut bytes = 0usize; + let revision = if let Some(source) = &cursor.split { + Some( + self.authority + .get_split_revision(identity, cursor.snapshot, source.revision, budget(), cancel) + .await + .map_err(split_error)?, + ) + } else { + None + }; + for _ in 0..5000 { + if cancel.is_cancelled() { + return Err(DatasetError::ReadCancelled.into()); + } + let remaining = deadline + .checked_duration_since(Instant::now()) + .ok_or(DatasetError::ReadLimitExceeded)?; + let mut limits = budget(); + limits.items = (size - ids.len()).min(20); + limits.timeout = remaining; + let page = self + .authority + .inspection_selection_page( + identity, + cursor.snapshot, + &cursor.spec.selection, + after, + limits, + cancel, + ) + .await + .map_err(inspection_error)?; + let mut work = InspectionWork::new(self.authority.store.as_ref(), cancel, limits) + .map_err(inspection_error)?; + for id in page.entries { + if let (Some(revision), Some(source)) = (&revision, &cursor.split) { + if !self + .authority + .frozen_split_contains(identity, revision, &source.name, &id, &mut work) + .await + .map_err(split_error)? + { + continue; + } + } + bytes = bytes + .checked_add(id.len()) + .ok_or(DatasetError::ReadLimitExceeded)?; + if bytes > 8 * 1024 * 1024 { + return Err(DatasetError::ReadLimitExceeded.into()); + } + ids.push(id); + } + after = page.after; + if ids.len() == size || after.is_none() { + return Ok((ids, after)); + } + } + Err(DatasetError::ReadLimitExceeded.into()) + } +} diff --git a/lib/crowdb-access-dataset/src/read_session.rs b/lib/crowdb-access-dataset/src/read_session.rs deleted file mode 100644 index 24e4c6e18..000000000 --- a/lib/crowdb-access-dataset/src/read_session.rs +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -//! Durable scoped progress. Creation and acknowledgement prepare the next token; fetch is read-only. - -mod pages; -mod planning; -mod progress; -mod split; -pub use split::{ReadPreparation, SessionSpecV2, SessionSplit}; - -use crate::{ - AuthorityError, CasOutcome, DatasetAuthority, DatasetError, DatasetIdentity, OperationId, Ordering, - ReadPlan, Selection, SnapshotId, -}; -use md5::{Digest, Md5}; -use serde::{Deserialize, Serialize}; -use std::sync::Arc; - -const TTL: u64 = 60; -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub struct SessionPartition { - pub worker: u16, - pub workers: u16, - pub rank: u16, - pub world_size: u16, -} -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub struct SessionShuffle { - pub seed: u64, - pub group_size: usize, -} -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub struct SessionSpec { - pub id: OperationId, - pub snapshot: String, - pub selection: Selection, - pub fields: Vec, - pub batch_size: usize, - pub partition: SessionPartition, - pub shuffle: Option, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct ReadSession { - pub id: String, - pub snapshot: String, - pub plan: String, - pub acknowledged: String, - pub expires_at: u64, - pub closed: bool, -} -#[derive(Clone, Serialize, Deserialize)] -struct SessionRecord { - version: u16, - id: OperationId, - fields: Vec, - batch_size: usize, - count: usize, - snapshot: SnapshotId, - plan: [u8; 16], - cursor: String, - position: usize, - issued: Option<(String, usize)>, - deadline: u64, - closed: bool, -} -impl SessionRecord { - fn prepare_next(&mut self) { - self.version = 3; - let end = self.position.saturating_add(self.batch_size).min(self.count); - self.issued - .get_or_insert_with(|| (OperationId::random().to_string(), end)); - } - fn public(&self) -> ReadSession { - ReadSession { - id: self.id.to_string(), - snapshot: self.snapshot.to_string(), - plan: hex_digest(self.plan), - acknowledged: self.cursor.clone(), - expires_at: self.deadline, - closed: self.closed, - } - } -} -fn hex_digest(value: [u8; 16]) -> String { - use std::fmt::Write; - value.iter().fold(String::with_capacity(32), |mut out, b| { - write!(&mut out, "{b:02x}").expect("string writing cannot fail"); - out - }) -} -fn session_key(identity: &DatasetIdentity, id: OperationId) -> Result, AuthorityError> { - let mut key = b"DS\0RS\0".to_vec(); - key.extend(identity.key()?); - key.extend(id.as_bytes()); - Ok(key) -} -fn encode(record: &SessionRecord) -> Result, AuthorityError> { - let bytes = bincode::serialize(record).map_err(|_| AuthorityError::Corrupt)?; - if bytes.len() > 8 * 1024 * 1024 { - return Err(DatasetError::ReadLimitExceeded.into()); - } - Ok(bytes) -} -fn decode(bytes: &[u8]) -> Result { - let record: SessionRecord = bincode::deserialize(bytes).map_err(|_| AuthorityError::Corrupt)?; - if !matches!(record.version, 2 | 3) || (record.version == 3 && record.issued.is_none()) { - return Err(AuthorityError::Corrupt); - } - Ok(record) -} - -pub struct SessionBatch { - pub snapshot: SnapshotId, - pub ids: Vec>, - pub fields: Vec, - pub next: String, - pub end: bool, -} -pub struct DatasetSessions { - authority: Arc, -} -impl DatasetSessions { - #[must_use] - pub fn new(authority: Arc) -> Self { - Self { authority } - } - async fn record( - &self, - identity: &DatasetIdentity, - id: OperationId, - ) -> Result<(Vec, SessionRecord), AuthorityError> { - let stored = self - .authority - .store - .get(&session_key(identity, id)?) - .await? - .ok_or(AuthorityError::NotFound)?; - let record = decode(&stored.bytes)?; - Ok((stored.bytes, record)) - } -} diff --git a/lib/crowdb-access-dataset/src/read_session/pages.rs b/lib/crowdb-access-dataset/src/read_session/pages.rs deleted file mode 100644 index f27e78ec3..000000000 --- a/lib/crowdb-access-dataset/src/read_session/pages.rs +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::{ - session_key, AuthorityError, CasOutcome, DatasetIdentity, DatasetSessions, OperationId, SessionSpec, - SnapshotId, -}; -use serde::{Deserialize, Serialize}; -const PAGE_SIZE: usize = 1024; -#[derive(Serialize, Deserialize)] -pub(super) struct BoundSpec { - version: u16, - pub spec: SessionSpec, - pub snapshot: SnapshotId, -} -fn key(identity: &DatasetIdentity, id: OperationId, suffix: &[u8]) -> Result, AuthorityError> { - let mut key = session_key(identity, id)?; - key.extend(suffix); - Ok(key) -} -fn page_key(identity: &DatasetIdentity, id: OperationId, index: usize) -> Result, AuthorityError> { - let mut suffix = Vec::from(b"\0P".as_slice()); - suffix.extend( - u64::try_from(index) - .map_err(|_| AuthorityError::Corrupt)? - .to_be_bytes(), - ); - key(identity, id, &suffix) -} -fn decode(bytes: &[u8]) -> Result { - let record: BoundSpec = bincode::deserialize(bytes).map_err(|_| AuthorityError::Corrupt)?; - if record.version != 1 { - return Err(AuthorityError::Corrupt); - } - Ok(record) -} -impl DatasetSessions { - pub(super) async fn read_spec( - &self, - identity: &DatasetIdentity, - id: OperationId, - ) -> Result { - let record = self - .authority - .store - .get(&key(identity, id, b"\0SPEC")?) - .await? - .ok_or(AuthorityError::NotFound)?; - decode(&record.bytes) - } - pub(super) async fn bind_spec( - &self, - identity: &DatasetIdentity, - spec: &SessionSpec, - snapshot: SnapshotId, - ) -> Result { - let bound = BoundSpec { - version: 1, - spec: spec.clone(), - snapshot, - }; - let bytes = bincode::serialize(&bound).map_err(|_| AuthorityError::Corrupt)?; - if bytes.len() > 8 * 1024 * 1024 { - return Err(crate::DatasetError::ReadLimitExceeded.into()); - } - match self - .authority - .store - .compare_exchange(&key(identity, spec.id, b"\0SPEC")?, None, &bytes) - .await? - { - CasOutcome::Applied(_) => Ok(bound), - CasOutcome::Conflict(Some(old)) => { - let old = decode(&old.bytes)?; - if old.spec != *spec { - return Err(AuthorityError::CursorConflict); - } - Ok(old) - } - CasOutcome::Conflict(None) => Err(AuthorityError::Corrupt), - } - } - pub(super) async fn save_ids( - &self, - identity: &DatasetIdentity, - id: OperationId, - ids: &[Vec], - ) -> Result<(), AuthorityError> { - for (index, page) in ids.chunks(PAGE_SIZE).enumerate() { - let bytes = bincode::serialize(page).map_err(|_| AuthorityError::Corrupt)?; - match self - .authority - .store - .compare_exchange(&page_key(identity, id, index)?, None, &bytes) - .await? - { - CasOutcome::Applied(_) => {} - CasOutcome::Conflict(Some(old)) if old.bytes == bytes => {} - CasOutcome::Conflict(_) => return Err(AuthorityError::CursorConflict), - } - } - Ok(()) - } - pub(super) async fn load_ids( - &self, - identity: &DatasetIdentity, - id: OperationId, - start: usize, - end: usize, - ) -> Result>, AuthorityError> { - let mut result = Vec::with_capacity(end.saturating_sub(start)); - for index in start / PAGE_SIZE..end.div_ceil(PAGE_SIZE) { - let value = self - .authority - .store - .get(&page_key(identity, id, index)?) - .await? - .ok_or(AuthorityError::Corrupt)?; - let page: Vec> = - bincode::deserialize(&value.bytes).map_err(|_| AuthorityError::Corrupt)?; - let low = start.saturating_sub(index * PAGE_SIZE); - let high = (end - index * PAGE_SIZE).min(page.len()); - result.extend_from_slice(page.get(low..high).ok_or(AuthorityError::Corrupt)?); - } - if result.len() != end - start { - return Err(AuthorityError::Corrupt); - } - Ok(result) - } -} diff --git a/lib/crowdb-access-dataset/src/read_session/planning.rs b/lib/crowdb-access-dataset/src/read_session/planning.rs deleted file mode 100644 index 110f119de..000000000 --- a/lib/crowdb-access-dataset/src/read_session/planning.rs +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::{ - encode, session_key, AuthorityError, CasOutcome, DatasetError, DatasetIdentity, DatasetSessions, Digest, - Md5, OperationId, Ordering, ReadPlan, ReadSession, SessionRecord, SessionSpec, SnapshotId, TTL, -}; - -impl DatasetSessions { - /// # Errors - /// Rejects invalid plans, conflicting identities, missing snapshots or storage failures. - pub async fn start( - &self, - identity: &DatasetIdentity, - spec: SessionSpec, - now: u64, - ) -> Result { - self.reserve_session_version(identity, spec.id, 1).await?; - match self.record(identity, spec.id).await { - Ok((_, _old)) => { - if self.read_spec(identity, spec.id).await?.spec != spec { - return Err(AuthorityError::CursorConflict); - } - return self.resume(identity, spec.id, now).await; - } - Err(AuthorityError::NotFound) => {} - Err(error) => return Err(error), - } - let snapshot = match self.read_spec(identity, spec.id).await { - Ok(old) => { - if old.spec != spec { - return Err(AuthorityError::CursorConflict); - } - old.snapshot - } - Err(AuthorityError::NotFound) => self.resolve_snapshot(identity, &spec.snapshot).await?, - Err(error) => return Err(error), - }; - let bound = self.bind_spec(identity, &spec, snapshot).await?; - let snapshot = bound.snapshot; - let ids = self.select_ids(identity, snapshot, &spec).await?; - self.save_ids(identity, spec.id, &ids).await?; - let fingerprint: [u8; 16] = Md5::digest( - bincode::serialize(&(identity.key()?, snapshot, &spec)).map_err(|_| AuthorityError::Corrupt)?, - ) - .into(); - let mut record = SessionRecord { - version: 2, - id: spec.id, - fields: spec.fields.clone(), - batch_size: spec.batch_size, - count: ids.len(), - snapshot, - plan: fingerprint, - cursor: OperationId::random().to_string(), - position: 0, - issued: None, - deadline: now.saturating_add(TTL), - closed: false, - }; - record.prepare_next(); - self.authority.check_snapshot_readable(identity, snapshot).await?; - match self - .authority - .store - .compare_exchange(&session_key(identity, spec.id)?, None, &encode(&record)?) - .await? - { - CasOutcome::Applied(_) => Ok(record.public()), - CasOutcome::Conflict(Some(_)) => self.resume(identity, spec.id, now).await, - CasOutcome::Conflict(None) => Err(AuthorityError::Corrupt), - } - } - pub(super) async fn resolve_snapshot( - &self, - identity: &DatasetIdentity, - value: &str, - ) -> Result { - let snapshot = match value { - "latest" => self - .authority - .latest(identity) - .await? - .ok_or(AuthorityError::SnapshotNotFound)?, - "stable" => self - .authority - .get_dataset(identity) - .await? - .stable - .ok_or(AuthorityError::SnapshotNotFound)?, - value => { - let mut id = [0; 16]; - if value.len() != 32 - || !value - .bytes() - .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) - { - return Err(DatasetError::InvalidManifest.into()); - } - for (i, byte) in id.iter_mut().enumerate() { - *byte = u8::from_str_radix(&value[i * 2..i * 2 + 2], 16) - .map_err(|_| DatasetError::InvalidManifest)?; - } - SnapshotId::from_bytes(&id)? - } - }; - Ok(snapshot) - } - async fn select_ids( - &self, - identity: &DatasetIdentity, - snapshot: SnapshotId, - spec: &SessionSpec, - ) -> Result>, AuthorityError> { - self.authority.get_snapshot(identity, snapshot).await?; - let plan = validate_spec(spec, snapshot)?; - let manifest = self.authority.get_manifest(identity, snapshot).await?; - if spec - .fields - .iter() - .any(|name| !manifest.schema.fields.iter().any(|f| &f.name == name)) - { - return Err(DatasetError::InvalidManifest.into()); - } - let ids = plan.scan_ids(&manifest)?; - if ids.len() > 100_000 { - return Err(DatasetError::ReadLimitExceeded.into()); - } - arrange_ids(spec, snapshot, ids) - } -} - -pub(super) fn validate_spec(spec: &SessionSpec, snapshot: SnapshotId) -> Result { - validate_delivery(spec)?; - let plan = ReadPlan { - snapshot, - selection: spec.selection.clone(), - ordering: Ordering::SampleId, - projection: spec.fields.clone(), - batch_size: spec.batch_size, - }; - plan.validate()?; - Ok(plan) -} -pub(super) fn arrange_ids( - spec: &SessionSpec, - snapshot: SnapshotId, - mut ids: Vec>, -) -> Result>, AuthorityError> { - let p = &spec.partition; - if let Some(shuffle) = &spec.shuffle { - if shuffle.group_size == 0 || shuffle.group_size > 1_000_000 { - return Err(DatasetError::InvalidManifest.into()); - } - for group in ids.chunks_mut(shuffle.group_size) { - group.sort_by_cached_key(|id| { - let mut hash = Md5::new(); - hash.update(shuffle.seed.to_le_bytes()); - hash.update(snapshot.as_bytes()); - hash.update(id); - <[u8; 16]>::from(hash.finalize()) - }); - } - } - let slot = usize::from(p.rank) * usize::from(p.workers) + usize::from(p.worker); - let slots = usize::from(p.world_size) * usize::from(p.workers); - ids = ids - .into_iter() - .enumerate() - .filter_map(|(i, id)| (i % slots == slot).then_some(id)) - .collect(); - Ok(ids) -} - -pub(super) fn validate_delivery(spec: &SessionSpec) -> Result<(), AuthorityError> { - let p = &spec.partition; - if p.workers == 0 - || p.world_size == 0 - || p.worker >= p.workers - || p.rank >= p.world_size - || spec.fields.len() > 256 - { - return Err(DatasetError::InvalidManifest.into()); - } - let mut names = std::collections::BTreeSet::new(); - if spec.batch_size == 0 - || spec.batch_size > crate::MAX_BATCH_SIZE - || spec - .fields - .iter() - .any(|name| name.is_empty() || !names.insert(name)) - || spec - .shuffle - .as_ref() - .is_some_and(|shuffle| shuffle.group_size == 0 || shuffle.group_size > 1_000_000) - { - return Err(DatasetError::InvalidManifest.into()); - } - Ok(()) -} diff --git a/lib/crowdb-access-dataset/src/read_session/progress.rs b/lib/crowdb-access-dataset/src/read_session/progress.rs deleted file mode 100644 index be8a3cabb..000000000 --- a/lib/crowdb-access-dataset/src/read_session/progress.rs +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::{ - decode, encode, session_key, AuthorityError, CasOutcome, DatasetError, DatasetIdentity, DatasetSessions, - OperationId, ReadSession, SessionBatch, TTL, -}; - -impl DatasetSessions { - /// Resumes acknowledged progress and refreshes its admission deadline. - /// # Errors - /// Rejects closed/reclaimed sessions or unavailable storage. - pub async fn resume( - &self, - identity: &DatasetIdentity, - id: OperationId, - now: u64, - ) -> Result { - let mut current = self.record(identity, id).await?; - for _ in 0..64 { - let (old, mut record) = current; - if record.closed { - return Err(DatasetError::ReadCancelled.into()); - } - self.authority - .require_committed_snapshot(identity, record.snapshot) - .await?; - self.authority - .check_snapshot_readable(identity, record.snapshot) - .await?; - record.deadline = record.deadline.max(now.saturating_add(TTL)); - record.prepare_next(); - let bytes = encode(&record)?; - if bytes == old { - return Ok(record.public()); - } - match self - .authority - .store - .compare_exchange(&session_key(identity, id)?, Some(&old), &bytes) - .await? - { - CasOutcome::Applied(_) => return Ok(record.public()), - CasOutcome::Conflict(Some(value)) => current = (value.bytes.clone(), decode(&value.bytes)?), - CasOutcome::Conflict(None) => return Err(AuthorityError::NotFound), - } - } - Err(AuthorityError::CursorConflict) - } - /// Fetch is read-only for prepared sessions and replays the same next token. - /// Only creation, explicit resume and acknowledgement refresh admission. - /// # Errors - /// Rejects disconnected/closed sessions or unavailable storage. - pub async fn next( - &self, - identity: &DatasetIdentity, - id: OperationId, - now: u64, - ) -> Result { - let mut current = self.record(identity, id).await?; - for _ in 0..64 { - let (old, mut r) = current; - if r.closed || now >= r.deadline { - return Err(DatasetError::ReadCancelled.into()); - } - self.authority - .check_snapshot_readable(identity, r.snapshot) - .await?; - if let Some((next, end)) = r.issued.clone() { - return Ok(SessionBatch { - snapshot: r.snapshot, - ids: self.load_ids(identity, id, r.position, end).await?, - fields: r.fields, - next, - end: end == r.count, - }); - } - // Old records may not have a prepared token yet. Upgrade once. - r.prepare_next(); - match self - .authority - .store - .compare_exchange(&session_key(identity, id)?, Some(&old), &encode(&r)?) - .await? - { - CasOutcome::Applied(_) => current = (encode(&r)?, r), - CasOutcome::Conflict(Some(value)) => current = (value.bytes.clone(), decode(&value.bytes)?), - CasOutcome::Conflict(None) => return Err(AuthorityError::NotFound), - } - } - Err(AuthorityError::CursorConflict) - } - /// # Errors - /// Rejects forged, stale or divergent cursors. Retrying accepted progress is idempotent. - pub async fn acknowledge( - &self, - identity: &DatasetIdentity, - id: OperationId, - expected: &str, - next: &str, - now: u64, - ) -> Result { - let mut current = self.record(identity, id).await?; - for _ in 0..64 { - let (old, mut r) = current; - if r.closed || now >= r.deadline { - return Err(DatasetError::ReadCancelled.into()); - } - if r.cursor == next { - return Ok(r.public()); - } - if r.cursor != expected || !r.issued.as_ref().is_some_and(|(token, _)| token == next) { - return Err(AuthorityError::CursorConflict); - } - let (_, position) = r.issued.take().ok_or(AuthorityError::CursorConflict)?; - r.position = position; - r.cursor = next.into(); - r.deadline = r.deadline.max(now.saturating_add(TTL)); - r.prepare_next(); - self.authority - .check_snapshot_readable(identity, r.snapshot) - .await?; - let bytes = encode(&r)?; - if bytes == old { - return Ok(r.public()); - } - match self - .authority - .store - .compare_exchange(&session_key(identity, id)?, Some(&old), &bytes) - .await? - { - CasOutcome::Applied(_) => return Ok(r.public()), - CasOutcome::Conflict(Some(value)) => current = (value.bytes.clone(), decode(&value.bytes)?), - CasOutcome::Conflict(None) => return Err(AuthorityError::NotFound), - } - } - Err(AuthorityError::CursorConflict) - } - /// Checks cancellation without advancing activity or acknowledged progress. - /// # Errors - /// Returns a closed/missing session or storage failure. - pub async fn check_open( - &self, - identity: &DatasetIdentity, - id: OperationId, - ) -> Result<(), AuthorityError> { - let (_, r) = self.record(identity, id).await?; - if r.closed { - return Err(DatasetError::ReadCancelled.into()); - } - Ok(()) - } - /// # Errors - /// Returns missing sessions or storage failures. Other readers remain admitted. - pub async fn close( - &self, - identity: &DatasetIdentity, - id: OperationId, - ) -> Result { - let mut current = self.record(identity, id).await?; - for _ in 0..64 { - let (old, mut r) = current; - r.closed = true; - let bytes = encode(&r)?; - if bytes == old { - return Ok(r.public()); - } - match self - .authority - .store - .compare_exchange(&session_key(identity, id)?, Some(&old), &bytes) - .await? - { - CasOutcome::Applied(_) => return Ok(r.public()), - CasOutcome::Conflict(Some(value)) => current = (value.bytes.clone(), decode(&value.bytes)?), - CasOutcome::Conflict(None) => return Err(AuthorityError::NotFound), - } - } - Err(AuthorityError::CursorConflict) - } -} diff --git a/lib/crowdb-access-dataset/src/read_session/split.rs b/lib/crowdb-access-dataset/src/read_session/split.rs deleted file mode 100644 index afbf64e40..000000000 --- a/lib/crowdb-access-dataset/src/read_session/split.rs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -mod binding; -mod finish; -mod planning; -mod state; - -use super::{ReadSession, SessionSpec}; -use crate::{ - split::{SplitName, SplitSource}, - SplitRevisionId, -}; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SessionSplit { - /// None resolves the current revision once when preparation begins. - pub revision: Option, - pub name: SplitName, -} -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SessionSpecV2 { - pub read: SessionSpec, - pub split: Option, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct ReadPreparation { - pub snapshot: crate::SnapshotId, - pub split: Option, - pub next_page: u64, - pub admitted: u64, - pub complete: bool, - pub session: Option, -} diff --git a/lib/crowdb-access-dataset/src/read_session/split/binding.rs b/lib/crowdb-access-dataset/src/read_session/split/binding.rs deleted file mode 100644 index 41a5b6418..000000000 --- a/lib/crowdb-access-dataset/src/read_session/split/binding.rs +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::{ - state::{budget, decode, encode, key, BoundSplitSpec}, - SessionSpecV2, -}; -use crate::{ - split::SplitSource, AuthorityError, DatasetError, DatasetIdentity, DatasetSessions, ReadCancellation, -}; - -impl DatasetSessions { - pub(super) async fn bind_split_read_spec( - &self, - identity: &DatasetIdentity, - spec: &SessionSpecV2, - _now: u64, - cancel: &ReadCancellation, - ) -> Result { - self.reserve_session_version(identity, spec.read.id, 2).await?; - match self.bound_split_read_spec(identity, spec.read.id, cancel).await { - Ok(bound) => { - if bound.spec != *spec { - return Err(AuthorityError::CursorConflict); - } - return Ok(bound); - } - Err(AuthorityError::NotFound) => {} - Err(error) => return Err(error), - } - let snapshot = self.resolve_snapshot(identity, &spec.read.snapshot).await?; - self.authority - .require_committed_snapshot(identity, snapshot) - .await?; - super::super::planning::validate_delivery(&spec.read)?; - let split = if let Some(split) = &spec.split { - let revision = match split.revision { - Some(revision) => revision, - None => self - .authority - .inspect_split_head(identity, snapshot) - .await - .map_err(split_error)? - .revision - .ok_or(AuthorityError::NotFound)?, - }; - let record = self - .authority - .get_split_revision(identity, snapshot, revision, budget(), cancel) - .await - .map_err(split_error)?; - if !record.splits.iter().any(|s| s.name == split.name) { - return Err(AuthorityError::NotFound); - } - Some(SplitSource { - snapshot, - revision, - name: split.name.clone(), - }) - } else { - None - }; - self.validate_split_read_schema(identity, snapshot, spec, cancel) - .await?; - self.authority.check_snapshot_readable(identity, snapshot).await?; - let bound = BoundSplitSpec { - version: 2, - spec: spec.clone(), - snapshot, - split, - }; - let stored_key = key(identity, spec.read.id, b"\0SPEC2")?; - match self - .immutable_session_metadata(&stored_key, &encode(&bound)?) - .await - { - Ok(()) => Ok(bound), - Err(AuthorityError::CursorConflict) => { - let winner = self.bound_split_read_spec(identity, spec.read.id, cancel).await?; - if winner.spec != *spec { - return Err(AuthorityError::CursorConflict); - } - Ok(winner) - } - Err(error) => Err(error), - } - } - pub(super) async fn bound_split_read_spec( - &self, - identity: &DatasetIdentity, - id: crate::OperationId, - cancel: &ReadCancellation, - ) -> Result { - let stored = self - .authority - .store - .get_bounded(&key(identity, id, b"\0SPEC2")?, budget(), cancel) - .await? - .ok_or(AuthorityError::NotFound)?; - let bound: BoundSplitSpec = decode(&stored.bytes)?; - if bound.version != 2 || bound.spec.read.id != id { - return Err(AuthorityError::Corrupt); - } - match (&bound.spec.split, &bound.split) { - (None, None) => {} - (Some(request), Some(source)) - if source.snapshot == bound.snapshot - && request.name == source.name - && request.revision.map_or(true, |r| r == source.revision) => {} - _ => return Err(AuthorityError::Corrupt), - } - Ok(bound) - } - async fn validate_split_read_schema( - &self, - identity: &DatasetIdentity, - snapshot: crate::SnapshotId, - spec: &SessionSpecV2, - cancel: &ReadCancellation, - ) -> Result<(), AuthorityError> { - let mut after = None; - let mut fields = Vec::new(); - loop { - let page = self - .authority - .inspection_schema_page(identity, snapshot, after, budget(), cancel) - .await - .map_err(inspection_error)?; - fields.extend(page.entries); - if fields.len() > 256 { - return Err(DatasetError::ReadLimitExceeded.into()); - } - after = page.after; - if after.is_none() { - break; - } - } - if spec - .read - .fields - .iter() - .any(|name| !fields.iter().any(|f| f.name == *name)) - { - return Err(DatasetError::InvalidManifest.into()); - } - crate::selection::EffectiveSelection::new(&spec.read.selection, &fields)?; - Ok(()) - } -} -pub(super) fn inspection_error(error: crate::inspection::InspectionError) -> AuthorityError { - match error { - crate::inspection::InspectionError::Authority(error) => error, - crate::inspection::InspectionError::Store(error) => error.into(), - crate::inspection::InspectionError::Cancelled => DatasetError::ReadCancelled.into(), - crate::inspection::InspectionError::Limit - | crate::inspection::InspectionError::NotPrepared - | crate::inspection::InspectionError::Unsupported(_) => DatasetError::ReadLimitExceeded.into(), - crate::inspection::InspectionError::Corrupt => AuthorityError::Corrupt, - } -} -pub(super) fn split_error(error: crate::split::SplitLifecycleError) -> AuthorityError { - match error { - crate::split::SplitLifecycleError::Authority(error) => error, - crate::split::SplitLifecycleError::Store(error) => error.into(), - crate::split::SplitLifecycleError::NotFound => AuthorityError::NotFound, - crate::split::SplitLifecycleError::Inspection(error) => inspection_error(error), - crate::split::SplitLifecycleError::Limit => DatasetError::ReadLimitExceeded.into(), - _ => AuthorityError::Corrupt, - } -} diff --git a/lib/crowdb-access-dataset/src/read_session/split/finish.rs b/lib/crowdb-access-dataset/src/read_session/split/finish.rs deleted file mode 100644 index 056b1ebb3..000000000 --- a/lib/crowdb-access-dataset/src/read_session/split/finish.rs +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::{ - binding::inspection_error, - state::{decode, page_key, BoundSplitSpec, IdPage, PreparationProgress}, - ReadPreparation, -}; -use crate::{ - inspection::work::InspectionWork, AuthorityError, CasOutcome, DatasetError, DatasetIdentity, - DatasetSessions, MetadataScanBudget, ReadCancellation, -}; -use sha2::{Digest, Sha256}; -use std::time::Duration; - -impl DatasetSessions { - pub(super) async fn finish_split_read( - &self, - identity: &DatasetIdentity, - bound: &BoundSplitSpec, - progress: &PreparationProgress, - now: u64, - cancel: &ReadCancellation, - ) -> Result { - let ids = self - .collect_split_read_ids(identity, bound, progress, cancel) - .await?; - let ids = super::super::planning::arrange_ids(&bound.spec.read, bound.snapshot, ids)?; - self.save_ids(identity, bound.spec.read.id, &ids).await?; - let record = split_session_record(identity, bound, ids.len(), now, false)?; - let key = super::super::session_key(identity, bound.spec.read.id)?; - match self - .authority - .store - .compare_exchange(&key, None, &super::super::encode(&record)?) - .await? - { - CasOutcome::Applied(_) | CasOutcome::Conflict(Some(_)) => { - self.prepared_split_read(identity, bound, progress, now).await - } - CasOutcome::Conflict(None) => Err(AuthorityError::Corrupt), - } - } - async fn collect_split_read_ids( - &self, - identity: &DatasetIdentity, - bound: &BoundSplitSpec, - progress: &PreparationProgress, - cancel: &ReadCancellation, - ) -> Result>, AuthorityError> { - let mut work = InspectionWork::new( - self.authority.store.as_ref(), - cancel, - MetadataScanBudget { - items: 20, - bytes: 16 * 1024 * 1024, - timeout: Duration::from_secs(30), - }, - ) - .map_err(inspection_error)?; - let mut ids = Vec::new(); - let mut binding = [0; 32]; - for index in 0..progress.next_page { - let stored = work - .get(&page_key(identity, bound.spec.read.id, index)?, 1024 * 1024) - .await - .map_err(inspection_error)? - .ok_or(AuthorityError::Corrupt)?; - let page: IdPage = decode(&stored.bytes)?; - let mut hash = Sha256::new(); - hash.update(binding); - hash.update(Sha256::digest(&stored.bytes)); - binding = hash.finalize().into(); - ids.extend(page.ids); - if ids.len() > 100_000 { - return Err(DatasetError::ReadLimitExceeded.into()); - } - } - if binding != progress.binding - || ids.len() as u64 != progress.admitted - || ids.windows(2).any(|pair| pair[0] >= pair[1]) - { - return Err(AuthorityError::Corrupt); - } - Ok(ids) - } - /// Cancels preparation or delivery and fences a late finalizer with a closed session record. - /// # Errors - /// Returns unknown preparation, corrupt metadata or storage failures. - pub async fn cancel_split_read( - &self, - identity: &DatasetIdentity, - id: crate::OperationId, - ) -> Result { - let cancel = ReadCancellation::new(); - let bound = self.bound_split_read_spec(identity, id, &cancel).await?; - let (old, mut progress) = self.split_preparation_progress(identity, id, &cancel).await?; - progress.cancelled = true; - self.replace_split_preparation(identity, id, old.as_ref(), &progress) - .await?; - let record = split_session_record(identity, &bound, 0, 0, true)?; - let key = super::super::session_key(identity, id)?; - match self - .authority - .store - .compare_exchange(&key, None, &super::super::encode(&record)?) - .await? - { - CasOutcome::Applied(_) | CasOutcome::Conflict(Some(_)) => self.close(identity, id).await, - CasOutcome::Conflict(None) => Err(AuthorityError::Corrupt), - } - } -} -fn split_session_record( - identity: &DatasetIdentity, - bound: &BoundSplitSpec, - count: usize, - now: u64, - closed: bool, -) -> Result { - let plan = md5::Md5::digest( - bincode::serialize(&( - "dataset-read-v2", - identity.key()?, - bound.snapshot, - &bound.split, - &bound.spec, - )) - .map_err(|_| AuthorityError::Corrupt)?, - ) - .into(); - let mut record = super::super::SessionRecord { - version: 2, - id: bound.spec.read.id, - fields: bound.spec.read.fields.clone(), - batch_size: bound.spec.read.batch_size, - count, - snapshot: bound.snapshot, - plan, - cursor: crate::OperationId::random().to_string(), - position: 0, - issued: None, - deadline: now.saturating_add(super::super::TTL), - closed, - }; - record.prepare_next(); - Ok(record) -} diff --git a/lib/crowdb-access-dataset/src/read_session/split/planning.rs b/lib/crowdb-access-dataset/src/read_session/split/planning.rs deleted file mode 100644 index 97b80802a..000000000 --- a/lib/crowdb-access-dataset/src/read_session/split/planning.rs +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::{ - binding::{inspection_error, split_error}, - state::{budget, decode, encode, page_key, BoundSplitSpec, IdPage, PreparationProgress}, - ReadPreparation, SessionSpecV2, -}; -use crate::{ - inspection::work::InspectionWork, AuthorityError, DatasetError, DatasetIdentity, DatasetSessions, - ReadCancellation, -}; -use sha2::{Digest, Sha256}; - -impl DatasetSessions { - /// Reads pinned preparation scope and progress without acknowledging any batch. - /// # Errors - /// Rejects unknown or cancelled preparations, retired owners and storage failures. - pub async fn split_read_status( - &self, - identity: &DatasetIdentity, - id: crate::OperationId, - now: u64, - cancel: &ReadCancellation, - ) -> Result { - let bound = self.bound_split_read_spec(identity, id, cancel).await?; - let (_, progress) = self.split_preparation_progress(identity, id, cancel).await?; - if progress.cancelled { - return Err(DatasetError::ReadCancelled.into()); - } - self.authority - .require_committed_snapshot(identity, bound.snapshot) - .await?; - match self.record(identity, id).await { - Ok(_) => self.prepared_split_read(identity, &bound, &progress, now).await, - Err(AuthorityError::NotFound) => Ok(preparation(&bound, &progress)), - Err(error) => Err(error), - } - } - /// Resolves aliases once and advances one bounded effective-selection/Split window. - /// Complete preparation freezes ordering and worker membership before payload reads. - /// # Errors - /// Rejects conflicting specs/pages, unavailable revisions, invalid rules and work limits. - pub async fn prepare_split_read( - &self, - identity: &DatasetIdentity, - spec: &SessionSpecV2, - expected_page: u64, - now: u64, - cancel: &ReadCancellation, - ) -> Result { - let result = self - .prepare_split_read_step(identity, spec, expected_page, now, cancel) - .await; - if cancel.is_cancelled() { - return Err(DatasetError::ReadCancelled.into()); - } - result - } - async fn prepare_split_read_step( - &self, - identity: &DatasetIdentity, - spec: &SessionSpecV2, - expected_page: u64, - now: u64, - cancel: &ReadCancellation, - ) -> Result { - if cancel.is_cancelled() { - return Err(DatasetError::ReadCancelled.into()); - } - let bound = self.bind_split_read_spec(identity, spec, now, cancel).await?; - let (old, progress) = self - .split_preparation_progress(identity, spec.read.id, cancel) - .await?; - if progress.cancelled { - return Err(DatasetError::ReadCancelled.into()); - } - self.authority - .require_committed_snapshot(identity, bound.snapshot) - .await?; - self.authority - .check_snapshot_readable(identity, bound.snapshot) - .await?; - match self.record(identity, spec.read.id).await { - Ok(_) => return self.prepared_split_read(identity, &bound, &progress, now).await, - Err(AuthorityError::NotFound) => {} - Err(error) => return Err(error), - } - if expected_page > progress.next_page { - return Err(AuthorityError::CursorConflict); - } - if progress.complete { - return self - .finish_split_read(identity, &bound, &progress, now, cancel) - .await; - } - if expected_page < progress.next_page { - return Ok(preparation(&bound, &progress)); - } - let page_key = page_key(identity, spec.read.id, expected_page)?; - let stored = self - .authority - .store - .get_bounded(&page_key, budget(), cancel) - .await?; - let page = if let Some(stored) = stored { - decode::(&stored.bytes)? - } else { - let page = self - .select_split_read_window(identity, &bound, &progress, cancel) - .await?; - self.immutable_session_metadata(&page_key, &encode(&page)?) - .await?; - page - }; - let bytes = encode(&page)?; - let mut next = progress.clone(); - next.admitted = next - .admitted - .checked_add(page.ids.len() as u64) - .ok_or(DatasetError::ReadLimitExceeded)?; - next.bytes = next - .bytes - .checked_add(bytes.len() as u64) - .ok_or(DatasetError::ReadLimitExceeded)?; - if next.admitted > 100_000 || next.bytes > 8 * 1024 * 1024 { - return Err(DatasetError::ReadLimitExceeded.into()); - } - let mut hash = Sha256::new(); - hash.update(next.binding); - hash.update(Sha256::digest(&bytes)); - next.binding = hash.finalize().into(); - next.next_page = next - .next_page - .checked_add(1) - .ok_or(DatasetError::ReadLimitExceeded)?; - next.after = page.after; - next.complete = next.after.is_none(); - self.replace_split_preparation(identity, spec.read.id, old.as_ref(), &next) - .await?; - if next.complete { - self.finish_split_read(identity, &bound, &next, now, cancel).await - } else { - Ok(preparation(&bound, &next)) - } - } - async fn select_split_read_window( - &self, - identity: &DatasetIdentity, - bound: &BoundSplitSpec, - progress: &PreparationProgress, - cancel: &ReadCancellation, - ) -> Result { - let mut page = self - .authority - .inspection_selection_page( - identity, - bound.snapshot, - &bound.spec.read.selection, - progress.after.clone(), - budget(), - cancel, - ) - .await - .map_err(inspection_error)?; - if let Some(source) = &bound.split { - let mut work = InspectionWork::new(self.authority.store.as_ref(), cancel, budget()) - .map_err(inspection_error)?; - let revision = self - .authority - .read_split_revision(identity, source.snapshot, source.revision, &mut work) - .await - .map_err(split_error)?; - let mut admitted = Vec::new(); - for id in page.entries { - if self - .authority - .frozen_split_contains(identity, &revision, &source.name, &id, &mut work) - .await - .map_err(split_error)? - { - admitted.push(id); - } - } - page.entries = admitted; - } - Ok(IdPage { - ids: page.entries, - after: page.after, - }) - } - pub(super) async fn prepared_split_read( - &self, - identity: &DatasetIdentity, - bound: &BoundSplitSpec, - progress: &PreparationProgress, - now: u64, - ) -> Result { - let mut preparation = preparation(bound, progress); - preparation.session = Some(self.resume(identity, bound.spec.read.id, now).await?); - preparation.complete = true; - Ok(preparation) - } -} -pub(super) fn preparation(bound: &BoundSplitSpec, progress: &PreparationProgress) -> ReadPreparation { - ReadPreparation { - snapshot: bound.snapshot, - split: bound.split.clone(), - next_page: progress.next_page, - admitted: progress.admitted, - complete: false, - session: None, - } -} diff --git a/lib/crowdb-access-dataset/src/read_session/split/state.rs b/lib/crowdb-access-dataset/src/read_session/split/state.rs deleted file mode 100644 index 7dc84dd7b..000000000 --- a/lib/crowdb-access-dataset/src/read_session/split/state.rs +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::SessionSpecV2; -use crate::{ - split::SplitSource, AuthorityError, CasOutcome, DatasetIdentity, DatasetSessions, MetadataScanBudget, - OperationId, ReadCancellation, SnapshotId, StoredValue, -}; -use serde::{Deserialize, Serialize}; -use std::time::Duration; - -#[derive(Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct BoundSplitSpec { - pub version: u16, - pub spec: SessionSpecV2, - pub snapshot: SnapshotId, - pub split: Option, -} -#[derive(Clone, Default, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct PreparationProgress { - pub next_page: u64, - pub after: Option>, - pub admitted: u64, - pub bytes: u64, - pub binding: [u8; 32], - pub complete: bool, - pub cancelled: bool, -} -#[derive(Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct IdPage { - pub ids: Vec>, - pub after: Option>, -} - -pub(super) fn key( - identity: &DatasetIdentity, - id: OperationId, - suffix: &[u8], -) -> Result, AuthorityError> { - let mut key = super::super::session_key(identity, id)?; - key.extend_from_slice(suffix); - Ok(key) -} -pub(super) fn page_key( - identity: &DatasetIdentity, - id: OperationId, - page: u64, -) -> Result, AuthorityError> { - let mut key = key(identity, id, b"\0SELECT2")?; - key.extend_from_slice(&page.to_be_bytes()); - Ok(key) -} -pub(super) fn budget() -> MetadataScanBudget { - MetadataScanBudget { - items: 20, - bytes: 1024 * 1024, - timeout: Duration::from_secs(2), - } -} -pub(super) fn encode(value: &T) -> Result, AuthorityError> { - crate::split::codec::encode(value).map_err(|_| crate::DatasetError::ReadLimitExceeded.into()) -} -pub(super) fn decode(bytes: &[u8]) -> Result { - crate::split::codec::decode(bytes).map_err(|_| AuthorityError::Corrupt) -} -impl DatasetSessions { - pub(in crate::read_session) async fn reserve_session_version( - &self, - identity: &DatasetIdentity, - id: OperationId, - version: u8, - ) -> Result<(), AuthorityError> { - let key = key(identity, id, b"\0VERSION")?; - if version == 2 - && self - .authority - .store - .get_bounded(&key, budget(), &ReadCancellation::new()) - .await? - .is_none() - { - let old_key = self::key(identity, id, b"\0SPEC")?; - if self - .authority - .store - .get_bounded(&old_key, budget(), &ReadCancellation::new()) - .await? - .is_some() - { - return Err(AuthorityError::CursorConflict); - } - } - self.immutable_session_metadata(&key, &[version]).await - } - pub(super) async fn immutable_session_metadata( - &self, - key: &[u8], - bytes: &[u8], - ) -> Result<(), AuthorityError> { - if let Some(existing) = self - .authority - .store - .get_bounded(key, budget(), &ReadCancellation::new()) - .await? - { - return if existing.bytes == bytes { - Ok(()) - } else { - Err(AuthorityError::CursorConflict) - }; - } - match self.authority.store.compare_exchange(key, None, bytes).await? { - CasOutcome::Applied(_) => Ok(()), - CasOutcome::Conflict(Some(value)) if value.bytes == bytes => Ok(()), - CasOutcome::Conflict(_) => Err(AuthorityError::CursorConflict), - } - } - pub(super) async fn split_preparation_progress( - &self, - identity: &DatasetIdentity, - id: OperationId, - cancel: &ReadCancellation, - ) -> Result<(Option, PreparationProgress), AuthorityError> { - let stored = self - .authority - .store - .get_bounded(&key(identity, id, b"\0PLAN2")?, budget(), cancel) - .await?; - let progress: PreparationProgress = stored - .as_ref() - .map(|s| decode(&s.bytes)) - .transpose()? - .unwrap_or_default(); - if progress.admitted > 100_000 - || progress.bytes > 8 * 1024 * 1024 - || (progress.complete && progress.after.is_some()) - { - return Err(AuthorityError::Corrupt); - } - Ok((stored, progress)) - } - pub(super) async fn replace_split_preparation( - &self, - identity: &DatasetIdentity, - id: OperationId, - old: Option<&StoredValue>, - next: &PreparationProgress, - ) -> Result<(), AuthorityError> { - let bytes = encode(next)?; - if old.is_some_and(|old| old.bytes == bytes) { - return Ok(()); - } - match self - .authority - .store - .compare_exchange( - &key(identity, id, b"\0PLAN2")?, - old.map(|s| s.bytes.as_slice()), - &bytes, - ) - .await? - { - CasOutcome::Applied(_) => Ok(()), - CasOutcome::Conflict(Some(value)) if value.bytes == bytes => Ok(()), - CasOutcome::Conflict(_) => Err(AuthorityError::CursorConflict), - } - } -} diff --git a/lib/crowdb-access-dataset/src/retention.rs b/lib/crowdb-access-dataset/src/retention.rs index 6ed955613..e70278ff5 100644 --- a/lib/crowdb-access-dataset/src/retention.rs +++ b/lib/crowdb-access-dataset/src/retention.rs @@ -1,10 +1,5 @@ -use std::collections::{HashMap, HashSet}; -use std::sync::{ - atomic::{AtomicU64, AtomicUsize, Ordering}, - Arc, -}; - use crate::{FieldLocator, ManifestRecord, SnapshotId, SnapshotRecord}; +use std::collections::{HashMap, HashSet}; /// Input and output of the bounded Dataset retention planner. #[derive(Clone, Debug, Eq, PartialEq)] @@ -20,81 +15,6 @@ pub struct RetentionPlan { /// can execute the returned candidates through their store's idempotent GC. pub struct RetentionPlanner; -/// Lock-free lease registry used by a reclaimer to protect active Dataset -/// reads. A released lease is immediately reclaimable once no other reader is -/// active; an abandoned lease expires after its configured TTL. -pub struct RetentionLeaseRegistry { - active: AtomicUsize, - deadline: AtomicU64, - ttl_seconds: u64, -} - -impl RetentionLeaseRegistry { - /// # Errors - /// Rejects a zero inactivity TTL. - pub fn new(ttl_seconds: u64) -> Result { - if ttl_seconds == 0 { - return Err(crate::DatasetError::InvalidManifest); - } - Ok(Self { - active: AtomicUsize::new(0), - deadline: AtomicU64::new(0), - ttl_seconds, - }) - } - - #[must_use] - pub fn acquire(&self, now_seconds: u64) -> RetentionLease<'_> { - self.active.fetch_add(1, Ordering::AcqRel); - self.deadline - .store(now_seconds.saturating_add(self.ttl_seconds), Ordering::Release); - RetentionLease { registry: self } - } - - #[must_use] - pub fn reclaim_allowed(&self, now_seconds: u64) -> bool { - self.active.load(Ordering::Acquire) == 0 || now_seconds >= self.deadline.load(Ordering::Acquire) - } - - #[must_use] - pub fn active(&self) -> usize { - self.active.load(Ordering::Acquire) - } -} - -pub struct RetentionLease<'a> { - registry: &'a RetentionLeaseRegistry, -} - -/// Owned lease used across async Dataset request boundaries. -pub struct RetentionLeaseHandle { - registry: Arc, -} - -impl RetentionLeaseHandle { - #[must_use] - pub fn new(registry: Arc, now_seconds: u64) -> Self { - registry.active.fetch_add(1, Ordering::AcqRel); - registry.deadline.store( - now_seconds.saturating_add(registry.ttl_seconds), - Ordering::Release, - ); - Self { registry } - } -} - -impl Drop for RetentionLeaseHandle { - fn drop(&mut self) { - self.registry.active.fetch_sub(1, Ordering::AcqRel); - } -} - -impl Drop for RetentionLease<'_> { - fn drop(&mut self) { - self.registry.active.fetch_sub(1, Ordering::AcqRel); - } -} - impl RetentionPlanner { /// Marks retained snapshots and their explicit parent chains, then returns /// Dataset-owned chunk locators from snapshots outside that reachability @@ -106,7 +26,6 @@ impl RetentionPlanner { snapshots: &[SnapshotRecord], manifests: &HashMap, retained: &HashSet, - active_lease: bool, external_locations: &HashSet>, ) -> Result { let by_id: HashMap<_, _> = snapshots @@ -124,13 +43,6 @@ impl RetentionPlanner { current = record.publication.parent; } } - if active_lease { - return Ok(RetentionPlan { - reachable_snapshots: reachable, - reclaimable_chunks: Vec::new(), - reclaimable_snapshots: Vec::new(), - }); - } let mut reclaimable = HashSet::new(); for (snapshot, manifest) in manifests { if reachable.contains(snapshot) { diff --git a/lib/crowdb-access-dataset/src/transport.rs b/lib/crowdb-access-dataset/src/transport.rs index b77722a2e..4c8cdbb4f 100644 --- a/lib/crowdb-access-dataset/src/transport.rs +++ b/lib/crowdb-access-dataset/src/transport.rs @@ -2,8 +2,7 @@ use std::sync::Arc; use crate::{ AuthorityError, DatasetAuthority, DatasetError, DatasetIdentity, DatasetReadRequest, DatasetReadResponse, - DatasetScanRequest, DatasetScanResponse, DeliveryWindow, ReadCursor, ReadLease, ReadSurface, - SampleShuffle, + DatasetScanRequest, DatasetScanResponse, DeliveryWindow, ReadCursor, ReadSurface, SampleShuffle, }; /// Shared bounded read adapter used by HTTP and native callers. @@ -15,7 +14,6 @@ pub struct DatasetReadService { authority: Arc, identity: DatasetIdentity, window: Arc, - lease: Arc, } #[derive(Debug, thiserror::Error)] @@ -29,46 +27,19 @@ pub enum ReadTransportError { } impl DatasetReadService { - /// Creates a service with a bounded number of batches in flight. - /// + /// Creates a service with a bounded number of independent requests in flight. /// # Errors - /// Returns `InvalidManifest` for a zero window. + /// Rejects a zero request window. pub fn new( authority: Arc, identity: DatasetIdentity, max_in_flight: usize, - now_seconds: u64, - ) -> Result { - Self::with_ttl( - authority, - identity, - max_in_flight, - now_seconds, - crate::DEFAULT_READ_LEASE_SECONDS, - ) - } - - /// Creates a service with an explicit inactivity TTL for deployments with - /// a configured lifecycle policy. - /// - /// # Errors - /// Returns `InvalidManifest` for a zero window or TTL. - pub fn with_ttl( - authority: Arc, - identity: DatasetIdentity, - max_in_flight: usize, - now_seconds: u64, - ttl_seconds: u64, ) -> Result { let window = DeliveryWindow::new(max_in_flight).ok_or(DatasetError::InvalidManifest)?; - if ttl_seconds == 0 { - return Err(DatasetError::InvalidManifest); - } Ok(Self { authority, identity, window: Arc::new(window), - lease: Arc::new(ReadLease::with_ttl(now_seconds, ttl_seconds)), }) } @@ -77,12 +48,8 @@ impl DatasetReadService { /// /// # Errors /// Returns validation, admission, cancellation, or authority failures. - pub async fn read( - &self, - request: DatasetReadRequest, - now_seconds: u64, - ) -> Result { - self.read_surface(ReadSurface::Http, request, now_seconds).await + pub async fn read(&self, request: DatasetReadRequest) -> Result { + self.read_surface(ReadSurface::Http, request).await } /// Reads the same request for either public surface. @@ -93,12 +60,8 @@ impl DatasetReadService { &self, surface: ReadSurface, request: DatasetReadRequest, - now_seconds: u64, ) -> Result { request.validate()?; - if self.lease.expired(now_seconds) { - return Err(DatasetError::ReadCancelled.into()); - } if !self.window.try_acquire() { return Err(ReadTransportError::WindowFull); } @@ -106,7 +69,6 @@ impl DatasetReadService { self.authority .check_snapshot_readable(&self.identity, request.snapshot) .await?; - self.lease.touch(now_seconds); let fields: Vec<&str> = request.fields.iter().map(String::as_str).collect(); let samples = self .authority @@ -119,7 +81,6 @@ impl DatasetReadService { ) .await; drop(permit); - self.lease.touch(now_seconds); Ok(DatasetReadResponse::from_batch(request.snapshot, samples?)) } @@ -127,12 +88,8 @@ impl DatasetReadService { /// /// # Errors /// Returns validation, admission, cursor, planning, or authority failures. - pub async fn scan( - &self, - request: DatasetScanRequest, - now_seconds: u64, - ) -> Result { - self.scan_surface(ReadSurface::Http, request, now_seconds).await + pub async fn scan(&self, request: DatasetScanRequest) -> Result { + self.scan_surface(ReadSurface::Http, request).await } /// Executes a bounded scan for a native or HTTP caller. @@ -143,12 +100,8 @@ impl DatasetReadService { &self, surface: ReadSurface, request: DatasetScanRequest, - now_seconds: u64, ) -> Result { request.plan.validate()?; - if self.lease.expired(now_seconds) { - return Err(DatasetError::ReadCancelled.into()); - } if !self.window.try_acquire() { return Err(ReadTransportError::WindowFull); } @@ -180,11 +133,7 @@ impl DatasetReadService { .await? } }; - let stored = self.authority.load_cursor(&self.identity, &request.plan).await?; - let cursor = request - .cursor - .or(stored) - .unwrap_or_else(|| ReadCursor::start(&request.plan)); + let cursor = request.cursor.unwrap_or_else(|| ReadCursor::start(&request.plan)); cursor.validate_for(&request.plan)?; let group = usize::try_from(cursor.group).map_err(|_| DatasetError::CursorMismatch)?; if group > windows.len() || (group == windows.len() && cursor.offset != 0) { @@ -212,12 +161,7 @@ impl DatasetReadService { .and_then(|value| u32::try_from(value).ok()) .ok_or(DatasetError::CursorMismatch)?; let next = cursor.confirm_batch(&request.plan, next_group, 0)?; - let expected = self.authority.load_cursor(&self.identity, &request.plan).await?; - self.authority - .persist_cursor(&self.identity, &request.plan, &next, expected.as_ref()) - .await?; drop(permit); - self.lease.touch(now_seconds); Ok(DatasetScanResponse { snapshot: request.plan.snapshot, samples, @@ -226,10 +170,9 @@ impl DatasetReadService { }) } - /// Cancels queued and future reads and releases the active lease. + /// Cancels queued and future requests on this local service. pub fn cancel(&self) { self.window.cancel(); - self.lease.release(); } #[must_use] @@ -242,11 +185,6 @@ impl DatasetReadService { self.window.is_cancelled() } - #[must_use] - pub fn lease_expired(&self, now_seconds: u64) -> bool { - self.lease.expired(now_seconds) - } - #[must_use] pub fn authority(&self) -> Arc { Arc::clone(&self.authority) diff --git a/lib/crowdb-access-dataset/tests/common/read_query.rs b/lib/crowdb-access-dataset/tests/common/read_query.rs new file mode 100644 index 000000000..8e699d589 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/common/read_query.rs @@ -0,0 +1,43 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_access_dataset::*; +pub(super) fn spec(name: &str) -> ReadSpec { + ReadSpec { + snapshot: "latest".into(), + selection: Selection::Not(Box::new(Selection::Explicit(vec![]))), + fields: vec!["marker".into()], + batch_size: 7, + partition: ReadPartition { + worker: 0, + workers: 1, + rank: 0, + world_size: 1, + }, + shuffle: None, + split: Some(ReadSplit { + revision: None, + name: split::SplitName::new(name).unwrap(), + }), + } +} +pub(super) async fn drain( + reads: &DatasetReads, + identity: &DatasetIdentity, + binding: ReadBinding, +) -> Vec> { + let mut cursor = binding.cursor; + let mut ids = Vec::new(); + for _ in 0..10000 { + let page = reads + .next(identity, &cursor, &ReadCancellation::new()) + .await + .unwrap(); + ids.extend(page.ids); + match page.next { + Some(next) => cursor = next, + None => return ids, + } + } + panic!("read failed to finish"); +} diff --git a/lib/crowdb-access-dataset/tests/common/split_session.rs b/lib/crowdb-access-dataset/tests/common/split_session.rs deleted file mode 100644 index 54bf3567e..000000000 --- a/lib/crowdb-access-dataset/tests/common/split_session.rs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use crowdb_access_dataset::*; -pub(super) fn spec(name: &str) -> SessionSpecV2 { - SessionSpecV2 { - read: SessionSpec { - id: OperationId::random(), - snapshot: "latest".into(), - selection: Selection::Not(Box::new(Selection::Explicit(vec![]))), - fields: vec!["marker".into()], - batch_size: 7, - partition: SessionPartition { - worker: 0, - workers: 1, - rank: 0, - world_size: 1, - }, - shuffle: None, - }, - split: Some(SessionSplit { - revision: None, - name: split::SplitName::new(name).unwrap(), - }), - } -} -pub(super) async fn prepare( - sessions: &DatasetSessions, - identity: &DatasetIdentity, - spec: &SessionSpecV2, -) -> ReadPreparation { - for page in 0..100 { - let result = sessions - .prepare_split_read(identity, spec, page, 10, &ReadCancellation::new()) - .await - .unwrap(); - if result.complete { - return result; - } - } - panic!("read preparation did not finish") -} -pub(super) async fn drain( - sessions: &DatasetSessions, - identity: &DatasetIdentity, - session: ReadSession, - id: OperationId, -) -> Vec> { - let mut cursor = session.acknowledged; - let mut ids = Vec::new(); - loop { - let batch = sessions.next(identity, id, 11).await.unwrap(); - ids.extend(batch.ids); - cursor = sessions - .acknowledge(identity, id, &cursor, &batch.next, 12) - .await - .unwrap() - .acknowledged; - if batch.end { - return ids; - } - } -} diff --git a/lib/crowdb-access-dataset/tests/core_test.rs b/lib/crowdb-access-dataset/tests/core_test.rs index cdc4fa4e1..5458eef05 100644 --- a/lib/crowdb-access-dataset/tests/core_test.rs +++ b/lib/crowdb-access-dataset/tests/core_test.rs @@ -10,8 +10,8 @@ use crowdb_access_dataset::{ SnapshotPublication, }; use crowdb_access_dataset::{ - DatasetReadRequest, DeliveryWindow, GroupShuffle, Ordering, ReadCursor, ReadFailure, ReadLease, - ReadLimits, ReadPlan, ReadSurface, RetryPolicy, SampleShuffle, Selection, ShuffleSpec, + DatasetReadRequest, DeliveryWindow, GroupShuffle, Ordering, ReadCursor, ReadFailure, ReadLimits, + ReadPlan, ReadSurface, RetryPolicy, SampleShuffle, Selection, ShuffleSpec, }; use dataset_store::TestDatasetStore as MemoryStore; use md5::{Digest, Md5}; @@ -441,17 +441,6 @@ fn cursor_is_bound_to_plan_and_advances_only_on_confirmation() { assert_eq!(next.validate_for(&wrong), Err(DatasetError::CursorMismatch)); } -#[test] -fn read_lease_expires_after_inactivity_and_release() { - let lease = ReadLease::new(100); - assert!(!lease.expired(159)); - assert!(lease.expired(160)); - lease.touch(200); - assert!(!lease.expired(259)); - lease.release(); - assert!(lease.expired(200)); -} - #[test] fn manifest_partitions_repeat_schema_and_keep_snapshot_binding() { let snapshot = SnapshotId::random(); @@ -1027,12 +1016,8 @@ async fn retention_stable_and_reclaim_protect_live_snapshots() { )); } -#[tokio::test] -async fn durable_cursor_survives_authority_restart_and_rejects_stale_writes() { - let store = Arc::new(MemoryStore::default()); - let authority = crowdb_access_dataset::DatasetAuthority::new(store.clone()); - let identity = DatasetIdentity::new(NamespacePath::root(), "cursor-store").unwrap(); - authority.create_dataset(identity.clone()).await.unwrap(); +#[test] +fn caller_checkpoint_is_portable_and_does_not_change_another_reader() { let plan = ReadPlan { snapshot: SnapshotId::random(), selection: Selection::Prefix(b"sample".to_vec()), @@ -1041,26 +1026,11 @@ async fn durable_cursor_survives_authority_restart_and_rejects_stale_writes() { batch_size: 2, }; let start = ReadCursor::start(&plan); - authority - .persist_cursor(&identity, &plan, &start, None) - .await - .unwrap(); let next = start.confirm_batch(&plan, 1, 2).unwrap(); - authority - .persist_cursor(&identity, &plan, &next, Some(&start)) - .await - .unwrap(); - let restarted = crowdb_access_dataset::DatasetAuthority::new(store); - assert_eq!( - restarted.load_cursor(&identity, &plan).await.unwrap(), - Some(next.clone()) - ); - assert!(matches!( - authority - .persist_cursor(&identity, &plan, &start, Some(&start)) - .await, - Err(AuthorityError::CursorConflict) - )); + let restored: ReadCursor = bincode::deserialize(&bincode::serialize(&next).unwrap()).unwrap(); + restored.validate_for(&plan).unwrap(); + assert_eq!(restored, next); + assert_eq!(start, ReadCursor::start(&plan)); } #[test] diff --git a/lib/crowdb-access-dataset/tests/read_query_cost_test.rs b/lib/crowdb-access-dataset/tests/read_query_cost_test.rs new file mode 100644 index 000000000..8b285706a --- /dev/null +++ b/lib/crowdb-access-dataset/tests/read_query_cost_test.rs @@ -0,0 +1,90 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/access_store.rs"] +mod access_store; +#[path = "common/publication.rs"] +mod publication; +use crowdb_access_dataset::*; +use std::sync::{atomic::Ordering, Arc}; + +#[tokio::test] +async fn binding_concurrent_fetch_and_cross_instance_replay_never_write() { + let store = access_store::TestAccessStore::shared(); + let authority = Arc::new(DatasetAuthority::new(store.clone())); + let identity = DatasetIdentity::new(NamespacePath::root(), "read-cost").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + authority + .publish_snapshot(&identity, None, publication::publication_manifest(1)) + .await + .unwrap(); + let writes = store.writes.load(Ordering::Relaxed); + let reads = DatasetReads::new(authority.clone()); + let spec = ReadSpec { + snapshot: "latest".into(), + selection: Selection::Prefix(b"sample-".to_vec()), + fields: vec!["marker".into()], + batch_size: 1, + partition: ReadPartition { + worker: 0, + workers: 1, + rank: 0, + world_size: 1, + }, + shuffle: None, + split: None, + }; + let bound = reads.bind(&identity, spec.clone()).await.unwrap(); + let other = reads.bind(&identity, spec).await.unwrap(); + assert_eq!(bound.cursor, other.cursor); + let cancel = ReadCancellation::new(); + let pages = + futures::future::join_all((0..96).map(|_| reads.next(&identity, &bound.cursor, &cancel))).await; + for page in &pages { + assert_eq!(page.as_ref().unwrap().ids, pages[0].as_ref().unwrap().ids); + } + let restarted = DatasetReads::new(authority); + restarted.next(&identity, &bound.cursor, &cancel).await.unwrap(); + assert_eq!(store.writes.load(Ordering::Relaxed), writes); +} + +#[tokio::test] +async fn legacy_scan_replays_only_the_caller_checkpoint_without_writes() { + let store = access_store::TestAccessStore::shared(); + let authority = Arc::new(DatasetAuthority::new(store.clone())); + let identity = DatasetIdentity::new(NamespacePath::root(), "scan-cost").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + let snapshot = authority + .publish_snapshot(&identity, None, publication::publication_manifest(1)) + .await + .unwrap(); + let writes = store.writes.load(Ordering::Relaxed); + let request = DatasetScanRequest { + plan: ReadPlan { + snapshot, + selection: Selection::Prefix(b"sample-".to_vec()), + ordering: crowdb_access_dataset::Ordering::SampleId, + projection: vec!["marker".into()], + batch_size: 1, + }, + shuffle: None, + cursor: None, + }; + let first = DatasetReadService::new(authority.clone(), identity.clone(), 1) + .unwrap() + .scan(request.clone()) + .await + .unwrap(); + let restarted = DatasetReadService::new(authority, identity, 1).unwrap(); + assert_eq!(restarted.scan(request.clone()).await.unwrap(), first); + let completed = restarted + .scan(DatasetScanRequest { + cursor: Some(first.cursor), + ..request + }) + .await + .unwrap(); + assert!(completed.samples.is_empty()); + assert!(completed.end); + assert_eq!(store.writes.load(Ordering::Relaxed), writes); +} diff --git a/lib/crowdb-access-dataset/tests/split_session_edges_test.rs b/lib/crowdb-access-dataset/tests/read_query_edges_test.rs similarity index 54% rename from lib/crowdb-access-dataset/tests/split_session_edges_test.rs rename to lib/crowdb-access-dataset/tests/read_query_edges_test.rs index 15f7a5077..f1d59da3a 100644 --- a/lib/crowdb-access-dataset/tests/split_session_edges_test.rs +++ b/lib/crowdb-access-dataset/tests/read_query_edges_test.rs @@ -1,6 +1,6 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -#[path = "common/split_session.rs"] +#[path = "common/read_query.rs"] #[allow(dead_code)] mod common; #[path = "common/publication.rs"] @@ -17,23 +17,22 @@ async fn empty_split_finishes_but_unknown_split_fails_without_manifest_reads() { split_store::publish_revision(&authority, &identity, &revision).await; let empty = revision.splits.iter().find(|s| s.count == 0).unwrap(); store.metadata_only.store(true, Ordering::Relaxed); - let sessions = DatasetSessions::new(Arc::new(authority)); + let reads = DatasetReads::new(Arc::new(authority)); let spec = common::spec(empty.name.as_str()); - let prepared = common::prepare(&sessions, &identity, &spec).await; - assert_eq!(prepared.admitted, 0); - let batch = sessions.next(&identity, spec.read.id, 11).await.unwrap(); - assert!(batch.ids.is_empty() && batch.end); - let missing = common::spec("unknown"); + let bound = reads.bind(&identity, spec).await.unwrap(); + let batch = reads + .next(&identity, &bound.cursor, &ReadCancellation::new()) + .await + .unwrap(); + assert!(batch.ids.is_empty() && batch.next.is_none()); assert!(matches!( - sessions - .prepare_split_read(&identity, &missing, 0, 10, &ReadCancellation::new()) - .await, + reads.bind(&identity, common::spec("unknown")).await, Err(AuthorityError::NotFound) )); } #[tokio::test] -async fn v2_session_accepts_empty_inline_equality_without_changing_legacy_plans() { +async fn query_accepts_empty_inline_equality_without_manifest_reads() { use md5::{Digest, Md5}; let (store, authority, identity, parent) = split_store::fixture().await; let mut manifest: ManifestRecord = bincode::deserialize(&publication::publication_manifest(2)).unwrap(); @@ -46,56 +45,44 @@ async fn v2_session_accepts_empty_inline_equality_without_changing_legacy_plans( .await .unwrap(); store.metadata_only.store(true, Ordering::Relaxed); - let sessions = DatasetSessions::new(Arc::new(authority)); + let reads = DatasetReads::new(Arc::new(authority)); let mut spec = common::spec("unused"); spec.split = None; - spec.read.selection = Selection::FieldMatch { + spec.selection = Selection::FieldMatch { field: "marker".into(), value: vec![], }; - let prepared = common::prepare(&sessions, &identity, &spec).await; - assert_eq!(prepared.admitted, 1); + let bound = reads.bind(&identity, spec).await.unwrap(); assert_eq!( - sessions.next(&identity, spec.read.id, 11).await.unwrap().ids, + reads + .next(&identity, &bound.cursor, &ReadCancellation::new()) + .await + .unwrap() + .ids, vec![b"sample-2".to_vec()] ); - assert!( - spec.read.selection.validate().is_err(), - "v1 equality must keep its original semantics" - ); } #[tokio::test] -async fn interrupted_preparation_preserves_progress_without_blocking_deletion() { +async fn cancellation_is_request_local_and_deletion_rejects_old_continuations() { let (store, authority, identity, parent) = split_store::fixture().await; let snapshot = split_store::publish_binary_samples(&authority, &identity, parent).await; let authority = Arc::new(authority); - let sessions = DatasetSessions::new(authority.clone()); + let reads = DatasetReads::new(authority.clone()); let mut spec = common::spec("unused"); spec.split = None; - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs(); - let first = sessions - .prepare_split_read(&identity, &spec, 0, now, &ReadCancellation::new()) - .await - .unwrap(); - assert!(!first.complete); + let binding = reads.bind(&identity, spec).await.unwrap(); store.cancel_scan.store(true, Ordering::Relaxed); let cancel = ReadCancellation::new(); assert!(matches!( - sessions - .prepare_split_read(&identity, &spec, first.next_page, now, &cancel) - .await, + reads.next(&identity, &binding.cursor, &cancel).await, Err(AuthorityError::Invalid(DatasetError::ReadCancelled)) )); assert!(cancel.is_cancelled()); - let resumed = sessions - .prepare_split_read(&identity, &spec, first.next_page, now, &ReadCancellation::new()) + reads + .next(&identity, &binding.cursor, &ReadCancellation::new()) .await .unwrap(); - assert!(resumed.next_page > first.next_page); let root = authority .prepare_snapshot_candidate( &identity, @@ -105,21 +92,21 @@ async fn interrupted_preparation_preserves_progress_without_blocking_deletion() ) .await .unwrap(); - let attempt = CommitAttempt { - version: 1, - id: OperationId::random(), - target: CommitTarget::Snapshot { snapshot: root }, - expected_head: authority.inspect_head(&identity).await.unwrap().generation, - }; authority - .publish_snapshot_candidate(&identity, &attempt) + .publish_snapshot_candidate( + &identity, + &CommitAttempt { + version: 1, + id: OperationId::random(), + target: CommitTarget::Snapshot { snapshot: root }, + expected_head: authority.inspect_head(&identity).await.unwrap().generation, + }, + ) .await .unwrap(); authority.begin_reclaim(&identity, snapshot).await.unwrap(); - cancel.cancel(); - assert!(sessions - .prepare_split_read(&identity, &spec, resumed.next_page, now, &cancel) + assert!(reads + .next(&identity, &binding.cursor, &ReadCancellation::new()) .await .is_err()); - authority.begin_reclaim(&identity, snapshot).await.unwrap(); } diff --git a/lib/crowdb-access-dataset/tests/read_query_test.rs b/lib/crowdb-access-dataset/tests/read_query_test.rs new file mode 100644 index 000000000..3ba8e4c2d --- /dev/null +++ b/lib/crowdb-access-dataset/tests/read_query_test.rs @@ -0,0 +1,141 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +#[path = "common/upload.rs"] +#[allow(dead_code)] +mod dataset_store; +#[path = "common/publication.rs"] +mod publication; +use crowdb_access_dataset::{ + DatasetAuthority, DatasetIdentity, DatasetReads, NamespacePath, ReadCancellation, ReadPartition, + ReadSpec, Selection, +}; +use dataset_store::TestStore; +use std::sync::Arc; +fn spec() -> ReadSpec { + ReadSpec { + snapshot: "latest".into(), + selection: Selection::Prefix(b"sample-".to_vec()), + fields: vec!["marker".into()], + batch_size: 1, + partition: ReadPartition { + worker: 0, + workers: 1, + rank: 0, + world_size: 1, + }, + shuffle: None, + split: None, + } +} +#[tokio::test] +async fn continuation_replays_and_rejects_cross_dataset_scope() { + let authority = Arc::new(DatasetAuthority::new(Arc::new(TestStore::default()))); + let identity = DatasetIdentity::new(NamespacePath::root(), "queries").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + let snapshot = authority + .publish_snapshot(&identity, None, publication::publication_manifest(1)) + .await + .unwrap(); + let reads = DatasetReads::new(authority.clone()); + let binding = reads.bind(&identity, spec()).await.unwrap(); + authority + .publish_snapshot(&identity, Some(snapshot), publication::publication_manifest(2)) + .await + .unwrap(); + let page = reads + .next(&identity, &binding.cursor, &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(page.snapshot, snapshot); + let restarted = DatasetReads::new(authority); + let replay = restarted + .next(&identity, &binding.cursor, &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(page.ids, replay.ids); + assert_eq!(page.next, replay.next); + let other = DatasetIdentity::new(NamespacePath::root(), "other").unwrap(); + assert!(restarted + .next(&other, &binding.cursor, &ReadCancellation::new()) + .await + .is_err()); + assert!(restarted + .next(&identity, "invalid", &ReadCancellation::new()) + .await + .is_err()); +} + +#[tokio::test] +async fn continuation_stays_small_without_copying_selected_ids() { + use crowdb_access_dataset::{ + DatasetStore, FieldDefinition, FieldLocator, FieldRecord, ManifestRecord, SampleRecord, SchemaRecord, + SnapshotId, + }; + use md5::{Digest, Md5}; + let store = Arc::new(TestStore::default()); + let authority = Arc::new(DatasetAuthority::new(store.clone())); + let identity = DatasetIdentity::new(NamespacePath::root(), "paged").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + let manifest = ManifestRecord { + version: 1, + snapshot: SnapshotId::random(), + schema: SchemaRecord { + version: 1, + fields: vec![FieldDefinition { + name: "marker".into(), + required: true, + }], + }, + samples: (0..3000) + .map(|i| SampleRecord { + sample_id: format!("sample-{i:04}-{}", "x".repeat(200)).into_bytes(), + fields: vec![FieldRecord { + name: "marker".into(), + value: FieldLocator::Inline { + value: vec![1], + md5: Md5::digest([1]).into(), + }, + }], + }) + .collect(), + }; + authority + .publish_snapshot(&identity, None, bincode::serialize(&manifest).unwrap()) + .await + .unwrap(); + let reads = DatasetReads::new(authority); + let opened = reads.bind(&identity, spec()).await.unwrap(); + assert!(opened.cursor.len() < 4096); + let batch = reads + .next(&identity, &opened.cursor, &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(batch.ids.len(), 1); + let next = batch.next.unwrap(); + assert!(next.len() < 4096); + assert_eq!( + reads + .next(&identity, &next, &ReadCancellation::new()) + .await + .unwrap() + .ids[0], + manifest.samples[1].sample_id + ); + assert!(store + .scan_page( + &crowdb_access_dataset::MetadataScan { + prefix: b"DS\0RS\0".to_vec(), + after: None, + budget: crowdb_access_dataset::MetadataScanBudget { + items: 20, + bytes: 1024 * 1024, + timeout: std::time::Duration::from_secs(2) + } + }, + &ReadCancellation::new() + ) + .await + .unwrap() + .entries + .is_empty()); +} diff --git a/lib/crowdb-access-dataset/tests/read_session_test.rs b/lib/crowdb-access-dataset/tests/read_session_test.rs deleted file mode 100644 index e72f2d043..000000000 --- a/lib/crowdb-access-dataset/tests/read_session_test.rs +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. -#[path = "common/upload.rs"] -#[allow(dead_code)] -mod dataset_store; -#[path = "common/publication.rs"] -mod publication; -use crowdb_access_dataset::{ - DatasetAuthority, DatasetIdentity, DatasetSessions, NamespacePath, OperationId, Selection, - SessionPartition, SessionSpec, -}; -use dataset_store::TestStore; -use std::sync::Arc; -fn spec() -> SessionSpec { - SessionSpec { - id: OperationId::random(), - snapshot: "latest".into(), - selection: Selection::Prefix(b"sample-".to_vec()), - fields: vec!["marker".into()], - batch_size: 1, - partition: SessionPartition { - worker: 0, - workers: 1, - rank: 0, - world_size: 1, - }, - shuffle: None, - } -} -#[tokio::test] -async fn fetch_ack_resume_and_close_are_scoped_and_snapshot_bound() { - let authority = Arc::new(DatasetAuthority::new(Arc::new(TestStore::default()))); - let identity = DatasetIdentity::new(NamespacePath::root(), "sessions").unwrap(); - authority.create_dataset(identity.clone()).await.unwrap(); - let first = authority - .publish_snapshot(&identity, None, publication::publication_manifest(1)) - .await - .unwrap(); - let sessions = DatasetSessions::new(authority.clone()); - let plan = spec(); - let other = spec(); - let opened = sessions.start(&identity, plan.clone(), 10).await.unwrap(); - sessions.start(&identity, other.clone(), 10).await.unwrap(); - authority - .publish_snapshot(&identity, Some(first), publication::publication_manifest(2)) - .await - .unwrap(); - let batch = sessions.next(&identity, plan.id, 11).await.unwrap(); - assert_eq!(batch.snapshot, first); - assert_eq!( - sessions.next(&identity, plan.id, 12).await.unwrap().next, - batch.next - ); - assert_eq!( - sessions - .resume(&identity, plan.id, 13) - .await - .unwrap() - .acknowledged, - opened.acknowledged - ); - assert!(sessions - .acknowledge(&identity, other.id, &opened.acknowledged, &batch.next, 14) - .await - .is_err()); - assert!(sessions - .acknowledge(&identity, plan.id, &opened.acknowledged, "forged", 14) - .await - .is_err()); - let progress = sessions - .acknowledge(&identity, plan.id, &opened.acknowledged, &batch.next, 14) - .await - .unwrap(); - assert_eq!( - sessions - .acknowledge(&identity, plan.id, &opened.acknowledged, &batch.next, 15) - .await - .unwrap() - .acknowledged, - progress.acknowledged - ); - assert!(sessions.next(&identity, plan.id, 100).await.is_err()); - let restarted = DatasetSessions::new(authority); - assert_eq!( - restarted - .resume(&identity, plan.id, 100) - .await - .unwrap() - .acknowledged, - progress.acknowledged - ); - restarted.close(&identity, plan.id).await.unwrap(); - assert!(restarted.resume(&identity, plan.id, 101).await.is_err()); - restarted.resume(&identity, other.id, 101).await.unwrap(); - assert_eq!( - restarted.next(&identity, other.id, 102).await.unwrap().snapshot, - first - ); -} - -#[tokio::test] -async fn progress_record_stays_small_when_selection_spans_id_pages() { - use crowdb_access_dataset::{ - DatasetStore, FieldDefinition, FieldLocator, FieldRecord, ManifestRecord, SampleRecord, SchemaRecord, - SnapshotId, - }; - use md5::{Digest, Md5}; - let store = Arc::new(TestStore::default()); - let authority = Arc::new(DatasetAuthority::new(store.clone())); - let identity = DatasetIdentity::new(NamespacePath::root(), "paged").unwrap(); - authority.create_dataset(identity.clone()).await.unwrap(); - let manifest = ManifestRecord { - version: 1, - snapshot: SnapshotId::random(), - schema: SchemaRecord { - version: 1, - fields: vec![FieldDefinition { - name: "marker".into(), - required: true, - }], - }, - samples: (0..3000) - .map(|i| SampleRecord { - sample_id: format!("sample-{i:04}-{}", "x".repeat(200)).into_bytes(), - fields: vec![FieldRecord { - name: "marker".into(), - value: FieldLocator::Inline { - value: vec![1], - md5: Md5::digest([1]).into(), - }, - }], - }) - .collect(), - }; - authority - .publish_snapshot(&identity, None, bincode::serialize(&manifest).unwrap()) - .await - .unwrap(); - let sessions = DatasetSessions::new(authority); - let plan = spec(); - let opened = sessions.start(&identity, plan.clone(), 10).await.unwrap(); - let mut key = b"DS\0RS\0".to_vec(); - key.extend(identity.key().unwrap()); - key.extend(plan.id.as_bytes()); - assert!(store.get(&key).await.unwrap().unwrap().bytes.len() < 1024); - let batch = sessions.next(&identity, plan.id, 11).await.unwrap(); - assert_eq!(batch.ids.len(), 1); - sessions - .acknowledge(&identity, plan.id, &opened.acknowledged, &batch.next, 12) - .await - .unwrap(); - assert!(store.get(&key).await.unwrap().unwrap().bytes.len() < 1024); - assert_eq!( - sessions.next(&identity, plan.id, 13).await.unwrap().ids[0], - manifest.samples[1].sample_id - ); -} diff --git a/lib/crowdb-access-dataset/tests/retention_test.rs b/lib/crowdb-access-dataset/tests/retention_test.rs index ceea27bb8..66914c846 100644 --- a/lib/crowdb-access-dataset/tests/retention_test.rs +++ b/lib/crowdb-access-dataset/tests/retention_test.rs @@ -1,31 +1,9 @@ use crowdb_access_dataset::{ FieldDefinition, FieldLocator, FieldRecord, ManifestRecord, OperationId, PublicationState, - RetentionLeaseRegistry, RetentionPlanner, SampleRecord, SchemaRecord, SnapshotId, SnapshotPublication, - SnapshotRecord, + RetentionPlanner, SampleRecord, SchemaRecord, SnapshotId, SnapshotPublication, SnapshotRecord, }; use std::collections::{HashMap, HashSet}; -#[test] -fn active_retention_lease_blocks_gc_until_release_and_ttl() { - let registry = RetentionLeaseRegistry::new(60).unwrap(); - let lease = registry.acquire(100); - assert!(registry.reclaim_allowed(1_000)); - drop(lease); - assert!(registry.reclaim_allowed(159)); - assert!(registry.reclaim_allowed(160)); -} - -#[test] -fn owned_read_lease_releases_and_expiry_is_observable() { - let registry = std::sync::Arc::new(RetentionLeaseRegistry::new(10).unwrap()); - let lease = crowdb_access_dataset::RetentionLeaseHandle::new(registry.clone(), 50); - assert_eq!(registry.active(), 1); - assert!(!registry.reclaim_allowed(55)); - drop(lease); - assert_eq!(registry.active(), 0); - assert!(registry.reclaim_allowed(55)); -} - #[test] fn retention_keeps_shared_and_external_chunks_until_unowned() { let old = SnapshotId::random(); @@ -64,6 +42,6 @@ fn retention_keeps_shared_and_external_chunks_until_unowned() { manifests.insert(old, manifest); let mut external = HashSet::new(); external.insert(b"external".to_vec()); - let plan = RetentionPlanner::plan(&[record], &manifests, &HashSet::new(), false, &external).unwrap(); + let plan = RetentionPlanner::plan(&[record], &manifests, &HashSet::new(), &external).unwrap(); assert_eq!(plan.reclaimable_chunks, vec![b"shared".to_vec()]); } diff --git a/lib/crowdb-access-dataset/tests/session_cost_test.rs b/lib/crowdb-access-dataset/tests/session_cost_test.rs deleted file mode 100644 index 410126211..000000000 --- a/lib/crowdb-access-dataset/tests/session_cost_test.rs +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -#[path = "common/access_store.rs"] -mod access_store; -#[path = "common/publication.rs"] -mod publication; -use crowdb_access_dataset::*; -use std::sync::{atomic::Ordering, Arc}; - -#[tokio::test] -async fn concurrent_fetch_and_replay_never_write_or_renew_progress() { - let store = access_store::TestAccessStore::shared(); - let authority = Arc::new(DatasetAuthority::new(store.clone())); - let identity = DatasetIdentity::new(NamespacePath::root(), "read-cost").unwrap(); - authority.create_dataset(identity.clone()).await.unwrap(); - authority - .publish_snapshot(&identity, None, publication::publication_manifest(1)) - .await - .unwrap(); - let sessions = DatasetSessions::new(authority.clone()); - let spec = SessionSpec { - id: OperationId::random(), - snapshot: "latest".into(), - selection: Selection::Prefix(b"sample-".to_vec()), - fields: vec!["marker".into()], - batch_size: 1, - partition: SessionPartition { - worker: 0, - workers: 1, - rank: 0, - world_size: 1, - }, - shuffle: None, - }; - let opened = sessions.start(&identity, spec.clone(), 10).await.unwrap(); - let writes = store.writes.load(Ordering::Relaxed); - let batches = futures::future::join_all((0..96).map(|_| sessions.next(&identity, spec.id, 69))).await; - let token = &batches[0].as_ref().unwrap().next; - for batch in &batches { - assert_eq!(&batch.as_ref().unwrap().next, token); - } - assert_eq!(store.writes.load(Ordering::Relaxed), writes); - assert!(sessions.next(&identity, spec.id, 70).await.is_err()); - let restarted = DatasetSessions::new(authority); - restarted.resume(&identity, spec.id, 70).await.unwrap(); - assert_eq!(&restarted.next(&identity, spec.id, 71).await.unwrap().next, token); - let writes = store.writes.load(Ordering::Relaxed); - restarted - .acknowledge(&identity, spec.id, &opened.acknowledged, token, 71) - .await - .unwrap(); - assert_eq!(store.writes.load(Ordering::Relaxed), writes + 1); - restarted - .acknowledge(&identity, spec.id, &opened.acknowledged, token, 71) - .await - .unwrap(); - restarted.next(&identity, spec.id, 72).await.unwrap(); - assert_eq!(store.writes.load(Ordering::Relaxed), writes + 1); - restarted.close(&identity, spec.id).await.unwrap(); - let writes = store.writes.load(Ordering::Relaxed); - restarted.close(&identity, spec.id).await.unwrap(); - assert_eq!(store.writes.load(Ordering::Relaxed), writes); -} diff --git a/lib/crowdb-access-dataset/tests/split_read_query_test.rs b/lib/crowdb-access-dataset/tests/split_read_query_test.rs new file mode 100644 index 000000000..5c9d87234 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/split_read_query_test.rs @@ -0,0 +1,92 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/read_query.rs"] +mod common; +#[path = "common/publication.rs"] +mod publication; +#[path = "common/split_store.rs"] +mod split_store; +use crowdb_access_dataset::*; +use std::sync::Arc; + +#[tokio::test] +async fn continuation_pins_snapshot_and_split_across_instances() { + let (_, authority, identity, parent) = split_store::fixture().await; + let snapshot = split_store::publish_binary_samples(&authority, &identity, parent).await; + let first = split_store::prepared(&authority, &identity, snapshot, None).await; + split_store::publish_revision(&authority, &identity, &first).await; + let authority = Arc::new(authority); + let reads = DatasetReads::new(authority.clone()); + let binding = reads.bind(&identity, common::spec("train")).await.unwrap(); + assert_eq!(binding.split.as_ref().unwrap().revision, first.revision); + let next = split_store::prepared(&authority, &identity, snapshot, Some(first.revision)).await; + split_store::publish_revision(&authority, &identity, &next).await; + authority + .publish_snapshot(&identity, Some(snapshot), publication::publication_manifest(2)) + .await + .unwrap(); + let batch = reads + .next(&identity, &binding.cursor, &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(batch.snapshot, snapshot); + let restarted = DatasetReads::new(authority); + assert_eq!( + restarted + .next(&identity, &binding.cursor, &ReadCancellation::new()) + .await + .unwrap() + .ids, + batch.ids + ); + let ids = common::drain(&restarted, &identity, binding).await; + assert_eq!( + ids.len() as u64, + first + .splits + .iter() + .find(|s| s.name.as_str() == "train") + .unwrap() + .count + ); +} + +#[tokio::test] +async fn split_read_filter_and_workers_partition_only_admitted_members() { + let (_, authority, identity, parent) = split_store::fixture().await; + let snapshot = split_store::publish_binary_samples(&authority, &identity, parent).await; + let revision = split_store::prepared(&authority, &identity, snapshot, None).await; + split_store::publish_revision(&authority, &identity, &revision).await; + let reads = DatasetReads::new(Arc::new(authority)); + let mut all = std::collections::BTreeSet::new(); + let mut total = 0; + for worker in 0..2 { + let mut spec = common::spec("train"); + spec.selection = Selection::Range { + start: vec![10], + end: vec![30], + }; + spec.partition.workers = 2; + spec.partition.worker = worker; + spec.shuffle = Some(ReadShuffle { + seed: 42, + group_size: 8, + }); + let prepared = reads.bind(&identity, spec.clone()).await.unwrap(); + for id in common::drain(&reads, &identity, prepared).await { + assert!((10..30).contains(&id[0])); + assert!(all.insert(id)); + total += 1; + } + } + let mut spec = common::spec("train"); + spec.selection = Selection::Range { + start: vec![10], + end: vec![30], + }; + let prepared = reads.bind(&identity, spec.clone()).await.unwrap(); + let expected = common::drain(&reads, &identity, prepared).await; + assert_eq!(total, expected.len()); + assert_eq!(all, expected.into_iter().collect()); +} diff --git a/lib/crowdb-access-dataset/tests/split_session_test.rs b/lib/crowdb-access-dataset/tests/split_session_test.rs deleted file mode 100644 index e44e60e31..000000000 --- a/lib/crowdb-access-dataset/tests/split_session_test.rs +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. -#[path = "common/split_session.rs"] -mod common; -#[path = "common/publication.rs"] -mod publication; -#[path = "common/split_store.rs"] -mod split_store; -use crowdb_access_dataset::*; -use std::sync::Arc; - -#[tokio::test] -async fn split_session_pins_both_aliases_and_replays_only_acknowledged_progress() { - let (_, authority, identity, parent) = split_store::fixture().await; - let snapshot = split_store::publish_binary_samples(&authority, &identity, parent).await; - let first = split_store::prepared(&authority, &identity, snapshot, None).await; - split_store::publish_revision(&authority, &identity, &first).await; - let authority = Arc::new(authority); - let sessions = DatasetSessions::new(authority.clone()); - let spec = common::spec("train"); - let started = sessions - .prepare_split_read(&identity, &spec, 0, 10, &ReadCancellation::new()) - .await - .unwrap(); - assert!(!started.complete); - assert_eq!(started.split.unwrap().revision, first.revision); - let next = split_store::prepared(&authority, &identity, snapshot, Some(first.revision)).await; - split_store::publish_revision(&authority, &identity, &next).await; - authority - .publish_snapshot(&identity, Some(snapshot), publication::publication_manifest(2)) - .await - .unwrap(); - let opened = common::prepare(&sessions, &identity, &spec).await; - assert_eq!(opened.snapshot, snapshot); - assert_eq!(opened.split.unwrap().revision, first.revision); - let session = opened.session.unwrap(); - let batch = sessions.next(&identity, spec.read.id, 11).await.unwrap(); - let restarted = DatasetSessions::new(authority); - assert_eq!( - restarted - .resume(&identity, spec.read.id, 12) - .await - .unwrap() - .acknowledged, - session.acknowledged - ); - assert_eq!( - restarted.next(&identity, spec.read.id, 13).await.unwrap().ids, - batch.ids - ); - let ids = common::drain(&restarted, &identity, session, spec.read.id).await; - assert_eq!( - ids.len() as u64, - first - .splits - .iter() - .find(|s| s.name.as_str() == "train") - .unwrap() - .count - ); - let changed = SessionSpecV2 { - split: Some(SessionSplit { - revision: Some(next.revision), - name: split::SplitName::new("train").unwrap(), - }), - ..spec - }; - assert!(matches!( - restarted - .prepare_split_read(&identity, &changed, 0, 14, &ReadCancellation::new()) - .await, - Err(AuthorityError::CursorConflict) - )); -} - -#[tokio::test] -async fn split_read_filter_and_workers_partition_only_admitted_members() { - let (_, authority, identity, parent) = split_store::fixture().await; - let snapshot = split_store::publish_binary_samples(&authority, &identity, parent).await; - let revision = split_store::prepared(&authority, &identity, snapshot, None).await; - split_store::publish_revision(&authority, &identity, &revision).await; - let sessions = DatasetSessions::new(Arc::new(authority)); - let mut all = std::collections::BTreeSet::new(); - let mut total = 0; - for worker in 0..2 { - let mut spec = common::spec("train"); - spec.read.selection = Selection::Range { - start: vec![10], - end: vec![30], - }; - spec.read.partition.workers = 2; - spec.read.partition.worker = worker; - spec.read.shuffle = Some(SessionShuffle { - seed: 42, - group_size: 8, - }); - let prepared = common::prepare(&sessions, &identity, &spec).await; - for id in common::drain(&sessions, &identity, prepared.session.unwrap(), spec.read.id).await { - assert!((10..30).contains(&id[0])); - assert!(all.insert(id)); - total += 1; - } - } - let mut spec = common::spec("train"); - spec.read.selection = Selection::Range { - start: vec![10], - end: vec![30], - }; - let prepared = common::prepare(&sessions, &identity, &spec).await; - let expected = common::drain(&sessions, &identity, prepared.session.unwrap(), spec.read.id).await; - assert_eq!(total, expected.len()); - assert_eq!(all, expected.into_iter().collect()); -} - -#[tokio::test] -async fn preparation_cancellation_and_version_collision_cannot_reopen_a_session() { - let (_, authority, identity, parent) = split_store::fixture().await; - let snapshot = split_store::publish_binary_samples(&authority, &identity, parent).await; - let revision = split_store::prepared(&authority, &identity, snapshot, None).await; - split_store::publish_revision(&authority, &identity, &revision).await; - let sessions = DatasetSessions::new(Arc::new(authority)); - let spec = common::spec("train"); - sessions - .prepare_split_read(&identity, &spec, 0, 10, &ReadCancellation::new()) - .await - .unwrap(); - assert!(sessions.start(&identity, spec.read.clone(), 10).await.is_err()); - sessions.cancel_split_read(&identity, spec.read.id).await.unwrap(); - assert!(sessions - .prepare_split_read(&identity, &spec, 1, 11, &ReadCancellation::new()) - .await - .is_err()); - assert!(sessions.resume(&identity, spec.read.id, 12).await.is_err()); - let mut legacy = common::spec("train"); - legacy.read.id = OperationId::random(); - sessions.start(&identity, legacy.read.clone(), 10).await.unwrap(); - assert!(sessions - .prepare_split_read(&identity, &legacy, 0, 10, &ReadCancellation::new()) - .await - .is_err()); -} diff --git a/lib/crowdb-access-dataset/tests/transport_test.rs b/lib/crowdb-access-dataset/tests/transport_test.rs index 23de6f18c..4a4de11c9 100644 --- a/lib/crowdb-access-dataset/tests/transport_test.rs +++ b/lib/crowdb-access-dataset/tests/transport_test.rs @@ -30,7 +30,6 @@ fn service() -> DatasetReadService { authority, DatasetIdentity::new(NamespacePath::root(), "transport").unwrap(), 1, - 10, ) .unwrap() } @@ -44,7 +43,7 @@ async fn transport_rejects_invalid_wire_request_before_admission() { fields: Vec::new(), }; assert!(matches!( - service.read(request, 10).await, + service.read(request).await, Err(crowdb_access_dataset::ReadTransportError::Invalid( DatasetError::InvalidManifest )) @@ -61,7 +60,7 @@ async fn transport_cancel_releases_window_and_rejects_future_reads() { sample_ids: vec![b"sample".to_vec()], fields: vec!["value".into()], }; - assert!(service.read(request, 10).await.is_err()); + assert!(service.read(request).await.is_err()); assert_eq!(service.in_flight(), 0); assert!(service.is_cancelled()); } diff --git a/lib/crowdb-sdk-native-python/README.md b/lib/crowdb-sdk-native-python/README.md index 41b3490b8..f116e185c 100644 --- a/lib/crowdb-sdk-native-python/README.md +++ b/lib/crowdb-sdk-native-python/README.md @@ -63,14 +63,14 @@ runtime.close() - `dataset.call(json, timeout=30)` uses a tagged `op` command and the existing Rust Dataset JSON schema. Commands cover create/open/snapshot/head/commit status, composition begin/status/edit/evaluate/prepare/publish/abandon/cleanup, - prepare/resume/acknowledge/close read, and upload create/complete/status/abort. + stateless read binding, and upload create/complete/status/abort. `upload_location` returns the authority's opaque location for a composition field. No command is retried automatically. -- `prepare_read` accepts `spec: SessionSpecV2` and `page`; continue its returned - page until complete. `next_batch(read_id)` fetches the issued batch. Call - `acknowledge` with the previously acknowledged cursor and `Batch.header.next` - only after application processing succeeds. Split/worker binding uses the - same shared session service as Rust and HTTP. +- `bind_read` accepts `spec: ReadSpec` and returns a `cursor`. Call + `next_batch(cursor)` to fetch; retrying the same cursor replays its batch. + Save `Batch.header.next` after successful processing. There is no remote + acknowledgement, TTL renewal or close. Snapshot/Split/worker binding is + identical in Rust and HTTP. - `upload_part(upload_id, index, bytes)` copies Python input once into final transport frame slots across allocations of at most 1 MiB and freezes them before submission. A logical upload part can still contain 4 MiB. Publication uses diff --git a/lib/crowdb-sdk-native-python/src/client.rs b/lib/crowdb-sdk-native-python/src/client.rs index ffe37afac..ee1206a99 100644 --- a/lib/crowdb-sdk-native-python/src/client.rs +++ b/lib/crowdb-sdk-native-python/src/client.rs @@ -102,16 +102,15 @@ impl Dataset { Ok(Batch::new(self.resources.clone(), read, cancel)) } - #[pyo3(signature=(read_id, timeout=30.0))] - fn next_batch(&self, py: Python<'_>, read_id: &[u8], timeout: f64) -> PyResult { + #[pyo3(signature=(cursor, timeout=30.0))] + fn next_batch(&self, py: Python<'_>, cursor: &str, timeout: f64) -> PyResult { let context = self.context(timeout)?; let cancel = context.cancellation.clone(); - let id = OperationId::from_bytes(read_id).map_err(|e| NativeError::new_err(e.to_string()))?; let read = py .detach(|| { self.resources .executor() - .block_on(self.client.next_batch(context, id)) + .block_on(self.client.next_batch(context, cursor)) }) .map_err(|e| failure(&e))?; Ok(Batch::new(self.resources.clone(), read, cancel)) diff --git a/lib/crowdb-sdk-native-python/src/control.rs b/lib/crowdb-sdk-native-python/src/control.rs index dc8f081d4..357636886 100644 --- a/lib/crowdb-sdk-native-python/src/control.rs +++ b/lib/crowdb-sdk-native-python/src/control.rs @@ -3,7 +3,7 @@ use crowdb_access_dataset::{ composition::{CompositionEdit, CompositionInput}, - CommitAttempt, DatasetUploads, OperationId, SessionSpecV2, SnapshotId, + CommitAttempt, DatasetUploads, OperationId, ReadSpec, SnapshotId, }; use crowdb_sdk_native::{ dataset::{CompositionStatus, DatasetClient}, @@ -55,20 +55,8 @@ pub enum Command { CleanupComposition { operation: OperationId, }, - PrepareRead { - spec: SessionSpecV2, - page: u64, - }, - ResumeRead { - read: OperationId, - }, - Acknowledge { - read: OperationId, - expected: String, - next: String, - }, - CloseRead { - read: OperationId, + BindRead { + spec: ReadSpec, }, CreateUpload { upload: OperationId, @@ -155,12 +143,7 @@ pub async fn execute(client: &DatasetClient, context: OperationContext, command: "physical_cleanup_deferred":status.physical_cleanup_deferred}), ) } - Command::PrepareRead { spec, page } => encoded(client.prepare_read(context, &spec, page).await?), - Command::ResumeRead { read } => encoded(client.resume_read(context, read).await?), - Command::Acknowledge { read, expected, next } => { - encoded(client.acknowledge(context, read, &expected, &next).await?) - } - Command::CloseRead { read } => encoded(client.close_read(context, read).await?), + Command::BindRead { spec } => encoded(client.bind_read(context, spec).await?), Command::CreateUpload { upload, length, diff --git a/lib/crowdb-sdk-native-python/tests/real_consumer.py b/lib/crowdb-sdk-native-python/tests/real_consumer.py index c85834c1b..af47969cf 100644 --- a/lib/crowdb-sdk-native-python/tests/real_consumer.py +++ b/lib/crowdb-sdk-native-python/tests/real_consumer.py @@ -34,61 +34,47 @@ def native_data(): return (b"native-producer" * 300000)[:(4 << 20) + 17] -def prepare(client, snapshot, worker=0, workers=1, ids=None, split=None): - read = identity() - spec = {"read": {"id": read, "snapshot": snapshot, +def bind(client, snapshot, worker=0, workers=1, ids=None, split=None): + spec = {"snapshot": snapshot, "selection": {"Explicit": [list(item) for item in ids] if ids is not None else [list(f"size-{i}".encode()) for i in range(8)]}, "fields": ["value"], "batch_size": 2, "partition": {"worker": worker, "workers": workers, "rank": 0, "world_size": 1}, - "shuffle": None}, "split": split} - page = 0 - for _ in range(32): - result = call(client, "prepare_read", spec=spec, page=page) - if result["complete"]: - return read, result["session"] - page = result["next_page"] - raise AssertionError("read preparation did not complete") + "shuffle": None, "split": split} + return call(client, "bind_read", spec=spec) -def session_parity(client, snapshot): +def continuation_parity(client, snapshot): selections = [] for worker in range(2): - read, session = prepare(client, snapshot, worker, 2) - cursor = session["acknowledged"] - batch = client.next_batch(bytes(read), timeout=60) + cursor = bind(client, snapshot, worker, 2)["cursor"] + batch = client.next_batch(cursor, timeout=60) header = json.loads(batch.header) first = drain(batch) - assert call(client, "resume_read", read=read)["acknowledged"] == cursor - replay = client.next_batch(bytes(read), timeout=60) + replay = client.next_batch(cursor, timeout=60) assert json.loads(replay.header) == header assert drain(replay) == first selected = [] while True: selected.extend(bytes(item) for item in header["sample_ids"]) - session = call(client, "acknowledge", read=read, expected=cursor, next=header["next"]) - cursor = session["acknowledged"] if header["end"]: break - batch = client.next_batch(bytes(read), timeout=60) + cursor = header["next"] + batch = client.next_batch(cursor, timeout=60) header = json.loads(batch.header) drain(batch) - assert call(client, "close_read", read=read)["closed"] selections.append(set(selected)) assert not selections[0].intersection(selections[1]) assert selections[0] | selections[1] == {f"size-{i}".encode() for i in range(8)} -def pressure_and_idle_renewal(client, snapshot): - read, session = prepare(client, snapshot) - batch = client.next_batch(bytes(read), timeout=120) - cursor = session["acknowledged"] - # Cross the existing inactivity TTL while a live producer is idle. - print("Holding a native batch across the 60-second active-read TTL", flush=True) +def pressure_and_idle_replay(client, snapshot): + cursor = bind(client, snapshot)["cursor"] + batch = client.next_batch(cursor, timeout=120) + print("Holding a native batch for 65 seconds; reads have no session TTL", flush=True) time.sleep(65) - drain(batch) - assert call(client, "resume_read", read=read)["acknowledged"] == cursor - call(client, "close_read", read=read) + first = drain(batch) + assert drain(client.next_batch(cursor, timeout=120)) == first runtime = native.NativeRuntime(os.environ["CROWDB_MANAGEMENT_SEEDS"].split(","), options=json.dumps(dict(held_bytes=128))) try: @@ -150,11 +136,10 @@ def publish_native(runtime, root): snapshot = state["uploads"]["candidate"] call(client, "publish_composition", operation=operation, attempt={"version": 1, "id": identity(), "target": {"Snapshot": {"snapshot": snapshot}}, "expected_head": None}) - split_read, _ = prepare(client, bytes(snapshot).hex(), ids=[b"one"], split={"revision": None, "name": "train"}) - split_batch = client.next_batch(bytes(split_read)) + split_binding = bind(client, bytes(snapshot).hex(), ids=[b"one"], split={"revision": None, "name": "train"}) + split_batch = client.next_batch(split_binding["cursor"]) assert json.loads(split_batch.header)["sample_ids"] == [list(b"one")] assert drain(split_batch)[0] == data - call(client, "close_read", read=split_read) for _ in range(32): if call(client, "cleanup_composition", operation=operation)["complete"]: break @@ -190,8 +175,8 @@ def main(): for i, size in enumerate(SIZES): batch = client.read_batch(bytes.fromhex(snapshot), [list(f"size-{i}".encode())], ["value"]) assert drain(batch).get(0, b"") == bytes([i]) * size - session_parity(client, snapshot) - pressure_and_idle_renewal(client, snapshot) + continuation_parity(client, snapshot) + pressure_and_idle_replay(client, snapshot) publish_native(runtime, root) batch = client.read_batch(bytes.fromhex(snapshot), [list(b"size-1")], ["value"]) held = batch.next() @@ -211,7 +196,7 @@ def main(): del held, view, batch gc.collect() assert json.loads(runtime.status())["held_bytes"] == 0 - print("native Python same-snapshot bytes, workers, ack replay, publication, cancellation and lifetime passed") + print("native Python same-snapshot bytes, workers, continuation replay, publication, cancellation and lifetime passed") finally: runtime.close() diff --git a/lib/crowdb-sdk-native/README.md b/lib/crowdb-sdk-native/README.md index 5b6d61535..b3c883f94 100644 --- a/lib/crowdb-sdk-native/README.md +++ b/lib/crowdb-sdk-native/README.md @@ -101,18 +101,16 @@ async fn read_object(seeds: Vec) -> Result<(), Box +// Licensed under the Apache License, Version 2.0. + +use super::DatasetClient; +use crate::{OperationContext, Result}; +use crowdb_access_dataset::{DatasetReads, ReadBinding, ReadSpec}; +impl DatasetClient { + /// Binds immutable versions into a client-owned continuation without writes. + /// # Errors + /// Returns invalid queries, unavailable metadata or cancellation. + pub async fn bind_read(&self, context: OperationContext, spec: ReadSpec) -> Result { + self.control(context, |scope| async move { + Ok(DatasetReads::new(scope.authority) + .bind(&scope.identity, spec) + .await?) + }) + .await + } +} diff --git a/lib/crowdb-sdk-native/src/dataset/read.rs b/lib/crowdb-sdk-native/src/dataset/read.rs index 8f698c5d8..da43b8b19 100644 --- a/lib/crowdb-sdk-native/src/dataset/read.rs +++ b/lib/crowdb-sdk-native/src/dataset/read.rs @@ -1,17 +1,14 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -mod admission; mod body; mod field; pub use body::{BatchBody, FieldBuffer}; -use super::{now, DatasetClient}; +use super::DatasetClient; use crate::runtime::RequestLease; use crate::{OperationContext, Result}; -use crowdb_access_dataset::{ - DatasetSessions, FieldLocator, OperationId, ReadCancellation, SampleView, SnapshotId, -}; +use crowdb_access_dataset::{DatasetReads, FieldLocator, ReadCancellation, SampleView, SnapshotId}; use std::sync::Arc; pub struct BatchRequest { @@ -56,18 +53,23 @@ impl DatasetClient { /// Returns missing data, invalid projection, admission or storage failure. pub async fn read_batch(&self, context: OperationContext, batch: BatchRequest) -> Result { let request = self.runtime.resources.admit(context)?; - self.batch(request, batch, None, true, None).await + self.batch(request, batch, None, true).await } - /// Fetches the issued batch; only `acknowledge` advances durable progress. + /// Fetches a caller-owned continuation. Retrying it replays the same batch. /// # Errors - /// Returns expired/closed sessions, selection, admission or storage failure. - pub async fn next_batch(&self, context: OperationContext, id: OperationId) -> Result { + /// Returns invalid continuations, deletion admission, bounded-work or storage errors. + pub async fn next_batch(&self, context: OperationContext, cursor: &str) -> Result { let request = self.runtime.resources.admit(context)?; - let sessions = DatasetSessions::new(Arc::clone(&self.authority)); + let reads = DatasetReads::new(Arc::clone(&self.authority)); let batch = request - .run(async { Ok(sessions.next(&self.identity, id, now()).await?) }) + .run(async { + Ok(reads + .next(&self.identity, cursor, &ReadCancellation::new()) + .await?) + }) .await?; + let end = batch.next.is_none(); self.batch( request, BatchRequest { @@ -75,9 +77,8 @@ impl DatasetClient { sample_ids: batch.ids, fields: batch.fields, }, - Some(batch.next), - batch.end, - Some(id), + batch.next, + end, ) .await } @@ -88,9 +89,15 @@ impl DatasetClient { batch: BatchRequest, next: Option, end: bool, - session: Option, ) -> Result { - let admission = admission::ReadAdmission::admit(self, &request, batch.snapshot, session).await?; + request + .run(async { + Ok(self + .authority + .check_snapshot_readable(&self.identity, batch.snapshot) + .await?) + }) + .await?; let samples = if batch.sample_ids.is_empty() { Vec::new() } else { @@ -123,7 +130,6 @@ impl DatasetClient { request, Arc::clone(&self.runtime.resources), Arc::clone(&self.uploads), - admission, ); Ok(ReadBatch { header, body }) } diff --git a/lib/crowdb-sdk-native/src/dataset/read/admission.rs b/lib/crowdb-sdk-native/src/dataset/read/admission.rs deleted file mode 100644 index e41356f76..000000000 --- a/lib/crowdb-sdk-native/src/dataset/read/admission.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::super::DatasetClient; -use crate::{runtime::RequestLease, Result}; -use crowdb_access_dataset::{DatasetAuthority, DatasetIdentity, DatasetSessions, OperationId, SnapshotId}; -use std::sync::Arc; - -pub(super) struct ReadAdmission { - authority: Arc, - identity: DatasetIdentity, - session: Option, -} -impl ReadAdmission { - pub(super) async fn admit( - client: &DatasetClient, - request: &RequestLease, - snapshot: SnapshotId, - session: Option, - ) -> Result { - request - .run(async { - Ok(client - .authority - .check_snapshot_readable(&client.identity, snapshot) - .await?) - }) - .await?; - Ok(Self { - authority: Arc::clone(&client.authority), - identity: client.identity.clone(), - session, - }) - } - pub(super) async fn check(&self) -> Result<()> { - if let Some(session) = self.session { - DatasetSessions::new(Arc::clone(&self.authority)) - .check_open(&self.identity, session) - .await?; - } - Ok(()) - } -} diff --git a/lib/crowdb-sdk-native/src/dataset/read/body.rs b/lib/crowdb-sdk-native/src/dataset/read/body.rs index b8648648f..fdf669a3b 100644 --- a/lib/crowdb-sdk-native/src/dataset/read/body.rs +++ b/lib/crowdb-sdk-native/src/dataset/read/body.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{admission::ReadAdmission, field::FieldReader, BatchHeader}; +use super::{field::FieldReader, BatchHeader}; use crate::{buffer::BufferLease, runtime::RequestLease, Error, Result, RuntimeResources}; use crowdb_access_dataset::{DatasetUploads, FieldLocator}; use crowdb_rpc_ffi::OwnedPayload; @@ -22,7 +22,6 @@ pub struct BatchBody { request: Option, resources: Arc, uploads: Arc, - admission: Option, sequence: u64, complete: bool, failure: Option, @@ -35,7 +34,6 @@ impl BatchBody { request: RequestLease, resources: Arc, uploads: Arc, - admission: ReadAdmission, ) -> Self { Self { header, @@ -45,7 +43,6 @@ impl BatchBody { request: Some(request), resources, uploads, - admission: Some(admission), sequence: 0, complete: false, failure: None, @@ -98,14 +95,12 @@ impl BatchBody { self.field = None; self.pending = None; self.locators.clear(); - self.admission = None; self.request = None; } async fn pull(&mut self) -> Result> { let request = self.request.as_ref().ok_or(Error::Cancelled)?; - let admission = self.admission.as_ref().ok_or(Error::Cancelled)?; - request.run(admission.check()).await?; + request.run(async { Ok(()) }).await?; loop { if self.field.is_none() { let Some((index, locator)) = self.locators.pop_front() else { diff --git a/lib/crowdb-sdk-native/src/dataset/session.rs b/lib/crowdb-sdk-native/src/dataset/session.rs deleted file mode 100644 index 8ae5c957a..000000000 --- a/lib/crowdb-sdk-native/src/dataset/session.rs +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::{now, DatasetClient}; -use crate::{OperationContext, Result}; -use crowdb_access_dataset::{ - DatasetSessions, OperationId, ReadCancellation, ReadPreparation, ReadSession, SessionSpecV2, -}; - -impl DatasetClient { - /// Advances one bounded preparation window; alias and Split binding are durable. - /// # Errors - /// Returns divergent preparation, selection/worker limits or storage failure. - pub async fn prepare_read( - &self, - context: OperationContext, - spec: &SessionSpecV2, - page: u64, - ) -> Result { - self.control(context, |scope| async move { - Ok(DatasetSessions::new(scope.authority) - .prepare_split_read(&scope.identity, spec, page, now(), &ReadCancellation::new()) - .await?) - }) - .await - } - - /// Resumes the acknowledged cursor. Buffer release never changes this cursor. - /// # Errors - /// Returns closed/missing session, cancellation or storage failure. - pub async fn resume_read(&self, context: OperationContext, id: OperationId) -> Result { - self.control(context, |scope| async move { - Ok(DatasetSessions::new(scope.authority) - .resume(&scope.identity, id, now()) - .await?) - }) - .await - } - - /// # Errors - /// Rejects forged/divergent cursors, expired sessions or storage failure. - pub async fn acknowledge( - &self, - context: OperationContext, - id: OperationId, - expected: &str, - next: &str, - ) -> Result { - self.control(context, |scope| async move { - Ok(DatasetSessions::new(scope.authority) - .acknowledge(&scope.identity, id, expected, next, now()) - .await?) - }) - .await - } - - /// Closes only this read and its active storage protection, without ack. - /// # Errors - /// Returns missing preparation, admission or storage failure. - pub async fn close_read(&self, context: OperationContext, id: OperationId) -> Result { - self.control(context, |scope| async move { - Ok(DatasetSessions::new(scope.authority) - .cancel_split_read(&scope.identity, id) - .await?) - }) - .await - } -} diff --git a/lib/crowdb-sdk-native/tests/dataset_read_test.rs b/lib/crowdb-sdk-native/tests/dataset_read_test.rs index 3fb90ac00..6fc3e9a82 100644 --- a/lib/crowdb-sdk-native/tests/dataset_read_test.rs +++ b/lib/crowdb-sdk-native/tests/dataset_read_test.rs @@ -8,7 +8,7 @@ mod metadata; #[path = "common/storage.rs"] mod storage; -use crowdb_access_dataset::{OperationId, Selection, SessionPartition, SessionSpec, SessionSpecV2}; +use crowdb_access_dataset::{OperationId, ReadPartition, ReadSpec, Selection}; use crowdb_sdk_native::{ dataset::{BatchRequest, FieldState}, Error, @@ -16,22 +16,19 @@ use crowdb_sdk_native::{ use dataset::{context, TestFixture}; use std::time::Duration; -fn spec() -> SessionSpecV2 { - SessionSpecV2 { - read: SessionSpec { - id: OperationId::random(), - snapshot: "latest".into(), - selection: Selection::Explicit(vec![vec![1], vec![2], vec![3]]), - fields: vec!["value".into(), "optional".into()], - batch_size: 1, - partition: SessionPartition { - worker: 0, - workers: 1, - rank: 0, - world_size: 1, - }, - shuffle: None, +fn spec() -> ReadSpec { + ReadSpec { + snapshot: "latest".into(), + selection: Selection::Explicit(vec![vec![1], vec![2], vec![3]]), + fields: vec!["value".into(), "optional".into()], + batch_size: 1, + partition: ReadPartition { + worker: 0, + workers: 1, + rank: 0, + world_size: 1, }, + shuffle: None, split: None, } } @@ -89,58 +86,31 @@ async fn deletion_rejects_new_requests_without_interrupting_an_admitted_body() { #[tokio::test] async fn fetching_and_releasing_owners_never_acknowledge_progress() { let fixture = TestFixture::new(1024).await; - let spec = spec(); - let mut preparation = fixture.client.prepare_read(context(2), &spec, 0).await.unwrap(); - for _ in 0..16 { - if preparation.complete { - break; - } - preparation = fixture - .client - .prepare_read(context(2), &spec, preparation.next_page) - .await - .unwrap(); - } - assert!(preparation.complete); - let cursor = preparation.session.unwrap().acknowledged; - let mut first = fixture.client.next_batch(context(3), spec.read.id).await.unwrap(); + let bound = fixture.client.bind_read(context(2), spec()).await.unwrap(); + let cursor = bound.cursor; + let mut first = fixture.client.next_batch(context(3), &cursor).await.unwrap(); assert_eq!(first.header.snapshot, fixture.snapshot); assert_eq!(first.header.fields[1].state, FieldState::Missing); let held = first.body.next().await.unwrap().unwrap().buffer; assert_eq!(held.as_slice(), &[0; 16]); assert!(first.body.next().await.unwrap().is_none()); drop(held); - assert_eq!( - fixture - .client - .resume_read(context(4), spec.read.id) - .await - .unwrap() - .acknowledged, - cursor - ); - let mut replay = fixture.client.next_batch(context(5), spec.read.id).await.unwrap(); + let mut replay = fixture.client.next_batch(context(5), &cursor).await.unwrap(); assert_eq!(replay.header.next, first.header.next); assert_eq!( replay.body.next().await.unwrap().unwrap().buffer.as_slice(), &[0; 16] ); assert!(replay.body.next().await.unwrap().is_none()); - fixture + let mut second = fixture .client - .acknowledge( - context(6), - spec.read.id, - &cursor, - first.header.next.as_ref().unwrap(), - ) + .next_batch(context(7), first.header.next.as_deref().unwrap()) .await .unwrap(); - let mut second = fixture.client.next_batch(context(7), spec.read.id).await.unwrap(); assert_eq!(second.header.sample_ids, vec![vec![2]]); assert_eq!(second.header.fields[1].state, FieldState::Tombstone); let retained = second.body.next().await.unwrap().unwrap().buffer; - fixture.client.close_read(context(8), spec.read.id).await.unwrap(); + second.body.cancel(); assert!(second.body.next().await.is_err()); fixture.runtime.shutdown().await.unwrap(); assert_eq!(retained.as_slice(), &[1; 16]); diff --git a/sdk/README.md b/sdk/README.md index c6bf5dfd7..638d63961 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -70,7 +70,7 @@ and iterators. Cancellation aborts the Rust operation. Closing a stream waits for producer cancellation and releases its admission; completed Python bytes remain valid. `refresh_bearer` updates the credential used by subsequent attempts. -For training, send a ReadPlan with `id`, `snapshot` (`latest`, `stable`, or an +For training, send a ReadPlan with `snapshot` (`latest`, `stable`, or an immutable ID), selection, projected fields, batch size, partition and nullable shuffle. Selection supports explicit IDs, prefix, range, keyword, field match and composition. Predicates evaluate inline manifest metadata; payloads are @@ -80,7 +80,7 @@ fetched only after server selection/shuffle/partition. from crowdb_dataset import DatasetIterable, WorkerPartition plan = { - "id": new_id(), "snapshot": "latest", + "snapshot": "latest", "selection": {"kind": "prefix", "value": sample_id(b"sample-")}, "fields": ["label"], "batch_size": 64, "shuffle": None, } @@ -90,21 +90,20 @@ with DatasetClient("http://localhost:9093", "training", "token") as client: batch.acknowledge() ``` -The iterable preserves its import name but now yields acknowledged batches. -Continuing without acknowledgement fails. It never downloads another worker's -samples and discards them. Save the read ID for resume; restarting with that -same plan restores acknowledged progress. Fetch alone never advances progress. -An expired connection can resume while its snapshot remains available; explicit -`close_read` permanently closes that handle. At most one batch is issued per -session; use independent sessions for concurrent workers. `next_batch` streams -are the interface for fields exceeding the convenience batch limit. +The iterable acknowledges locally and makes no progress RPC. Continuing without +acknowledgement fails. Save the cursor returned by `batch.acknowledge()` after +processing succeeds; pass it as `checkpoint` to a new iterable to resume. +`bind_read(plan)` pins immutable versions and returns the initial cursor. +`next_batch(cursor)` replays that cursor on any server. Drop or cancel stops only +the current stream; there is no read ID, session TTL or remote close. Use streams +for fields exceeding the convenience batch limit. ## Rust `DatasetClient` is cloneable; clones share the pool and atomic admission. It exposes create/open, upload parts/status/complete/abort, file transfer, prepare/publish/status, snapshot/manifest/listing, stable, retain/release, -reclaim/status, batch streams, read plans and explicit progress/resume/close. +reclaim/status, batch streams and stateless read continuations. `UploadOptions` controls part size and concurrency. File convenience defaults to 1-MiB parts and four transfers, bounded by the configured request capacity. Use 4-MiB parts for uploads up to the v1 limit of 10000 parts (39.0625 GiB). @@ -154,12 +153,12 @@ It contains no benchmark driver or timing threshold. disappear; surviving samples may inherit omitted optional fields through parents. Explicit tombstones hide inherited fields, including required fields that are explicitly removed. Required field names remain in the schema. -- JSON/manifest metadata is bounded to 8 MiB; inline values to 64 KiB. Read plans - admit at most 100000 selected IDs and keep immutable 1024-ID pages separately - from the small acknowledged cursor. Close/cancel is scoped; leases expire - after 60 seconds without activity and renew during active streaming. +- JSON/manifest metadata is bounded to 8 MiB; inline values to 64 KiB. Stateless + reads keep no selected-ID pages or progress on the server. A shuffle group has + at most 100000 IDs and 8 MiB of ID bytes; the cursor is bounded to 256 KiB before + encoding. Small batches within a shuffle group reconstruct that bounded group. - Prepared candidates persist until publication or explicit reclaim. Reclaim - preserves head/stable/retained/ancestor/live-read data. Deferred cleanup reports + preserves head/stable/retained/ancestor data and a deletion grace period. Deferred cleanup reports pending; shared ranges do not count as reclaimed whole chunks. Local disk cache/checkpoint staging, direct chunk access, RDMA, storage-side @@ -202,10 +201,10 @@ Existing `DatasetClient` methods retain their v1 contracts. `samples` return lazy `PageCursor`s. Each `next_page` fetches at most 100 items; it never eagerly collects the Dataset. Persist `continuation()` for scoped resume. Dropping the cursor stops further requests. -- Read plans pin Snapshot and optional Split revision once. `prepare_read` advances - a bounded window; `next_batch` reuses verified streaming framing. Neither preview - nor a successful fetch acknowledges data. Call `acknowledge` only after application - processing succeeds. `close_read` releases the plan's source protection. +- `bind_read` pins Snapshot and optional Split revision without writes; + `next_batch(cursor)` uses verified streaming framing. Save the next cursor after + successful processing. The required capability is `stateless-reads`; old session + routes are retired. No remote acknowledgement or cleanup call is necessary. - `upload_file` performs bounded digest and transfer passes. `upload_reader` accepts caller-owned local/remote readers plus exact length and MD5, checking existing parts against replayed bytes. Reopen the same source from the beginning on retry. @@ -213,11 +212,11 @@ Existing `DatasetClient` methods retain their v1 contracts. This is a reader adapter, not a built-in remote storage/catalog connector. - `abandon_composition` and `cleanup_composition` are distinct operations. Run cleanup steps until complete; physical payload deletion remains deferred. Snapshot reclaim - similarly reports its current phase and respects latest/stable/retention/leases. + similarly reports its current phase and respects latest/stable/retention/deletion grace. The `split_control` example exercises local file and caller-owned stream ingestion, initial Split publication, independent revision publication, historical Split reads, -frozen child inheritance, paged inspection and explicit acknowledgement. Run it +frozen child inheritance, paged inspection and continuation replay. Run it against a configured Dataset listener with `CROWDB_DATASET_ENDPOINT`, `CROWDB_DATASET_NAME` and `CROWDB_DATASET_READ_TOKEN` set. @@ -226,7 +225,7 @@ v2 operations through Rust. `SplitSource` identifies a frozen revision/name and `decimal(n)` rejects floats, booleans and out-of-range integers. Request dictionaries use the OpenAPI v2 field names; omitted required nullable preconditions fail. `pages(kind, ...)` is lazy in both synchronous and asynchronous clients. Use -`next_batch` as a context manager, then explicitly acknowledge after processing. +`next_batch` as a context manager, then save the returned continuation after processing. Cancellation aborts the Rust operation; clients inherited across fork remain invalid. `tests/split_consumer.py` is a complete installed-wheel consumer and includes a diff --git a/sdk/crates/crowdb-sdk-dataset/examples/payload.rs b/sdk/crates/crowdb-sdk-dataset/examples/payload.rs index faf63f933..43fc8cf9a 100644 --- a/sdk/crates/crowdb-sdk-dataset/examples/payload.rs +++ b/sdk/crates/crowdb-sdk-dataset/examples/payload.rs @@ -5,8 +5,8 @@ use base64::{engine::general_purpose::STANDARD, Engine}; use bytes::Bytes; use crowdb_sdk_dataset::{ - wire::BatchEvent, Acknowledge, DatasetClient, Error, FieldDefinition, FieldValue, Manifest, Publication, - ReadPlan, Sample, Selection, UploadRequest, WorkerPartition, + wire::BatchEvent, DatasetClient, Error, FieldDefinition, FieldValue, Manifest, Publication, ReadPlan, + Sample, Selection, UploadRequest, WorkerPartition, }; use md5::{Digest, Md5}; use std::{collections::BTreeMap, path::PathBuf}; @@ -157,7 +157,6 @@ async fn verify( let mut owned = Vec::new(); for worker in 0..2 { let plan = ReadPlan { - id: uuid::Uuid::new_v4().simple().to_string(), snapshot: "latest".into(), selection: Selection::Prefix { value: STANDARD.encode(b"sample-"), @@ -172,11 +171,11 @@ async fn verify( }, shuffle: None, }; - let session = client.start_read(&plan).await?; - assert_eq!(session.snapshot, snapshot); - let mut cursor = session.acknowledged; + let bound = client.bind_read(&plan).await?; + assert_eq!(bound.snapshot, snapshot); + let mut cursor = bound.cursor; loop { - let mut stream = client.next_batch(&plan.id).await?; + let mut stream = client.next_batch(&cursor).await?; let mut header = None; while let Some(event) = stream.next_event().await? { if let BatchEvent::Header(h) = event { @@ -185,18 +184,11 @@ async fn verify( } let header = header.ok_or("batch header")?; owned.extend(header.sample_ids.iter().cloned()); - assert_eq!(client.progress(&plan.id).await?.cursor, cursor); - let ack = Acknowledge { - expected: cursor.clone(), - next: header.next.clone().ok_or("issued cursor")?, - }; - cursor = client.acknowledge(&plan.id, &ack).await?.cursor; - assert_eq!(client.acknowledge(&plan.id, &ack).await?.cursor, cursor); if header.end { break; } + cursor = header.next.clone().ok_or("next cursor")?; } - client.close_read(&plan.id).await?; } owned.sort(); owned.dedup(); diff --git a/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs b/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs index 1bfa2a9f2..46fa6e98b 100644 --- a/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs +++ b/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs @@ -310,8 +310,7 @@ async fn read_split( snapshot: &v2::OpaqueId, revision: &v2::OpaqueId, ) -> Result<()> { - let mut request = v2::ReadPlan { - id: v2::OpaqueId::random(), + let request = v2::ReadPlan { snapshot: snapshot.to_string(), selection: v2::Predicate::All, fields: vec!["value".into()], @@ -322,49 +321,24 @@ async fn read_split( revision: Some(revision.clone()), name: "train/a b".into(), }), - expected_page: 0.into(), }; - let mut ready = false; - for _ in 0..100 { - let p = client.prepare_read(&request).await?; - if p.complete { - ready = true; - break; - } - request.expected_page = p.next_page; - } - assert!(ready); - assert_eq!( - client.read_status(&request.id).await?.split.unwrap().revision, - *revision - ); - let before = client.progress(&request.id).await?; - let mut stream = client.next_batch(&request.id).await?; + let bound = client.bind_read(&request).await?; + assert_eq!(bound.split.as_ref().unwrap().revision, *revision); + let mut stream = client.next_batch(&bound.cursor).await?; let mut data = Vec::new(); - let mut next = None; while let Some(event) = stream.next_event().await? { - match event { - BatchEvent::Data(bytes) => data.extend_from_slice(&bytes), - BatchEvent::Header(header) => next.clone_from(&header.next), - _ => {} + if let BatchEvent::Data(bytes) = event { + data.extend_from_slice(&bytes); } } assert_eq!(data, b"data"); - assert_eq!(client.progress(&request.id).await?.cursor, before.cursor); - let next = next.unwrap(); - assert_eq!( - client - .acknowledge( - &request.id, - &crowdb_sdk_dataset::Acknowledge { - expected: before.cursor, - next: next.clone() - } - ) - .await? - .cursor, - next - ); - assert!(client.close_read(&request.id).await?.closed); + let mut replay = client.next_batch(&bound.cursor).await?; + let mut again = Vec::new(); + while let Some(event) = replay.next_event().await? { + if let BatchEvent::Data(bytes) = event { + again.extend_from_slice(&bytes); + } + } + assert_eq!(again, data); Ok(()) } diff --git a/sdk/crates/crowdb-sdk-dataset/src/lib.rs b/sdk/crates/crowdb-sdk-dataset/src/lib.rs index 8179d95e7..9a4b44a8e 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/lib.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/lib.rs @@ -17,6 +17,6 @@ pub use crowdb_sdk_core::Error; pub use models::*; pub use read::BatchStream; -mod session; +mod query; pub use transfer::UploadOptions; diff --git a/sdk/crates/crowdb-sdk-dataset/src/models.rs b/sdk/crates/crowdb-sdk-dataset/src/models.rs index ae2b533d9..20346eb27 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/models.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/models.rs @@ -158,7 +158,6 @@ pub enum Selection { } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct ReadPlan { - pub id: String, pub snapshot: String, pub selection: Selection, pub fields: Vec, @@ -167,22 +166,8 @@ pub struct ReadPlan { pub shuffle: Option, } #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct ReadSession { - pub id: String, +pub struct ReadBinding { pub snapshot: String, - pub plan: String, - pub acknowledged: String, - pub expires_at: u64, - pub closed: bool, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Acknowledge { - pub expected: String, - pub next: String, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Progress { - pub plan: String, pub cursor: String, } #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/sdk/crates/crowdb-sdk-dataset/src/query.rs b/sdk/crates/crowdb-sdk-dataset/src/query.rs new file mode 100644 index 000000000..a65b795a6 --- /dev/null +++ b/sdk/crates/crowdb-sdk-dataset/src/query.rs @@ -0,0 +1,53 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::{BatchStream, DatasetClient, Error, ReadBinding, ReadPlan, Reclaim, SnapshotPage}; +impl DatasetClient { + /// Pins immutable versions without registering a read or saving progress. + /// # Errors + /// Returns invalid queries, missing snapshots or transport failures. + pub async fn bind_read(&self, plan: &ReadPlan) -> Result { + self.call("POST", &["reads", "bind"], Some(plan)).await + } + /// The caller retains the current cursor until processing succeeds. + /// # Errors + /// Returns invalid continuations, deletion admission, capacity or service failures. + pub async fn next_batch(&self, cursor: &str) -> Result { + Ok(BatchStream::new( + self.http + .stream( + "POST", + &self.path(&["reads", "next"]), + Some(&serde_json::json!({"cursor":cursor})), + ) + .await?, + )) + } + /// # Errors + /// Returns protected/missing snapshots or service failures. Deferred cleanup remains pending. + pub async fn reclaim(&self, id: &str) -> Result { + self.call("POST", &["snapshots", id, "reclaim"], Some(&())).await + } + /// # Errors + /// Returns missing datasets or service failures. + pub async fn reclaim_status(&self, id: &str) -> Result { + self.call::<(), _>("GET", &["snapshots", id, "reclaim"], None) + .await + } + /// Returns one bounded ancestry page; `after` is the preceding page's next token. + /// # Errors + /// Returns invalid bounds/cursors or service failures. + pub async fn list_snapshots(&self, after: Option<&str>, limit: usize) -> Result { + if !(1..=1000).contains(&limit) { + return Err(Error::Invalid("snapshot page size".into())); + } + let size = limit.to_string(); + let mut query = vec![("limit", size.as_str())]; + if let Some(after) = after { + query.push(("after", after)); + } + self.http + .json_query("GET", &self.path(&["snapshots"]), &query) + .await + } +} diff --git a/sdk/crates/crowdb-sdk-dataset/src/session.rs b/sdk/crates/crowdb-sdk-dataset/src/session.rs deleted file mode 100644 index 812684d09..000000000 --- a/sdk/crates/crowdb-sdk-dataset/src/session.rs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use crate::{ - Acknowledge, BatchStream, DatasetClient, Error, Progress, ReadPlan, ReadSession, Reclaim, SnapshotPage, -}; -impl DatasetClient { - /// # Errors - /// Returns invalid plans, conflicts, capacity or service failures. - pub async fn start_read(&self, plan: &ReadPlan) -> Result { - self.call("POST", &["read-plans"], Some(plan)).await - } - /// # Errors - /// Returns closed/reclaimed sessions or transport failures. - pub async fn resume_read(&self, id: &str) -> Result { - self.call::<(), _>("GET", &["read-plans", id], None).await - } - /// # Errors - /// Returns cancellation, integrity, capacity or service failures. - pub async fn next_batch(&self, id: &str) -> Result { - Ok(BatchStream::new( - self.http - .stream("POST", &self.path(&["read-plans", id, "next"]), Some(&())) - .await?, - )) - } - /// # Errors - /// Returns stale/forged progress or service/transport failures. - pub async fn acknowledge(&self, id: &str, ack: &Acknowledge) -> Result { - self.call("PUT", &["read-plans", id, "progress"], Some(ack)).await - } - /// # Errors - /// Returns missing/closed sessions or service failures. - pub async fn progress(&self, id: &str) -> Result { - self.call::<(), _>("GET", &["read-plans", id, "progress"], None) - .await - } - /// # Errors - /// Returns missing sessions or service failures. Cancels only this session. - pub async fn close_read(&self, id: &str) -> Result { - self.call("DELETE", &["read-plans", id], Some(&())).await - } - /// # Errors - /// Returns protected/missing snapshots or service failures. Deferred cleanup remains pending. - pub async fn reclaim(&self, id: &str) -> Result { - self.call("POST", &["snapshots", id, "reclaim"], Some(&())).await - } - /// # Errors - /// Returns missing datasets or service failures. - pub async fn reclaim_status(&self, id: &str) -> Result { - self.call::<(), _>("GET", &["snapshots", id, "reclaim"], None) - .await - } - /// Returns one bounded ancestry page; `after` is the preceding page's next token. - /// # Errors - /// Returns invalid bounds/cursors or service failures. - pub async fn list_snapshots(&self, after: Option<&str>, limit: usize) -> Result { - if !(1..=1000).contains(&limit) { - return Err(Error::Invalid("snapshot page size".into())); - } - let size = limit.to_string(); - let mut query = vec![("limit", size.as_str())]; - if let Some(after) = after { - query.push(("after", after)); - } - self.http - .json_query("GET", &self.path(&["snapshots"]), &query) - .await - } -} diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2/read.rs b/sdk/crates/crowdb-sdk-dataset/src/v2/read.rs index 8d6d3b8ec..38a228443 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/v2/read.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/v2/read.rs @@ -19,7 +19,6 @@ pub struct Shuffle { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ReadPlan { - pub id: OpaqueId, /// Concrete Snapshot ID, latest or stable. The server pins it at first admission. pub snapshot: String, pub selection: Predicate, @@ -30,25 +29,11 @@ pub struct ReadPlan { pub shuffle: Option, #[serde(deserialize_with = "required_option")] pub split: Option, - pub expected_page: Decimal, } #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct ReadSession { - pub id: OpaqueId, - pub snapshot: OpaqueId, - pub plan: String, - pub acknowledged: String, - pub expires_at: Decimal, - pub closed: bool, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct ReadPreparation { +pub struct ReadBinding { pub snapshot: OpaqueId, #[serde(deserialize_with = "required_option")] pub split: Option, - pub next_page: Decimal, - pub admitted: Decimal, - pub complete: bool, - #[serde(deserialize_with = "required_option")] - pub session: Option, + pub cursor: String, } diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2/requests.rs b/sdk/crates/crowdb-sdk-dataset/src/v2/requests.rs index 777c7f815..5ab9d734e 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/v2/requests.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/v2/requests.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::{ ControlClient, CreateUpload, ExactMatrix, FieldPreview, InspectionPreparation, MatrixRequest, OpaqueId, - ReadPlan, ReadPreparation, ReadSession, SampleId, SnapshotSummary, UploadPart, UploadStatus, + ReadBinding, ReadPlan, SampleId, SnapshotSummary, UploadPart, UploadStatus, }; use crate::{BatchStream, Error}; use bytes::Bytes; @@ -154,62 +154,27 @@ impl ControlClient { .map_err(|error| error.with_identity(id.as_str()))?; Ok(serde_json::from_slice(&bytes)?) } - /// One bounded plan window. Keep the same plan identity and advance only the returned page. + /// Pins immutable versions into a client-owned continuation, without read state. /// # Errors - /// Returns capability, stale preparation, selection or service errors. - pub async fn prepare_read(&self, request: &ReadPlan) -> Result { - self.call("split-reads", "POST", &["read-plans"], Some(request)) - .await - .map_err(|error| error.with_identity(request.id.as_str())) - } - /// # Errors - /// Returns capability, missing preparation or service errors. - pub async fn read_status(&self, id: &OpaqueId) -> Result { - self.call::<(), _>("split-reads", "GET", &["read-plans", id.as_str()], None) + /// Returns capability, query, admission or service failures. + pub async fn bind_read(&self, request: &ReadPlan) -> Result { + self.call("stateless-reads", "POST", &["reads", "bind"], Some(request)) .await } + /// Dropping cancels only this HTTP stream. Retrying a cursor replays its batch. /// # Errors - /// Returns capability, closed preparation or service errors. - pub async fn close_read(&self, id: &OpaqueId) -> Result { - self.call::<(), _>("split-reads", "DELETE", &["read-plans", id.as_str()], None) - .await - } - /// Only an explicit acknowledgement advances delivery. Dropping cancels the HTTP stream. - /// # Errors - /// Returns capability, closed session, bounded stream or service errors. - pub async fn next_batch(&self, id: &OpaqueId) -> Result { - self.require("split-reads").await?; + /// Returns invalid continuations, deletion admission, bounded work or service failures. + pub async fn next_batch(&self, cursor: &str) -> Result { + self.require("stateless-reads").await?; Ok(BatchStream::new_v2( self.inner .http - .stream::<()>("POST", &self.path(&["read-plans", id.as_str(), "next"]), None) + .stream( + "POST", + &self.path(&["reads", "next"]), + Some(&serde_json::json!({"cursor":cursor})), + ) .await?, )) } - /// # Errors - /// Returns capability, stale acknowledgement or service errors. - pub async fn acknowledge( - &self, - id: &OpaqueId, - request: &crate::Acknowledge, - ) -> Result { - self.call( - "split-reads", - "PUT", - &["read-plans", id.as_str(), "progress"], - Some(request), - ) - .await - } - /// # Errors - /// Returns capability, closed session or service errors. - pub async fn progress(&self, id: &OpaqueId) -> Result { - self.call::<(), _>( - "split-reads", - "GET", - &["read-plans", id.as_str(), "progress"], - None, - ) - .await - } } diff --git a/sdk/crates/crowdb-sdk-python/src/control.rs b/sdk/crates/crowdb-sdk-python/src/control.rs index 0e4c0179b..16393729f 100644 --- a/sdk/crates/crowdb-sdk-python/src/control.rs +++ b/sdk/crates/crowdb-sdk-python/src/control.rs @@ -54,11 +54,7 @@ pub(super) async fn dispatch(client: DatasetClient, name: &str, args: &str) -> R "complete_upload" => encode(&client.complete_upload(&id()?).await?), "abort_upload" => encode(&client.abort_upload(&id()?).await?), "upload_file" => encode(&client.upload_file(text(&value, "path")?, &id()?).await?), - "prepare_read" => encode(&client.prepare_read(&request(&value)?).await?), - "read_status" => encode(&client.read_status(&id()?).await?), - "close_read" => encode(&client.close_read(&id()?).await?), - "acknowledge" => encode(&client.acknowledge(&id()?, &request(&value)?).await?), - "progress" => encode(&client.progress(&id()?).await?), + "bind_read" => encode(&client.bind_read(&request(&value)?).await?), _ => Err(Error::Invalid("unknown v2 SDK operation".into())), } } diff --git a/sdk/crates/crowdb-sdk-python/src/operation.rs b/sdk/crates/crowdb-sdk-python/src/operation.rs index d18c6b360..13108008c 100644 --- a/sdk/crates/crowdb-sdk-python/src/operation.rs +++ b/sdk/crates/crowdb-sdk-python/src/operation.rs @@ -128,9 +128,9 @@ async fn dispatch(c: DatasetClient, name: &str, args: &str) -> Result { + "bind_read" => { return Ok(serde_json::to_string( - &c.start_read(&serde_json::from_str(args)?).await?, + &c.bind_read(&serde_json::from_str(args)?).await?, )?) } _ => {} @@ -180,13 +180,6 @@ async fn dispatch(c: DatasetClient, name: &str, args: &str) -> Result download(&c, &v).await, - "resume_read" => output!(c.resume_read(id()?).await), - "close_read" => output!(c.close_read(id()?).await), - "acknowledge" => output!( - c.acknowledge(id()?, &serde_json::from_value(v["ack"].clone())?) - .await - ), - "progress" => output!(c.progress(id()?).await), "reclaim" => output!(c.reclaim(id()?).await), "reclaim_status" => output!(c.reclaim_status(id()?).await), "list_snapshots" => output!( diff --git a/sdk/crates/crowdb-sdk-python/src/stream.rs b/sdk/crates/crowdb-sdk-python/src/stream.rs index 232ec63e1..d9bc49e48 100644 --- a/sdk/crates/crowdb-sdk-python/src/stream.rs +++ b/sdk/crates/crowdb-sdk-python/src/stream.rs @@ -139,7 +139,7 @@ impl NativeStream { let mut stream = match kind.as_str() { "batch" => client.stream_batch(&serde_json::from_str(&args)?).await?, "next" => client.next_batch(&args).await?, - "v2-next" => client.control().next_batch(&args.parse()?).await?, + "v2-next" => client.control().next_batch(&args).await?, _ => return Err(Error::Invalid("stream operation".into())), }; while let Some(event) = stream.next_event().await? { diff --git a/sdk/crowdb_dataset/client.py b/sdk/crowdb_dataset/client.py index dd3a0ef38..9616da618 100644 --- a/sdk/crowdb_dataset/client.py +++ b/sdk/crowdb_dataset/client.py @@ -190,11 +190,7 @@ def upload_file(self, path, id, *, part_size=1024 * 1024, concurrency=4): return self._call("upload_file", {"path": os.fspath(path), "id": id, "part_size": part_size, "concurrency": concurrency}) def download_to_path(self, snapshot, sample, field, path): return self._call("download_to_path", {"snapshot": snapshot, "sample": sample, "field": field, "path": os.fspath(path)}) - def start_read(self, plan): return self._call("start_read", plan) - def resume_read(self, id): return self._call("resume_read", {"id": id}) - def close_read(self, id): return self._call("close_read", {"id": id}) - def acknowledge(self, id, expected, next): return self._call("acknowledge", {"id": id, "ack": {"expected": expected, "next": next}}) - def progress(self, id): return self._call("progress", {"id": id}) + def bind_read(self, plan): return self._call("bind_read", plan) def reclaim(self, id): return self._call("reclaim", {"id": id}) def reclaim_status(self, id): return self._call("reclaim_status", {"id": id}) @@ -202,9 +198,9 @@ def stream_batch(self, request): self._check() return BatchStream(self._native.stream("batch", _encode(request))) - def next_batch(self, id): + def next_batch(self, cursor): self._check() - return BatchStream(self._native.stream("next", id)) + return BatchStream(self._native.stream("next", cursor)) def get_batch(self, request, max_bytes=8 * 1024 * 1024): with self.stream_batch(request) as stream: @@ -288,7 +284,7 @@ async def upload_part(self, id, index, data): raise def stream_batch(self, request): return AsyncBatchStream(self._client.stream_batch(request)) - def next_batch(self, id): return AsyncBatchStream(self._client.next_batch(id)) + def next_batch(self, cursor): return AsyncBatchStream(self._client.next_batch(cursor)) async def get_batch(self, request, max_bytes=8 * 1024 * 1024): fields, chunks, size, header = [], [], 0, None async with self.stream_batch(request) as stream: diff --git a/sdk/crowdb_dataset/control.py b/sdk/crowdb_dataset/control.py index 08ec01bd5..34f3b6c90 100644 --- a/sdk/crowdb_dataset/control.py +++ b/sdk/crowdb_dataset/control.py @@ -93,14 +93,10 @@ def upload_file(self, path, id): return self._call("upload_file", {"path": os.fs def upload_part(self, id, index, data): self._owner._check() return _result(self._owner._native.part_v2(id, index, data), id) - def prepare_read(self, request): return self._call("prepare_read", {"request": request}) - def read_status(self, id): return self._call("read_status", {"id": id}) - def close_read(self, id): return self._call("close_read", {"id": id}) - def progress(self, id): return self._call("progress", {"id": id}) - def acknowledge(self, id, expected, next): return self._call("acknowledge", {"id": id, "request": {"expected": expected, "next": next}}) - def next_batch(self, id): + def bind_read(self, request): return self._call("bind_read", {"request": request}) + def next_batch(self, cursor): self._owner._check() - return BatchStream(self._owner._native.stream("v2-next", id)) + return BatchStream(self._owner._native.stream("v2-next", cursor)) def page(self, kind, *, snapshot=None, source=None, after=None, limit=20): """One bounded page; source is a SplitSource or its exact wire dictionary.""" @@ -147,7 +143,7 @@ async def upload_part(self, id, index, data): await asyncio.to_thread(pending.result) raise - def next_batch(self, id): return AsyncBatchStream(ControlClient(self._owner).next_batch(id)) + def next_batch(self, cursor): return AsyncBatchStream(ControlClient(self._owner).next_batch(cursor)) async def pages(self, kind, **scope): while True: diff --git a/sdk/crowdb_dataset/iterable.py b/sdk/crowdb_dataset/iterable.py index eb26ae380..1c99e38ed 100644 --- a/sdk/crowdb_dataset/iterable.py +++ b/sdk/crowdb_dataset/iterable.py @@ -18,38 +18,42 @@ def as_dict(self): class DatasetBatch: - def __init__(self, batch, client, session, expected): + """A processed batch and its client-owned checkpoint; acknowledgement makes no RPC.""" + def __init__(self, batch): self.header, self.fields = batch.header, batch.fields - self._client, self._session, self._expected = client, session, expected self.acknowledged = False def acknowledge(self): - progress = self._client.acknowledge(self._session, self._expected, self.header["next"]) self.acknowledged = True - return progress + return self.header["next"] class DatasetIterable: - """Yield bounded DatasetBatch objects; acknowledge each before requesting more. + """Bounded batches with local acknowledgement. Save checkpoint after processing. - Tensor decoding/collation belongs above this adapter. Large fields use - next_batch streams directly. Unacknowledged batches replay on resume. + Supplying a previous checkpoint replays the corresponding immutable batch. + Closing/dropping an iterator needs no remote cleanup. """ - def __init__(self, client, plan, *, partition=None, prefetch=1): + def __init__(self, client, plan, *, partition=None, prefetch=1, checkpoint=None): if prefetch != 1: - raise ValueError("this acknowledged iterator supports one batch in flight") + raise ValueError("this iterator supports one batch in flight") self.client, self.plan = client, dict(plan) self.plan["partition"] = (partition or WorkerPartition()).as_dict() + self.checkpoint = checkpoint + self.complete = False def __iter__(self): - session = self.client.start_read(self.plan) - expected = session["acknowledged"] + if self.complete: + return + if self.checkpoint is None: + self.checkpoint = self.client.bind_read(self.plan)["cursor"] while True: - with self.client.next_batch(session["id"]) as stream: - batch = DatasetBatch(stream.batch(), self.client, session["id"], expected) + with self.client.next_batch(self.checkpoint) as stream: + batch = DatasetBatch(stream.batch()) yield batch if not batch.acknowledged: raise RuntimeError("acknowledge the consumed batch before requesting the next batch") - expected = batch.header["next"] - if batch.header["end"]: + self.checkpoint = batch.header["next"] + self.complete = batch.header["end"] + if self.complete: return diff --git a/sdk/test_iterable.py b/sdk/test_iterable.py index b30237a49..787ecc91a 100644 --- a/sdk/test_iterable.py +++ b/sdk/test_iterable.py @@ -7,30 +7,32 @@ class TestStream: def __enter__(self): return self def __exit__(self, *_): pass - def batch(self): return Batch({"next": "issued", "end": True}, [b"owned"]) + def batch(self): return Batch({"next": None, "end": True}, [b"owned"]) class TestClient: - def start_read(self, plan): + def bind_read(self, plan): self.plan = plan - return {"id": "session", "acknowledged": "old"} - def next_batch(self, id): return TestStream() - def acknowledge(self, id, expected, next): - self.progress = (id, expected, next) - return {"cursor": next} + return {"cursor": "old"} + def next_batch(self, cursor): + self.cursor = cursor + return TestStream() class IterableTest(unittest.TestCase): def test_partition_is_sent_before_payload_and_ack_is_explicit(self): client = TestClient() - iterator = iter(DatasetIterable(client, {}, partition=WorkerPartition(worker=1, workers=2))) + reader = DatasetIterable(client, {}, partition=WorkerPartition(worker=1, workers=2)) + iterator = iter(reader) batch = next(iterator) self.assertEqual(client.plan["partition"]["worker"], 1) self.assertFalse(hasattr(client, "progress")) self.assertEqual(batch.fields, [b"owned"]) batch.acknowledge() - self.assertEqual(client.progress, ("session", "old", "issued")) + self.assertEqual(reader.checkpoint, "old") + self.assertFalse(hasattr(client, "progress")) self.assertEqual(list(iterator), []) + self.assertTrue(reader.complete) def test_continuing_without_acknowledgement_is_rejected(self): iterator = iter(DatasetIterable(TestClient(), {})) diff --git a/sdk/tests/real_consumer.py b/sdk/tests/real_consumer.py index 13a8bb789..714213b51 100644 --- a/sdk/tests/real_consumer.py +++ b/sdk/tests/real_consumer.py @@ -38,17 +38,16 @@ def write(): assert client.open()["latest"] == info["latest"] published = client.publish(publication) assert published["id"] == prepared["id"] - plan = {"id": new_id(), "snapshot": "latest", + plan = {"snapshot": "latest", "selection": {"kind": "prefix", "value": sample_id(b"python-")}, "fields": ["label"], "batch_size": 1, "shuffle": None, "partition": WorkerPartition().as_dict()} - session = client.start_read(plan) - with client.next_batch(plan["id"]) as stream: + binding = client.bind_read(plan) + with client.next_batch(binding["cursor"]) as stream: issued = stream.batch().header - assert client.progress(plan["id"])["cursor"] == session["acknowledged"] state_file.write_text(json.dumps({"publication": publication, "snapshot": published["id"], "upload": upload_id, - "read": plan["id"], "acknowledged": session["acknowledged"], "issued": issued})) + "cursor": binding["cursor"], "issued": issued})) print("installed Python SDK uploaded and published real Dataset bytes", flush=True) @@ -70,15 +69,11 @@ def verify(): assert client.publish(state["publication"])["id"] == snapshot assert client.operation(state["publication"]["operation"])["id"] == snapshot assert client.upload_status(state["upload"])["state"] == "claimed" - resumed = client.resume_read(state["read"]) - assert resumed["snapshot"] == snapshot and resumed["acknowledged"] == state["acknowledged"] - with client.next_batch(state["read"]) as stream: + with client.next_batch(state["cursor"]) as stream: replay = stream.batch().header + assert replay["snapshot"] == snapshot assert replay["sample_ids"] == state["issued"]["sample_ids"] assert replay["next"] == state["issued"]["next"] - progress = client.acknowledge(state["read"], state["acknowledged"], replay["next"]) - assert client.resume_read(state["read"])["acknowledged"] == progress["cursor"] - client.close_read(state["read"]) destination = root / "python-output.bin" client.download_to_path(snapshot, sample_id(b"python-0"), "value", destination) with destination.open("rb") as stream: actual = hashlib.file_digest(stream, "md5").digest() @@ -86,12 +81,11 @@ def verify(): assert actual == expected owned = [] for worker in range(2): - plan = {"id": new_id(), "snapshot": "latest", "selection": {"kind": "prefix", "value": sample_id(b"python-")}, "fields": ["label"], "batch_size": 1, "shuffle": None} + plan = {"snapshot": "latest", "selection": {"kind": "prefix", "value": sample_id(b"python-")}, "fields": ["label"], "batch_size": 1, "shuffle": None} for batch in DatasetIterable(client, plan, partition=WorkerPartition(worker=worker, workers=2)): owned.extend(batch.header["sample_ids"]) assert batch.fields[0] in (b"\0", b"\1", b"\2", b"\3") batch.acknowledge() - client.close_read(plan["id"]) assert len(owned) == len(set(owned)) == 4 client.retain(snapshot) client.release(snapshot) diff --git a/sdk/tests/split_consumer.py b/sdk/tests/split_consumer.py index 7cad9445c..0ce40d094 100644 --- a/sdk/tests/split_consumer.py +++ b/sdk/tests/split_consumer.py @@ -64,25 +64,19 @@ async def verify_async(endpoint, name, token, snapshot, revision): async with AsyncDatasetClient(endpoint, name, token) as owner: client = owner.control() assert (await client.lifecycle(snapshot))["snapshot"] == snapshot - plan = {"id": new_id(), "snapshot": snapshot, "selection": {"kind": "all"}, + plan = {"snapshot": snapshot, "selection": {"kind": "all"}, "fields": ["value"], "batch_size": 1, "partition": {"worker": 0, "workers": 1, "rank": 0, "world_size": 1}, - "shuffle": None, "split": {"revision": revision, "name": "train/a b"}, "expected_page": "0"} - for _ in range(100): - result = await client.prepare_read(plan) - if result["complete"]: break - plan["expected_page"] = result["next_page"] - else: raise AssertionError("read preparation budget") - before = await client.progress(plan["id"]) + "shuffle": None, "split": {"revision": revision, "name": "train/a b"}} + binding = await client.bind_read(plan) + assert binding["split"]["revision"] == revision chunks, header = [], None - async with client.next_batch(plan["id"]) as stream: + async with client.next_batch(binding["cursor"]) as stream: async for kind, value in stream: if kind == "data": chunks.append(value) if kind == "header": header = value assert b"".join(chunks) == b"external source" - assert await client.progress(plan["id"]) == before - await client.acknowledge(plan["id"], before["cursor"], header["next"]) - assert (await client.close_read(plan["id"]))["closed"] + assert header["snapshot"] == snapshot pages = [page async for page in client.pages("samples", snapshot=snapshot, limit=1)] assert sum(len(page["entries"]) for page in pages) == 1 From e102d80eb86a23243661856592a58329634dafa9 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 09:37:39 +0800 Subject: [PATCH 092/258] Remove persistent table read projections and reuse upload descriptors --- .../src/dataset/api/batch.rs | 26 +- .../src/dataset/api/v2/preview.rs | 6 +- .../tests/dataset_batch_cost_test.rs | 113 ++ .../tests/dataset_preview_contract_test.rs | 8 +- .../dataset/design-crowdb-access-dataset.md | 5 + .../iceberge/design-crowdb-iceberg.md | 24 +- doc/working/plan-access-simplification.md | 1143 +++++++---------- lib/crowdb-access-dataset/src/lib.rs | 2 +- .../src/payload/reader.rs | 25 +- lib/crowdb-access-dataset/src/upload.rs | 18 +- lib/crowdb-access-dataset/src/upload/read.rs | 88 ++ .../src/upload/stream.rs | 34 +- .../tests/common/upload.rs | 9 + .../tests/upload_read_cost_test.rs | 130 ++ .../src/metadata_projection.rs | 1 - .../src/metadata_projection/validated.rs | 81 -- lib/crowdb-access-iceberg/src/table/load.rs | 57 +- .../tests/common/store.rs | 47 +- .../tests/table_projection_test.rs | 146 +-- .../src/dataset/read/field.rs | 12 +- .../tests/dataset_payload_test.rs | 49 + 21 files changed, 1006 insertions(+), 1018 deletions(-) create mode 100644 app/crowdb-access-server/tests/dataset_batch_cost_test.rs create mode 100644 lib/crowdb-access-dataset/src/upload/read.rs create mode 100644 lib/crowdb-access-dataset/tests/upload_read_cost_test.rs delete mode 100644 lib/crowdb-access-iceberg/src/metadata_projection/validated.rs diff --git a/app/crowdb-access-server/src/dataset/api/batch.rs b/app/crowdb-access-server/src/dataset/api/batch.rs index 95a3f2ea6..164cf18c9 100644 --- a/app/crowdb-access-server/src/dataset/api/batch.rs +++ b/app/crowdb-access-server/src/dataset/api/batch.rs @@ -4,7 +4,7 @@ use super::{error::ApiError, read_json, snapshot, ApiResult}; use crate::dataset::{DatasetBody, DatasetHttpService}; use base64::{engine::general_purpose::STANDARD, Engine}; -use crowdb_access_dataset::{DatasetIdentity, FieldLocator, SampleView, SnapshotId}; +use crowdb_access_dataset::{DatasetIdentity, FieldLocator, SampleView, SnapshotId, UploadReadDescriptor}; use futures::stream; use http_body_util::{BodyExt, StreamBody}; use hyper::{ @@ -123,6 +123,7 @@ impl DatasetHttpService { locators, index: 0, part: 0, + upload: None, pending: Some(Bytes::from(prefix)), finished: false, }; @@ -174,6 +175,7 @@ struct BatchState { locators: Vec, index: usize, part: u32, + upload: Option, pending: Option, finished: bool, } @@ -217,13 +219,24 @@ impl BatchState { self.part = 1; self.data_frame(bytes, digest) } - FieldLocator::Chunk { location, length, .. } => { - let uploads = self.uploads.as_ref().ok_or_else(ApiError::unsupported)?; - let metadata = uploads.metadata(location).await?; - if u64::from(self.part) >= length.div_ceil(u64::from(metadata.part_size)) { + FieldLocator::Chunk { + location, + length, + md5, + } => { + if self.upload.is_none() { + let uploads = self.uploads.as_ref().ok_or_else(ApiError::unsupported)?; + let descriptor = uploads.open_read(location).await?; + if descriptor.length() != *length || descriptor.md5() != *md5 { + return Err(ApiError::integrity()); + } + self.upload = Some(descriptor); + } + let descriptor = self.upload.as_ref().ok_or_else(ApiError::integrity)?; + if self.part >= descriptor.part_count() { return self.end_field(); } - let (bytes, digest) = uploads.read_part_verified(location, self.part).await?; + let (bytes, digest) = descriptor.read_part_verified(self.part).await?; self.part += 1; self.data_frame(bytes, digest) } @@ -242,6 +255,7 @@ impl BatchState { fn end_field(&mut self) -> Result { self.index += 1; self.part = 0; + self.upload = None; frame(3, &Bytes::new()) } } diff --git a/app/crowdb-access-server/src/dataset/api/v2/preview.rs b/app/crowdb-access-server/src/dataset/api/v2/preview.rs index 642f6439a..5b0e2c8b5 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/preview.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/preview.rs @@ -120,14 +120,14 @@ impl DatasetHttpService { crowdb_access_dataset::DatasetUploads::upload_id(identity, &location) .map_err(|_| ApiError::unsupported())?; let uploads = self.uploads.as_ref().ok_or_else(ApiError::unsupported)?; - let metadata = uploads.metadata(&location).await?; - if metadata.length != length || metadata.md5 != md5 { + let metadata = uploads.open_read(&location).await?; + if metadata.length() != length || metadata.md5() != md5 { return Err(ApiError::integrity()); } let mut value = Vec::new(); let mut part = 0; while value.len() < limit && (value.len() as u64) < length { - let (bytes, _) = uploads.read_part_verified(&location, part).await?; + let (bytes, _) = metadata.read_part_verified(part).await?; let count = bytes.len().min(limit - value.len()); value.extend_from_slice(&bytes[..count]); part += 1; diff --git a/app/crowdb-access-server/tests/dataset_batch_cost_test.rs b/app/crowdb-access-server/tests/dataset_batch_cost_test.rs new file mode 100644 index 000000000..c6a2a3f6d --- /dev/null +++ b/app/crowdb-access-server/tests/dataset_batch_cost_test.rs @@ -0,0 +1,113 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/dataset_v2.rs"] +#[allow(dead_code)] +mod common; +#[path = "common/dataset.rs"] +mod dataset; + +use common::response; +use dataset::{TestDatasetHttp, TestStore}; +use md5::{Digest, Md5}; +use reqwest::Client; +use serde_json::json; +use std::sync::{atomic::Ordering, Arc}; + +#[tokio::test] +async fn multipart_fields_use_one_upload_read_each_and_no_eof_query() { + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start_payload(store.clone()).await; + let client = Client::new(); + let root = format!( + "{}/api/dataset/v1/namespaces/dataset-ns/datasets/cost", + test.origin + ); + response(client.put(&root), 200).await; + let upload = "11111111111111111111111111111111"; + let data = b"abcdefghij"; + response( + client.post(format!("{root}/uploads")).json(&json!({ + "id":upload,"length":10,"part_size":4,"md5":hex::encode(Md5::digest(data)) + })), + 200, + ) + .await; + for (index, part) in data.chunks(4).enumerate() { + response( + client + .put(format!("{root}/uploads/{upload}/parts/{index}")) + .header("content-type", "application/octet-stream") + .body(part.to_vec()), + 200, + ) + .await; + } + response(client.post(format!("{root}/uploads/{upload}/complete")), 200).await; + let published = response( + client.post(format!("{root}/snapshots/publish")).json(&json!({ + "operation":"22222222222222222222222222222222", "parent":null, + "manifest":{"schema":[{"name":"value","required":true}],"samples":[ + {"id":"YQ==","fields":{"value":{"kind":"payload","upload":upload}}}, + {"id":"Yg==","fields":{"value":{"kind":"payload","upload":upload}}} + ]} + })), + 200, + ) + .await; + let reads = store.upload_reads.load(Ordering::Relaxed); + let writes = store.mutations.load(Ordering::Relaxed); + let reply = client + .post(format!("{root}/reads")) + .bearer_auth("secret") + .json(&json!({ + "snapshot":published["id"],"sample_ids":["YQ==","Yg=="],"fields":["value"] + })) + .send() + .await + .unwrap(); + let status = reply.status(); + let bytes = reply.bytes().await.unwrap(); + assert_eq!(status, 200, "{}", String::from_utf8_lossy(&bytes)); + assert_fields(&bytes, &[data.as_slice(), data.as_slice()]); + assert_eq!(store.upload_reads.load(Ordering::Relaxed) - reads, 8); + assert_eq!(store.mutations.load(Ordering::Relaxed), writes); + drop(client); + test.stop().await; +} + +fn assert_fields(mut bytes: &[u8], expected: &[&[u8]]) { + assert_eq!(&bytes[..8], b"CDBDS001"); + bytes = &bytes[8..]; + let mut field = Vec::new(); + let mut index = 0; + let mut finished = false; + while !bytes.is_empty() { + assert!(!finished); + let kind = bytes[0]; + let length = u32::from_be_bytes(bytes[1..5].try_into().unwrap()) as usize; + let body = &bytes[5..5 + length]; + bytes = &bytes[5 + length..]; + match kind { + 1 => { + let _: serde_json::Value = serde_json::from_slice(body).unwrap(); + } + 2 => { + assert_eq!(&body[..16], Md5::digest(&body[16..]).as_slice()); + field.extend_from_slice(&body[16..]); + } + 3 => { + assert_eq!(field, expected[index]); + index += 1; + field.clear(); + } + 0 => { + assert_eq!(index, expected.len()); + assert!(field.is_empty()); + finished = true; + } + _ => panic!("unexpected frame {kind}: {}", String::from_utf8_lossy(body)), + } + } + assert!(finished); +} diff --git a/app/crowdb-access-server/tests/dataset_preview_contract_test.rs b/app/crowdb-access-server/tests/dataset_preview_contract_test.rs index 468e9c846..6aff90dbe 100644 --- a/app/crowdb-access-server/tests/dataset_preview_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_preview_contract_test.rs @@ -69,7 +69,8 @@ async fn exact_matrix_and_inline_preview_do_not_create_read_sessions() { #[tokio::test] async fn payload_preview_caps_bytes_across_parts_and_distinguishes_missing_tombstone() { - let test = TestDatasetHttp::start_payload(Arc::new(TestStore::default())).await; + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start_payload(store.clone()).await; let client = Client::new(); let root = format!( "{}/api/dataset/v1/namespaces/dataset-ns/datasets/preview", @@ -91,6 +92,7 @@ async fn payload_preview_caps_bytes_across_parts_and_distinguishes_missing_tombs root.replace("/v1/", "/v2/"), published["id"].as_str().unwrap() ); + let upload_reads = store.upload_reads.load(std::sync::atomic::Ordering::Relaxed); let preview = response( client .get(&url) @@ -105,6 +107,10 @@ async fn payload_preview_caps_bytes_across_parts_and_distinguishes_missing_tombs assert_eq!(preview["length"], "100000"); assert_eq!(preview["truncated"], true); assert_eq!(preview["md5"], hex::encode(Md5::digest(&data))); + assert_eq!( + store.upload_reads.load(std::sync::atomic::Ordering::Relaxed) - upload_reads, + 3 + ); for (field, state) in [("missing", "missing"), ("deleted", "tombstone")] { let p = response( client diff --git a/doc/design/access-server/dataset/design-crowdb-access-dataset.md b/doc/design/access-server/dataset/design-crowdb-access-dataset.md index 4a72b370f..652a9ce0a 100644 --- a/doc/design/access-server/dataset/design-crowdb-access-dataset.md +++ b/doc/design/access-server/dataset/design-crowdb-access-dataset.md @@ -284,6 +284,11 @@ HTTP EOF with no trailing bytes. Each active transfer owns local admission credits through delivery/drop. Snapshot reads perform no persistent activity updates or heartbeat writes. Cancellation stops the producer and returns credits; retained SDK-owned buffers keep their byte credits until released. +For an uploaded field, HTTP and native readers validate one immutable upload +description per request, then fetch each part record once. Part indices, lengths +and payload digests remain checked; EOF does not reload metadata. The description +is request-owned and registers no remote reader or retention. New requests still +perform admission, and reclaimed parts fail instead of producing successful EOF. The internal legacy scan adapter also returns a caller-owned cursor without loading or persisting progress. Requests have no shared inactivity TTL. diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg.md b/doc/design/access-server/iceberge/design-crowdb-iceberg.md index 15aeb8d18..7177f0a79 100644 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg.md +++ b/doc/design/access-server/iceberge/design-crowdb-iceberg.md @@ -47,17 +47,14 @@ Standard Iceberg metadata is the recoverable table state. CROWDB may maintain derived indexes or projections for scale, but they are disposable and cannot become a second table authority. -Generation-local metadata projections preserve raw top-level JSON children in -bounded pages. REFS loads construct them only after canonical metadata passes -the complete bounded parser. A versioned validation receipt binds the selected -head, exact parser limits and projection root; a generic JSON projection alone -cannot stand in for metadata validation. REFS loads may reuse this validated -representation without decoding the complete object graph. They still read and -verify canonical storage and recheck the namespace and head before responding. -Missing, partial, corrupt, unknown-version or differently bounded projections -fall back to the canonical parser. ALL responses preserve exact canonical bytes. -Projection construction is optional and never participates in commit proofs or -head publication. No cross-generation cache or deduplication is implied. +Table ALL/REFS loads read and verify canonical storage, run the complete bounded +metadata parser, and recheck the namespace and selected head before responding. +ALL preserves exact canonical bytes; REFS filters snapshots from the same validated +content. Table loads do not read or write projection pages, validation receipts, +or other persistent cache state, including on repeated and conditional requests. +Optional generation-local JSON projections used by file-level selection remain +disposable derived data with canonical fallback; they cannot replace table +validation, participate in commit proofs, or become file/publication authority. Iceberg metadata stores bounded logical records and opaque data references. Physical chunk placement and storage topology remain below the access boundary. @@ -153,8 +150,9 @@ Metadata files use useful public relative prefixes such as `metadata/.metadata.json`, `metadata/.avro` and `statistics/.puffin`. These are defaults, not restrictions on safe client-chosen paths. The optional generation-bound projections described above -accelerate metadata reads; they are not duplicate file authorities. Parser, -collection and response budgets remain explicit even for chunk-backed JSON. +support file-level selection; they are not used by table ALL/REFS loads or duplicate +file authorities. Parser, collection and response budgets remain explicit even for +chunk-backed JSON. An unsupported oversize request fails clearly rather than silently truncating snapshots. ALL and REFS keep their standard response semantics; internal projection paging does not introduce private REST pagination fields. diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 0aea6ea6b..fc60f3c0e 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -3,707 +3,442 @@ # Access Server Simplification Plan -Upstream: [Dataset SDK](../backlog/R220-dataset-fat-client-plan.md), -[Access design](../design/access-server/design-crowdb-access-server.md). - -Goal: reduce Access request RPCs, copies, CPU work and shared contention; remove -coordination without a necessary business contract and retain justified authority. -This is a persistent cleanup inventory requested by the user; remove completed -items after recording their verified outcome in the owning design. - -## Agreed Dataset deletion contract - -- Mark deletion durably, reject new requests, then reclaim after a persisted - grace deadline. No Access Server/client acknowledgements or reader registry. -- Requests admitted before the mark may continue during the grace period. - Reclamation may interrupt unusually long reads; return an error, never a - successful truncated response or a different snapshot. -- Default grace: five minutes. It is a delay, not a promise that a running job - can issue new requests forever. Retries/restarts must not reset the deadline. -- Retention, live snapshot ownership, publication fences, physical allocation - identity, in-flight I/O and caller-owned buffers remain distinct protections. -- A read checks authoritative availability at admission. A stale cache is not - sufficient to admit a new read after deletion. Read paths perform no snapshot - activity writes, heartbeats or asynchronous release writes. - -## Approved stateless read contract (2026-10-24) - -- The user explicitly removed persisted read sessions. This supersedes the - earlier inventory's proposals to retain durable ack/resume/remote close. -- Requests carry the query and continuation. The continuation binds Dataset, - immutable Snapshot, Split revision, projection, worker partition and shuffle. - Processing checkpoints belong to the caller; fetching/releasing a buffer does - not commit progress anywhere. Reusing a continuation replays its batch. -- Remove session records, persisted selected-ID pages, preparation progress, - version reservations, TTL renewal, ack/close routes and per-frame state reads. - Do not replace them with a required in-memory session or sticky routing. -- Keep bounded work, cancellation, integrity and buffer ownership. New requests - still check snapshot deletion. A continuation is not an authorization grant. -- Review remaining persistent mutations by use case and total RPC cost before - changing their business semantics. Strict nonempty namespace deletion remains - approved; durable publication and shared data ownership need separate review. - -## Execution - -- [ ] **Confirm remaining persistence contracts**: review P1–P14 below before - changing resource/recovery semantics. In particular confirm disposable draft - work, upload restart behavior, exact retry responses and global quotas. Strict - nonempty namespace deletion is already decided and remains unchanged. -- [ ] **Reduce remaining request costs**: execute the still-applicable findings - 1–13 below; session-only findings are superseded by the completed removal. Measure total reads/mutations and serial - dependency depth, not only CAS counts. Consume returned CAS observations on - justified retries; preserve explicit expected-version failures. -- [ ] **Measure real services**: rebuild matching sources and run mixed-size/restart - acceptance after the confirmed simplifications. No throughput claim from static RPC counts. - -### Verified session cleanup - -- Replaced persisted Dataset read sessions with `DatasetReads` and caller-owned - continuation tokens. Removed remote ack/resume/close, reader TTL/configuration, - selected-ID/progress pages and legacy cursor storage. HTTP v1/v2 binding uses - `POST reads/bind`; `POST reads/next` replays a continuation. Existing v1 direct - batch reads remain `POST reads`. Native and Python callers use the same shared - query semantics, with client-side processing checkpoints. -- Added zero-write regressions for bind, 96 concurrent fetches, cross-instance - replay and legacy scan continuation. Covered immutable Snapshot/Split binding, - worker coverage, cancellation, deletion admission and bounded token size. -- Passed Dataset core all-target tests and the final focused query regressions; - Access Server and native SDK all-target tests; independent HTTP SDK tests; - affected all-feature clippy, independent SDK clippy and both fmt checks. - Parsed both OpenAPI documents and resolved every local reference. -- The HTTP SDK wheel was rebuilt and installed into an isolated environment. - Its Split/read round trip passed against the real HTTP listener, with the - consumer script copied outside the repository. -- A full Access run initially exhausted disk during linking, before tests; - deleting only rebuildable incremental cache restored space. A subsequent - payload test exposed a conflicting `reads` route; the independent `reads/bind` - route fixed it, and the complete Access suite then passed. -- This validates session removal, not full R220 performance acceptance. The - container mixed-size/restart workload and local-HCA comparison remain pending; - an OCI image from baseline `878ee901` does not contain these changes. - -The review below describes baseline `878ee901`. Its session preservation and -remote-close alternatives are superseded by the approved contract above; those -entries remain evidence of the cost being removed, not current requirements. - -## Remaining persistence review (2026-10-24) - -Scope: Access Server adapters plus Dataset, S3 and Iceberg metadata repositories. -`persistent` here means durable application metadata, including read-triggered -cache fills. A local atomic, resource permit or transport connection is not a -persistent read session. Chunk allocation/WAL/Paxos durability remains storage -infrastructure and is outside this Access semantic cleanup. Counts are logical -metadata operations; nested validation/replication and payload IO are additional. -These items are review proposals, not authorization to remove business guarantees. - -- **P1 — Iceberg GET can persist an optional projection**. `TableLoader::representation` - in `lib/crowdb-access-iceberg/src/table/load.rs` calls `ProjectionStore::put_document` - on a refs-mode cache miss. `metadata_projection/repository.rs` writes each child - page and root with create-CAS; `validated.rs` rereads the root and writes a - validation receipt. For P child pages this fill alone adds `(P + 2)C + 1G`, - on top of reads, parsing and hashing. Concurrent GETs repeat that durable work. - This is derived data, not publication authority. Proposal: remove synchronous - persistent fills from reads; compare a bounded local cache with publication-time - construction. Review decision: whether a durable projection is needed at all. -- **P2 — Dataset Composition workflow**. `composition/{begin,edit,state,evaluation, - evaluate_step,prepare,cleanup}.rs` stores draft revisions, edit receipts, - evaluation windows/rows, candidate pointers and cleanup progress. Use case: - editing a large derived Dataset and recovering across Access restarts without - redoing all preparation. Each window/row/receipt adds separate create-CAS and - progress replacement, with validation GETs; this is not one publication CAS. - Proposal: client-owned editing specification and replayable/disposable - preparation, retaining only immutable candidate data plus publication authority. - Decision: may an unfinished draft/evaluation be lost and restarted? If yes, - remove durable editing/progress first; do not remove committed candidates. -- **P3 — Dataset Split workflow**. `split/{lifecycle,state,work,evaluation,prepare, - recovery,cleanup,initial}.rs` similarly persists edits, evaluation progress, - overlap examples, membership work pages and fences. Published membership and - revision identity are business data; draft progress is auxiliary recovery state. - Proposal: compute from client-owned rules, publish immutable revision once, - restart incomplete evaluation instead of journaling every window. Same decision - as P2; retain frozen historical membership and explicit currentSplit CAS. -- **P4 — Dataset uploads**. `upload.rs`, `upload/{parts,index,ownership,retirement, - reclaim}.rs` retain upload descriptor, part intent/result/index, completion, - owner list, retired-owner IDs and abort/cleanup status. Trigger: multi-request - payload upload, retry after lost storage replies, sharing one payload across - candidates/Snapshots. Create currently costs `1C`; part acceptance adds intent, - part receipt and index mutations plus repeated state reads. Owner mutation is - `1G + C` attempts on one upload; observed conflicts must feed the next attempt. - Permanent retired-owner IDs can grow, and the part index serializes writers. - Proposal: separate immutable part results from upload-wide state; assess replacing - the growing owner list/index with per-owner/per-part records and bounded discovery. - Decisions: is resumable upload across restarts required, and may an incomplete - upload be discarded? Shared published payloads must not be reclaimed accidentally. -- **P5 — S3 multipart**. `metadata/multipart_repository{,/parts,/completion, - /publication,/terminal}.rs` stores upload identity, immutable parts/index, - completion/abort state and publication evidence. These implement S3 multipart - operations, not login/read sessions. Baseline normal new part is `7G + 4C` at - the helper boundary (finding 5); payload and replica IO are extra. First simplify - redundant state reloads and pass observed/applied values through helpers. - Removing upload durability changes resume/list/complete semantics and requires - explicit review; don't silently break S3 clients while removing Dataset reads. -- **P6 — Iceberg multipart and durable admission credits**. `file/multipart_repository/` - and `file/multipart_credits{,/settle}.rs` contain upload state plus a catalog-wide - sessions/byte reservation and pending-credit journal. Such a shared counter - serializes otherwise independent transfers. The current Access file upload path - uses its active upload-head flow; legacy durable-credit code must be traced by - caller before attributing its cost to active requests (finding 7). - Proposal: remove unreachable credit machinery; use bounded local producer - admission for resource protection. Decision for any reachable global budget: - are hard cross-server quotas a required contract, or is per-server admission enough? -- **P7 — Iceberg file writer recovery**. `file/{writer,write_intent,injection}.rs` - and writer checkpoints persist physical ownership/offsets, exact write intents, - completed file records and adoption receipts. Current active part publication - costs `1G + 1C` after admission; intent/checkpoint paths have separate costs. - Purpose: distinguish committed bytes from readable but unconfirmed bytes after - timeout/crash, and avoid adopting recycled storage. Proposal: retain physical - generation/fencing and immutable completion; review whether incomplete writer - checkpoints can be discarded/replayed. Acceptance must inject lost storage replies. -- **P8 — Iceberg HTTP retry ledger and body copies**. `operation/{retry,result,payload}.rs` - and Access `iceberg/{table_write,namespace_write}` persist request binding, - original body pages, result body and terminal status. `RetryLedger::begin` checks - context and locates a binding before create-CAS; finish checks context, locates - the binding, reads/writes result and updates binding again. Large bodies add - page reads/writes; replay may repair a status marker. Boolean CAS loses the - conflict observation and adds a confirmation GET. Proposal: first reuse observed - values; then merge retry evidence with the actual mutation journal or commit - record. Decision: must every retry reproduce the original response, or may an - ambiguous reply return a conflict/unknown outcome for client reconciliation? -- **P9 — Iceberg namespace/table journals and reservations**. `namespace/{journal, - reservation,admission,publication,drop_fence,drop_finish,update_recovery}.rs`, - `table/lifecycle/` and `commit/create/` persist name reservations, parent entries, - namespace/table generations and recovery phases. Multiple records/RPCs implement - multi-key create/drop/rename recovery. **Nonempty namespace deletion must fail**, - already confirmed by the user; do not ask again or replace it with eventual - emptiness. Review phase journals and duplicated context reads independently. - Removing a journal must still preserve strict parent admission and name uniqueness. -- **P10 — Dataset publication and explicit resource controls**. `authority.rs`, - `authority/publication/`, `split/{head,publication,recovery}.rs` persist Dataset - identity, immutable Manifest/Snapshot/Split, latest/stable pointer, retention - marker, commit attempt/receipt and visibility evidence. User mutations need - durable results; head CAS prevents a stale expected generation from publishing. - Extra attempt/receipt/visibility records are separately reviewable overhead. - Proposal: minimize the commit record set while preserving lost-reply outcome - discovery; evaluate whether the separate retention API is actually used. - Dropping persistent resource data is not justified by removing read sessions. -- **P11 — S3 object/bucket publication**. `publication.rs`, `bucket.rs`, - `metadata/{store,namespace}.rs` store object version/location/user metadata, - bucket identity and deletion markers. Unconditional PUT/DELETE should use the - overwrite contract without a preliminary GET/CAS; conditional operations retain - their expected-version check. Bucket tombstone replay still has an avoidable - bool-CAS/put-if-absent sequence (finding 4). These are resource writes; review - unnecessary mutation stages rather than replacing persistence with volatile state. -- **P12 — Dataset inspection indexes**. `inspection/{index,preparation,legacy, - retirement,reclaim}.rs` store schema/member/field projections, summary and - preparation/reclamation progress. They enable bounded reads without decoding a - full Manifest, but build cost is serial and multiplied per row (finding 9). - Explicit legacy preparation remains a mutation. Proposal: keep useful immutable - indexes with publication; compare batched/co-located writes and restarting - interrupted builds, removing auxiliary progress where redo is acceptable. -- **P13 — Cleanup/GC tasks and tombstones**. Dataset `authority/access/`, upload - cleanup and Split/Composition cleanup; Iceberg `gc/{repository,budget,worker}`; - S3 multipart terminal cleanup persist claim/phase/cursors and deletion fences. - Mark + delayed reclamation is already approved for snapshots. Proposal: use - idempotent bounded scans and local cursors where restart only repeats work; - review durable global counters and per-page progress. Keep enough physical - identity/fencing that a delayed delete cannot free a newly reused range. - Decision: accept recomputation after cleanup-worker restart and approximate - progress counters? Distinguish that from a repeated destructive operation. -- **P14 — Catalog lifecycle and credential authority**. Iceberg - `catalog/{repository,storage}.rs` persists active context/root/epoch; Access - `credentials{,/ensure}.rs` persists user credentials in Group-0. These are - administrative identity/revocation boundaries, not per-read sessions. Review - repeated root/context GETs and redundant initialization writes; deleting these - records would change authentication or catalog-reset safety. S3 session-token - credential fields alone are not evidence of server-side session persistence. - -Execution priority after confirmation: P1 read-triggered writes; P2/P3 disposable -workflows; P8 duplicate retry persistence; P4/P5 upload amplification; P6/P13 global -coordination. Every item must name its required invariant, exact normal/conflict/ -unknown-outcome RPC path, and a smaller alternative before implementation. - -### Cost still present in the new stateless read - -- No persistent reader record or selected-ID page is written. Continuations do - not eliminate snapshot/index/locator reads, payload IO or digest work. -- A partially consumed shuffle group is reconstructed; small `batch_size` with - very large `group_size` can therefore increase metadata RPCs. Repeated selection - windows also reload schema/membership validation. This is an explicit remaining - performance review item, not a claimed throughput improvement. Compare bounded - client-side planning/group buffering against server re-evaluation before choosing - a default; do not reintroduce mandatory read-session storage to cache that work. -- Query, payload admission and inspection can repeat deletion/visibility checks. - Consolidate within a request only after preserving the new-request deletion - boundary. Native/HTTP streaming no longer reads a session record per frame. -- Legacy `DatasetReadService::scan_surface` still reconstructs all planned windows - from the Manifest before selecting one cursor window. Removing its persisted - cursor does not remove that CPU/memory amplification; migrate callers to the - bounded inspection path or replace the planner in a separate measured change. -- Old `DS\\0RS\\0` and cursor records are no longer consulted or written. Physical - purge needs a separately scoped maintenance operation and coordinated removal - of old writers; no bulk deletion was executed during this cleanup. - -## Performance review: counting rules and current evidence - -- Static review baseline: `878ee901`. The prior implementation passed Dataset, - Access Server, S3 and native component/integration suites, namespace admission - and drop suites, workspace all-feature clippy and fmt. Its local OCI image - built successfully. Full mixed-size workload/restart was not run on that - revision before the user requested this broader review; do not report it as - accepted performance. -- `G` means one Access metadata point read, `B` one bounded read, `S` one scan, - `C` one conditional mutation, and `W` one unconditional mutation. `C(None)` - is create-if-absent, not a read-modify-write retry. Counts below describe the - named function path, not an entire HTTP request unless explicitly stated. -- These are logical storage calls, not physical wire-packet counts. Dataset `B` - currently becomes a one-key `scan_bounded`; routing, range crossings, topology - discovery, replica reads, Paxos and WAL can add messages and work. Measure - actual client RPC sends and server replication separately. -- A definite CAS conflict returns the value observed when the mutation was - evaluated. It can already be stale by the time it arrives. Use it immediately - to recompute an allowed retry; repeated competition can still fail, so preserve - the existing request deadline and bounded retry budget. Never wait for three - failures before using it. The three-conflict test injects competing writes to - verify this behavior, not a production batching/backoff rule. -- A lost reply is not a definite conflict. It may require exact-operation replay - or a confirmation read. An explicit expected revision/selected generation must - return a conflict when invalidated, rather than silently applying user intent - to a different base. -- Add request-scoped traces/counters in test wrappers and existing metrics, not - per-RPC production logging or a global profiling lock. Count GET/B/scan/CAS/ - PUT/delete, repeated keys, retries, bytes, concurrent operations, and serial - dependency stages. Separately record copied bytes, allocations/retained bytes, - digest/codec CPU, queue wait, active requests, wakeups and cleanup lag. - -## Findings and proposed corrections - -### 1. Repeated snapshot admission within one Dataset batch — confirmed - -- Use case: read the next batch of a persisted session, with deletion rejecting - new requests and admitted reads allowed to finish during grace. -- `DatasetSessions::next` reads session + deletion state + selected ID pages: - `2G + P G`, where P is the number of ID pages. Native `ReadAdmission::admit` - and HTTP `ReadAdmission::acquire` read deletion state again. For a nonempty - batch, `ready_inspection -> inspection_snapshot_committed` reads the same - deletion state a third time through `B`, before separate commit proof reads. -- Proposed: establish one request-local, snapshot-bound admission result and - reuse it across session/locator/body setup. Public standalone entry points - must still admit independently. Do not cache mutable deletion permission - across requests or confuse it with immutable publication proof. -- Verify: count one authoritative deletion check per admitted request; race - deletion at each boundary, direct/session and HTTP/native parity, missing and - unpublished snapshots, retained bodies and failure after physical cleanup. - -### 2. Remote session GET on each body pull — confirmed; contract-sensitive - -- Use case: a separate client closes a session while another client is reading - its current body. Existing code deliberately observes that close remotely. -- Native `BatchBody::pull -> ReadAdmission::check -> check_open` performs `1G` - on each pull. HTTP `BatchState::next` does the same before even returning - already-held `pending` bytes: frame header, payload, field end and stream end - can each incur a session GET. This traffic grows with delivery fragmentation, - not only logical requests or payload size. Ordinary direct batches have no - session ID and skip this cost. -- Proposed: first eliminate duplicate checks within one logical delivery step - if its existing cancellation boundary permits it. Separately evaluate whether - remote close must interrupt an admitted body, or can reject the next request - while local cancel stops the current body. The latter removes per-pull RPCs - but is a distinct contract change, not implied by snapshot deletion approval. -- Verify: current remote-close tests, empty/pending/EOF/capacity-wait cases, - plus RPC counts versus frame count. No new persistent reader registration. - -### 3. Upload metadata repeatedly reloaded while reading — confirmed - -- Use case: read an immutable completed upload referenced by a snapshot field. -- Native `FieldReader` loads upload metadata once, but every - `read_part_stream` loads it again and then loads the part: `1 + 2P` metadata - GETs for P parts, excluding snapshot/session/locator work. -- HTTP `BatchState` loads metadata before each part, and - `read_part_verified` loads it again before the part record. Its normal chunk - branch costs `3P + 1` GETs including the final end-of-field metadata probe, - excluding per-frame session checks and payload reads. -- Proposed: pass a validated, request-owned upload descriptor to part readers; - validate part index/length/digest against it. Reclamation may make later part - I/O fail; it must never yield successful truncation or another upload's bytes. - Keep mutable admission separate from immutable shape and physical identity. -- Also remove upload `create/complete/abort` readbacks that only reconstruct - state already returned by a successful mutation or conflict; assemble parts - for response only when the response needs them. -- Verify: per-field/per-part call budgets, shared owners, corruption, abort and - physical reclamation races; held buffers remain valid after completion. - -### 4. CAS observations still discarded by retry paths — confirmed - -- Dataset `upload/index.rs::index_part` does `GET -> CAS` on every attempt. - Concurrent different part numbers update one 128-part index page; merging - into the returned page is necessary to preserve both, another GET is not. -- Dataset `upload/ownership.rs::{abort,cleanup,cleanup_part}` loops reload the - same upload/part after confirmed CAS conflicts. Retain CAS to prevent abort - racing a claim, a delayed write intent overwriting reclaim, or an owner update - being lost. Recompute state from each returned observation. -- Dataset `authority/access/reclaim.rs::{fence_reclaim,retire_snapshot_record}` - and `split/recovery.rs::invalidate_split_head` do the same. The head nonce - change is a real publisher fence even if latest/stable IDs are unchanged; - it is not removable as a no-op. Reuse observed head and preserve receipt help. -- Dataset `read_session/planning.rs::start` and `read_session/split/finish.rs` - discard a session-create conflict value before entering `resume`; existing - start also reads a record and then resumes through another read. Pass state - through instead. Existing progress and upload-owner retry loops already use - observations; keep them as the expected behavior. -- S3 `bucket.rs::create_bucket` discards a tombstone replacement CAS result - through the bool wrapper, then performs another put-if-absent. Decode and use - the returned active mapping/tombstone immediately. Keep name uniqueness CAS. -- Iceberg `RetryLedger::begin` restarts `ledger_locate` after a failed bool CAS; - use the returned occupant, looking up an overflow key only when the competing - operation actually belongs to a different collision slot. `finish` also GETs - result bytes already supplied by a definite conflict. -- Verify: inject different numbers of actual conflicting writes, assert one - initial read and zero same-key re-GETs solely for each definite conflict; - preserve terminal/missing outcomes, explicit preconditions and lost replies. - -### 5. S3 multipart helpers discard known state — confirmed - -- Use case: publish a new part in an existing open upload; simultaneous parts, - complete and abort share that upload's durable pending/accounting state. -- With no existing part/generation, no pending work, no conflict and no error, - `put_stream_part` currently performs **7G + 4C** before returning: - load index/session (2G), read part (1G), create immutable generation (1C), - reserve session (1C), settle reload index/session (2G), read generation (1G), - read pointer (1G), publish pointer (1C), clear pending (1C). -- `load_identity` reads an index even though the session key is derivable and - `MultipartSessionRecord::decode` checks bucket/object/upload identity. Assess - whether index presence is an independent contract or only listing metadata; - remove it from point operations if the session alone is authoritative. -- Carry the known session, generation and pointer into settling; CAS still - protects against a competing writer. Make `exchange` return typed observed - state through its callers instead of a bool that forces `load` again. -- `publish_completion` has an extra object GET after definite CAS conflict. - `finish_publication`, `abort` and `settle_pending_part` likewise reload the - session after `exchange` discards the observation. Uncertain errors retain - separate confirmation logic. Do not silently overwrite a changed predecessor. -- The four conditional writes have separate current responsibilities. Reducing - them further requires proving quota accounting, same-part replacement and - freeze/abort behavior, not simply replacing them with unconditional PUTs. -- Verify: call counts for new/replaced/replayed parts, concurrent distinct/same - parts, complete-vs-abort, generation orphan cleanup and every lost-write reply. - -### 6. Iceberg phase machines reload their own progress — confirmed - -- Use case: namespace creation/drop/rename/property update and table lifecycle - operations survive Access restart and helpers finishing another request. -- `NamespaceJournal::advance`, table lifecycle/create/commit journals and - `CatalogRepository::cas` reduce CAS to bool. Their driver loops then load - progress from storage after both successful advancement and conflicts. - `NamespaceJournal::load` also checks ActiveRoot, so one reload is at least - `2G`, before payload and authority reads. Preserve applied or observed typed - state within the running request; restart still loads durable progress. -- `NamespaceRepository::load` costs `2 + 2D` GETs for an existing D-component - namespace: two context checks plus mapping and authority for each component. - Nested callers often resolve the same namespace/context again. Identify each - required clear/drop boundary before introducing request-local reuse; a cached - Ready result cannot grant indefinite mutation permission. -- `cleanup_marker` GETs the exact expected value before issuing a conditional - replacement. The CAS itself rejects a changed value. Compare direct-CAS and - existing-marker replay costs; carry known observations when available. -- Preserve the user's strict nonempty-delete guarantee. Parent admission, - name reservation and publication proof cannot be removed merely because the - recovery path has too many RPCs. Simplify state propagation first. -- Verify: count per phase; concurrent create/drop/rename, nonempty rejection, - recreation, epoch change, helper races and lost replies at every durable step. - -### 7. Active Iceberg paths versus legacy work — confirmed distinction - -- `publish_direct_commit` already conditionally publishes one table head and - examines the observed head on conflict. Keep optimistic table requirements; - do not add its legacy multi-phase publisher to normal commit cost estimates. -- Active streamed `put_stream_part` uses one part GET plus one part CAS after - admission, and returns a conflict instead of silently rebasing replacement. - Inspect its callers for duplicate session/context lookup; do not introduce - catalog-wide quota writes already absent from this path. -- Legacy multipart `part` checks `load` before and after reading a part; each - `load` itself reads context + session, giving `5G` for this helper. Completion - can nest further load/exchange checks. `exchange` checks context before and - after CAS. Track these as legacy/recovery costs, with a migration/removal - option if there are no live callers; never attribute them to streamed traffic. -- Native injection selects an immutable receipt, then publishes file authority; - both have request/content identity and deletion replay constraints. Preserve - winner selection while passing known descriptors and admission through. -- Verify: active route tracing, file replacement/deletion, retry identity, - direct-file no-KV reads, old-session recovery and catalog clear/rotation. - -### 8. Fewer writes can still mean more RPCs — confirmed trade-off - -- Previous cleanup added preflight reads to Dataset immutable session metadata - and Composition/Split receipt helping. A new value now costs `G/B + C` - instead of just `C`; an existing identical value costs a read instead of a - failed conditional mutation. Zero-write replay tests alone miss this trade-off. -- `reserve_session_version(version=2)` can read VERSION, check legacy SPEC, then - call `immutable_session_metadata`, which reads VERSION again. Carry that first - observation and avoid redundant preflights. -- Iceberg `PayloadStore::put_page` similarly performs GET then create-if-absent; - a new P-page payload costs `P G + P C`, serially. Journal validation rereads - payloads which the same request may just have produced. -- Proposed: distinguish definitely-new writes from replay/help paths using - existing request-owned evidence, not a new durable flag. Compare direct - create-if-absent, known-value reuse and read-before-write with total round trips - and server work. Immutable CAS preserves identity collisions and cleanup seals; - plain PUT is allowed only if divergent overwrite/resurrection is impossible. -- Verify: first request versus repeated helpers, collisions, late writers after - cleanup, bounds and total GET+CAS counts as well as persistent mutation counts. - -### 9. Serial fine-grained metadata I/O — confirmed shape, cost to measure - -- `resolve_inspection_batch_locators` serially walks samples and fields: one - membership read per sample, a field metadata lookup per ancestor searched, - then a separate locator/inline-value lookup for each present value. Costs grow - with samples × fields × ancestry depth. Schema and snapshot proofs add work. -- These bounded point reads use scans in `chunk_store/metadata.rs`, not a native - bounded point-get RPC. Review protocol support before substituting an unbounded - GET; byte/deadline bounds must remain enforced before materialization. -- `persist_inspection_rows` writes schema, membership, field metadata and value - rows one at a time. For F schema fields, N samples and E present sample-field - entries it emits `F + N + 2E` conditional row writes, then 3 summary/index - writes, excluding manifest/publication records. Session ID pages and Iceberg - payload pages are also written serially. -- Proposed: batch compatible point operations or use bounded concurrent reads/ - writes with a shared byte/work budget; reuse immutable request-local schemas - and proofs. Consider co-locating metadata/locators only after measuring byte - amplification and defining migration. Preserve output order and missing-field - semantics; do not turn a sparse projection into an unbounded Manifest read. -- Verify: samples/fields/ancestry/page-count sweeps, RPC count and dependency - depth, missing/tombstone fields, byte budgets and cancellation under concurrency. - -### 10. Payload copies and full-part materialization — confirmed - -- HTTP Dataset upload collects an entire part before writing. Byte-mode - `ChunkPayloadStorage::read` concatenates receive chunks into `BytesMut` before - HTTP delivery. Inline HTTP delivery uses `Bytes::copy_from_slice` even though - the request already owns the value. Keep the native owner-backed path intact. -- Compatibility `UploadedChunkReader::read` adds another concatenation and then - `output.to_vec()`, copying the assembled result again. Trace its active callers - before prioritizing; it is not the normal native streaming reader. -- Generic S3 coalescing copies frames into a fresh Vec per receive window; - native direct-receive is a separate path. HTTP read-ahead/native fallback and - mixed-size partial frames need separate measurements, not a zero-copy label - applied to all traffic. -- Proposed: move existing owners into outputs, stream retained payload owners - where framing allows, avoid concatenation just to change container type, and - keep bounded aggregation only where measurements justify it. A header-copy - and payload-copy have different costs; count bytes, not only copy calls. -- Verify: address/owner lifetime, partial frames and unaligned sizes, caller-held - buffers, cancel/drop/shutdown, peak physical allocation and explicit fallback. - -### 11. Digest, serialization and allocation CPU — confirmed work, profile first - -- `DatasetUploads::complete` serially rereads every accepted part. Each - `verified_part` verifies its MD5, then completion hashes the bytes again for - whole-upload MD5. Part MD5s cannot simply be combined into whole-upload MD5; - removing this read requires durable trusted whole-content evidence, not - trusting a caller checksum or weakening integrity. -- Native `VerifiedUploadPart::next` and `FieldReader::delivered` both hash - delivered bytes for different scopes. When a field is exactly one verified - part, investigate sharing the certificate; multipart fields still require - their whole-field validation. Measure hash CPU separately from transport. -- Upload/session/draft CAS serializes records repeatedly; owner revocation lists - grow to 4096 entries, part-index pages to 128 receipts. Conflicts repeat - decoding/encoding of whole records. Record encoded bytes and CPU by size and - contention; evaluate narrower records only if lifecycle atomicity survives. -- Inspection stores JSON field/locator values and clones sample/field metadata - for response headers. Profile parsing, allocations and inline byte encoding, - then pre-size or move owned values where justified. An Arc/Bytes clone does - not imply a payload copy; do not optimize it based on syntax alone. -- Verify: CPU samples, allocations/request and bytes processed by each digest - or codec; singleton/multipart, small/large and high-owner-count cases. - -### 12. Upload expiry throughput and coupled cleanup — confirmed - -- Current `start_cleanup` wakes every 15 seconds, runs snapshot cleanup first, - then one upload page. `expire` scans 20 keys from `DS\0U\0`, including part - and index records; it also revisits claimed and cleanup-complete authorities. -- With R stable scanned records, the nominal full-pass scheduling budget is - `15 * ceil(R / 20)` seconds in the no-overrun case. At 100,000 records that is - about 20.8 hours, before I/O delay. Five-minute grace is a minimum delay, not - an upper cleanup bound. Removing the registry eliminated foreground writes - but did not establish acceptable large-scale cleanup latency. -- One expired upload can enumerate up to 10,000 parts and clean them serially; - the two-second scan timeout does not bound the entire cleanup step. A slow - upload delays the next snapshot sweep on the same task. Multiple Access - instances can also duplicate scans and cleanup work. -- Proposed: separate bounded progress budgets for discovery and reclamation; - consume multiple pages while within time/bytes/I/O budget, skip child ranges - where supported, retain incremental cleanup progress and yield fairly. - Compare per-object expiry indexing with full scans only after accounting for - any extra foreground write; do not restore one global registry contention key. -- Verify: sparse expired uploads among live/cleaned records, maximum-part uploads, - repeated restarts, corrupt entries, multiple cleaners, cleanup lag and effect - on foreground p99/CPU/metadata RPCs. Restart cursors must not lose obligations. - -### 13. Admission, scheduling and lower-layer costs — measurement tasks - -- Review count-based versus byte-based admission for mixed sizes, queue residence - and capacity retained by slow consumers; differentiate overload rejection from - insufficient in-flight work. Do not increase capacity just to raise benchmark - throughput or reintroduce unbounded prefetched buffers/tasks. -- Native Dataset pulls one field stream at a time. Measure whether bounded - prefetch hides storage RTT under real mixed sizes, and whether it increases - retained-memory pressure or harms small-request p99. One thread can have - multiple pending operations; the load generator's request/response discipline - must be recorded. -- S3 native allocation credit has a 1-ms retry timer when other producers/CQ - completions can release arena credit; Dataset metadata cancellation polls at - 10 ms. Measure wakeup overhead, stalls and cancellation latency before changing - notification plumbing. Preserve missed-wakeup safety and avoid a new lock. -- Profile allocation arenas, serialization/hash time, scheduler occupancy, - shared writer queues, batching dwell time, transport in-flight WR/byte depth, - CQ processing, disk durability and metadata replication separately. Attribute - a lower-layer bottleneck with traces before expanding this Access change into - a storage or RDMA redesign. -- Include authentication/signing, error/logging paths and metrics overhead in - endpoint CPU profiles. Retain authorization/integrity checks; consolidate - duplicate computation or reuse already-validated request-local evidence. -- No direct production Mutex/RwLock was found in the audited Access libraries; - that does not establish lock-free behavior in dependencies, FFI or the kernel. - -## CAS use cases and disposition - -- **Ordinary reads, fetch replay, payload delivery**: no persistent reader CAS. - Keep only request admission, selected immutable identity and physical integrity; - repeated GETs remain subject to findings 1–3 and 9. -- **Ordinary S3 object PUT/DELETE**: final object publication/deletion already - uses one unconditional mutation; HEAD uses one object GET after bucket - resolution. Do not add preflight CAS. Bucket names still need uniqueness and - generation protection across delete/recreate. -- **Snapshot/table/Split head publication**: competing publishers select one - current generation, respecting an explicit expected base. Keep one conditional - publication point and exact lost-reply proof; do not silently rebase user edits. -- **Upload owners, claim versus reclaim**: two snapshots may reference one - upload. Keep an atomic owner/state update so removing one owner cannot delete - another's live data; permanent owner revocation rejects delayed claims. -- **Part intent/accept/index/complete**: prevent same-part writers, delayed I/O - and abort/complete from publishing inconsistent data. Keep the necessary - atomic state transitions; returned observations and known state eliminate - redundant reads. Index-page sharing is limited to one upload and merits a - separate contention measurement for parallel part uploads. -- **Session ack/resume/close**: same-session cursor/closed/deadline changes must - not overwrite each other. CAS is scoped to that handle; independent sessions - must not share mutable reader state. Normal fetch needs none. -- **Draft edits/checkpoints/cleanup seals**: explicit revision checks protect - concurrent edits; seals reject a late writer after cleanup. Reuse known - immutable receipts without inventing another mutable coordination record. -- **Iceberg namespace parent/name/lifecycle**: preserve strict nonempty delete, - generation isolation and recovery. Ordinary table creation already avoids a - parent mutation. Reduce supporting RPCs without removing the required boundary. -- **Idempotency bindings/immutable records**: create-if-absent selects one payload - per operation/name and detects divergent identity reuse. Plain PUT is not a - generic substitute; prove sole ownership, identical bytes and no cleanup seal - before removing the condition. A failed create-if-absent still costs an RPC. -- **GC progress and conditional deletion**: prevent stale workers from overwriting - newer progress or deleting a reused identity. Background workers need bounded - work and fair progress, not reader leases. Legacy catalog credit accounting - remains only where old records require settlement. -- Remote CAS inventory covers Dataset authority/publication/access, uploads, - sessions, Composition/Split/inspection; S3 bucket and multipart repositories; - Iceberg catalog/namespace/commit/table lifecycle, files/multipart/injection, - retry/payload/projection and GC. Shared `crowdb-access-multipart` helpers do - arithmetic/validation rather than issuing storage RPCs. Local delivery/arena - atomics and metrics are excluded from remote CAS counts. Dynamic counts for - every route and failure branch are still an execution task, not a static proof. - -## Performance verification matrix - -- Run direct read, session start/next/ack/resume/close; upload create/new-part/ - replay/complete/abort; Composition/Split edit/evaluate/publish/recover/cleanup; - S3 ordinary objects and multipart; Iceberg active file/table/namespace paths, - legacy recovery and GC. Include auth/admission entry costs in endpoint totals. -- Vary payload sizes, samples and fields, ancestry, part count, owner count, - concurrency and retained buffers. Separate one shared key from independent - objects. Include no contention, definite conflicts, response loss and restart. -- For each change record before/after logical storage calls, actual routed RPCs, - serial stages, bytes, CPU, allocations, p50/p99 and backlog. Keep byte/digest - correctness and resource limits fixed. Batch operations may lower RPC count - without lowering backend I/O; report both. -- The unchanged full HTTP/native Rust/native Python workload and restart check - remain the final acceptance. Do not compare phases from different source - images or count an interrupted/partial run as a throughput result. Physical - multi-host validation remains in its separate plan. - -## Decisions applied to the inventory - -- Dataset sessions retain explicit durable acknowledgement and remote close. - Normal fetch/replay is read-only; start/ack prepares the next token. Only - start/ack/resume extends admission. Legacy unprepared sessions upgrade once. - See [Dataset read flow](../design/access-server/dataset/design-crowdb-access-dataset.md#102-read-flow). -- Composition/Split retain draft-scoped recovery checkpoints and cleanup fences - for exact lost-reply recovery and late-writer exclusion. Source activity leases - are absent; receipt helping avoids redundant CAS. No reader-shared journal was - found. See [control lifecycle](../design/access-server/dataset/design-dataset-control.md). -- Dataset uploads no longer register in shared tail/pages or persist a scan - cursor. Expiry directly scans upload authorities with local continuation. - Permanent owner revocation protects other live snapshots; bounded part indexes - remain to avoid one metadata point read per part during listing/completion. - See [upload flow](../design/access-server/dataset/design-crowdb-access-dataset.md#101-upload-and-publication-flow). -- Iceberg catalog-wide credits are legacy recovery only. New streamed sessions - already use independent session limits; no foreground reserve/release was - found. Keep legacy recovery so old outstanding records can settle. -- **User decision, 2026-10-24**: nonempty namespace deletion must fail. Keep the - parent admission protocol where required by creation/rename/drop recovery; - do not replace it with cascading deletion. Ordinary table creation already - avoids a parent authority mutation. See [Iceberg authority](../design/access-server/iceberge/design-crowdb-iceberg.md#2-authority-model). -- Ordinary Iceberg commits already use one head CAS; the legacy multi-phase - publisher is not the normal HTTP path. Preserve GC/publication fencing because - it prevents deleting referenced files, rather than only interrupting a reader. -- Iceberg streamed parts already publish independently by part number. S3 keeps - upload-local accounting and the completion/part-pointer fence; independent - uploads do not contend on one global record. Confirmed S3 CAS conflicts now - use returned observed bytes, and part retry retains its latest loaded state. - See [S3 data flow](../design/access-server/s3/design-crowdb-access-s3.md). -- Dataset session and upload-owner retry loops use returned observed state. - Confirmed stale-base retries and uncertain transport reconciliation remain - distinct. Explicit revision preconditions still return conflict; retrying an - application mutation against a new base is not universally valid. - -## Files and focused checks - -- Dataset: `read_session/`, `upload/`, `composition/`, `split/state.rs`; Access - Server Dataset cleanup wiring and publication; Dataset design/spec/OpenAPI. -- S3: `metadata/store.rs`, `metadata/multipart_repository{.rs,/parts.rs}`; - owning design. Iceberg design corrections document existing foreground paths. -- Cost regressions: `session_cost_test`, `draft_cost_test`, - `upload_registry_test`; lifecycle regressions retain restart, shared-owner, - lost-response, invalid-content and cancellation assertions. -- Integration: affected Access Server/native/S3 suites; Iceberg namespace drop - and admission tests retain the explicitly selected nonempty-delete contract. -- Gates: Rust fmt and workspace clippy through Pixi. Full workload and restart - use `tools/sdk/dataset_workload.py` with its unchanged defaults and same-source - image; physical multi-host acceptance remains in its independent plan. - -## Inspection coverage and exclusions - -- Inspected route ownership and coordination sites in - `app/crowdb-access-server/src/` and production sources of - `lib/crowdb-access-{dataset,s3,iceberg,multipart}/src/`, including shared credentials, - admission, streaming, Dataset lifecycle, S3 metadata and Iceberg catalogs. - This inventory identifies candidates; it is not a measured proof that every - listed write is unnecessary. -- No direct `Mutex`/`RwLock` usages found in those five production source trees. - Dependencies and FFI may contain locks; audit them only when profiling points - there. Atomic resource counters are not automatically removal candidates. -- S3 ordinary object PUT/DELETE already use unconditional point mutations; - ordinary GET has no persistent reader registration. Bucket-name deletion CAS, - explicitly conditional writes and multipart completion have different scopes. -- Iceberg `FileReader` verifies physical directory ownership and block digests - without registering each reader. Preserve stale/reused-location rejection. -- Credentials, immutable content publication, explicit retention, shared payload - owners, authorization and physical allocation identity are not expendable - merely because they persist state or use CAS. - -## Verification for each cleanup - -- Record the old guarantee, accepted replacement semantics, conflict scope, - persistent GET/write counts and reader/request lifetime before implementation. -- Unit/integration: independent concurrent operations, deletion races, retries, - restart, partial results, retained buffers and shared references. -- E2E: existing HTTP/native Dataset parity and mixed-size workload; use matched - source/builds. For other domains run the affected existing protocol suites. -- Run affected tests and Rust fmt/clippy through `pixi run`. Do not increase - retry bounds, weaken integrity assertions or call workload reduction a fix. +Upstream: [Dataset SDK](../backlog/R220-dataset-fat-client-plan.md)、 +[Access design](../design/access-server/design-crowdb-access-server.md)。 + +目标：减少 Access 请求的 RPC、持久化、复制、CPU 和共享竞争，保留确有业务用途的约束。 +这是用户要求长期维护的清理计划；完成项转入验证记录和所属设计后，从待办中移除。 + +## 跟踪方式与当前状态 + +- `[ ]` 未完成，`[~]` 正在执行（最多一项），`[x]` 已验证；任务 ID 保持稳定。 +- P1–P14 对应持久化审查；F1–F10 对应其他性能问题；V1–V4 是验收任务。 +- D1–D3 的原则已由用户确认；已记录决定不等于实现完成。 + R1–R4 跟踪具体写入反例审查，D4–D5 仅在出现实际业务需求时讨论；不预设保留中间状态。 +- 每项实现完成需记录：实际修改、正常/冲突/结果未知路径、验证命令和结果。 + 静态发现、提出方案、代码完成、性能验收不能互相代替。 +- 当前实施基线为 `489ae507`；旧性能审查来自 `878ee901`， + 涉及已删除 session 的结论已移除。下面的成本数字只描述注明的函数路径，需动态复核。 + +## 已确定的约束 + +- Dataset 读取无服务端 session：无持久化 reader、选中 ID 页、进度、续租、ack/close。 + continuation 固定 Dataset、Snapshot、Split revision、投影、分区和 shuffle； + 客户端处理成功后保存 next，重用原 token 可重放，不依赖固定 Access 实例。 +- Snapshot 删除：持久化标记，拒绝新请求，默认等待五分钟后回收。 + 重试/重启不重置期限；已准入流可继续，过久的流允许失败，不能截断后报成功。 + Access Server 不登记读者、不互相确认删除、不异步写释放记录。 +- **非空 Iceberg namespace 删除必须失败**，已由用户决定，不再提出放宽选项。 +- definite CAS conflict 立即使用返回的 observed value 计算合法重试，无“三次后才更新”。 + 返回值可能随后又变旧，因此仍需期限/重试上限；transport timeout 不等于 CAS 失败。 + 明确 expected revision 的业务操作不能擅自改 base 重放。 +- 新请求仍须鉴权并检查删除状态；同一请求内可复用已验证结果。 + 跨请求缓存可变准入结果必须证明失效语义，不能只依靠 TTL。 +- 已提交业务数据、共享 payload 存活引用、物理分配代际和校验仍须正确。 + 每项持久化都要说明用途，但删除 session 不等于删除业务资源本身。 +- **用户决定（2026-10-24）**：服务端不保存编辑、计算、上传的中间过程；客户端控制流程， + 失败后重新请求。允许 MPU，恢复粒度为完整 part，不支持 part 内断点续传。 + 不用新的内存会话、固定实例路由或持久化 job 替代旧 session。 +- 已完成 part、最终文件/Manifest 和正式引用属于数据结果；步骤号、处理偏移、工作页、 + 输入/输出重放账本、pending journal 等不得仅为恢复请求过程而保存。 + 确有其他用途的记录必须列出具体使用场景和最小方案，先 review，再保留或新增。 +- 中途生成但未被正式引用的文件/part 交给 GC；删除过程状态之前必须验证 GC 能独立发现 + 并回收它们，且不会误删待发布或已有 owner 的内容。不能假设现有 GC 已具备此能力。 +- **用户决定（2026-10-24）**：未发布 MPU part 有明确可提交期限；过期拒绝新提交， + 客户端重传。GC 在期限后再等待安全期回收；已发布资源不能按该临时期限回收。 + 期限必须随 part 结果返回，最终提交验证期限；本地时钟检查不能单独证明迟到发布安全。 +- 幂等按操作效果判断：同目标同内容的普通覆盖可直接重试，不要求原响应重放、 + operation ledger 或多次查询来“证明幂等”。有反例按 R1–R4 单独 review； + 不把“最小提交回执 + 查询结果”设成所有写 API 的默认成本。 +- 热路径不新增锁；不得用无界缓存、无界并发或降低校验来换吞吐。 + +## 已确认的设计决定与具体反例审查 + +### D1 — 客户端控制 Composition / Split，失败重新计算（已确认） + +**旧流程及开销**：begin 写草稿 → edit 保存修订/回执 → evaluate 保存每页工作和中间行 → +CAS 推进进度 → prepare → publish。它让服务端能接着上次编辑/计算，却给每页增加 GET、 +持久化和草稿竞争。这个恢复合同不再需要，不能仅降低检查点频率后继续保留。 + +**目标流程**：客户端保存规则/修改和调用顺序 → 提交独立完整、资源有界的请求 → +服务端计算并生成最终所需的内容 → 发布最终引用。失败由客户端重试，服务端不恢复过程。 +大输入和分页由客户端组织，不要求一次塞进内存；输出文件可以先存在，再由最终发布引用。 +这些文件不是服务端步骤回执或隐藏的工作状态；未引用内容由 GC 清理。 + +**实施边界**：删除 durable draft/edit/evaluation/work-progress 等过程记录与相应续算 API； +已发布 Snapshot/Split 成员继续固定，不能因为重试改变历史结果。迟到请求与 GC/发布竞争 +需要单独证明安全；若认为必须保留某个封口/阶段记录，提交 R4 场景，不能默认保留。 +影响 P2/P3/P12/P13，决策不再等待确认。 + +### D2 — 无服务端上传续传，MPU 按 part 由客户端控制（已确认） + +**旧流程及开销**：upload descriptor → part intent → 物理写入 → part 结果/索引 → +complete → owner 引用。这里混合了实际数据结果与“服务端记住做到了哪一步”的状态， +还有重复 GET、共享索引页竞争和过程恢复记录。 + +**目标流程**：客户端决定分片与 part 标识 → 每次请求完整写一个 part → 返回完整 part 的 +标识/长度/校验等结果 → 客户端收集结果并提交最终清单 → 发布最终资源。 +某个 part 断了就重传该 part，不恢复其字节偏移；客户端无需服务端保存步骤/上传进度。 +其他成功 part 可继续由客户端引用，换 Access 也不能依赖原实例的会话。 +响应丢失时同一 part 的重写优先按相同目标/内容实现，不先加查询和去重账本。 + +**实施边界**：已完成 part 的数据和必要资源元数据可以存在；upload-wide pending、 +逐步恢复 journal、part 内 checkpoint 不因 MPU 而自动获得保留理由。 +S3 ListParts 等协议入口若需要索引，必须解释资源查询用途、最小记录与成本，不能据此 +保留全部旧会话状态机。GC 负责失败的半成品和未被最终引用的完整 part。 +影响 P4/P5/P6/P7/P13，跨重启续传不再是默认合同或待选建议。 + +### D3 — 按实际操作效果选择重试方式，不默认保存幂等账本（原则已确认） + +用户认可普通覆盖写相同内容的逻辑幂等。此前“所有写操作至少保存提交证据、允许查询结果” +的统一建议撤回。先列目标对象、输入、并发写者、失败点和重复执行后果，再判断是否需要 +条件写或其他约束；需要条件写不等于需要持久化过程/响应账本。 +现有测试要求原错误/原响应重放，是旧实现合同，不能单凭测试把该行为保留下来。 + +#### R1 — 普通覆盖写：通常无需额外去重；并发语义要说清楚 + +**流程**：客户端向同一 key 写 A，响应丢失，再向同一 key 写 A。没有夹入其他修改时， +最终逻辑内容仍是 A。S3 `publish_versioned` 当前直接调用一次 `store.put`， +说明完成资源发布不必经过统一 RetryLedger。相同逻辑内容不要求物理地址/修订号不变； +重试产生的未引用物理副本交给 GC。 + +**边界例子**：A 已写入 → 另一个客户端写 B → A 的客户端重试 A，最终内容回到 A。 +若 API 是无条件覆盖，这是新的有效覆盖，不需要为此引入账本；若 API 承诺不得覆盖 +并发更新，就应由请求携带 expected version 做条件写。问题是 API 合同，不是同内容本身。 +若每次覆盖还创建可枚举历史版本或其他业务副作用，则需要对该实际入口单独 review； +不要把尚未发现的副作用当作默认事实。 + +- [ ] **R1 逐路由确定覆盖合同**：普通 S3/Dataset part/文件发布分别记录是否无条件覆盖、 + 是否有额外可见副作用。可重复覆盖的入口删除过程/响应去重和额外确认 GET；保持内容校验。 + Files: S3 `publication.rs`、Dataset upload/MPU、Iceberg file 入口。 + +#### R2 — expected-base 发布：CAS 是必要约束，历史成功查询未必必要 + +**具体流程**：请求固定 H0，希望发布 H1；CAS 已成功但回复丢失；另一个写者又把 H1 +推进到 H2。原请求重试 H0→H1 时，CAS 返回 H2，直接覆盖 H1 会丢掉后续写者的更新。 +这不同于无条件 PUT。当前 Iceberg `publish_direct_commit` 已用 CAS 返回值判断相同 +candidate；观察到后续代际时可能返回 Uncertain，没有必要再盲目 GET 同一个 head。 + +**初步判断**：保留请求的 expected base/候选身份，重试同一条件写；observed 就是候选时 +可以认定结果一致，observed 已变化时按真实条件失败/无法判断历史结果返回，不能暗中 rebase。 +CAS 不再满足只证明“本次条件不成立”，不能谎称“第一次一定没成功”。 +不为每次请求添加历史查询或回执；只有某个具体调用者必须识别被后续提交遮住的历史成功， +才提出该场景需要的最小方案及代价，交用户 review。 + +- [ ] **R2 审查条件发布的回复丢失窗口**：用 H0→H1→H2 测试区分条件失败、相同结果与历史未知； + 列出确实依赖历史精确结果的调用者，没有这种需求就移除相应 ledger/response replay。 + Files: Iceberg `commit/direct.rs`、Dataset `authority/publication/`、Split head、SDK。 + +#### R3 — rename / 严格非空删除：涉及多个资源，不是一个 key 的覆盖 + +**具体流程**：Iceberg rename A→B 当前涉及目标名称 reservation、表 head、目标 mapping、 +源 mapping 和父 namespace admission。代码先改权威 head，再发布目标名、删除源名； +在任一边界宕机，重试不能只凭“A 不在了”就认定原操作成功，也不能误删重建后的 A。 +并发创建子对象与删除父 namespace 的检查/执行之间，也不能只做 GET empty 后无条件删除。 + +**初步判断**：这是必须审查的多资源一致性场景，不是保留整个 journal 的结论。 +先比较以单个权威记录确定最终状态、从资源事实重建派生 mapping、请求携带原资源代际、 +以及已有存储原子能力，明确各方案能否保持非空删除失败和名称唯一。 +若仍主张某个中间记录不可避免，必须展示具体失败时间线、无法从最终事实修复的原因和 +最小持久化集合，先 review；不能用“恢复方便”作为理由。 + +- [ ] **R3 提交多资源操作的最小方案**：按 rename/create/drop 分别列失败点、实际读取可见性、 + 新请求如何处理和无 journal 替代方案；有保留中间记录的需求时单独确认。 + Files: Iceberg `table/lifecycle/{operation,completion,reservation}.rs`、`namespace/`。 + +#### R4 — 去掉过程状态后，GC 是否还能发现所有孤儿？这是实现前置检查 + +**具体流程**：物理文件已写出 → 尚未保存 part/最终引用时进程退出。若 GC 只扫描 upload +记录，就可能根本看不到这份数据。另一个窗口是文件已生成但尚未发布，GC 判断无 owner， +同时迟到请求又尝试发布；仅“当前无 owner”不自动意味着可以安全回收。 +现有 `DatasetUploads::expire` 扫 `DS\0U\0`，并从 upload record 进入 abort/cleanup， +不能凭这个入口证明没有 upload record 的物理孤儿也能被清理。 + +**初步判断**：先证明底层分配/文件清单及引用信息足以发现孤儿，建立等待期和发布/回收边界， +并测试物理身份复用。扫描游标是本地优化，不恢复服务端业务步骤；必要的文件身份/时间等 +资源事实与操作 checkpoint 分开审查。不能为补 GC 缺口直接把已否定的上传/计算 journal 加回来。 + +**已核实的缺口（2026-10-24）**：`ChunkdbClient::list_chunks` 能枚举 chunk；共享 writer +过期后在 `acknowledged_cursor` 封口，保留已经确认的字节。`FrameHeaderPrefix` 有大小和 +写入时间，没有对象 owner；`reclaim_location` 只能回收调用者给定的物理范围。 +Dataset expiry 依赖 upload/part 记录，Iceberg discovery/write sweep 依赖 file/intent 记录， +都不是“封口 chunk 范围减去全部存活引用”的独立扫描器。删除写前 intent 后，写出 payload +而尚未保存结果就宕机会产生不可发现的范围。tree 内部 orphan 回收不能代替用户 payload GC。 + +**已确认的期限与剩余工程证明**：未发布 part 过期后重传；不支持无限期稍后提交。 +仅在 Access 开始请求时检查期限仍有竞态：检查通过 → 发布 RPC 延迟 → GC 回收 → 迟到发布。 +必须在最终权威发布边界拒绝过期候选，或以资源代际/封口隔离迟到发布；等待若干分钟本身 +不能证明不存在无限延迟请求。此处没有授权新增逐请求 journal，也不先删除唯一孤儿发现链路。 + +`DatasetAuthority::fence_reclaim` 已通过修改 head nonce 作废旧发布 CAS； +`commit_snapshot_attempt` 在读取 head 后检查 Snapshot 准入，再做 expected-head CAS。 +这是可复用的隔离机制，但当前只针对指定 Snapshot，不能直接当作任意过期 part 的发布屏障。 +后续须证明全体引用发布入口都经过该边界、扫描与新提交的顺序、跨 Dataset 共享引用的范围； +S3 普通对象是直接 PUT，不能未经设计就套用 Dataset head 机制或给每次 PUT 增加 CAS。 + +- [ ] **R4 验证无过程记录的孤儿发现**：注入文件生成前后、part 回复前后、最终引用前后宕机； + 证明无 owner 内容最终清掉、已发布/共享内容不误删。若缺底层发现能力，先补独立 GC， + 需要新增持久化信息时说明用途供 review。Files: Dataset expiry/reclaim、Iceberg file/GC、底层文件/分配枚举接口。 +- [ ] **R4.1 建立有界物理范围发现**：枚举已封口用户数据 chunk，按完整 frame 边界建立候选， + 与最终 Manifest/文件引用求差；包含写成功但结果记录缺失、共享 chunk 混合存活和孤儿、 + writer 失效和物理身份复用。不得把整个 shared chunk 当作一个可删除文件。 + Files: chunk-client reclamation、chunkdb ListChunks、protocol frame、Dataset/Iceberg GC。 +- [ ] **R4.2 实现提交期限和回收隔离**：part 结果携带期限，提交权威边界验证；覆盖期限前准入 + 但期限后落地的 RPC、GC 与最终引用并发、成功发布后的长期存活。若需新增持久化资源事实， + 先列出最小字段和每请求 RPC 成本供 review。Files: Dataset publication/upload、Iceberg file、GC。 + +### D4 — 全局硬配额，还是每台 Access 的资源准入？仅在发现活跃需求后讨论 + +**场景和流程**：旧 Iceberg multipart credit 在 catalog-wide 记录上预留 sessions/bytes， +写 pending journal，再创建 upload，结束时归还。不同上传会竞争同一个计数器；它实现的是 +跨实例总量约束，而本地 semaphore/字节预算只保护一台机器。两者不能直接等价替换。 +当前新上传前台路径未使用该全局 reserve/release；旧记录恢复仍可能依赖它。 + +**判断与建议**：先追踪入口和历史记录，删除确证无调用、无恢复责任的旧逻辑； +不因为看到了类名就把旧路径开销算进新上传。只有确有硬配额需求时才讨论全局协调方案。 + +- [ ] **D4 条件决策**：P6 若证明仍有跨实例硬配额需求，再确认是否允许按实例限制， + 接受实例数增加导致总上限增加；若无需求，记录无需产品决策及调查证据。 + +### D5 — GC 是否有完成时限和精确恢复进度要求？先测清理积压 + +**场景和流程**：Snapshot 标记后等待五分钟，只保证不早于期限回收；实际回收由后台扫描。 +当前调度每 15 秒做 snapshot cleanup，再扫一页 upload（20 个 key，包含子记录）。 +100,000 条稳定记录理想扫描周期已约 20.8 小时；单个大 upload 还可能串行清理大量 part。 +持久化每页进度有写入成本，完全重扫则在重启后重复工作。 + +**判断与建议**：默认用有界、公平、可幂等重做的清理，允许本地游标和进度估计； +保留删除标记、物理身份和不能遗失的清理义务。重试删除必须验证代际，不能删到复用后的数据。 +若实测重扫已影响容量或明确要求多久清完，再提出 expiry index 等资源发现方案及前台写放大供 review，不能自行恢复过程检查点。 + +- [ ] **D5 条件决策**：P13 测得积压后，如需保证清理完成时限或重启精确续扫，再确认具体目标； + 五分钟 grace 本身不意味着“五分钟内清完”。若无此需求，按可重复计算的实现处理。 + +## 无需重新讨论的工程判断 + +- P1 投影是可重建派生数据，当前失败就回退 canonical 内容；从 GET 中去掉同步持久化 + 不改变表的权威数据合同。先验证冷读/热读成本，再选无缓存、本地有界缓存或发布时构建。 +- confirmed CAS conflict 使用 observed value、已成功写入后的状态直传、请求内复用 + immutable descriptor、减少不必要复制，均可在现有合同下进行，不必逐项请求产品批准。 +- namespace 非空删除、名称唯一性、共享 payload 引用、物理代际、已提交数据持久性、 + 鉴权撤销边界不重新投票。若发现其实现复杂，先提出保持这些约束的更小方案。 +- 尚未证明性能收益或未确认调用路径属于调查任务，不当作必须由用户回答的问题。 + +## 持久化清理任务 + +每项先记录正常/冲突/结果未知流程和必要不变量，再修改。D1/D2 和 D3 原则已经确定， +据此删除过程状态；任何需要保留中间状态的具体例外先提交 R1–R4 review。R4 是去掉旧过程记录的验收前置条件。 + +### P1 — Iceberg GET 派生投影 + +P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内容，不读写持久化投影。 + +- [ ] **P1.2 验证剩余解析 CPU 成本**：当前选用无投影缓存；旧路径无论命中与否都读 canonical， + 额外访问持久化投影只节省部分解析。零投影 RPC 已有回归，尚未测混合大小的冷/热 CPU 和 + p99。与 F9 一起测量；只有解析成为瓶颈再比较本地有界缓存/发布时构建，明确上限和失效身份。 + Files: Iceberg `table/load.rs`、相关 table/projection 测试及真实负载。 + +### P2 — Dataset Composition（执行已确认 D1） + +- [ ] **P2.1 分清输入、计算进度与候选内容**：逐项列出 draft/edit receipt/work page/row/candidate/ + cleanup seal 的用途、写入次数和调用者；按 D1 更新 API/SDK 合同，移除服务端续算接口。Files: Dataset `composition/{begin,edit,state,evaluation,evaluate_step,prepare,cleanup}.rs`。 +- [ ] **P2.2 删除服务端编辑/计算过程**：删除 durable 编辑/进度阶段，客户端失败重新请求； + 保持发布前不可见、expected head 及源 Snapshot 被删时的明确错误。孤儿清理通过 R4 验证，例外封口记录先 review。Files: `composition/`、Access v2 composition、SDK 和恢复测试。 + +### P3 — Dataset Split（执行已确认 D1） + +- [ ] **P3.1 分开临时评估和已发布成员**：盘点 edits/evaluation/overlap examples/work pages； + 保留历史 revision 和冻结成员，不能重算时偷偷改变既有 train/test。Files: Dataset `split/{lifecycle,state,work,evaluation,prepare,initial}.rs`。 +- [ ] **P3.2 实现选定 Split 流程**：按 D1 删除辅助进度与检查点，客户端重试，保留 currentSplit + 发布 CAS、覆盖/重叠校验和旧 revision 读取。Files: `split/{publication,recovery,cleanup}.rs`、v2/SDK Split 和恢复测试。 + +### P4 — Dataset 上传、索引和 owners（执行已确认 D2） + +- [ ] **P4.1 缩减 upload/part 状态与回读**：列出 create/intent/write/accept/index/complete/abort + 的必要资源事实与过程状态；删除后者，改成客户端控制完整 part/最终清单； + 成功结果直接返回，不为续传保存进度。需要保留 intent 的具体反例先按 R4 review。Files: Dataset `upload.rs`、`upload/{parts,index,ownership}.rs`。 +- [ ] **P4.2 修复 CAS 冲突重读**：`index_part`、`abort/cleanup/cleanup_part` 直接使用 observation； + 合并不同 part 的索引更新时保留双方结果。claim/release 已正确使用 observation，不重复改写。Files: `upload/{index,ownership,retirement}.rs`。 +- [ ] **P4.3 缩小 owner/part 争用范围**：测 128-part 索引页、增长的 retired owners 的争用和编码； + 比较更小记录，证明 claim-vs-reclaim 和迟到 claim 不破坏共享引用。Files: `upload/{index,ownership,retirement,reclaim}.rs`、相关并发/丢回复测试。 + +### P5 — S3 multipart（客户端控制 part；协议资源与过程状态分开审查） + +- [ ] **P5.1 精简 MPU 持久化与重复装载**：先说明 UploadId/ListParts/Complete 所需资源事实， + 删除无用过程阶段；复核旧实现新 part `7G + 4C` 路径，把已知 session/generation/ + pointer 传给 settle；验证 identity index 是否只是列表元数据，不能未经证明删除权威检查。Files: S3 `metadata/multipart_repository{.rs,/parts.rs}`。 +- [ ] **P5.2 保留 typed CAS 结果**：exchange 返回 applied/observed，消除 complete/abort/settle + 对同 key 的 conflict 后 GET；同时覆盖同 part 替换、不同 part 并发、complete/abort 和不确定写入。Files: S3 `metadata/multipart_repository/{completion,publication,terminal}.rs`。 + +### P6 — Iceberg 旧 multipart / 全局 credits + +- [ ] **P6.1 追踪活跃与恢复入口**：新 part 前台已是准入后 `1G + 1C`，旧 `part` helper 可有 + `5G`；分别计数。列出旧记录兼容/结算责任，不能把“无前台调用”当成“可以删”。Files: Iceberg `file/multipart{,_credits,_recovery,_repository}.rs` 及子模块、Access file 路由。 +- [ ] **P6.2 删除确证无用的协调**：没有调用/恢复义务的代码可删；有旧记录则给出迁移/结算方案。 + 只有仍有全局硬配额需求才触发 D4。Files: `file/multipart_admission.rs`、`file/multipart_credits/settle.rs` 及恢复测试。 + +### P7 — Iceberg 物理写入（执行 D2；必要安全例外先 review） + +- [ ] **P7.1 区分安全事实与续传检查点**：逐一标明 ownership/generation、write intent、偏移、 + completed file、adoption receipt 防止的失败；可读 bytes 不等于已确认提交。Files: Iceberg `file/{writer,write_intent,injection}.rs` 和 writer 子模块。 +- [ ] **P7.2 精简可重做阶段**：按 D2 删除未完成偏移/步骤恢复，复用请求已知 descriptor； + 保留复用存储拒绝能力；若认为须保留 intent/checkpoint，先给出具体 R4 反例和替代方案。用丢失写回复、旧实例迟到、重启/回收注入验证。Files: 同 P7.1 及 file recovery 测试。 + +### P8 — Iceberg RetryLedger（执行 D3；按 route 判定必要性） + +- [ ] **P8.1 清掉确定冲突后的查询**：begin 使用返回 binding，仅真实碰撞才查 overflow；finish + 使用已返回 result，不重新 GET。Files: Iceberg `operation/{retry,ledger,result}.rs`。 +- [ ] **P8.2 按实际用途删除重试账本**：按 route 对照业务 journal 与 HTTP binding/input/result，判定实际重试效果； + 普通覆盖删除 ledger/body/状态，条件或多资源操作按 R2/R3 处理；不默认保留“最小回执”。迁移测试和 SDK。Files: `operation/{retry,result,payload}.rs`、Access `iceberg/{table_write,namespace_write}`。 + +### P9 — Iceberg namespace / table 多阶段操作 + +- [ ] **P9.1 直传阶段状态与上下文**：journal CAS 后复用 applied/observed，避免重新 load + + ActiveRoot 检查；逐个标明必须重新检查的 clear/drop 边界。现有 D 层 namespace load + 静态成本为 `2 + 2D` GET。Files: Iceberg `namespace/`、`table/lifecycle/`、`commit/create/`、`catalog/repository.rs`。 +- [ ] **P9.2 删除或替代阶段 journal**：画 create/drop/rename 的失败点与恢复流程；保持非空删除 + 失败、名称唯一、代际隔离。按 R3 先审查无 journal 方案，确需中间状态再 review，不能用 eventual emptiness 简化。Files: 同 P9.1 及 namespace admission/drop/recovery 测试。 + +### P10 — Dataset 发布与显式资源控制 + +- [ ] **P10.1 区分发布事实与辅助回执**：区分 immutable Manifest/Snapshot/Split、latest/stable、retention + 与 attempt/receipt/visibility 的重复证明；最终发布 CAS 保持 expected base。Files: Dataset `authority.rs`、`authority/publication/`、`split/{head,publication,recovery}.rs`。 +- [ ] **P10.2 精简额外标记**：依据 P10.1、D3 删除不必要恢复写；有必要回执反例先按 R2 review；retention API 查调用者和用途， + 不在无替代合同下删除用户 pin。不默认承诺 head 再次推进后仍可查询原操作的精确结果。Files: 同 P10.1、publication/retention 测试。 + +### P11 — S3 object / bucket 发布 + +- [ ] **P11.1 修复 bucket tombstone CAS 回路**：直接消费 observed active mapping/tombstone， + 不接无意义 put-if-absent；保留名称唯一和删除重建代际。Files: S3 `bucket.rs`、`metadata/store.rs`。 +- [ ] **P11.2 锁定普通对象最小路径**：普通 PUT/DELETE 已为无条件 mutation，不加 GET/CAS； + 单独验证 conditional 请求和 multipart 发布，记录包括 bucket resolution 的端到端成本。Files: S3 `publication.rs`、`metadata/namespace.rs`、对象/桶测试。 + +### P12 — Dataset inspection 索引 + +- [ ] **P12.1 降低索引写入粒度**：当前 F 个 schema field、N 个 sample、E 个非空 sample-field + 产生 `F + N + 2E` 行写和 3 个 summary/index 写，不含 Manifest/publication。 + 比较批量与有界并发，保留顺序、预算和未完整索引不可见性。Files: Dataset `inspection/{index,preparation}.rs`。 +- [ ] **P12.2 精简索引准备/回收检查点**：保留服务必需的不可变索引，按 D1 删除辅助恢复进度，失败重做； + 未发布索引/候选纳入 R4 孤儿清理。显式 legacy preparation 仍是 mutation，读 miss 不自动持久化构建。Files: `inspection/{legacy,retirement,reclaim}.rs` 及索引恢复测试。 + +### P13 — GC / expiry / cleanup + +- [ ] **P13.1 拆开有界发现与回收**：复核 15 秒/20 key 调度，避免一个最多 10,000-part 的 + upload 串行清理阻塞 snapshot sweep；分时间/字节/IO 预算公平推进。Files: Access Dataset `start_cleanup`、Dataset `upload/{expiry,ownership,reclaim}.rs`。 +- [ ] **P13.2 审查 cleanup 持久化**：将 claim/phase/cursor/tombstone 分为可重做进度与不可丢的 + 义务/代际；即使重扫昂贵也先优化发现/预算；新增持久化信息必须 review，用途和额外前台写均写明。Files: Dataset `authority/access/`、Split/Composition cleanup、Iceberg `gc/`、S3 terminal cleanup。 +- [ ] **P13.3 量化回收滞后**：稀疏过期、大 upload、多 cleaner、频繁重启、损坏记录下测 backlog、 + 前台 p99 和孤儿收敛；如需确定完成时限再讨论 D5。Files: 相关 cleanup 测试、负载工具。 + +### P14 — Catalog 与凭据 + +- [ ] **P14.1 审查 root/context/初始化回读**：记录正常请求内重复 GET 和初始化写，复用已验证 + immutable 事实，保留撤销和 catalog reset 的准入边界。凭据持久化不是 reader session。Files: Iceberg `catalog/{repository,storage}.rs`、Access `credentials.rs`、`credentials/ensure.rs`。 + +## 其他性能任务 + +- [ ] **F1 请求级成本计数**：记录 GET/bounded GET/scan/CAS/PUT/delete、重复 key、冲突与 + unknown、真实 routed RPC、串行阶段和复制字节；用测试 wrapper/已有 metrics，不加生产 + 每 RPC 日志或 profiling 锁。Files: Dataset/S3/Iceberg 测试 store、现有指标、`tools/sdk/`。 +- [ ] **F2 合并一次请求的删除准入**：`DatasetReads::next`、HTTP/native batch 和 inspection + 仍可重复查询删除/提交状态。建立 snapshot-bound 请求内结果，独立入口仍需准入；不跨请求 + 缓存删除许可。Files: Dataset `read_query/`、`inspection/`、Access `dataset/api/batch.rs`、native `dataset/read.rs`。 +- [ ] **F3 避免无状态读取重复规划**：大 shuffle group + 小 batch 会反复重建同一 group； + selection 窗口还重复加载 schema/membership。比较有界客户端 group 缓存、请求内规划复用 + 与 token 大小；不恢复持久化 session。Files: Dataset `read_query/page.rs`、`inspection/evaluate.rs`、SDK iterable。 +- [ ] **F4 替换 legacy scan 全量规划**：`scan_surface` 每页重建所有窗口，造成 CPU/内存放大； + 追踪调用者后迁移到有界 inspection 或增量 planner。Files: Dataset `transport.rs`、`planner/`、legacy API 和 scan 测试。 +- [ ] **F6 批量或并发 locator 读取**：membership + 每 sample/field/ancestor 元数据 + value + 读取目前串行；bounded point read 在 chunk store 内还转成单 key scan。保留 materialize 前 + 字节/期限限制、结果顺序和 missing/tombstone。Files: Dataset `inspection/`、`chunk_store/metadata.rs`、相关 RPC 支持。 +- [ ] **F7 比较预读与 create-CAS 总成本**：新 immutable 页的 GET+CAS 可能比直接 CAS 更贵， + replay 则可能相反；比较新写/重放比例，复用本请求已知值，不能无证明改普通 PUT。Files: Dataset Composition/Split receipts、Iceberg `operation/payload.rs`、journal validation。 +- [ ] **F8 减少 payload 复制**：审查 HTTP 收整 part、`BytesMut` 拼接、inline copy、兼容 + UploadedChunkReader 的拼接后 `to_vec` 和 S3 coalescing；先查活跃调用，再移交 owner 或流式 + 发送。Files: Dataset payload/upload、Access HTTP batch、S3/native 接收路径。验证部分帧、非整齐 size、held buffer 和实际复制字节。 +- [ ] **F9 分解摘要/编码/分配 CPU**：complete 重读 part 并做 part/whole MD5，native part 与 + field 还各有摘要；MD5 不能直接拼合，只有同范围可信校验证据才可复用。测大 owner/part-index + 编码、JSON/base64、header clone 和 allocations。Files: Dataset upload、native `dataset/read/field.rs`、inspection response。 +- [ ] **F10 混合大小准入与调度**：测 count/bytes 限制、慢消费者占用、单线程多个 in-flight、 + 有界 prefetch、batch 等待、1 ms arena 重试和 10 ms cancellation 轮询。区分 Access、CQ/WR、 + replication/disk 瓶颈，保留鉴权/校验，禁止仅增大容量刷 TPS。Files: native runtime/arena、Dataset cancellation、现有 workload/transport 指标。 + +## 执行顺序与验收 checkbox + +- 第一批：F1 建成本基线，P1 去掉 GET 写入；R4 验证孤儿发现/回收，R1–R3 确定各写入场景。 +- 第二批：按已确认 D1/D2 实施 P2/P3、P4/P5/P7，按 R1–R3 处理 P8.2/P9.2/P10； + 只有确有中间状态必要性的例外等待单独 review。涉及孤儿的删除先通过 R4。 +- 第三批：在保留下来的真实路径上做 P4.2/P5.2/P8.1/P9.1、P11、F2/F7 等 RPC 优化（F5 已完成）； + 路径若随状态删除则移除对应优化任务，避免先优化即将删除的 journal。 + F3/F4/F6/F8/F9/F10、P12/P13 按测量瓶颈排序；P6/P14 可独立审查。 +- 每个可独立验证的改动完成后运行对应检查，不等所有任务完成才验证。 + +- [ ] **V1 单元/组件与成本回归**：正常请求、1/多次真实 CAS 冲突、缺失/终态、丢回复、重启、 + 独立对象并发与同 key 竞争；新增无过程记录的孤儿回收、part 整体重传、 + 无条件覆盖与 expected-base 重试测试。断言总 GET+写，不仅是零写。Files: Dataset `tests/*cost*`、`read_query*`、upload/Composition/Split/inspection；S3/Iceberg 对应 repository 测试。 +- [ ] **V2 HTTP/native/SDK 集成**：各受影响 route 的鉴权、错误、协议/SDK 一致性；保留删除准入、 + 迟到写者、非空 namespace、共享 owner、held buffer、cancel/drop 和损坏内容测试。Files: Access/native tests、`sdk/tests/`、native Python consumer。 +- [ ] **V3 工程门禁与文档**：每批执行适用 fmt/clippy/测试，全部经 `pixi run`；同步 owning + design/spec/OpenAPI/SDK。只修改文档时检查 diff、引用和任务状态，不为此跑整套服务测试。 +- [ ] **V4 同源真实负载验收**：重新构建当前源码的镜像，运行现有 HTTP/native Rust/native Python + 混合 size、restart 和本机 HCA 对比；固定核心、并发、请求完成口径、数据集和完整性校验。 + 记录 p50/p99、TPS、CPU/RSS、RPC、复制/摘要、清理积压，不缩小负载掩盖失败。 + Files: `tools/sdk/dataset_workload.py`、相关 examples、`e2e-test/`、本机 HCA runner。 + 物理多主机验证继续留在独立计划，不并回此处。 + +## 成本口径与集中代码索引 + +- `G` 是一个逻辑 point GET，`B` 是有界读，`S` 是 scan，`C` 是条件 mutation，`W` 是无条件 mutation。 + `C(None)` 是 create-if-absent，也消耗 RPC。上述数字不含嵌套路由/复制/WAL/物理 IO， + 单函数静态数不能当端到端 RTT 或 TPS 结果；批量 RPC 也未必减少后端 IO。 +- CAS 必要性按用例证明：发布 head 的 expected base、名称唯一、part/finalize/abort、 + 共享引用 vs 回收、迟到写者隔离。明确失败冲突可复用 observation；unknown 按该操作语义处理，不默认追加查询。 +- Dataset 路径前缀：`lib/crowdb-access-dataset/src/`；重点 `read_query/`、`upload/`、 + `composition/`、`split/`、`inspection/`、`authority/`、`transport.rs`。 +- S3 路径前缀：`lib/crowdb-access-s3/src/`；重点 `bucket.rs`、`publication.rs`、`metadata/`。 +- Iceberg 路径前缀：`lib/crowdb-access-iceberg/src/`；重点 `table/`、`metadata_projection/`、 + `operation/`、`namespace/`、`commit/`、`catalog/`、`file/`、`gc/`。 +- 入口/客户端：`app/crowdb-access-server/src/`、`lib/crowdb-sdk-native/src/`、 + `lib/crowdb-sdk-native-python/`、`sdk/`。检查现有调用者后再删除代码，不把 legacy 算作活跃路径。 +- 先前检查以上 Access production trees 未发现直接 Mutex/RwLock；不代表依赖、FFI、内核无锁。 + 下层改造应由 profiling 证据触发。共享 `crowdb-access-multipart` 主要做计算/验证，不发 storage RPC。 + +## 已完成与验证记录 + +- **2026-10-24，P1.1**：删除 TableLoader 持久化投影 GET、miss 填充和验证回执代码。 + 冷读、重复读、32 个并发读取及 conditional GET 均无投影读取和 CAS；refs/all、ETag、 + parser limits、损坏拒绝和 namespace/head 变化检查保留。历史投影不参与表读取，未批量删除。 + file-level 通用 projection 仍有独立调用者，不在这批删除。 +- **2026-10-24，F5**：增加请求内 `UploadReadDescriptor`，私有字段绑定原 location/storage； + HTTP batch、native、preview、兼容 range reader 均复用一次只读描述信息。 + P 个 part 的上传元数据 GET 从 native `1 + 2P`、HTTP batch `3P + 1` 降为 `1 + P`； + 这是逻辑元数据调用数，不含 Snapshot/locator 准入、路由复制和物理 payload IO。 + 不持久化、不登记 reader、不跨请求缓存准入；继续验证 part index/length/digest， + 已回收 part 报错，已交付 buffer 保持有效。HTTP 增加 locator 与描述信息的 length/md5 一致性检查。 +- 本批功能回归共 **59 个测试通过**：Iceberg `table_projection_test`、`metadata_projection_test`、 + `table_load_test`（21）；Access `iceberg_table_http_test`、`iceberg_table_acceptance_test`（14）； + Dataset `upload_read_cost_test`、`upload_stream_test`、`upload_test`（12）； + Access `dataset_batch_cost_test`、`dataset_preview_contract_test`、`dataset_read_query_contract_test`（5）； + native `dataset_payload_test`、`dataset_read_test`（7）。均通过 `pixi run cargo test`； + HTTP/native 集成先执行 `pixi run clean-env`，完成短尾 part、跨 part range、校验损坏、 + 回收、取消、held buffer 和零写/RPC 次数验证。 +- 本批门禁：四个受影响 crate 的 `pixi run cargo clippy --all-targets --all-features -- -D warnings`、 + workspace fmt 和 `git diff --check`。无新锁、无 unsafe 例外、无 HTTP wire 协议变化。 + 测试辅助代码的依赖/格式 lint 已修正；HTTP 测试释放客户端后再关闭 listener。 +- **R4 未完成**：已定位物理孤儿发现缺口，并记录用户确认的未发布 part 有效期。 + 尚未新增范围 GC 或删除上传/Composition/Split 的过程状态，不能将本批读取优化算作该清理完成。 + 尚无当前源码的 HCA/TPS 对比，不声称吞吐提升比例。 + + +- `489ae507`：移除 Dataset 持久化读取 session、远端 ack/resume/close、TTL/配置和 legacy + cursor storage；HTTP 使用 `reads/bind`、`reads/next`，原 v1 直接 `reads` 保留。 + native/Python 同步迁移，读取进度归客户端；旧 session key 不再读写，尚未物理批量清除。 +- 已通过 Dataset 全目标及最终 query 专项、Access Server 全目标、native SDK 全目标、独立 + HTTP SDK 测试、安装后 Python wheel 的真实 HTTP Split/read 测试、相关 all-feature clippy、 + SDK clippy、两个 workspace fmt，以及 OpenAPI 解析/本地引用检查。 +- 零写回归覆盖 bind、96 个并发 fetch、跨实例重放和 legacy scan；另有 Snapshot/Split 固定、 + worker 覆盖、删除/取消和 token 大小验证。每帧 session GET 已随旧实现删除。 +- 曾出现链接磁盘不足和 `reads` 路由冲突：前者仅清理可重建 incremental 缓存，后者改为 + 独立 `reads/bind`；修复后完整 Access suite 通过。这不是未处理的测试阻塞。 +- 以上是功能与局部成本证据。全量混合 size/restart、本机 HCA 性能验收仍未完成； + `878ee901` 的旧 OCI 镜像不能用于声称当前改动已验收，R220 尚未完成。 diff --git a/lib/crowdb-access-dataset/src/lib.rs b/lib/crowdb-access-dataset/src/lib.rs index 8b8b89ed6..3f7ca1ea7 100644 --- a/lib/crowdb-access-dataset/src/lib.rs +++ b/lib/crowdb-access-dataset/src/lib.rs @@ -34,11 +34,11 @@ mod wire; pub use payload::{ ChunkPayloadStorage, PayloadDurability, PayloadStorage, PayloadStream, UploadedChunkReader, }; -pub use upload::VerifiedUploadPart; pub use upload::{ DatasetUploads, SnapshotUploadCleanup, UploadError, UploadPart, UploadReclaimError, UploadRecord, UploadState, MAX_UPLOAD_PART_BYTES, }; +pub use upload::{UploadReadDescriptor, VerifiedUploadPart}; pub use cursor::ReadCursor; pub use delivery::DeliveryWindow; diff --git a/lib/crowdb-access-dataset/src/payload/reader.rs b/lib/crowdb-access-dataset/src/payload/reader.rs index 4df2a993f..c0d125aa2 100644 --- a/lib/crowdb-access-dataset/src/payload/reader.rs +++ b/lib/crowdb-access-dataset/src/payload/reader.rs @@ -24,27 +24,28 @@ impl ChunkReader for UploadedChunkReader { range: Option>, cancel: &ReadCancellation, ) -> Result, ChunkReadError> { - let record = self.uploads.metadata(location).await.map_err(|e| map_error(&e))?; - let range = range.unwrap_or(0..record.length); - if range.start > range.end || range.end > record.length || range.end - range.start > 8 * 1024 * 1024 { + let record = self + .uploads + .open_read(location) + .await + .map_err(|e| map_error(&e))?; + let range = range.unwrap_or(0..record.length()); + if range.start > range.end || range.end > record.length() || range.end - range.start > 8 * 1024 * 1024 + { return Err(ChunkReadError::Failed("bounded field range required".into())); } let mut output = BytesMut::new(); - let first = range.start / u64::from(record.part_size); - let end = range.end.div_ceil(u64::from(record.part_size)); + let first = range.start / u64::from(record.part_size()); + let end = range.end.div_ceil(u64::from(record.part_size())); for index in first..end { if cancel.is_cancelled() { return Err(ChunkReadError::Cancelled); } - let bytes = self - .uploads - .read_part( - location, - u32::try_from(index).map_err(|_| ChunkReadError::Truncated)?, - ) + let (bytes, _) = record + .read_part_verified(u32::try_from(index).map_err(|_| ChunkReadError::Truncated)?) .await .map_err(|e| map_error(&e))?; - let offset = index * u64::from(record.part_size); + let offset = index * u64::from(record.part_size()); let start = usize::try_from(range.start.saturating_sub(offset)).map_err(|_| ChunkReadError::Truncated)?; let finish = usize::try_from((range.end - offset).min(bytes.len() as u64)) diff --git a/lib/crowdb-access-dataset/src/upload.rs b/lib/crowdb-access-dataset/src/upload.rs index 285517c52..127ee1a48 100644 --- a/lib/crowdb-access-dataset/src/upload.rs +++ b/lib/crowdb-access-dataset/src/upload.rs @@ -7,6 +7,8 @@ mod expiry; mod index; mod ownership; mod parts; +mod read; +pub use read::UploadReadDescriptor; mod retirement; mod stream; pub use stream::VerifiedUploadPart; @@ -232,21 +234,9 @@ impl DatasetUploads { location: &[u8], index: u32, ) -> Result<(Bytes, [u8; 16]), UploadError> { - let session = self.session(location).await?; - if !matches!(session.state, UploadState::Complete | UploadState::Claimed) { - return Err(UploadError::Conflict); - } - let entry = self - .store - .get(&part_key(location, index)) - .await? - .ok_or(UploadError::NotFound)?; - let entry = decode_part(&entry.bytes)?; - if entry.state != PartState::Accepted { - return Err(UploadError::Conflict); - } - Ok((self.verified_part(&entry.part).await?, entry.part.md5)) + self.open_read(location).await?.read_part_verified(index).await } + async fn verified_part(&self, part: &UploadPart) -> Result { let bytes = self.payloads.read(&part.locations).await?; if bytes.len() as u64 != part.length || digest(&bytes) != part.md5 { diff --git a/lib/crowdb-access-dataset/src/upload/read.rs b/lib/crowdb-access-dataset/src/upload/read.rs new file mode 100644 index 000000000..06fa550fa --- /dev/null +++ b/lib/crowdb-access-dataset/src/upload/read.rs @@ -0,0 +1,88 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; + +use super::{decode_part, part_key, DatasetUploads, PartState, UploadError, UploadPart, UploadState}; + +/// Immutable upload description for one admitted read. Own this in the request, +/// not in a cross-request admission cache. It does not register or pin a reader; +/// concurrent reclamation may still make a part read fail. +pub struct UploadReadDescriptor { + pub(super) uploads: DatasetUploads, + location: Vec, + length: u64, + part_size: u32, + parts: u32, + md5: [u8; 16], +} + +impl DatasetUploads { + /// Loads and validates the upload once for all parts of this read request. + /// # Errors + /// Rejects missing, malformed or uncommitted uploads. + pub async fn open_read(&self, location: &[u8]) -> Result { + let record = self.session(location).await?; + if !matches!(record.state, UploadState::Complete | UploadState::Claimed) { + return Err(UploadError::Conflict); + } + Ok(UploadReadDescriptor { + uploads: self.clone(), + location: location.to_vec(), + length: record.length, + part_size: record.part_size, + parts: super::part_count(&record), + md5: record.md5, + }) + } +} + +impl UploadReadDescriptor { + #[must_use] + pub fn length(&self) -> u64 { + self.length + } + + #[must_use] + pub fn md5(&self) -> [u8; 16] { + self.md5 + } + + #[must_use] + pub fn part_count(&self) -> u32 { + self.parts + } + + pub(crate) fn part_size(&self) -> u32 { + self.part_size + } + + pub(super) async fn part(&self, index: u32) -> Result { + if index >= self.part_count() { + return Err(UploadError::Invalid); + } + let value = self + .uploads + .store + .get(&part_key(&self.location, index)) + .await? + .ok_or(UploadError::NotFound)?; + let entry = decode_part(&value.bytes)?; + let start = u64::from(index) * u64::from(self.part_size); + if entry.state != PartState::Accepted + || entry.part.index != index + || entry.part.length != (self.length - start).min(u64::from(self.part_size)) + { + return Err(UploadError::Integrity); + } + Ok(entry.part) + } + + /// Returns a bounded part and its verified digest without rereading the upload. + /// # Errors + /// Rejects invalid boundaries, reclaimed parts and corrupt or unavailable data. + pub async fn read_part_verified(&self, index: u32) -> Result<(Bytes, [u8; 16]), UploadError> { + let part = self.part(index).await?; + Ok((self.uploads.verified_part(&part).await?, part.md5)) + } +} diff --git a/lib/crowdb-access-dataset/src/upload/stream.rs b/lib/crowdb-access-dataset/src/upload/stream.rs index a184fefa2..3593c68d0 100644 --- a/lib/crowdb-access-dataset/src/upload/stream.rs +++ b/lib/crowdb-access-dataset/src/upload/stream.rs @@ -4,7 +4,7 @@ use crowdb_rpc_ffi::OwnedPayload; use md5::{Digest, Md5}; -use super::{decode_part, part_key, DatasetUploads, PartState, UploadError, UploadPart, UploadState}; +use super::{DatasetUploads, UploadError, UploadPart, UploadReadDescriptor}; use crate::PayloadStream; /// An accepted upload part with incremental integrity checks. Emitted owners @@ -26,28 +26,20 @@ impl DatasetUploads { location: &[u8], index: u32, ) -> Result { - let session = self.session(location).await?; - if !matches!(session.state, UploadState::Complete | UploadState::Claimed) { - return Err(UploadError::Conflict); - } - let entry = self - .store - .get(&part_key(location, index)) - .await? - .ok_or(UploadError::NotFound)?; - let entry = decode_part(&entry.bytes)?; - let start = u64::from(index) * u64::from(session.part_size); - if entry.state != PartState::Accepted - || entry.part.index != index - || start >= session.length - || entry.part.length != (session.length - start).min(u64::from(session.part_size)) - { - return Err(UploadError::Integrity); - } - let stream = self.payloads.read_stream(&entry.part.locations).await?; + self.open_read(location).await?.read_part_stream(index).await + } +} + +impl UploadReadDescriptor { + /// Opens a part using this request's immutable upload description. + /// # Errors + /// Rejects invalid boundaries, reclaimed parts and unavailable storage. + pub async fn read_part_stream(&self, index: u32) -> Result { + let part = self.part(index).await?; + let stream = self.uploads.payloads.read_stream(&part.locations).await?; Ok(VerifiedUploadPart { stream: Some(stream), - expected: entry.part, + expected: part, delivered: 0, digest: Md5::new(), failed: false, diff --git a/lib/crowdb-access-dataset/tests/common/upload.rs b/lib/crowdb-access-dataset/tests/common/upload.rs index bc8473d61..ab4b1e31c 100644 --- a/lib/crowdb-access-dataset/tests/common/upload.rs +++ b/lib/crowdb-access-dataset/tests/common/upload.rs @@ -18,10 +18,17 @@ use std::{ #[derive(Default)] pub struct TestStore { values: ArcSwap, StoredValue>>, + pub reads: AtomicU64, + pub upload_reads: AtomicU64, + pub mutations: AtomicU64, } #[async_trait] impl DatasetStore for TestStore { async fn get(&self, key: &[u8]) -> Result, StoreError> { + self.reads.fetch_add(1, Ordering::Relaxed); + if key.starts_with(b"DS\0U\0") { + self.upload_reads.fetch_add(1, Ordering::Relaxed); + } Ok(self.values.load().get(key).cloned()) } async fn get_bounded( @@ -92,6 +99,7 @@ impl DatasetStore for TestStore { expected: Option<&[u8]>, value: &[u8], ) -> Result { + self.mutations.fetch_add(1, Ordering::Relaxed); loop { let old = self.values.load_full(); let current = old.get(key).cloned(); @@ -113,6 +121,7 @@ impl DatasetStore for TestStore { } } async fn delete(&self, key: &[u8]) -> Result<(), StoreError> { + self.mutations.fetch_add(1, Ordering::Relaxed); loop { let old = self.values.load_full(); let mut next = (*old).clone(); diff --git a/lib/crowdb-access-dataset/tests/upload_read_cost_test.rs b/lib/crowdb-access-dataset/tests/upload_read_cost_test.rs new file mode 100644 index 000000000..be28dcf27 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/upload_read_cost_test.rs @@ -0,0 +1,130 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/payload_stream.rs"] +mod payload_stream; + +use bytes::Bytes; +use crowdb_access_dataset::{DatasetIdentity, DatasetStore, DatasetUploads, NamespacePath, OperationId}; +use md5::{Digest, Md5}; +use payload_stream::{TestPayloadStreamStorage, TestStore}; +use std::sync::{atomic::Ordering, Arc}; + +async fn fixture() -> (DatasetUploads, Vec, Arc) { + let store = Arc::new(TestStore::default()); + let uploads = DatasetUploads::new(store.clone(), Arc::new(TestPayloadStreamStorage::default())); + let identity = DatasetIdentity::new(NamespacePath::root(), "cost").unwrap(); + let location = DatasetUploads::location(&identity, OperationId::random()).unwrap(); + uploads + .create(&location, 10, 4, Md5::digest(b"abcdefghij").into(), 100) + .await + .unwrap(); + for (index, part) in b"abcdefghij".chunks(4).enumerate() { + uploads + .put_part( + &location, + u32::try_from(index).unwrap(), + Bytes::copy_from_slice(part), + 1, + ) + .await + .unwrap(); + } + uploads.complete(&location, 2).await.unwrap(); + (uploads, location, store) +} + +#[tokio::test] +async fn buffered_and_streaming_reads_use_one_description_plus_one_get_per_part() { + let (uploads, location, store) = fixture().await; + for streaming in [false, true] { + let reads = store.reads.load(Ordering::Relaxed); + let writes = store.mutations.load(Ordering::Relaxed); + let descriptor = uploads.open_read(&location).await.unwrap(); + assert_eq!(descriptor.length(), 10); + assert_eq!(descriptor.md5(), <[u8; 16]>::from(Md5::digest(b"abcdefghij"))); + assert_eq!(descriptor.part_count(), 3); + let mut bytes = Vec::new(); + for index in 0..descriptor.part_count() { + if streaming { + let mut part = descriptor.read_part_stream(index).await.unwrap(); + while let Some(payload) = part.next().await.unwrap() { + bytes.extend_from_slice(payload.as_ref()); + } + assert!(part.next().await.unwrap().is_none()); + } else { + let (part, digest) = descriptor.read_part_verified(index).await.unwrap(); + assert_eq!(digest, <[u8; 16]>::from(Md5::digest(&part))); + bytes.extend_from_slice(&part); + } + } + assert_eq!(bytes, b"abcdefghij"); + // EOF and invalid indices must not cause another metadata RPC. + assert!(descriptor.read_part_verified(3).await.is_err()); + assert!(descriptor.read_part_stream(u32::MAX).await.is_err()); + assert_eq!(store.reads.load(Ordering::Relaxed) - reads, 4); + assert_eq!(store.mutations.load(Ordering::Relaxed), writes); + } +} + +#[tokio::test] +async fn a_descriptor_does_not_pin_reclaimed_data_or_admit_new_reads() { + let (uploads, location, store) = fixture().await; + let descriptor = uploads.open_read(&location).await.unwrap(); + let mut part = descriptor.read_part_stream(0).await.unwrap(); + let held = part.next().await.unwrap().unwrap(); + uploads.abort(&location).await.unwrap(); + assert!(uploads.open_read(&location).await.is_err()); + let reads = store.reads.load(Ordering::Relaxed); + assert!(descriptor.read_part_stream(1).await.is_err()); + assert!(descriptor.read_part_verified(1).await.is_err()); + assert_eq!(store.reads.load(Ordering::Relaxed) - reads, 2); + assert_eq!(held.as_ref(), b"ab"); +} + +#[tokio::test] +async fn cached_description_still_checks_part_index_length_and_digest() { + for damage in ["index", "length", "md5"] { + let (uploads, location, store) = fixture().await; + let descriptor = uploads.open_read(&location).await.unwrap(); + let mut key = location.clone(); + key.extend_from_slice(b"\0P"); + key.extend_from_slice(&1_u32.to_be_bytes()); + let old = store.get(&key).await.unwrap().unwrap(); + let mut entry: serde_json::Value = serde_json::from_slice(&old.bytes).unwrap(); + match damage { + "index" => entry["part"]["index"] = 0.into(), + "length" => entry["part"]["length"] = 3.into(), + _ => entry["part"]["md5"][0] = (entry["part"]["md5"][0].as_u64().unwrap() ^ 1).into(), + } + store + .compare_exchange(&key, Some(&old.bytes), &serde_json::to_vec(&entry).unwrap()) + .await + .unwrap(); + assert!(descriptor.read_part_verified(1).await.is_err(), "{damage}"); + match descriptor.read_part_stream(1).await { + Err(_) => assert_ne!(damage, "md5"), + Ok(mut part) => loop { + match part.next().await { + Ok(Some(_)) => {} + Ok(None) => panic!("corrupt {damage} incorrectly completed"), + Err(_) => break, + } + }, + } + } +} + +#[tokio::test] +async fn bounded_compatibility_reader_reuses_description_across_part_boundaries() { + use crowdb_access_dataset::{ChunkReader, ReadCancellation, UploadedChunkReader}; + let (uploads, location, store) = fixture().await; + let reader = UploadedChunkReader::new(uploads); + let before = store.reads.load(Ordering::Relaxed); + let bytes = reader + .read(&location, Some(2..9), &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(bytes, b"cdefghi"); + assert_eq!(store.reads.load(Ordering::Relaxed) - before, 4); +} diff --git a/lib/crowdb-access-iceberg/src/metadata_projection.rs b/lib/crowdb-access-iceberg/src/metadata_projection.rs index 6a38f0edc..79f9a0d9c 100644 --- a/lib/crowdb-access-iceberg/src/metadata_projection.rs +++ b/lib/crowdb-access-iceberg/src/metadata_projection.rs @@ -2,7 +2,6 @@ mod model; mod repository; -mod validated; use model::ProjectionIdentity; pub use model::{MAX_PROJECTION_BYTES, PROJECTION_PAGE_BYTES, PROJECTION_VERSION}; diff --git a/lib/crowdb-access-iceberg/src/metadata_projection/validated.rs b/lib/crowdb-access-iceberg/src/metadata_projection/validated.rs deleted file mode 100644 index 379148ae6..000000000 --- a/lib/crowdb-access-iceberg/src/metadata_projection/validated.rs +++ /dev/null @@ -1,81 +0,0 @@ -use std::collections::BTreeMap; - -use sha2::{Digest, Sha256}; - -use crate::{ - file::FileRecord, - record::StorageRecord, - table::{TableHead, TableMetadataDocument, TableMetadataLimits}, -}; - -use super::{model::Root, ProjectionIdentity, ProjectionStore, MAX_PROJECTION_BYTES}; - -impl ProjectionStore { - pub(crate) async fn put_document( - &self, - record: &FileRecord, - document: &TableMetadataDocument, - limits: TableMetadataLimits, - ) -> bool { - let head = document.selected_head(); - if record.file != head.metadata_file - || record.location != head.metadata_location - || record.digest != head.metadata_digest - || !self.put(record, head.generation, document.canonical()).await - { - return false; - } - let identity = ProjectionIdentity::new(record, head.generation); - let Some(key) = identity.key(0, 0) else { - return false; - }; - let Ok(Some(root)) = self.store.get(&key).await else { - return false; - }; - let Some(receipt) = receipt(head, limits, &root.bytes) else { - return false; - }; - self.put_page(identity, 0, 1, &receipt).await - } - - pub(crate) async fn document_fields( - &self, - record: &FileRecord, - head: &TableHead, - limits: TableMetadataLimits, - ) -> Option>> { - if record.length > MAX_PROJECTION_BYTES as u64 || record.length > limits.bytes as u64 { - return None; - } - let identity = ProjectionIdentity::new(record, head.generation); - let encoded = self.store.get(&identity.key(0, 0)?).await.ok()??; - let root = Root::decode(&encoded.bytes, identity, record.length)?; - let stored = self.store.get(&identity.key(0, 1)?).await.ok()??; - if stored.bytes != receipt(head, limits, &encoded.bytes)? { - return None; - } - let mut fields = BTreeMap::new(); - for (name, child) in &root.children { - fields.insert(name.clone(), self.read_child(identity, child).await?); - } - Some(fields) - } -} - -fn receipt(head: &TableHead, limits: TableMetadataLimits, root: &[u8]) -> Option> { - limits.validate().ok()?; - let mut digest = Sha256::new(); - digest.update(b"crowdb-iceberg-validated-metadata-v1"); - digest.update(StorageRecord::TableHead(Box::new(head.clone())).encode().ok()?); - for limit in [ - limits.bytes, - limits.values, - limits.depth, - limits.string_bytes, - limits.collection_entries, - ] { - digest.update(u64::try_from(limit).ok()?.to_be_bytes()); - } - digest.update(root); - Some(digest.finalize().to_vec()) -} diff --git a/lib/crowdb-access-iceberg/src/table/load.rs b/lib/crowdb-access-iceberg/src/table/load.rs index 5c153db1e..ef1b6e7e2 100644 --- a/lib/crowdb-access-iceberg/src/table/load.rs +++ b/lib/crowdb-access-iceberg/src/table/load.rs @@ -10,7 +10,6 @@ use super::{TableHead, TableMetadataDocument, TableMetadataError, TableMetadataL use crate::{ catalog::{Capabilities, CatalogContext, CatalogError, FormatAction}, file::FileBlockStore, - metadata_projection::ProjectionStore, namespace::{NamespaceIdentifier, NamespaceRepository, NamespaceStore}, }; @@ -48,9 +47,6 @@ pub struct TableLoader { tables: TableRepository, blocks: Arc, limits: TableMetadataLimits, - projections: ProjectionStore, - #[cfg(feature = "test-util")] - projection_hits: std::sync::atomic::AtomicUsize, } impl TableLoader { @@ -62,12 +58,9 @@ impl TableLoader { ) -> Self { Self { namespaces: NamespaceRepository::new(store.clone()), - projections: ProjectionStore::new(store.clone(), blocks.clone()), tables: TableRepository::new(store), blocks, limits, - #[cfg(feature = "test-util")] - projection_hits: std::sync::atomic::AtomicUsize::new(0), } } @@ -164,7 +157,8 @@ impl TableLoader { let canonical = super::metadata::read_table_metadata_bytes(self.blocks.clone(), &selected, self.limits).await?; let etag = etag(&selected.head, mode); - let metadata = self.representation(&selected, canonical, mode).await?; + let document = TableMetadataDocument::parse(canonical, &selected.head, self.limits)?; + let metadata = representation(&document, mode)?; if metadata.len() > self.limits.bytes { return Err(TableMetadataError::Bounds.into()); } @@ -186,42 +180,6 @@ impl TableLoader { }) } - #[cfg(feature = "test-util")] - #[must_use] - pub fn projection_hits_for_tests(&self) -> usize { - self.projection_hits.load(std::sync::atomic::Ordering::Relaxed) - } - - async fn representation( - &self, - selected: &super::SelectedTable, - canonical: Vec, - mode: SnapshotLoadingMode, - ) -> Result, TableMetadataError> { - if mode == SnapshotLoadingMode::Refs { - if let Some(fields) = self - .projections - .document_fields(&selected.metadata, &selected.head, self.limits) - .await - { - if let Ok(metadata) = projected_representation(&fields, &canonical) { - #[cfg(feature = "test-util")] - self.projection_hits - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - return Ok(metadata); - } - } - } - let document = TableMetadataDocument::parse(canonical, &selected.head, self.limits)?; - let metadata = representation(&document, mode)?; - if mode == SnapshotLoadingMode::Refs { - self.projections - .put_document(&selected.metadata, &document, self.limits) - .await; - } - Ok(metadata) - } - async fn check_namespace( &self, context: CatalogContext, @@ -262,17 +220,6 @@ fn representation( refs_representation(fields, document.canonical()) } -fn projected_representation( - fields: &BTreeMap>, - canonical: &[u8], -) -> Result, TableMetadataError> { - let fields = fields - .iter() - .map(|(name, bytes)| Ok((name.as_str(), serde_json::from_slice::<&RawValue>(bytes)?))) - .collect::, serde_json::Error>>()?; - refs_representation(fields, canonical) -} - fn refs_representation( fields: BTreeMap<&str, &RawValue>, canonical: &[u8], diff --git a/lib/crowdb-access-iceberg/tests/common/store.rs b/lib/crowdb-access-iceberg/tests/common/store.rs index b8bb553a0..b9c680bd2 100644 --- a/lib/crowdb-access-iceberg/tests/common/store.rs +++ b/lib/crowdb-access-iceberg/tests/common/store.rs @@ -22,6 +22,8 @@ pub struct TestStore { pub gc_delete_reply_loss: AtomicBool, pub writes: AtomicUsize, pub reads: AtomicUsize, + pub cas_attempts: AtomicUsize, + pub projection_reads: AtomicUsize, pub file_scans: AtomicUsize, pub fencing_delay_ms: AtomicUsize, pub fencing_barrier: Option>, @@ -66,6 +68,23 @@ impl TestStore { } } + async fn pause_file_location(&self, key: &[u8]) { + if matches!( + crowdb_access_iceberg::key::IcebergKey::decode(key), + Ok(crowdb_access_iceberg::key::IcebergKey::Catalog { + scope: crowdb_access_iceberg::key::CatalogScope::FileLocation, + .. + }) + ) { + self.pause_file_mapping().await; + if let Some(barrier) = &self.file_mapping_barrier { + if self.file_mapping_visits.fetch_add(1, Ordering::SeqCst) < 2 { + barrier.wait().await; + } + } + } + } + fn lose_file_reply(&self, key: &[u8]) -> bool { if matches!( StorageKey::decode(key), @@ -189,6 +208,18 @@ impl CatalogStore for TestStore { tokio::task::yield_now().await; } self.reads.fetch_add(1, Ordering::SeqCst); + if matches!( + StorageKey::decode(key), + Ok(StorageKey::Legacy(IcebergKey::Catalog { + scope: CatalogScope::MetadataProjection, + .. + }) | StorageKey::NativeTable(TableStorageKey { + scope: TableStorageScope::Projection, + .. + })) + ) { + self.projection_reads.fetch_add(1, Ordering::SeqCst); + } Ok(self.values.load().get(key).cloned()) } @@ -199,6 +230,7 @@ impl CatalogStore for TestStore { value: &[u8], identity: ClientRequestId, ) -> Result { + self.cas_attempts.fetch_add(1, Ordering::SeqCst); identity.validate().unwrap(); if self.yield_operations { tokio::task::yield_now().await; @@ -208,20 +240,7 @@ impl CatalogStore for TestStore { } self.pause_table_head(key, expected, value, false).await; self.pause_stage_transition(key, expected).await; - if matches!( - crowdb_access_iceberg::key::IcebergKey::decode(key), - Ok(crowdb_access_iceberg::key::IcebergKey::Catalog { - scope: crowdb_access_iceberg::key::CatalogScope::FileLocation, - .. - }) - ) { - self.pause_file_mapping().await; - if let Some(barrier) = &self.file_mapping_barrier { - if self.file_mapping_visits.fetch_add(1, Ordering::SeqCst) < 2 { - barrier.wait().await; - } - } - } + self.pause_file_location(key).await; if expected.is_none() { if let Ok(crowdb_access_iceberg::record::StorageRecord::TableMapping(mapping)) = crowdb_access_iceberg::key::IcebergKey::decode(key) diff --git a/lib/crowdb-access-iceberg/tests/table_projection_test.rs b/lib/crowdb-access-iceberg/tests/table_projection_test.rs index 4f225e120..a6cb2fda3 100644 --- a/lib/crowdb-access-iceberg/tests/table_projection_test.rs +++ b/lib/crowdb-access-iceberg/tests/table_projection_test.rs @@ -19,7 +19,6 @@ use crowdb_access_iceberg::{ table::{SnapshotLoadingMode, TableLoad, TableLoader}, }; use fixture::TestTable; -use sha2::{Digest, Sha256}; async fn load(fixture: &TestTable, loader: &TableLoader, mode: SnapshotLoadingMode) -> Vec { let TableLoad::Loaded { metadata, .. } = loader @@ -49,20 +48,46 @@ fn projection(key: &[u8]) -> bool { } #[tokio::test] -async fn validated_refs_reuse_pages_but_still_verify_canonical_storage() { - let fixture = TestTable::new().await; - let loader = fixture.loader(); +async fn cold_repeated_and_concurrent_loads_never_write_or_read_projection_state() { + let fixture = Arc::new(TestTable::new().await); + let loader = Arc::new(fixture.loader()); + let store = &fixture.fixture.store; + let writes = store.writes.load(Ordering::SeqCst); + let attempts = store.cas_attempts.load(Ordering::SeqCst); + let before = store.values.load_full(); let expected = load(&fixture, &loader, SnapshotLoadingMode::Refs).await; - assert_eq!(loader.projection_hits_for_tests(), 0); - let reads = fixture.blocks.reads.load(Ordering::SeqCst); - assert_eq!(load(&fixture, &loader, SnapshotLoadingMode::Refs).await, expected); - assert_eq!(loader.projection_hits_for_tests(), 1); - assert!(fixture.blocks.reads.load(Ordering::SeqCst) > reads); + let canonical_reads = fixture.blocks.reads.load(Ordering::SeqCst); + let mut readers = tokio::task::JoinSet::new(); + for _ in 0..32 { + let fixture = fixture.clone(); + let loader = loader.clone(); + readers.spawn(async move { load(&fixture, &loader, SnapshotLoadingMode::Refs).await }); + } + while let Some(result) = readers.join_next().await { + assert_eq!(result.unwrap(), expected); + } + assert!(fixture.blocks.reads.load(Ordering::SeqCst) > canonical_reads); assert_eq!( load(&fixture, &loader, SnapshotLoadingMode::All).await, fixture.bytes ); - assert_eq!(loader.projection_hits_for_tests(), 1); + assert!(matches!( + loader + .load( + fixture.fixture.context, + &fixture.parent.identifier, + &fixture.head.name, + SnapshotLoadingMode::Refs, + Some("*") + ) + .await + .unwrap(), + TableLoad::NotModified { .. } + )); + assert_eq!(store.writes.load(Ordering::SeqCst), writes); + assert_eq!(store.cas_attempts.load(Ordering::SeqCst), attempts); + assert_eq!(store.projection_reads.load(Ordering::SeqCst), 0); + assert!(Arc::ptr_eq(&before, &store.values.load_full())); fixture.blocks.corrupt_reads.store(true, Ordering::SeqCst); assert!(loader .load( @@ -74,66 +99,48 @@ async fn validated_refs_reuse_pages_but_still_verify_canonical_storage() { ) .await .is_err()); - assert_eq!(loader.projection_hits_for_tests(), 1); } #[tokio::test] -async fn every_missing_or_corrupt_projection_page_and_receipt_falls_back_exactly() { +async fn existing_or_corrupt_projection_pages_are_ignored_without_repair() { + use crowdb_access_iceberg::{file::FileRepository, metadata_projection::ProjectionStore}; let fixture = TestTable::new().await; let loader = fixture.loader(); let expected = load(&fixture, &loader, SnapshotLoadingMode::Refs).await; - let original = fixture.fixture.store.values.load_full(); + let record = FileRepository::new(fixture.fixture.store.clone()) + .load(fixture.fixture.context, &fixture.head.metadata_location) + .await + .unwrap() + .unwrap(); + assert!( + ProjectionStore::new(fixture.fixture.store.clone(), fixture.blocks.clone()) + .put(&record, fixture.head.generation, &fixture.bytes) + .await + ); + let store = &fixture.fixture.store; + let original = store.values.load_full(); let keys: Vec<_> = original.keys().filter(|key| projection(key)).cloned().collect(); - assert!(keys.len() > 2); - for key in &keys { + assert!(!keys.is_empty()); + let attempts = store.cas_attempts.load(Ordering::SeqCst); + assert_eq!(load(&fixture, &loader, SnapshotLoadingMode::Refs).await, expected); + for key in keys { for remove in [false, true] { let mut damaged = (*original).clone(); if remove { - damaged.remove(key); + damaged.remove(&key); } else { - damaged.get_mut(key).unwrap().bytes[0] ^= 1; + damaged.get_mut(&key).unwrap().bytes[0] ^= 1; } - fixture.fixture.store.values.store(Arc::new(damaged)); + store.values.store(Arc::new(damaged)); assert_eq!(load(&fixture, &loader, SnapshotLoadingMode::Refs).await, expected); - assert_eq!(loader.projection_hits_for_tests(), 0); - assert_eq!( - load(&fixture, &loader, SnapshotLoadingMode::All).await, - fixture.bytes - ); } } + assert_eq!(store.cas_attempts.load(Ordering::SeqCst), attempts); + assert_eq!(store.projection_reads.load(Ordering::SeqCst), 0); } #[tokio::test] -async fn projection_identity_and_version_cannot_replace_validation_receipt() { - let fixture = TestTable::new().await; - let loader = fixture.loader(); - let expected = load(&fixture, &loader, SnapshotLoadingMode::Refs).await; - let original = fixture.fixture.store.values.load_full(); - let root_key = original - .keys() - .find(|key| projection(key) && key.ends_with(&[0; 4])) - .unwrap(); - for field in ["version", "generation", "digest", "table", "length"] { - let mut damaged = (*original).clone(); - let root = damaged.get_mut(root_key).unwrap(); - let mut body: serde_json::Value = serde_json::from_slice(&root.bytes[32..]).unwrap(); - if matches!(field, "digest" | "table") { - body[field][0] = serde_json::json!(body[field][0].as_u64().unwrap() ^ 1); - } else { - body[field] = serde_json::json!(999); - } - let body = serde_json::to_vec(&body).unwrap(); - root.bytes = Sha256::digest(&body).to_vec(); - root.bytes.extend(body); - fixture.fixture.store.values.store(Arc::new(damaged)); - assert_eq!(load(&fixture, &loader, SnapshotLoadingMode::Refs).await, expected); - assert_eq!(loader.projection_hits_for_tests(), 0); - } -} - -#[tokio::test] -async fn receipt_never_relaxes_parser_limits_or_selected_head_binding() { +async fn loads_preserve_parser_limits_and_selected_head_binding() { let mut fixture = TestTable::new().await; load(&fixture, &fixture.loader(), SnapshotLoadingMode::Refs).await; for limit in 0..5 { @@ -156,7 +163,6 @@ async fn receipt_never_relaxes_parser_limits_or_selected_head_binding() { ) .await .is_err()); - assert_eq!(loader.projection_hits_for_tests(), 0); } fixture.head.table_uuid = Some(uuid::Uuid::new_v4()); fixture.publish_head().await; @@ -171,41 +177,10 @@ async fn receipt_never_relaxes_parser_limits_or_selected_head_binding() { ) .await .is_err()); - assert_eq!(loader.projection_hits_for_tests(), 0); -} - -#[tokio::test] -async fn optional_projection_write_failures_do_not_fail_canonical_loads() { - let fixture = TestTable::new().await; - let original = fixture.fixture.store.values.load_full(); - let loader = fixture.loader(); - let expected = load(&fixture, &loader, SnapshotLoadingMode::Refs).await; - let count = fixture - .fixture - .store - .values - .load() - .keys() - .filter(|key| projection(key)) - .count(); - for boundary in 1..=count { - fixture.fixture.store.values.store(original.clone()); - let writes = fixture.fixture.store.writes.load(Ordering::SeqCst); - fixture - .fixture - .store - .fail_after - .store(writes + boundary, Ordering::SeqCst); - assert_eq!( - load(&fixture, &fixture.loader(), SnapshotLoadingMode::Refs).await, - expected - ); - fixture.fixture.store.fail_after.store(0, Ordering::SeqCst); - } } #[tokio::test] -async fn warm_projections_cannot_hide_head_or_namespace_changes_during_reads() { +async fn loads_reject_head_or_namespace_changes_during_reads() { for change_head in [false, true] { let fixture = TestTable::new().await; let loader = fixture.loader(); @@ -245,6 +220,5 @@ async fn warm_projections_cannot_hide_head_or_namespace_changes_during_reads() { crowdb_access_iceberg::catalog::CatalogError::Conflict )) )); - assert_eq!(loader.projection_hits_for_tests(), 1); } } diff --git a/lib/crowdb-sdk-native/src/dataset/read/field.rs b/lib/crowdb-sdk-native/src/dataset/read/field.rs index 8feede6bd..05df5b937 100644 --- a/lib/crowdb-sdk-native/src/dataset/read/field.rs +++ b/lib/crowdb-sdk-native/src/dataset/read/field.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use crate::{runtime::RequestLease, Error, Result}; -use crowdb_access_dataset::{DatasetUploads, FieldLocator, UploadRecord, VerifiedUploadPart}; +use crowdb_access_dataset::{DatasetUploads, FieldLocator, UploadReadDescriptor, VerifiedUploadPart}; use crowdb_rpc_ffi::OwnedPayload; use md5::{Digest, Md5}; @@ -10,7 +10,7 @@ enum Source { Inline(Option), Upload { location: Vec, - metadata: Option, + metadata: Option, part: u32, stream: Option>, }, @@ -77,22 +77,22 @@ impl FieldReader { } => { if metadata.is_none() { let record = request - .run(async { Ok(uploads.metadata(location).await?) }) + .run(async { Ok(uploads.open_read(location).await?) }) .await?; - if record.length != self.length || record.md5 != self.expected { + if record.length() != self.length || record.md5() != self.expected { return Err(Error::Integrity); } *metadata = Some(record); } let record = metadata.as_ref().ok_or(Error::Integrity)?; loop { - if u64::from(*part) >= record.length.div_ceil(u64::from(record.part_size)) { + if *part >= record.part_count() { break None; } if stream.is_none() { *stream = Some( request - .run(async { Ok(Box::new(uploads.read_part_stream(location, *part).await?)) }) + .run(async { Ok(Box::new(record.read_part_stream(*part).await?)) }) .await?, ); } diff --git a/lib/crowdb-sdk-native/tests/dataset_payload_test.rs b/lib/crowdb-sdk-native/tests/dataset_payload_test.rs index bffea1cd4..8af294dd1 100644 --- a/lib/crowdb-sdk-native/tests/dataset_payload_test.rs +++ b/lib/crowdb-sdk-native/tests/dataset_payload_test.rs @@ -143,3 +143,52 @@ async fn maximum_upload_part_keeps_every_producer_allocation_through_durable_wri assert_eq!(views.concat(), data); runtime.shutdown().await.unwrap(); } + +#[tokio::test] +async fn multipart_field_streams_a_short_last_part_and_preserves_held_buffers() { + let backend = write_storage::TestWriteStorage::new(); + let runtime = backend.runtime(metadata::client().await); + let identity = DatasetIdentity::new(NamespacePath::root(), "multipart").unwrap(); + let dataset = DatasetClient::new(&runtime, identity.clone()).unwrap(); + dataset.create(context(1)).await.unwrap(); + let upload = OperationId::random(); + let digest = Md5::digest(b"retained").into(); + dataset + .create_upload(context(2), upload, 8, 3, digest, u64::MAX) + .await + .unwrap(); + for (index, part) in b"retained".chunks(3).enumerate() { + let buffer = runtime.prepare_framed(part).unwrap(); + dataset + .upload_framed_part(context(3), upload, u32::try_from(index).unwrap(), buffer) + .await + .unwrap(); + } + dataset.complete_upload(context(4), upload).await.unwrap(); + let location = DatasetUploads::location(&identity, upload).unwrap(); + let snapshot = composition::publish(&dataset, location, digest).await; + let mut read = dataset + .read_batch( + context(5), + BatchRequest { + snapshot, + sample_ids: vec![b"one".to_vec()], + fields: vec!["value".into()], + }, + ) + .await + .unwrap(); + let mut held = Vec::new(); + while let Some(part) = read.body.next().await.unwrap() { + held.push(part.buffer); + } + assert!(read.body.is_complete()); + drop(read); + runtime.shutdown().await.unwrap(); + assert_eq!( + held.iter() + .flat_map(|buffer| buffer.as_slice().iter().copied()) + .collect::>(), + b"retained" + ); +} From 6abf973e69318a172dd91159d75fb4f0ce2fcd3f Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 10:10:46 +0800 Subject: [PATCH 093/258] Remove redundant access reads and reuse bucket conflict values --- .../src/dataset/api/batch.rs | 4 +- .../src/dataset/api/v2/preview.rs | 1 - .../tests/dataset_preview_contract_test.rs | 5 + .../tests/dataset_read_query_contract_test.rs | 21 ++ .../dataset/design-crowdb-access-dataset.md | 12 ++ .../s3/design-crowdb-access-s3.md | 6 + doc/working/plan-access-simplification.md | 45 +++- .../src/authority/publication/legacy.rs | 79 ++++--- .../src/authority/publication/visibility.rs | 22 +- .../src/read_query/binding.rs | 1 - .../src/read_query/page.rs | 3 - .../tests/common/access_store.rs | 7 + .../tests/common/upload.rs | 7 + .../tests/legacy_read_cost_test.rs | 176 ++++++++++++++++ .../tests/read_query_cost_test.rs | 4 + lib/crowdb-access-s3/src/bucket.rs | 33 +-- .../tests/bucket_cas_cost_test.rs | 195 ++++++++++++++++++ .../tests/common/bucket_client.rs | 67 ++++++ lib/crowdb-sdk-native/src/dataset/read.rs | 16 +- 19 files changed, 643 insertions(+), 61 deletions(-) create mode 100644 lib/crowdb-access-dataset/tests/legacy_read_cost_test.rs create mode 100644 lib/crowdb-access-s3/tests/bucket_cas_cost_test.rs create mode 100644 lib/crowdb-access-s3/tests/common/bucket_client.rs diff --git a/app/crowdb-access-server/src/dataset/api/batch.rs b/app/crowdb-access-server/src/dataset/api/batch.rs index 164cf18c9..d782118c4 100644 --- a/app/crowdb-access-server/src/dataset/api/batch.rs +++ b/app/crowdb-access-server/src/dataset/api/batch.rs @@ -82,7 +82,9 @@ impl DatasetHttpService { let id = snapshot(&request.snapshot)?; let ids = request.ids()?; let authority = self.reads.authority(); - authority.check_snapshot_readable(identity, id).await?; + if ids.is_empty() || request.sample_id_limit.is_none() { + authority.check_snapshot_readable(identity, id).await?; + } let samples = if ids.is_empty() { Vec::new() } else if request.sample_id_limit.is_some() { diff --git a/app/crowdb-access-server/src/dataset/api/v2/preview.rs b/app/crowdb-access-server/src/dataset/api/v2/preview.rs index 5b0e2c8b5..2f7823775 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/preview.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/preview.rs @@ -73,7 +73,6 @@ impl DatasetHttpService { } read_body(request, false).await?; let authority = self.reads.authority(); - authority.check_snapshot_readable(identity, snapshot).await?; let mut rows = authority .resolve_inspection_batch_locators( identity, diff --git a/app/crowdb-access-server/tests/dataset_preview_contract_test.rs b/app/crowdb-access-server/tests/dataset_preview_contract_test.rs index 6aff90dbe..6f473e12d 100644 --- a/app/crowdb-access-server/tests/dataset_preview_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_preview_contract_test.rs @@ -93,6 +93,7 @@ async fn payload_preview_caps_bytes_across_parts_and_distinguishes_missing_tombs published["id"].as_str().unwrap() ); let upload_reads = store.upload_reads.load(std::sync::atomic::Ordering::Relaxed); + let admissions = store.admission_reads.load(std::sync::atomic::Ordering::Relaxed); let preview = response( client .get(&url) @@ -107,6 +108,10 @@ async fn payload_preview_caps_bytes_across_parts_and_distinguishes_missing_tombs assert_eq!(preview["length"], "100000"); assert_eq!(preview["truncated"], true); assert_eq!(preview["md5"], hex::encode(Md5::digest(&data))); + assert_eq!( + store.admission_reads.load(std::sync::atomic::Ordering::Relaxed) - admissions, + 1 + ); assert_eq!( store.upload_reads.load(std::sync::atomic::Ordering::Relaxed) - upload_reads, 3 diff --git a/app/crowdb-access-server/tests/dataset_read_query_contract_test.rs b/app/crowdb-access-server/tests/dataset_read_query_contract_test.rs index a6ba3165d..ed4ea8eb9 100644 --- a/app/crowdb-access-server/tests/dataset_read_query_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_read_query_contract_test.rs @@ -37,6 +37,27 @@ async fn split_read_continuation_replays_without_remote_progress_or_cleanup() { drop(client); test.stop().await; } + +#[tokio::test] +async fn unsplit_delivery_has_no_outer_duplicate_admission_reads() { + use std::sync::atomic::Ordering; + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start(store.clone()).await; + let client = Client::new(); + let (root, _) = common::fixture(&client, &test.origin).await; + let mut request = plan(); + request["split"] = Value::Null; + let binding = response(client.post(format!("{root}/reads/bind")).json(&request), 200).await; + let before = store.admission_reads.load(Ordering::Relaxed); + let writes = store.mutations.load(Ordering::Relaxed); + let page = batch(&client, &root, &binding["cursor"]).await; + assert_eq!(page["sample_ids"].as_array().unwrap().len(), 3); + // Selection and delivery still each own one bounded authority check. + assert_eq!(store.admission_reads.load(Ordering::Relaxed) - before, 2); + assert_eq!(store.mutations.load(Ordering::Relaxed), writes); + drop(client); + test.stop().await; +} fn plan() -> Value { json!({"snapshot":"latest","selection":{"kind":"all"},"fields":["value"],"batch_size":3, "partition":{"worker":0,"workers":1,"rank":0,"world_size":1},"shuffle":null, diff --git a/doc/design/access-server/dataset/design-crowdb-access-dataset.md b/doc/design/access-server/dataset/design-crowdb-access-dataset.md index 652a9ce0a..ed520d069 100644 --- a/doc/design/access-server/dataset/design-crowdb-access-dataset.md +++ b/doc/design/access-server/dataset/design-crowdb-access-dataset.md @@ -292,6 +292,18 @@ perform admission, and reclaimed parts fail instead of producing successful EOF. The internal legacy scan adapter also returns a caller-owned cursor without loading or persisting progress. Requests have no shared inactivity TTL. +Binding reuses its committed-Snapshot admission. Inspection-backed selection, +preview and nonempty batch delivery perform admission inside their bounded +resolver; wrappers do not repeat the same deletion lookup. Empty and legacy +batch entry points retain independent admission. Selection and delivery remain +separate bounded operations, each with its own admission. + +Legacy Snapshots without a stored visibility proof are checked against head and +anchor ancestry using bounded, cancellable reads. The traversal validates the +Snapshot identity and publication state, rejects cycles and never fills a +persistent proof cache. Existing proofs remain readable; modern unpublished +candidates cannot use legacy ancestry as evidence of publication. + ### 10.3 Retention and reclaim flow Reclaim atomically fences new reads, invalidates stale head publishers with a diff --git a/doc/design/access-server/s3/design-crowdb-access-s3.md b/doc/design/access-server/s3/design-crowdb-access-s3.md index a07b694d2..a488f9130 100644 --- a/doc/design/access-server/s3/design-crowdb-access-s3.md +++ b/doc/design/access-server/s3/design-crowdb-access-s3.md @@ -42,6 +42,12 @@ to stable identities, while object keys form an ordered namespace within a bucket identity. Object records select complete immutable data generations and carry the logical information needed for S3 reads and integrity checks. +Bucket creation first attempts create-if-absent. A returned active mapping is +the result; a tombstone is replaced with a fresh bucket identity through CAS. +Definite conflicts reuse the returned mapping directly, with bounded attempts +and no confirmation GET. Missing observations permit create-if-absent again. +Transport errors propagate without treating an unknown result as a conflict. + S3 is authoritative for bucket and object visibility, overwrite behavior, listing position, multipart publication, deletion, and S3 credentials. It stores opaque data references rather than physical storage topology. diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index fc60f3c0e..8b2ed7355 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -165,6 +165,28 @@ Dataset expiry 依赖 upload/part 记录，Iceberg discovery/write sweep 依赖 都不是“封口 chunk 范围减去全部存活引用”的独立扫描器。删除写前 intent 后，写出 payload 而尚未保存结果就宕机会产生不可发现的范围。tree 内部 orphan 回收不能代替用户 payload GC。 +**继续核查（2026-10-24）**：生产 `chunkdb_rpc_service/mutations.rs::handle_delete_range` +明确返回 `Unimplemented`，没有更新 used-range/bitmap；`reclaim_location` 将其当作 Deferred。 +因此测试替身返回 Reclaimed 不能证明实际范围已释放。底层依赖是 +[R95](../backlog/R95-chunkdb-chunk-range-delete.md)（范围资格与孤儿发现）及 +[R92](../backlog/R92-chunkdb-in-chunk-gc.md)（混存 strip 的物理回收，当前仅占位设计）。 +不能用“把 RPC 改为成功”或把共享 chunk 整块删除替代实现；R95 旧文档的 reader/session +约束需按本计划已确认的删除 grace、part 期限重新整理。R168 的逐对象 cleanup record +也不是本计划自动批准的新状态。涉及新持久化资源字段先列出用途、写入频率和替代方案。 + +**需要 review 的物理资源事实（尚未实施）**：R95 若按范围逐步回收，需要在 chunk/strip +元数据中保留已经判定死亡的精确区间及布局代际。它表示“哪些物理字节已经不可再引用”， +不是 upload 的步骤、进度或 owner session；由后台 GC 按 chunk 合并写，前台上传不追加 +cleanup/journal RPC。现有 `usage_bitmap` 描述的粒度不能直接用于任意字节范围，不能把 +一个 frame 的删除向邻居取整。部分死亡 strip 暂时只记录死亡事实，释放整块分配或搬迁 +存活部分后才能报告物理 reclaimed；R92 负责后者及逻辑地址到新物理布局的映射。 + +失败流程：GC 先完成发布隔离与活引用检查 → 以 chunk revision 条件更新死亡范围/布局 → +物理释放。进程在二者间退出时，从资源事实重做释放；丢回复时重复操作不得伤及新代际。 +如果不保留死亡范围，每次必须重新从全部最终引用计算；可以先只释放完全无引用的整 chunk， +但长期混存 chunk 的孤儿空间不会收敛，不能据此宣称 R4/R92 完成。这个容量与实现复杂度 +取舍需要用户确认后再扩充底层持久化结构；独立的 RPC/CAS 清理不受此决策影响。 + **已确认的期限与剩余工程证明**：未发布 part 过期后重传；不支持无限期稍后提交。 仅在 Access 开始请求时检查期限仍有竞态：检查通过 → 发布 RPC 延迟 → GC 回收 → 迟到发布。 必须在最终权威发布边界拒绝过期候选，或以资源代际/封口隔离迟到发布；等待若干分钟本身 @@ -308,8 +330,6 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 ### P11 — S3 object / bucket 发布 -- [ ] **P11.1 修复 bucket tombstone CAS 回路**：直接消费 observed active mapping/tombstone， - 不接无意义 put-if-absent；保留名称唯一和删除重建代际。Files: S3 `bucket.rs`、`metadata/store.rs`。 - [ ] **P11.2 锁定普通对象最小路径**：普通 PUT/DELETE 已为无条件 mutation，不加 GET/CAS； 单独验证 conditional 请求和 multipart 发布，记录包括 bucket resolution 的端到端成本。Files: S3 `publication.rs`、`metadata/namespace.rs`、对象/桶测试。 @@ -405,6 +425,27 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 ## 已完成与验证记录 +- **2026-10-24，P11.1**：bucket 首次 PutIfAbsent 后，tombstone 冲突直接沿 observed CAS； + active 直接返回，missing 才重新 PutIfAbsent。名称唯一、删除重建新代际和八次上限保留； + unknown 直接报错，无额外确认 GET。连续四次 tombstone 冲突总计五次 mutation，零 GET。 + `bucket_cas_cost_test`、`bucket_namespace_test`、`publication_store_test` 共 12 项通过。 +- **2026-10-24，F2.1**：移除 bind、next、preview、非空 HTTP/native inspection batch 外层的 + 重复删除准入 GET；空/legacy batch 仍独立准入。bind 和 preview 各一次删除标记读取， + 96 个并发 next 共 96 次；无 Split 的 HTTP next 加交付由四次降到两次，零 mutation。 + selection→delivery 及大 group 多页复用尚未完成，仍由 F2/F3 跟踪。 + Dataset `read_query_cost_test`、`read_query_edges_test`、`inspection_read_test`、`read_query_test`、 + `split_read_query_test` 共 10 项；Access read-query/preview/batch cost 与 native read/payload + 共 13 项通过，包含删除后拒绝 continuation 和跨实例重放。 +- **2026-10-24，P10.3**：旧 Snapshot 无 LC proof 时共用有界只读 ancestry 判断，不再读时补写。 + 现存 LC 继续验证；目标 Snapshot 身份/Published 状态必须正确，新候选不得走旧格式旁路。 + inspection 共用调用者的总字节/时间/取消预算；已有 head 直接传入，不重复 GET。 + `legacy_read_cost_test`、`legacy_publication_test`、`inspection_legacy_test`、 + `candidate_publication_test`、`inspection_read_test` 共 17 项通过；覆盖重复/重建 authority + 读取零写、Prepared/Aborted/身份损坏/循环拒绝、预算/取消和新候选不可见。 +- 上述测试均经 `pixi run cargo test`，HTTP/native 测试先执行 `pixi run clean-env`。 + Dataset、S3、Access Server、native SDK 的 all-target/all-feature clippy、workspace fmt + 通过；没有新增锁、unsafe、过程记录或 wire 字段。成本是逻辑调用数，不是 TPS 实测。 + - **2026-10-24，P1.1**：删除 TableLoader 持久化投影 GET、miss 填充和验证回执代码。 冷读、重复读、32 个并发读取及 conditional GET 均无投影读取和 CAS；refs/all、ETag、 parser limits、损坏拒绝和 namespace/head 变化检查保留。历史投影不参与表读取，未批量删除。 diff --git a/lib/crowdb-access-dataset/src/authority/publication/legacy.rs b/lib/crowdb-access-dataset/src/authority/publication/legacy.rs index 080c3a949..f2e7a495f 100644 --- a/lib/crowdb-access-dataset/src/authority/publication/legacy.rs +++ b/lib/crowdb-access-dataset/src/authority/publication/legacy.rs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::super::{head::decode_head, VersionedRecord}; +use crate::inspection::{work::InspectionWork, InspectionError}; use crate::{ key, AuthorityError, CasOutcome, DatasetAuthority, DatasetError, DatasetIdentity, HeadRecord, MetadataScanBudget, PublicationState, ReadCancellation, SnapshotId, SnapshotRecord, StoredValue, @@ -76,62 +77,69 @@ impl DatasetAuthority { if self.snapshot_candidate(identity, snapshot).await?.is_some() { return Err(AuthorityError::SnapshotNotFound); } - if self.legacy_snapshot_visible(identity, snapshot).await? { - let bytes = serde_json::to_vec(&LegacyVisibility { version: 1, snapshot }) - .map_err(|_| AuthorityError::Corrupt)?; - return self - .put_commit_record(&visibility_key(identity, snapshot)?, &bytes) - .await; + let cancel = ReadCancellation::new(); + let mut work = InspectionWork::new( + self.store.as_ref(), + &cancel, + MetadataScanBudget { + items: 64, + bytes: 16 * 1024 * 1024, + timeout: Duration::from_secs(2), + }, + ) + .map_err(legacy_read_error)?; + let head = work + .get(&key::head(identity)?, 8192) + .await + .map_err(legacy_read_error)?; + if self + .legacy_snapshot_visible(identity, snapshot, head.as_ref(), &mut work) + .await? + { + return Ok(()); } Err(AuthorityError::SnapshotNotFound) } - async fn legacy_snapshot_visible( + pub(super) async fn legacy_snapshot_visible( &self, identity: &DatasetIdentity, target: SnapshotId, + head: Option<&StoredValue>, + work: &mut InspectionWork<'_>, ) -> Result { let mut roots = Vec::new(); - if let Some(head) = self.store.get(&key::head(identity)?).await? { + if let Some(head) = head { let head = decode_head(&head.bytes)?; roots.extend([head.latest, head.stable].into_iter().flatten()); } - if let Some(anchor) = self.store.get(&anchor_key(identity)?).await? { + if let Some(anchor) = work + .get(&anchor_key(identity)?, 2048) + .await + .map_err(legacy_read_error)? + { let anchor = decode_anchor(&anchor.bytes)?; roots.extend([anchor.head.latest, anchor.head.stable].into_iter().flatten()); } - let mut remaining = 16 * 1024 * 1024usize; let mut seen = std::collections::HashSet::new(); for root in roots { let mut current = Some(root); let mut chain = std::collections::HashSet::new(); while let Some(snapshot) = current { - if snapshot == target { - return Ok(true); - } if !chain.insert(snapshot) { return Err(AuthorityError::AncestryCycle); } if !seen.insert(snapshot) { break; } - if seen.len() > 64 || remaining == 0 { + if seen.len() > 64 { return Err(DatasetError::ReadLimitExceeded.into()); } - let stored = self - .store - .get_bounded( - &key::snapshot(identity, snapshot)?, - MetadataScanBudget { - items: 1, - bytes: remaining, - timeout: Duration::from_secs(2), - }, - &ReadCancellation::new(), - ) - .await? + let stored = work + .get(&key::snapshot(identity, snapshot)?, 16 * 1024 * 1024) + .await + .map_err(legacy_read_error)? .ok_or(AuthorityError::Corrupt)?; - remaining = remaining.saturating_sub(stored.bytes.len()); let record: VersionedRecord = bincode::DefaultOptions::new() .with_fixint_encoding() .with_limit(stored.bytes.len() as u64) @@ -142,9 +150,14 @@ impl DatasetAuthority { return Err(AuthorityError::Corrupt); } let record = record.payload; - if record.publication.state == PublicationState::Prepared { + if record.publication.snapshot != snapshot + || record.publication.state == PublicationState::Prepared + { return Err(AuthorityError::Corrupt); } + if snapshot == target { + return Ok(record.publication.state == PublicationState::Published); + } current = record.publication.parent; } } @@ -186,3 +199,13 @@ pub(super) fn validate_visibility(bytes: &[u8], snapshot: SnapshotId) -> Result< } Ok(()) } + +fn legacy_read_error(error: InspectionError) -> AuthorityError { + match error { + InspectionError::Authority(error) => error, + InspectionError::Store(error) => error.into(), + InspectionError::Limit => DatasetError::ReadLimitExceeded.into(), + InspectionError::Cancelled => DatasetError::ReadCancelled.into(), + _ => AuthorityError::Corrupt, + } +} diff --git a/lib/crowdb-access-dataset/src/authority/publication/visibility.rs b/lib/crowdb-access-dataset/src/authority/publication/visibility.rs index f8b834296..f72434a60 100644 --- a/lib/crowdb-access-dataset/src/authority/publication/visibility.rs +++ b/lib/crowdb-access-dataset/src/authority/publication/visibility.rs @@ -48,10 +48,10 @@ impl DatasetAuthority { } return Ok(true); } - if let Some(value) = work + let head = work .get(&crate::key::head(identity).map_err(AuthorityError::from)?, 8192) - .await? - { + .await?; + if let Some(value) = &head { if head_attempt(&value.bytes)?.is_some_and(|a| a.target == (CommitTarget::Snapshot { snapshot })) { return Ok(true); @@ -67,7 +67,21 @@ impl DatasetAuthority { super::legacy::validate_visibility(&value.bytes, snapshot)?; return Ok(true); } - Ok(false) + // A prepared modern candidate is not a legacy publication, even when + // a malformed old head happens to name it. + if work + .get( + &super::candidate::candidate_key(identity, snapshot).map_err(AuthorityError::from)?, + 4096, + ) + .await? + .is_some() + { + return Ok(false); + } + Ok(self + .legacy_snapshot_visible(identity, snapshot, head.as_ref(), work) + .await?) } } diff --git a/lib/crowdb-access-dataset/src/read_query/binding.rs b/lib/crowdb-access-dataset/src/read_query/binding.rs index 702bc9868..f4a377501 100644 --- a/lib/crowdb-access-dataset/src/read_query/binding.rs +++ b/lib/crowdb-access-dataset/src/read_query/binding.rs @@ -29,7 +29,6 @@ impl DatasetReads { self.authority .require_committed_snapshot(identity, snapshot) .await?; - self.authority.check_snapshot_readable(identity, snapshot).await?; let split = if let Some(request) = &mut spec.split { let revision = match request.revision { Some(revision) => revision, diff --git a/lib/crowdb-access-dataset/src/read_query/page.rs b/lib/crowdb-access-dataset/src/read_query/page.rs index 29ce73815..a0ca25021 100644 --- a/lib/crowdb-access-dataset/src/read_query/page.rs +++ b/lib/crowdb-access-dataset/src/read_query/page.rs @@ -22,9 +22,6 @@ impl DatasetReads { cancel: &ReadCancellation, ) -> Result { let mut cursor = Continuation::decode(identity, token)?; - self.authority - .check_snapshot_readable(identity, cursor.snapshot) - .await?; let (mut ids, after) = tokio::time::timeout(Duration::from_secs(2), self.group(identity, &cursor, cancel)) .await diff --git a/lib/crowdb-access-dataset/tests/common/access_store.rs b/lib/crowdb-access-dataset/tests/common/access_store.rs index 053ac8db7..1ab59931c 100644 --- a/lib/crowdb-access-dataset/tests/common/access_store.rs +++ b/lib/crowdb-access-dataset/tests/common/access_store.rs @@ -15,6 +15,7 @@ pub struct TestAccessStore { inner: dataset_store::TestDatasetStore, pub writes: AtomicU64, pub reads: AtomicU64, + pub admission_reads: AtomicU64, pub upload_conflicts: AtomicU64, } impl TestAccessStore { @@ -26,6 +27,9 @@ impl TestAccessStore { impl DatasetStore for TestAccessStore { async fn get(&self, key: &[u8]) -> Result, StoreError> { self.reads.fetch_add(1, Ordering::Relaxed); + if key.starts_with(b"DS\0L\0") { + self.admission_reads.fetch_add(1, Ordering::Relaxed); + } tokio::task::yield_now().await; self.inner.get(key).await } @@ -35,6 +39,9 @@ impl DatasetStore for TestAccessStore { budget: MetadataScanBudget, cancel: &ReadCancellation, ) -> Result, StoreError> { + if key.starts_with(b"DS\0L\0") { + self.admission_reads.fetch_add(1, Ordering::Relaxed); + } self.inner.get_bounded(key, budget, cancel).await } async fn scan_page( diff --git a/lib/crowdb-access-dataset/tests/common/upload.rs b/lib/crowdb-access-dataset/tests/common/upload.rs index ab4b1e31c..23b26dabd 100644 --- a/lib/crowdb-access-dataset/tests/common/upload.rs +++ b/lib/crowdb-access-dataset/tests/common/upload.rs @@ -20,12 +20,16 @@ pub struct TestStore { values: ArcSwap, StoredValue>>, pub reads: AtomicU64, pub upload_reads: AtomicU64, + pub admission_reads: AtomicU64, pub mutations: AtomicU64, } #[async_trait] impl DatasetStore for TestStore { async fn get(&self, key: &[u8]) -> Result, StoreError> { self.reads.fetch_add(1, Ordering::Relaxed); + if key.starts_with(b"DS\0L\0") { + self.admission_reads.fetch_add(1, Ordering::Relaxed); + } if key.starts_with(b"DS\0U\0") { self.upload_reads.fetch_add(1, Ordering::Relaxed); } @@ -37,6 +41,9 @@ impl DatasetStore for TestStore { budget: crowdb_access_dataset::MetadataScanBudget, cancel: &crowdb_access_dataset::ReadCancellation, ) -> Result, crowdb_access_dataset::StoreError> { + if key.starts_with(b"DS\0L\0") { + self.admission_reads.fetch_add(1, Ordering::Relaxed); + } budget.validate()?; if key.is_empty() || key.len() >= 64 * 1024 { return Err(crowdb_access_dataset::StoreError::Rejected); diff --git a/lib/crowdb-access-dataset/tests/legacy_read_cost_test.rs b/lib/crowdb-access-dataset/tests/legacy_read_cost_test.rs new file mode 100644 index 000000000..62755e21a --- /dev/null +++ b/lib/crowdb-access-dataset/tests/legacy_read_cost_test.rs @@ -0,0 +1,176 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/access_store.rs"] +mod access_store; +#[path = "common/legacy.rs"] +mod legacy; +#[path = "common/publication.rs"] +mod publication; + +use crowdb_access_dataset::*; +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; + +async fn fixture() -> ( + Arc, + DatasetIdentity, + SnapshotId, + SnapshotId, +) { + let store = access_store::TestAccessStore::shared(); + let authority = DatasetAuthority::new(store.clone()); + let identity = DatasetIdentity::new(NamespacePath::root(), "legacy-read").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + let root = authority + .publish_snapshot(&identity, None, publication::publication_manifest(1)) + .await + .unwrap(); + let child = authority + .publish_snapshot(&identity, Some(root), publication::publication_manifest(2)) + .await + .unwrap(); + legacy::emulate_legacy(store.as_ref(), &authority, &identity, &[root, child], child).await; + (store, identity, root, child) +} + +fn budget(bytes: usize) -> MetadataScanBudget { + MetadataScanBudget { + items: 20, + bytes, + timeout: Duration::from_secs(2), + } +} + +#[tokio::test] +async fn repeated_legacy_authority_and_inspection_reads_never_fill_visibility_records() { + let (store, identity, root, child) = fixture().await; + let writes = store.writes.load(Ordering::Relaxed); + for _ in 0..2 { + let authority = DatasetAuthority::new(store.clone()); + for snapshot in [root, child] { + authority + .require_committed_snapshot(&identity, snapshot) + .await + .unwrap(); + let summary = authority + .snapshot_inspection_summary(&identity, snapshot, budget(1 << 20), &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(summary.snapshot, snapshot); + let key = [ + b"DS\0LC\0".as_slice(), + identity.key().unwrap().as_slice(), + snapshot.as_bytes(), + ] + .concat(); + assert!(store.get(&key).await.unwrap().is_none()); + } + } + assert_eq!(store.writes.load(Ordering::Relaxed), writes); +} + +async fn replace_snapshot( + store: &dyn DatasetStore, + identity: &DatasetIdentity, + snapshot: SnapshotId, + update: impl FnOnce(&mut SnapshotRecord), +) { + let key = [ + b"DS\0S\0".as_slice(), + identity.key().unwrap().as_slice(), + snapshot.as_bytes(), + ] + .concat(); + let stored = store.get(&key).await.unwrap().unwrap(); + let (version, mut record): (u16, SnapshotRecord) = bincode::deserialize(&stored.bytes).unwrap(); + update(&mut record); + assert!(matches!( + store + .compare_exchange( + &key, + Some(&stored.bytes), + &bincode::serialize(&(version, record)).unwrap() + ) + .await + .unwrap(), + CasOutcome::Applied(_) + )); +} + +#[tokio::test] +async fn legacy_fallback_rejects_unpublished_corrupt_and_cyclic_authority_without_writes() { + for case in 0..4 { + let (store, identity, _, child) = fixture().await; + replace_snapshot(store.as_ref(), &identity, child, |record| match case { + 0 => record.publication.state = PublicationState::Prepared, + 1 => record.publication.state = PublicationState::Aborted, + 2 => record.publication.snapshot = SnapshotId::random(), + _ => record.publication.parent = Some(child), + }) + .await; + let target = if case == 3 { SnapshotId::random() } else { child }; + let authority = DatasetAuthority::new(store.clone()); + let writes = store.writes.load(Ordering::Relaxed); + assert!( + authority + .require_committed_snapshot(&identity, target) + .await + .is_err(), + "case {case}" + ); + assert!( + authority + .snapshot_inspection_summary(&identity, target, budget(1 << 20), &ReadCancellation::new()) + .await + .is_err(), + "case {case}" + ); + assert_eq!(store.writes.load(Ordering::Relaxed), writes); + } +} + +#[tokio::test] +async fn legacy_inspection_fallback_obeys_aggregate_budget_and_cancellation() { + let (store, identity, root, _) = fixture().await; + let authority = DatasetAuthority::new(store.clone()); + let writes = store.writes.load(Ordering::Relaxed); + assert!(authority + .snapshot_inspection_summary(&identity, root, budget(32), &ReadCancellation::new()) + .await + .is_err()); + let cancelled = ReadCancellation::new(); + cancelled.cancel(); + assert!(authority + .snapshot_inspection_summary(&identity, root, budget(1 << 20), &cancelled) + .await + .is_err()); + assert_eq!(store.writes.load(Ordering::Relaxed), writes); +} + +#[tokio::test] +async fn unpublished_modern_candidate_cannot_fall_back_to_legacy_visibility() { + let (store, identity, _, _) = fixture().await; + let authority = DatasetAuthority::new(store.clone()); + let snapshot = authority + .prepare_snapshot_candidate( + &identity, + None, + publication::publication_manifest(3), + OperationId::random(), + ) + .await + .unwrap(); + let writes = store.writes.load(Ordering::Relaxed); + assert!(authority + .require_committed_snapshot(&identity, snapshot) + .await + .is_err()); + assert!(authority + .snapshot_inspection_summary(&identity, snapshot, budget(1 << 20), &ReadCancellation::new()) + .await + .is_err()); + assert_eq!(store.writes.load(Ordering::Relaxed), writes); +} diff --git a/lib/crowdb-access-dataset/tests/read_query_cost_test.rs b/lib/crowdb-access-dataset/tests/read_query_cost_test.rs index 8b285706a..8f8167de9 100644 --- a/lib/crowdb-access-dataset/tests/read_query_cost_test.rs +++ b/lib/crowdb-access-dataset/tests/read_query_cost_test.rs @@ -34,15 +34,19 @@ async fn binding_concurrent_fetch_and_cross_instance_replay_never_write() { shuffle: None, split: None, }; + let admissions = store.admission_reads.load(Ordering::Relaxed); let bound = reads.bind(&identity, spec.clone()).await.unwrap(); + assert_eq!(store.admission_reads.load(Ordering::Relaxed) - admissions, 1); let other = reads.bind(&identity, spec).await.unwrap(); assert_eq!(bound.cursor, other.cursor); let cancel = ReadCancellation::new(); + let admissions = store.admission_reads.load(Ordering::Relaxed); let pages = futures::future::join_all((0..96).map(|_| reads.next(&identity, &bound.cursor, &cancel))).await; for page in &pages { assert_eq!(page.as_ref().unwrap().ids, pages[0].as_ref().unwrap().ids); } + assert_eq!(store.admission_reads.load(Ordering::Relaxed) - admissions, 96); let restarted = DatasetReads::new(authority); restarted.next(&identity, &bound.cursor, &cancel).await.unwrap(); assert_eq!(store.writes.load(Ordering::Relaxed), writes); diff --git a/lib/crowdb-access-s3/src/bucket.rs b/lib/crowdb-access-s3/src/bucket.rs index e5884b235..b95a85e06 100644 --- a/lib/crowdb-access-s3/src/bucket.rs +++ b/lib/crowdb-access-s3/src/bucket.rs @@ -52,6 +52,7 @@ pub async fn create_bucket( name: &[u8], ) -> Result { let key = MetadataKey::bucket_name(tenant, name)?; + let mut expected = None; for _ in 0..8 { let record = BucketNameRecord { tenant: tenant.clone(), @@ -59,20 +60,26 @@ pub async fn create_bucket( bucket_id: generator.next_id(), tombstone: false, }; - match store.put_if_absent(key.clone(), record.encode()).await? { - PutIfAbsentOutcome::Inserted { .. } => return Ok(record.bucket_id), - PutIfAbsentOutcome::Existing(existing) => { - let observed = BucketNameRecord::decode(&existing.value)?; - if !observed.tombstone { - return Ok(observed.bucket_id); - } - if store - .compare_exchange(key.clone(), existing.value, record.encode()) - .await? - { - return Ok(record.bucket_id); - } + let observed = if let Some(expected) = expected.take() { + let (applied, observed) = store + .compare_exchange_observed(key.clone(), expected, record.encode()) + .await?; + if applied { + return Ok(record.bucket_id); } + observed + } else { + match store.put_if_absent(key.clone(), record.encode()).await? { + PutIfAbsentOutcome::Inserted { .. } => return Ok(record.bucket_id), + PutIfAbsentOutcome::Existing(existing) => Some(existing), + } + }; + if let Some(existing) = observed { + let observed = BucketNameRecord::decode(&existing.value)?; + if !observed.tombstone { + return Ok(observed.bucket_id); + } + expected = Some(existing.value); } } Err(MetadataStoreError::Result("repeated bucket-name conflict").into()) diff --git a/lib/crowdb-access-s3/tests/bucket_cas_cost_test.rs b/lib/crowdb-access-s3/tests/bucket_cas_cost_test.rs new file mode 100644 index 000000000..d9ee696f4 --- /dev/null +++ b/lib/crowdb-access-s3/tests/bucket_cas_cost_test.rs @@ -0,0 +1,195 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/bucket_client.rs"] +mod bucket_client; + +use async_trait::async_trait; +use crowdb_access_s3::{ + bucket::{create_bucket, BucketIdGenerator}, + metadata::{BucketId, BucketNameRecord, ChunkKvMetadataStore, MetadataKey, TenantId}, +}; +use crowdb_chunk_kv_client::{ChunkKvTransport, Result}; +use crowdb_protocol::chunk_kv::*; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +enum TestReply { + Conflict(Option), + Applied, + Unknown, +} +struct TestTransport { + replies: Vec, + calls: AtomicUsize, +} + +#[async_trait] +impl ChunkKvTransport for TestTransport { + async fn point(&self, _: &str, request: &PointRequest) -> Result { + let index = self.calls.fetch_add(1, Ordering::Relaxed); + let tenant = TenantId::new(b"tenant".to_vec()).unwrap(); + assert_eq!( + request.operation.key(), + MetadataKey::bucket_name(&tenant, b"bucket").unwrap() + ); + let expected = index.checked_sub(1).and_then(|i| match &self.replies[i] { + TestReply::Conflict(record) => record.as_ref(), + _ => panic!("operation retried after success or unknown response"), + }); + let bytes = match (&request.operation, expected) { + (PointOperation::PutIfAbsent { value, .. }, None) => value, + ( + PointOperation::CompareExchange { + condition: RpcCompareCondition::Value(actual), + value, + .. + }, + Some(expected), + ) => { + assert_eq!( + actual, + &expected.encode(), + "must retry against the latest returned observation" + ); + value + } + _ => panic!("unexpected extra RPC or stale base: {:?}", request.operation), + }; + let candidate = BucketNameRecord::decode(bytes).unwrap(); + assert!(!candidate.tombstone); + assert_eq!( + candidate.bucket_id, + BucketId::new([u8::try_from(index + 1).unwrap(); 16]) + ); + let result = match self.replies.get(index).expect("unexpected extra RPC") { + TestReply::Conflict(record) => OperationResult::Mutation { + applied: false, + revision: None, + observed: record.as_ref().map(|record| RpcValue { + key: request.operation.key().to_vec(), + value: record.encode(), + revision: index as u64 + 1, + }), + }, + TestReply::Applied => OperationResult::Mutation { + applied: true, + revision: Some(100), + observed: None, + }, + TestReply::Unknown => OperationResult::Value(None), + }; + Ok(ChunkKvResponse { + map_revision: 1, + journal_position: None, + result: Ok(result), + }) + } + async fn seek(&self, _: &str, _: &SeekRequest) -> Result { + panic!("unexpected seek") + } + async fn scan(&self, _: &str, _: &ScanRequest) -> Result { + panic!("unexpected scan") + } +} + +#[derive(Default)] +struct TestIds(AtomicUsize); +impl BucketIdGenerator for TestIds { + fn next_id(&self) -> BucketId { + BucketId::new([u8::try_from(self.0.fetch_add(1, Ordering::Relaxed) + 1).unwrap(); 16]) + } +} + +fn record(id: u8, tombstone: bool) -> BucketNameRecord { + BucketNameRecord { + tenant: TenantId::new(b"tenant".to_vec()).unwrap(), + name: b"bucket".to_vec(), + bucket_id: BucketId::new([id; 16]), + tombstone, + } +} + +async fn run(replies: Vec, expected: Option) { + let count = replies.len(); + let transport = Arc::new(TestTransport { + replies, + calls: AtomicUsize::new(0), + }); + let client = Arc::new(bucket_client::client(transport.clone())); + client.refresh_catalog().await.unwrap(); + let result = create_bucket( + &ChunkKvMetadataStore::new(client), + &TestIds::default(), + &record(64, true).tenant, + b"bucket", + ) + .await; + match expected { + Some(id) => assert_eq!(result.unwrap(), id), + None => assert!(result.is_err()), + } + assert_eq!(transport.calls.load(Ordering::Relaxed), count); +} + +#[tokio::test] +async fn new_or_existing_bucket_needs_one_conditional_rpc() { + run(vec![TestReply::Applied], Some(BucketId::new([1; 16]))).await; + run( + vec![TestReply::Conflict(Some(record(64, false)))], + Some(BucketId::new([64; 16])), + ) + .await; +} + +#[tokio::test] +async fn every_tombstone_conflict_reuses_observed_without_get_or_repeated_create() { + let mut replies: Vec<_> = (64..68) + .map(|id| TestReply::Conflict(Some(record(id, true)))) + .collect(); + replies.push(TestReply::Applied); + run(replies, Some(BucketId::new([5; 16]))).await; +} + +#[tokio::test] +async fn concurrent_creator_wins_without_an_extra_rpc() { + run( + vec![ + TestReply::Conflict(Some(record(64, true))), + TestReply::Conflict(Some(record(65, false))), + ], + Some(BucketId::new([65; 16])), + ) + .await; +} + +#[tokio::test] +async fn missing_mapping_retries_create_and_never_reuses_an_old_bucket_identity() { + run( + vec![ + TestReply::Conflict(Some(record(64, true))), + TestReply::Conflict(None), + TestReply::Applied, + ], + Some(BucketId::new([3; 16])), + ) + .await; +} + +#[tokio::test] +async fn conflict_budget_and_unknown_response_do_not_trigger_confirmation_reads() { + run( + (64..72) + .map(|id| TestReply::Conflict(Some(record(id, true)))) + .collect(), + None, + ) + .await; + run( + vec![TestReply::Conflict(Some(record(64, true))), TestReply::Unknown], + None, + ) + .await; +} diff --git a/lib/crowdb-access-s3/tests/common/bucket_client.rs b/lib/crowdb-access-s3/tests/common/bucket_client.rs new file mode 100644 index 000000000..daf4c15c4 --- /dev/null +++ b/lib/crowdb-access-s3/tests/common/bucket_client.rs @@ -0,0 +1,67 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use crowdb_chunk_kv_client::{ + ChunkKvClient, ChunkKvRangeCatalogSource, ChunkKvTransport, ClientConfig, Result, +}; +use crowdb_protocol::{chunk_kv::*, chunk_stream::StreamName}; +use std::{sync::Arc, time::Duration}; + +struct TestCatalog(ChunkKvRangeCatalogHead, Vec); +#[async_trait] +impl ChunkKvRangeCatalogSource for TestCatalog { + async fn load(&self) -> Result<(ChunkKvRangeCatalogHead, Vec)> { + Ok((self.0.clone(), self.1.clone())) + } +} + +pub fn client(transport: Arc) -> ChunkKvClient { + let mut page = ChunkKvRangeCatalogPage { + generation: 1, + page_index: 0, + entries: vec![ChunkKvRangeCatalogEntry { + partition_id: Id128 { high: 1, low: 1 }, + range: KeyRange { + start: Vec::new(), + end: None, + }, + owner: OwnerDescriptor { + instance_id: 1, + rpc_endpoint: "owner".into(), + }, + owner_epoch: 1, + state: ChunkKvRangeCatalogPartitionState::Serving, + artifact: PartitionArtifact { + tree_id: 1, + stream_name: StreamName { high: 1, low: 1 }, + tail_overlay: None, + }, + transition_id: None, + }], + checksum: [0; 32], + }; + page.seal().expect("catalog page"); + let mut head = ChunkKvRangeCatalogHead { + generation: 1, + previous_generation: None, + pages: vec![ChunkKvRangeCatalogPageRef { + page_generation: 1, + page_index: 0, + first_key: Vec::new(), + page_checksum: page.checksum, + }], + checksum: [0; 32], + }; + head.seal().expect("catalog head"); + ChunkKvClient::new( + ClientConfig { + operation_timeout: Duration::from_secs(1), + retry_backoff: Duration::from_millis(1), + ..ClientConfig::default() + }, + Arc::new(TestCatalog(head, vec![page])), + transport, + ) + .expect("client") +} diff --git a/lib/crowdb-sdk-native/src/dataset/read.rs b/lib/crowdb-sdk-native/src/dataset/read.rs index da43b8b19..a3d225bcd 100644 --- a/lib/crowdb-sdk-native/src/dataset/read.rs +++ b/lib/crowdb-sdk-native/src/dataset/read.rs @@ -90,15 +90,15 @@ impl DatasetClient { next: Option, end: bool, ) -> Result { - request - .run(async { - Ok(self - .authority - .check_snapshot_readable(&self.identity, batch.snapshot) - .await?) - }) - .await?; let samples = if batch.sample_ids.is_empty() { + request + .run(async { + Ok(self + .authority + .check_snapshot_readable(&self.identity, batch.snapshot) + .await?) + }) + .await?; Vec::new() } else { request From 571014316942784d2c253ef58beaa18ac18df181 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 10:15:18 +0800 Subject: [PATCH 094/258] Reuse dataset selection planning within each read request --- .../dataset/design-crowdb-access-dataset.md | 5 + doc/working/plan-access-simplification.md | 15 +- .../src/inspection/evaluate.rs | 118 ++++++++---- .../src/read_query/page.rs | 26 ++- .../tests/common/access_store.rs | 15 ++ .../tests/read_query_planning_test.rs | 181 ++++++++++++++++++ 6 files changed, 315 insertions(+), 45 deletions(-) create mode 100644 lib/crowdb-access-dataset/tests/read_query_planning_test.rs diff --git a/doc/design/access-server/dataset/design-crowdb-access-dataset.md b/doc/design/access-server/dataset/design-crowdb-access-dataset.md index ed520d069..1cba2546d 100644 --- a/doc/design/access-server/dataset/design-crowdb-access-dataset.md +++ b/doc/design/access-server/dataset/design-crowdb-access-dataset.md @@ -263,6 +263,11 @@ The token binds Dataset, Snapshot, Split, selection, projection, worker partitio shuffle seed/group size and position. It is a cursor, not an authorization grant. Selection uses bounded inspection windows without fetching a full Manifest. +Within one next request, selection windows share admission, immutable summaries, +schema and compiled predicates. Explicit IDs are validated once. This prepared +query is local to the request and binds its authority, Dataset and Snapshot; +independent page requests still perform their own admission. Per-window storage +budgets and the overall group deadline remain enforced. Split filtering precedes deterministic shuffle and worker partitioning. The cursor carries a group start and offset, not the Dataset's selected IDs. A partly consumed shuffle group is reconstructed from immutable metadata; this bounds memory but diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 8b2ed7355..857320df2 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -174,7 +174,7 @@ Dataset expiry 依赖 upload/part 记录，Iceberg discovery/write sweep 依赖 约束需按本计划已确认的删除 grace、part 期限重新整理。R168 的逐对象 cleanup record 也不是本计划自动批准的新状态。涉及新持久化资源字段先列出用途、写入频率和替代方案。 -**需要 review 的物理资源事实（尚未实施）**：R95 若按范围逐步回收，需要在 chunk/strip +**已确认的物理资源事实（2026-10-24，待实施）**：R95 若按范围逐步回收，需要在 chunk/strip 元数据中保留已经判定死亡的精确区间及布局代际。它表示“哪些物理字节已经不可再引用”， 不是 upload 的步骤、进度或 owner session；由后台 GC 按 chunk 合并写，前台上传不追加 cleanup/journal RPC。现有 `usage_bitmap` 描述的粒度不能直接用于任意字节范围，不能把 @@ -184,8 +184,8 @@ cleanup/journal RPC。现有 `usage_bitmap` 描述的粒度不能直接用于任 失败流程：GC 先完成发布隔离与活引用检查 → 以 chunk revision 条件更新死亡范围/布局 → 物理释放。进程在二者间退出时，从资源事实重做释放；丢回复时重复操作不得伤及新代际。 如果不保留死亡范围，每次必须重新从全部最终引用计算；可以先只释放完全无引用的整 chunk， -但长期混存 chunk 的孤儿空间不会收敛，不能据此宣称 R4/R92 完成。这个容量与实现复杂度 -取舍需要用户确认后再扩充底层持久化结构；独立的 RPC/CAS 清理不受此决策影响。 +但长期混存 chunk 的孤儿空间不会收敛，不能据此宣称 R4/R92 完成。用户已确认采用精确范围和布局代际的持久化资源状态，继续补齐范围回收； +不采用仅回收整 chunk 的替代方案，不再等待此项确认。 **已确认的期限与剩余工程证明**：未发布 part 过期后重传；不支持无限期稍后提交。 仅在 Access 开始请求时检查期限仍有竞态：检查通过 → 发布 RPC 延迟 → GC 回收 → 迟到发布。 @@ -425,6 +425,15 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 ## 已完成与验证记录 +- **2026-10-24，F3.1**：同一 next 多窗口使用请求局部的 selection 准备结果，绑定 + authority/Dataset/Snapshot/规则，复用准入、summary、schema 和 explicit-ID 校验。 + 81 个候选、五个窗口、两个 explicit ID：独立逐页调用实测 25 bounded GET + 10 scan， + 请求内复用实测 5 bounded GET + 6 scan；零 mutation。新请求重新准入，不共享缓存或 session。 + 小 batch 跨请求重建 group 仍未处理。每窗口预算、group 两秒/5000 页/8 MiB ID 和取消保留。 + `read_query_planning_test`、`read_query_cost_test`、`read_query_edges_test`、`split_read_query_test`、 + `inspection_pages_test`、`selection_v2_test` 共 16 项通过；HTTP read-query 与 native read + 共 7 项通过。三 crate all-target/all-feature clippy、workspace fmt 通过，均经 pixi。 + - **2026-10-24，P11.1**：bucket 首次 PutIfAbsent 后，tombstone 冲突直接沿 observed CAS； active 直接返回，missing 才重新 PutIfAbsent。名称唯一、删除重建新代际和八次上限保留； unknown 直接报错，无额外确认 GET。连续四次 tombstone 冲突总计五次 mutation，零 GET。 diff --git a/lib/crowdb-access-dataset/src/inspection/evaluate.rs b/lib/crowdb-access-dataset/src/inspection/evaluate.rs index 80adec094..447ec0400 100644 --- a/lib/crowdb-access-dataset/src/inspection/evaluate.rs +++ b/lib/crowdb-access-dataset/src/inspection/evaluate.rs @@ -3,7 +3,7 @@ use super::{ content::validate_page_budget, cursor::CursorScope, key, work::InspectionWork, FieldMetadata, - FieldStorageKind, InspectionError, InspectionPage, + FieldStorageKind, InspectionError, InspectionPage, SnapshotSummary, }; use crate::{ selection::EffectiveSelection, AuthorityError, DatasetAuthority, DatasetError, DatasetIdentity, @@ -28,15 +28,27 @@ impl DatasetAuthority { ) -> Result>, InspectionError> { validate_page_budget(budget)?; let mut work = InspectionWork::new(self.store.as_ref(), cancel, budget)?; - let summary = self.ready_inspection(identity, snapshot, &mut work).await?; + self.prepare_inspection_selection(identity, snapshot, selection, &mut work) + .await? + .page(after, budget.items, &mut work) + .await + } + + pub(crate) async fn prepare_inspection_selection<'a>( + &'a self, + identity: &'a DatasetIdentity, + snapshot: SnapshotId, + selection: &'a Selection, + work: &mut InspectionWork<'_>, + ) -> Result, InspectionError> { + let summary = self.ready_inspection(identity, snapshot, work).await?; let schema = self - .inspection_schema_definitions(identity, &summary, &mut work) + .inspection_schema_definitions(identity, &summary, work) .await?; let compiled = EffectiveSelection::new(selection, &schema).map_err(AuthorityError::from)?; let prefix = key::scope(identity, snapshot, b'M').map_err(AuthorityError::from)?; let query = serde_json::to_vec(selection).map_err(|_| InspectionError::Corrupt)?; let cursor = CursorScope::new(&prefix, &query); - let after = cursor.decode(after)?; for id in &compiled.explicit_ids { let mut key = prefix.clone(); key::component(&mut key, id); @@ -50,34 +62,13 @@ impl DatasetAuthority { return Err(InspectionError::Corrupt); } } - let members = self - .inspection_member_window(identity, snapshot, after, budget.items, &mut work) - .await?; - let mut summaries = BTreeMap::from([(snapshot, summary)]); - let mut entries = Vec::new(); - for id in members.entries { - let mut values = BTreeMap::new(); - for field in &compiled.fields { - if let Some(metadata) = self - .effective_field_metadata(identity, snapshot, &id, field, &mut summaries, &mut work) - .await? - { - if metadata.storage == FieldStorageKind::Inline { - values.insert( - field.clone(), - self.inspection_inline_value(identity, &id, &metadata, &mut work) - .await?, - ); - } - } - } - if compiled.matches(&id, &values) { - entries.push(id); - } - } - Ok(InspectionPage { - entries, - after: cursor.encode(members.after)?, + Ok(PreparedInspectionSelection { + authority: self, + identity, + snapshot, + compiled, + cursor, + summaries: BTreeMap::from([(snapshot, summary)]), }) } @@ -110,3 +101,66 @@ impl DatasetAuthority { Ok(locator) } } + +/// Local to one bounded request; its authority and immutable query cannot be replaced. +pub(crate) struct PreparedInspectionSelection<'a> { + authority: &'a DatasetAuthority, + identity: &'a DatasetIdentity, + snapshot: SnapshotId, + compiled: EffectiveSelection<'a>, + cursor: CursorScope, + summaries: BTreeMap, +} +impl PreparedInspectionSelection<'_> { + pub(crate) async fn page( + &mut self, + after: Option>, + items: usize, + work: &mut InspectionWork<'_>, + ) -> Result>, InspectionError> { + let members = self + .authority + .inspection_member_window( + self.identity, + self.snapshot, + self.cursor.decode(after)?, + items, + work, + ) + .await?; + let mut entries = Vec::new(); + for id in members.entries { + let mut values = BTreeMap::new(); + for field in &self.compiled.fields { + if let Some(metadata) = self + .authority + .effective_field_metadata( + self.identity, + self.snapshot, + &id, + field, + &mut self.summaries, + work, + ) + .await? + { + if metadata.storage == FieldStorageKind::Inline { + values.insert( + field.clone(), + self.authority + .inspection_inline_value(self.identity, &id, &metadata, work) + .await?, + ); + } + } + } + if self.compiled.matches(&id, &values) { + entries.push(id); + } + } + Ok(InspectionPage { + entries, + after: self.cursor.encode(members.after)?, + }) + } +} diff --git a/lib/crowdb-access-dataset/src/read_query/page.rs b/lib/crowdb-access-dataset/src/read_query/page.rs index a0ca25021..ef155b221 100644 --- a/lib/crowdb-access-dataset/src/read_query/page.rs +++ b/lib/crowdb-access-dataset/src/read_query/page.rs @@ -113,6 +113,18 @@ impl DatasetReads { } else { None }; + let mut selection_work = + InspectionWork::new(self.authority.store.as_ref(), cancel, budget()).map_err(inspection_error)?; + let mut selection = self + .authority + .prepare_inspection_selection( + identity, + cursor.snapshot, + &cursor.spec.selection, + &mut selection_work, + ) + .await + .map_err(inspection_error)?; for _ in 0..5000 { if cancel.is_cancelled() { return Err(DatasetError::ReadCancelled.into()); @@ -123,16 +135,8 @@ impl DatasetReads { let mut limits = budget(); limits.items = (size - ids.len()).min(20); limits.timeout = remaining; - let page = self - .authority - .inspection_selection_page( - identity, - cursor.snapshot, - &cursor.spec.selection, - after, - limits, - cancel, - ) + let page = selection + .page(after, limits.items, &mut selection_work) .await .map_err(inspection_error)?; let mut work = InspectionWork::new(self.authority.store.as_ref(), cancel, limits) @@ -160,6 +164,8 @@ impl DatasetReads { if ids.len() == size || after.is_none() { return Ok((ids, after)); } + selection_work = InspectionWork::new(self.authority.store.as_ref(), cancel, limits) + .map_err(inspection_error)?; } Err(DatasetError::ReadLimitExceeded.into()) } diff --git a/lib/crowdb-access-dataset/tests/common/access_store.rs b/lib/crowdb-access-dataset/tests/common/access_store.rs index 1ab59931c..0aa22c324 100644 --- a/lib/crowdb-access-dataset/tests/common/access_store.rs +++ b/lib/crowdb-access-dataset/tests/common/access_store.rs @@ -16,6 +16,9 @@ pub struct TestAccessStore { pub writes: AtomicU64, pub reads: AtomicU64, pub admission_reads: AtomicU64, + pub bounded_reads: AtomicU64, + pub schema_scans: AtomicU64, + pub member_scans: AtomicU64, pub upload_conflicts: AtomicU64, } impl TestAccessStore { @@ -39,6 +42,7 @@ impl DatasetStore for TestAccessStore { budget: MetadataScanBudget, cancel: &ReadCancellation, ) -> Result, StoreError> { + self.bounded_reads.fetch_add(1, Ordering::Relaxed); if key.starts_with(b"DS\0L\0") { self.admission_reads.fetch_add(1, Ordering::Relaxed); } @@ -49,6 +53,17 @@ impl DatasetStore for TestAccessStore { request: &MetadataScan, cancel: &ReadCancellation, ) -> Result { + if request.prefix.starts_with(b"DS\0I\0") { + match request.prefix.last() { + Some(b'S') => { + self.schema_scans.fetch_add(1, Ordering::Relaxed); + } + Some(b'M') => { + self.member_scans.fetch_add(1, Ordering::Relaxed); + } + _ => {} + } + } self.inner.scan_page(request, cancel).await } async fn compare_exchange( diff --git a/lib/crowdb-access-dataset/tests/read_query_planning_test.rs b/lib/crowdb-access-dataset/tests/read_query_planning_test.rs new file mode 100644 index 000000000..60e8b8868 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/read_query_planning_test.rs @@ -0,0 +1,181 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/access_store.rs"] +mod access_store; +#[path = "common/publication.rs"] +mod publication; +use crowdb_access_dataset::*; +use std::sync::{atomic::Ordering, Arc}; + +async fn fixture() -> ( + Arc, + Arc, + DatasetIdentity, +) { + let store = access_store::TestAccessStore::shared(); + let authority = Arc::new(DatasetAuthority::new(store.clone())); + let identity = DatasetIdentity::new(NamespacePath::root(), "planning-cost").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + let mut manifest: ManifestRecord = bincode::deserialize(&publication::publication_manifest(1)).unwrap(); + let sample = manifest.samples[0].clone(); + manifest.samples = (0..81) + .map(|i| SampleRecord { + sample_id: format!("sample-{i:03}").into_bytes(), + ..sample.clone() + }) + .collect(); + authority + .publish_snapshot(&identity, None, bincode::serialize(&manifest).unwrap()) + .await + .unwrap(); + (store, authority, identity) +} + +fn spec(selection: Selection) -> ReadSpec { + ReadSpec { + snapshot: "latest".into(), + selection, + fields: vec!["marker".into()], + batch_size: 7, + partition: ReadPartition { + worker: 0, + workers: 1, + rank: 0, + world_size: 1, + }, + shuffle: Some(ReadShuffle { + seed: 42, + group_size: 81, + }), + split: None, + } +} + +fn costs(store: &access_store::TestAccessStore) -> [u64; 5] { + [ + &store.admission_reads, + &store.bounded_reads, + &store.schema_scans, + &store.member_scans, + &store.writes, + ] + .map(|counter| counter.load(Ordering::Relaxed)) +} +fn delta(store: &access_store::TestAccessStore, before: [u64; 5]) -> [u64; 5] { + let after = costs(store); + std::array::from_fn(|i| after[i] - before[i]) +} + +#[tokio::test] +async fn multi_window_shuffle_prepares_once_and_new_requests_recheck() { + let (store, authority, identity) = fixture().await; + let reads = DatasetReads::new(authority.clone()); + let selection = Selection::Or(vec![ + Selection::Prefix(b"sample-".to_vec()), + Selection::Explicit(vec![b"sample-000".to_vec(), b"sample-080".to_vec()]), + ]); + let bound = reads.bind(&identity, spec(selection.clone())).await.unwrap(); + let before = costs(&store); + let mut after = None; + let mut reference_ids = Vec::new(); + loop { + let page = authority + .inspection_selection_page( + &identity, + bound.snapshot, + &selection, + after, + MetadataScanBudget { + items: 20, + bytes: 1024 * 1024, + timeout: std::time::Duration::from_secs(2), + }, + &ReadCancellation::new(), + ) + .await + .unwrap(); + reference_ids.extend(page.entries); + after = page.after; + if after.is_none() { + break; + } + } + assert_eq!(reference_ids.len(), 81); + // Standalone page calls each perform their own admission and query preparation. + assert_eq!(delta(&store, before), [5, 25, 5, 5, 0]); + let before = costs(&store); + let page = reads + .next(&identity, &bound.cursor, &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(page.ids.len(), 7); + assert!(page.ids.iter().all(|id| reference_ids.contains(id))); + // Three admission/projection reads + two explicit IDs, one schema and five member windows. + assert_eq!(delta(&store, before), [1, 5, 1, 5, 0]); + + let restarted = DatasetReads::new(authority); + let before = costs(&store); + let replay = restarted + .next(&identity, &bound.cursor, &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(replay.ids, page.ids); + assert_eq!(replay.next, page.next); + assert_eq!(delta(&store, before), [1, 5, 1, 5, 0]); + + let before = costs(&store); + let second = restarted + .next(&identity, &page.next.unwrap(), &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(second.ids.len(), 7); + assert!(second.ids.iter().all(|id| !page.ids.contains(id))); + assert_eq!(delta(&store, before), [1, 5, 1, 5, 0]); +} + +#[tokio::test] +async fn empty_filtered_windows_keep_advancing_without_replanning() { + let (store, authority, identity) = fixture().await; + let reads = DatasetReads::new(authority); + let bound = reads + .bind( + &identity, + spec(Selection::And(vec![ + Selection::Explicit(vec![b"sample-080".to_vec()]), + Selection::FieldMatch { + field: "marker".into(), + value: vec![1], + }, + ])), + ) + .await + .unwrap(); + let before = costs(&store); + let page = reads + .next(&identity, &bound.cursor, &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(page.ids, vec![b"sample-080".to_vec()]); + assert!(page.next.is_none()); + // One explicit ID plus metadata and verified inline value for each of 81 candidates. + assert_eq!(delta(&store, before), [1, 4 + 2 * 81, 1, 5, 0]); +} + +#[tokio::test] +async fn unknown_explicit_id_fails_before_any_member_window() { + let (store, authority, identity) = fixture().await; + let reads = DatasetReads::new(authority); + let bound = reads + .bind(&identity, spec(Selection::Explicit(vec![b"missing".to_vec()]))) + .await + .unwrap(); + let before = costs(&store); + assert!(matches!( + reads + .next(&identity, &bound.cursor, &ReadCancellation::new()) + .await, + Err(AuthorityError::Invalid(DatasetError::SampleNotFound)) + )); + assert_eq!(delta(&store, before), [1, 4, 1, 0, 0]); +} From 4920cd0e8da8723a7c71b8d48778f92492623a49 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 10:30:09 +0800 Subject: [PATCH 095/258] Persist exact chunk range retirement with explicit physical completion --- app/crowdb-chunkdb/src/lifecycle/handler.rs | 2 + .../src/lifecycle/handler/range_retirement.rs | 24 ++ .../service/chunkdb_rpc_service/mutations.rs | 33 +- .../src/service/chunkdb_rpc_service/wire.rs | 2 + app/crowdb-chunkdb/src/storage.rs | 3 + .../src/storage/range_retirement.rs | 139 +++++++ .../src/storage/range_retirement/record.rs | 116 ++++++ .../tests/range_retirement_test.rs | 357 ++++++++++++++++++ doc/backlog/R95-chunkdb-chunk-range-delete.md | 54 ++- doc/design/chunkdb/design-crowdb-chunkdb.md | 23 +- doc/working/plan-access-simplification.md | 29 +- lib/crowdb-chunk-client/src/reclamation.rs | 13 +- .../tests/reclamation_test.rs | 29 +- .../src/rpc_transport.rs | 5 +- .../src/fb_wrappers/chunkdb.rs | 3 + lib/crowdb-protocol/src/fbs/chunkdb.fbs | 2 + lib/crowdb-protocol/src/types/chunkdb.rs | 7 +- .../tests/chunkdb_wrappers_test.rs | 18 + 18 files changed, 817 insertions(+), 42 deletions(-) create mode 100644 app/crowdb-chunkdb/src/lifecycle/handler/range_retirement.rs create mode 100644 app/crowdb-chunkdb/src/storage/range_retirement.rs create mode 100644 app/crowdb-chunkdb/src/storage/range_retirement/record.rs create mode 100644 app/crowdb-chunkdb/tests/range_retirement_test.rs diff --git a/app/crowdb-chunkdb/src/lifecycle/handler.rs b/app/crowdb-chunkdb/src/lifecycle/handler.rs index 8afeea280..237421a8b 100644 --- a/app/crowdb-chunkdb/src/lifecycle/handler.rs +++ b/app/crowdb-chunkdb/src/lifecycle/handler.rs @@ -90,6 +90,7 @@ impl From for LifecycleError { fn from(error: StoreError) -> Self { match error { StoreError::Conflict => Self::StateConflict, + StoreError::InvalidRange(message) => Self::InvalidRequest(message), other => Self::Storage(other), } } @@ -132,6 +133,7 @@ mod append; #[path = "lock_map.rs"] mod lock_map; mod recovery; +mod range_retirement; mod reservation; mod seal; diff --git a/app/crowdb-chunkdb/src/lifecycle/handler/range_retirement.rs b/app/crowdb-chunkdb/src/lifecycle/handler/range_retirement.rs new file mode 100644 index 000000000..9ab965475 --- /dev/null +++ b/app/crowdb-chunkdb/src/lifecycle/handler/range_retirement.rs @@ -0,0 +1,24 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{LifecycleError, LifecycleHandler}; +use crowdb_protocol::chunkdb::rpc::{DeleteChunkRangeRequest, DeleteChunkRangeResponse}; + +impl LifecycleHandler { + pub async fn delete_chunk_range( + &self, + request: &DeleteChunkRangeRequest, + ) -> Result { + let id = request + .chunk_id + .ok_or_else(|| LifecycleError::InvalidRequest("missing chunk identity".into()))?; + self.check_range(&id)?; + self.store + .retire_chunk_range(request) + .await + .map_err(|error| match error { + crate::storage::StoreError::ChunkNotFound => LifecycleError::ChunkNotFound, + other => other.into(), + }) + } +} diff --git a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs index 7b96e0081..d52c5fea9 100644 --- a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs +++ b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs @@ -994,13 +994,14 @@ impl ChunkdbRpcService { &self, req: ServerRequest, server: &Arc, - _request: RequestGuard, + mut request: RequestGuard, ) { let req_id = req.request_id; let create_nano = req.rpc_create_nano; let msg_type = FBMsgType::EDeleteChunkRangeResponse.0 as u16; let conn_handle_usize = req.conn_handle as usize; + let handler = Arc::clone(&self.handler); let server = Arc::clone(server); self.spawn(async move { let Ok(fb_req) = flatbuffers::root::(req.control()) else { @@ -1031,16 +1032,32 @@ impl ChunkdbRpcService { ); return; }; - let _ = (chunk_id, fb_req.chunk_offset(), fb_req.chunk_size()); - // The used-range lifecycle must exist before this RPC can safely - // reclaim data. Until then it must never mutate data. + let result = handler + .delete_chunk_range(&crowdb_protocol::chunkdb::rpc::DeleteChunkRangeRequest { + chunk_id: Some(chunk_id), + chunk_offset: fb_req.chunk_offset(), + chunk_size: fb_req.chunk_size(), + expected_modify_ts: fb_req.expected_modify_ts(), + }) + .await; + let (code, message, range_start, range_end, reclaimed) = match result { + Ok(result) => { + request.mark_success(); + (FBChunkdbRetCode::Success, None, 0, 0, result.reclaimed) + } + Err(error) => { + let (code, message, start, end) = map_error(&error); + (code, Some(message), start, end, false) + } + }; let ctrl = build_delete_range_response( req_id, create_nano, - FBChunkdbRetCode::Unimplemented, - Some("chunk range deletion is not implemented"), - 0, - 0, + code, + message.as_deref(), + range_start, + range_end, + reclaimed, ); submit_fb_response( &server, diff --git a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/wire.rs b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/wire.rs index 4f01a2961..d42cd699e 100644 --- a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/wire.rs +++ b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/wire.rs @@ -599,6 +599,7 @@ pub(super) fn build_delete_range_response( error_msg: Option<&str>, range_start: u32, range_end: u32, + reclaimed: bool, ) -> (Vec, usize) { let mut fbb = FlatBufferBuilder::new(); let err_off = error_msg.map(|m| fbb.create_string(m)); @@ -611,6 +612,7 @@ pub(super) fn build_delete_range_response( error_msg: err_off, range_start, range_end, + reclaimed, }, ); fbb.finish(off, None); diff --git a/app/crowdb-chunkdb/src/storage.rs b/app/crowdb-chunkdb/src/storage.rs index ae3fe7254..7fa124866 100644 --- a/app/crowdb-chunkdb/src/storage.rs +++ b/app/crowdb-chunkdb/src/storage.rs @@ -7,6 +7,7 @@ //! routed KV group. Chunk records have one explicit, strictly decoded envelope. mod chunk_codec; +pub mod range_retirement; pub mod reservation; mod scope; use chunk_codec::{decode_chunk, encode_chunk}; @@ -29,6 +30,8 @@ use crowdb_protocol::chunk_domain::ChunkDomain; /// Storage error. #[derive(Debug, thiserror::Error)] pub enum StoreError { + #[error("invalid retired chunk range: {0}")] + InvalidRange(String), #[error("chunk not found")] ChunkNotFound, #[error("chunk already exists")] diff --git a/app/crowdb-chunkdb/src/storage/range_retirement.rs b/app/crowdb-chunkdb/src/storage/range_retirement.rs new file mode 100644 index 000000000..1ac9501d7 --- /dev/null +++ b/app/crowdb-chunkdb/src/storage/range_retirement.rs @@ -0,0 +1,139 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Exact dead byte ranges, co-located with and fenced by the owning chunk. + +mod record; +pub use record::{ChunkRangeRetirement, RetiredByteRange}; + +use super::{chunk_key, decode_chunk, encode_chunk, ChunkStore, Result, StoreError}; +use bytes::Bytes; +use crowdb_kv_client::{BatchOp, GetOutcome, ReadMode}; +use crowdb_protocol::chunk_domain::ChunkDomain; +use crowdb_protocol::chunkdb::rpc::{ChunkState, DeleteChunkRangeRequest, DeleteChunkRangeResponse}; +use crowdb_protocol::common::ChunkId; + +impl ChunkStore { + /// Records an already-qualified dead range without freeing physical blocks. + /// The caller must establish absence of live/future references before calling. + /// Conflicts and unknown results are returned without confirmation reads. + pub async fn retire_chunk_range( + &self, + request: &DeleteChunkRangeRequest, + ) -> Result { + let id = request + .chunk_id + .ok_or_else(|| invalid("missing chunk identity"))?; + let end = u64::from(request.chunk_offset) + u64::from(request.chunk_size); + if request.chunk_size == 0 || request.expected_modify_ts == 0 { + return Err(invalid("empty range or missing qualification revision")); + } + let route = self.route(&id)?; + let authority = self.capture_authority(&id)?; + let key = chunk_key(&id); + let GetOutcome::Found { value, revision } = self + .kv + .get( + route.kv_store_id, + route.kv_group_id, + &key, + ReadMode::Linearizable, + None, + ) + .await + .map_err(|error| StoreError::Kv(error.to_string()))? + else { + return Err(StoreError::ChunkNotFound); + }; + let chunk = decode_chunk(&value)?; + if chunk.id != Some(id) + || ChunkDomain::for_chunk(&id) != Some(ChunkDomain::UserData) + || chunk.chunk_type != i32::from(id.high.to_be_bytes()[0]) + || chunk.writer_epoch == 0 + || end > chunk.acknowledged_cursor + { + return Err(invalid("range must belong to a sealed shared user-data chunk")); + } + match ChunkState::try_from(chunk.state) { + Ok(ChunkState::Deleted) => { + return Ok(DeleteChunkRangeResponse { + reclaimed: chunk.strips.is_empty() && chunk.cleanup_intents.is_empty(), + }) + } + Ok(ChunkState::Sealed) => {} + _ => return Err(invalid("shared writer is not sealed")), + } + let mut record = self + .get_chunk_range_retirement(&id) + .await? + .unwrap_or_else(|| ChunkRangeRetirement::new(&chunk)); + record.validate(&chunk)?; + let range = RetiredByteRange { + start: u64::from(request.chunk_offset), + end, + }; + if record.contains(range) { + // Repeating a proven retirement is safe after later layout changes. + return Ok(DeleteChunkRangeResponse { reclaimed: false }); + } + if chunk.modify_ts != request.expected_modify_ts { + return Err(StoreError::Conflict); + } + record.insert(range)?; + record.revision = chunk.modify_ts; + // Re-put the unchanged chunk as the atomic batch precondition. Retirement + // does not change physical layout or invalidate the foreground layout cache. + let ops = [ + BatchOp::Put { + key: Bytes::from(key.clone()), + value: Bytes::from(encode_chunk(&chunk)), + }, + BatchOp::Put { + key: Bytes::from(resource_key(&id)), + value: Bytes::from(record.encode()?), + }, + ]; + self.check_submission(&id, authority.as_ref())?; + match self + .kv + .batch_write_cas(route.kv_store_id, route.kv_group_id, &ops, &key, revision) + .await + { + Ok(_) => Ok(DeleteChunkRangeResponse { reclaimed: false }), + Err(crowdb_kv_client::Error::CasFailed { .. } | crowdb_kv_client::Error::CasBusy) => { + Err(StoreError::Conflict) + } + Err(error) => Err(StoreError::Kv(error.to_string())), + } + } + + /// Loads physical retirement metadata; ordinary chunk reads do not fetch it. + pub async fn get_chunk_range_retirement(&self, id: &ChunkId) -> Result> { + let route = self.route(id)?; + match self + .kv + .get( + route.kv_store_id, + route.kv_group_id, + &resource_key(id), + ReadMode::Linearizable, + None, + ) + .await + .map_err(|error| StoreError::Kv(error.to_string()))? + { + GetOutcome::NotFound => Ok(None), + GetOutcome::Found { value, .. } => ChunkRangeRetirement::decode(&value).map(Some), + } + } +} + +fn resource_key(id: &ChunkId) -> Vec { + let mut key = b"/chunk-ranges/".to_vec(); + key.extend_from_slice(&id.high.to_be_bytes()); + key.extend_from_slice(&id.low.to_be_bytes()); + key +} +fn invalid(message: &str) -> StoreError { + StoreError::InvalidRange(message.into()) +} diff --git a/app/crowdb-chunkdb/src/storage/range_retirement/record.rs b/app/crowdb-chunkdb/src/storage/range_retirement/record.rs new file mode 100644 index 000000000..a9f382525 --- /dev/null +++ b/app/crowdb-chunkdb/src/storage/range_retirement/record.rs @@ -0,0 +1,116 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{invalid, Result, StoreError}; +use bincode::Options; +use crowdb_protocol::{chunkdb::rpc::Chunk, common::ChunkId}; +use serde::{Deserialize, Serialize}; + +const MAX_RANGES: usize = 4096; +const MAX_RECORD_BYTES: u64 = 128 * 1024; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct RetiredByteRange { + pub start: u64, + pub end: u64, +} + +#[derive(Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct ChunkRangeRetirement { + version: u16, + chunk_id: Option, + writer_epoch: u64, + create_ts_ms: u64, + sealed_ts_ms: u64, + sealed_bytes: u64, + pub revision: u64, + ranges: Vec, +} + +impl ChunkRangeRetirement { + #[must_use] + pub fn ranges(&self) -> &[RetiredByteRange] { + &self.ranges + } + + pub(super) fn new(chunk: &Chunk) -> Self { + Self { + version: 1, + chunk_id: chunk.id, + writer_epoch: chunk.writer_epoch, + create_ts_ms: chunk.create_ts_ms, + sealed_ts_ms: chunk.sealed_ts_ms, + sealed_bytes: chunk.acknowledged_cursor, + revision: chunk.modify_ts, + ranges: Vec::new(), + } + } + pub(super) fn validate(&self, chunk: &Chunk) -> Result<()> { + self.validate_shape()?; + if self.chunk_id != chunk.id + || self.writer_epoch != chunk.writer_epoch + || self.create_ts_ms != chunk.create_ts_ms + || self.sealed_ts_ms != chunk.sealed_ts_ms + || self.sealed_bytes != chunk.acknowledged_cursor + { + return Err(invalid("retirement resource identity corruption")); + } + if self.revision > chunk.modify_ts { + return Err(StoreError::Conflict); + } + Ok(()) + } + fn validate_shape(&self) -> Result<()> { + if self.version != 1 + || self.chunk_id.is_none() + || self.writer_epoch == 0 + || self.sealed_bytes == 0 + || self.revision == 0 + || self.ranges.len() > MAX_RANGES + || self + .ranges + .iter() + .any(|r| r.start >= r.end || r.end > self.sealed_bytes) + || self.ranges.windows(2).any(|pair| pair[0].end >= pair[1].start) + { + return Err(invalid("retirement resource format or interval corruption")); + } + Ok(()) + } + pub(super) fn contains(&self, range: RetiredByteRange) -> bool { + self.ranges + .iter() + .any(|r| r.start <= range.start && r.end >= range.end) + } + pub(super) fn insert(&mut self, range: RetiredByteRange) -> Result<()> { + let mut ranges = self.ranges.clone(); + ranges.push(range); + ranges.sort_unstable_by_key(|r| r.start); + let mut merged: Vec = Vec::with_capacity(ranges.len()); + for range in ranges { + if let Some(previous) = merged.last_mut().filter(|previous| previous.end >= range.start) { + previous.end = previous.end.max(range.end); + } else { + merged.push(range); + } + } + if merged.len() > MAX_RANGES { + return Err(invalid("retirement range budget exceeded")); + } + self.ranges = merged; + Ok(()) + } + pub(super) fn encode(&self) -> Result> { + bincode::serialize(self).map_err(|error| StoreError::Serde(error.to_string())) + } + pub(super) fn decode(bytes: &[u8]) -> Result { + let record: Self = bincode::DefaultOptions::new() + .with_fixint_encoding() + .with_limit(MAX_RECORD_BYTES) + .reject_trailing_bytes() + .deserialize(bytes) + .map_err(|error| StoreError::Serde(error.to_string()))?; + record.validate_shape()?; + Ok(record) + } +} diff --git a/app/crowdb-chunkdb/tests/range_retirement_test.rs b/app/crowdb-chunkdb/tests/range_retirement_test.rs new file mode 100644 index 000000000..032bafabd --- /dev/null +++ b/app/crowdb-chunkdb/tests/range_retirement_test.rs @@ -0,0 +1,357 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/runtime_context.rs"] +#[allow(dead_code)] +mod runtime_context; +#[path = "common/slot_groups.rs"] +#[allow(dead_code)] +mod slot_groups; + +use crowdb_chunkdb::routing::{BindingCache, BindingTable}; +use crowdb_chunkdb::storage::{range_retirement::RetiredByteRange, ChunkStore, StoreError}; +use crowdb_protocol::chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, ChunkType, DeleteChunkRangeRequest}; +use crowdb_protocol::common::ChunkId; +use std::sync::Arc; + +fn store(test: &slot_groups::TestGroups) -> ChunkStore { + ChunkStore::new(Arc::clone(&test.kv), bindings()) +} +fn bindings() -> BindingCache { + let bindings = BindingCache::new(); + bindings + .replace(BindingTable::new( + ChunkSlotBootstrap { + service_instances: vec![1], + storage_groups: vec![ChunkStorageGroup { + store_id: 0, + group_id: 1, + }], + } + .storage_map() + .unwrap(), + )) + .unwrap(); + bindings +} +fn chunk() -> Chunk { + Chunk { + id: Some(ChunkId { + high: 5 << 56, + low: 1, + }), + modify_ts: 1, + state: ChunkState::Sealed as i32, + chunk_type: ChunkType::S3 as i32, + writer_epoch: 7, + acknowledged_cursor: 8192, + sealed_length: 8, + create_ts_ms: 10, + sealed_ts_ms: 20, + ..Default::default() + } +} +fn request(chunk: &Chunk, offset: u32, size: u32) -> DeleteChunkRangeRequest { + DeleteChunkRangeRequest { + chunk_id: chunk.id, + chunk_offset: offset, + chunk_size: size, + expected_modify_ts: chunk.modify_ts, + } +} + +#[tokio::test] +async fn exact_union_survives_restart_without_changing_layout_or_claiming_capacity() { + let test = slot_groups::TestGroups::start().await; + let store = store(&test); + let chunk = chunk(); + let id = chunk.id.unwrap(); + store.put_chunk(&chunk).await.unwrap(); + for (start, size) in [(134, 234), (300, 200), (500, 1), (1025, 2047)] { + assert!( + !store + .retire_chunk_range(&request(&chunk, start, size)) + .await + .unwrap() + .reclaimed + ); + } + let expected = store.get_chunk_range_retirement(&id).await.unwrap().unwrap(); + assert_eq!( + expected.ranges(), + [ + RetiredByteRange { start: 134, end: 501 }, + RetiredByteRange { + start: 1025, + end: 3072 + } + ] + ); + assert_eq!(store.get_chunk(&id).await.unwrap(), chunk); + + let restarted = crate::store(&test); + assert!( + !restarted + .retire_chunk_range(&request(&chunk, 134, 367)) + .await + .unwrap() + .reclaimed + ); + assert_eq!( + restarted.get_chunk_range_retirement(&id).await.unwrap().unwrap(), + expected + ); + assert_eq!(restarted.get_chunk(&id).await.unwrap(), chunk); +} + +#[tokio::test] +async fn changed_layout_rejects_new_ranges_but_known_retirement_replays() { + let test = slot_groups::TestGroups::start().await; + let store = store(&test); + let mut chunk = chunk(); + let id = chunk.id.unwrap(); + store.put_chunk(&chunk).await.unwrap(); + let retired = request(&chunk, 0, 10); + let stale = request(&chunk, 20, 10); + store.retire_chunk_range(&retired).await.unwrap(); + let before = store.get_chunk_range_retirement(&id).await.unwrap(); + chunk.modify_ts += 1; + store.put_chunk(&chunk).await.unwrap(); + assert!(matches!( + store.retire_chunk_range(&stale).await, + Err(StoreError::Conflict) + )); + assert!(!store.retire_chunk_range(&retired).await.unwrap().reclaimed); + assert_eq!(store.get_chunk_range_retirement(&id).await.unwrap(), before); + store.retire_chunk_range(&request(&chunk, 20, 10)).await.unwrap(); + assert_eq!( + store + .get_chunk_range_retirement(&id) + .await + .unwrap() + .unwrap() + .revision, + 2 + ); +} + +#[tokio::test] +async fn concurrent_retirements_are_atomic_and_conflicts_never_lose_a_range() { + let test = slot_groups::TestGroups::start().await; + let store = store(&test); + let chunk = chunk(); + store.put_chunk(&chunk).await.unwrap(); + let (a, b) = (request(&chunk, 0, 10), request(&chunk, 20, 10)); + let (first, second) = tokio::join!(store.retire_chunk_range(&a), store.retire_chunk_range(&b)); + for (result, req) in [(first, a), (second, b)] { + match result { + Ok(result) => assert!(!result.reclaimed), + Err(StoreError::Conflict) => { + store.retire_chunk_range(&req).await.unwrap(); + } + error => panic!("unexpected retirement outcome: {error:?}"), + } + } + assert_eq!( + store + .get_chunk_range_retirement(&chunk.id.unwrap()) + .await + .unwrap() + .unwrap() + .ranges(), + [ + RetiredByteRange { start: 0, end: 10 }, + RetiredByteRange { start: 20, end: 30 } + ] + ); +} + +#[tokio::test] +async fn invalid_ranges_active_writers_and_system_chunks_never_create_retirement() { + let test = slot_groups::TestGroups::start().await; + let store = store(&test); + for case in 0..7 { + let mut chunk = chunk(); + chunk.id.as_mut().unwrap().low += case; + let mut req = request(&chunk, 0, 1); + match case { + 0 => req.chunk_size = 0, + 1 => req.chunk_offset = 8192, + 2 => req.expected_modify_ts = 0, + 3 => chunk.state = ChunkState::Active as i32, + 4 => { + chunk.id.as_mut().unwrap().high = 1 << 56; + req.chunk_id = chunk.id; + } + 5 => chunk.writer_epoch = 0, + 6 => { + req.chunk_offset = u32::MAX; + req.chunk_size = u32::MAX; + } + _ => unreachable!(), + } + store.put_chunk(&chunk).await.unwrap(); + assert!( + matches!( + store.retire_chunk_range(&req).await, + Err(StoreError::InvalidRange(_)) + ), + "case {case}" + ); + assert!(store + .get_chunk_range_retirement(&chunk.id.unwrap()) + .await + .unwrap() + .is_none()); + assert_eq!(store.get_chunk(&chunk.id.unwrap()).await.unwrap(), chunk); + } +} + +#[tokio::test] +async fn reused_physical_identity_cannot_adopt_old_retired_ranges() { + let test = slot_groups::TestGroups::start().await; + let store = store(&test); + let mut chunk = chunk(); + store.put_chunk(&chunk).await.unwrap(); + store.retire_chunk_range(&request(&chunk, 0, 10)).await.unwrap(); + chunk.writer_epoch += 1; + chunk.modify_ts += 1; + store.put_chunk(&chunk).await.unwrap(); + assert!(matches!( + store.retire_chunk_range(&request(&chunk, 0, 10)).await, + Err(StoreError::InvalidRange(_)) + )); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn production_rpc_requires_qualification_and_reports_retirement_as_deferred() { + use crowdb_chunkdb::range_guard::RangeGuard; + use crowdb_chunkdb_client::{ChunkdbClientError, ChunkdbRpcTransport}; + use crowdb_protocol::{chunk_domain::ChunkDomain, port::alloc::alloc_test_port, ServicePort}; + let test = slot_groups::TestGroups::start().await; + let store = store(&test); + let chunk = chunk(); + store.put_chunk(&chunk).await.unwrap(); + let guard = Arc::new(RangeGuard::new()); + guard + .install( + &ChunkSlotBootstrap { + service_instances: vec![1], + storage_groups: vec![ChunkStorageGroup { + store_id: 0, + group_id: 1, + }], + } + .service_map() + .unwrap(), + 1, + ) + .unwrap(); + let (context, stop) = runtime_context::runtime_context(Arc::clone(&test.kv), bindings(), guard); + let runtime = context.start(ChunkDomain::UserData).await.unwrap(); + let service = Arc::new(runtime.rpc_service); + let server = Arc::new(crowdb_rpc_ffi::RpcServer::new(None)); + let port = alloc_test_port(ServicePort::ChunkdbRpc); + server.listen("127.0.0.1", i32::from(port)).unwrap(); + service.register_handlers(&server); + server.start(); + let endpoint = format!("http://127.0.0.1:{port}"); + let client = ChunkdbRpcTransport::new(); + let mut req = request(&chunk, 134, 234); + req.expected_modify_ts = 0; + assert!(matches!( + client.send_delete_chunk_range(&endpoint, &req).await, + Err(ChunkdbClientError::InvalidArgument(_)) + )); + assert!(store + .get_chunk_range_retirement(&chunk.id.unwrap()) + .await + .unwrap() + .is_none()); + req.expected_modify_ts = chunk.modify_ts; + for _ in 0..2 { + assert!( + !client + .send_delete_chunk_range(&endpoint, &req) + .await + .unwrap() + .reclaimed + ); + } + assert_eq!( + store + .get_chunk_range_retirement(&chunk.id.unwrap()) + .await + .unwrap() + .unwrap() + .ranges(), + [RetiredByteRange { start: 134, end: 368 }] + ); + assert_eq!(store.get_chunk(&chunk.id.unwrap()).await.unwrap(), chunk); + req.chunk_id.as_mut().unwrap().low += 1; + assert!(matches!( + client.send_delete_chunk_range(&endpoint, &req).await, + Err(ChunkdbClientError::NotFound(_)) + )); + server.stop(); + stop.send(true).unwrap(); + for worker in runtime.handles { + tokio::time::timeout(std::time::Duration::from_secs(5), worker) + .await + .unwrap() + .unwrap(); + } +} + +#[tokio::test] +async fn interval_budget_rejects_growth_but_allows_coalescing_without_losing_live_gaps() { + let test = slot_groups::TestGroups::start().await; + let store = store(&test); + let mut chunk = chunk(); + chunk.acknowledged_cursor = 1024 * 1024; + chunk.sealed_length = 1024; + store.put_chunk(&chunk).await.unwrap(); + let id = chunk.id.unwrap(); + let mut key = b"/chunk-ranges/".to_vec(); + key.extend_from_slice(&id.high.to_be_bytes()); + key.extend_from_slice(&id.low.to_be_bytes()); + let ranges: Vec<_> = (0..4096) + .map(|i| RetiredByteRange { + start: i * 2, + end: i * 2 + 1, + }) + .collect(); + // Version-one resource fixture at the maximum interval count. + let encoded = bincode::serialize(&( + 1_u16, + chunk.id, + chunk.writer_epoch, + chunk.create_ts_ms, + chunk.sealed_ts_ms, + chunk.acknowledged_cursor, + chunk.modify_ts, + &ranges, + )) + .unwrap(); + test.kv.put(0, 1, &key, &encoded, None).await.unwrap(); + assert!(matches!( + store.retire_chunk_range(&request(&chunk, 8192, 1)).await, + Err(StoreError::InvalidRange(_)) + )); + assert_eq!( + store + .get_chunk_range_retirement(&id) + .await + .unwrap() + .unwrap() + .ranges(), + ranges + ); + store.retire_chunk_range(&request(&chunk, 1, 1)).await.unwrap(); + let record = store.get_chunk_range_retirement(&id).await.unwrap().unwrap(); + assert_eq!(record.ranges().len(), 4095); + assert_eq!(record.ranges()[0], RetiredByteRange { start: 0, end: 3 }); + assert_eq!(record.ranges()[1], RetiredByteRange { start: 4, end: 5 }); + assert_eq!(store.get_chunk(&id).await.unwrap(), chunk); +} diff --git a/doc/backlog/R95-chunkdb-chunk-range-delete.md b/doc/backlog/R95-chunkdb-chunk-range-delete.md index 9839ad963..a961230a3 100644 --- a/doc/backlog/R95-chunkdb-chunk-range-delete.md +++ b/doc/backlog/R95-chunkdb-chunk-range-delete.md @@ -14,40 +14,51 @@ add metadata writes to the upload path and still miss those pre-record crashes. ## Solution 1. Complete `DeleteChunkRange(chunk_id, offset, size)` in the chunkdb protocol, - client and server. The existing stub remains explicitly not implemented - until range validation, used-bitmap updates, idempotency and in-chunk GC can - prove that the exact physical range is safe to retire. + client and server. A request carries the observed chunk revision. Only sealed + user-data shared chunks accept exact byte ranges; stale layouts and active + writers are rejected. Persist merged dead intervals and the sealed physical + identity as resource metadata, atomically fenced by the chunk revision. + This background resource state is approved; per-upload process records are + not. A successful retirement is distinct from physical reclamation: return + an explicit unreclaimed result until blocks have actually been released. + Never round a dead frame outward to the usage bitmap's coarser granularity. 2. Use a bounded, chunk-centered scanner to find old chunks and ranges whose bytes have no live owner. Do not create per-MPU cleanup intents, candidate records or a durable cleanup queue. Start with a configurable age threshold of one day; a candidate must be older than the threshold after its last write. Age alone never authorizes deletion. 3. Compare each candidate against published S3 objects, Iceberg file - descriptors, active multipart sessions and parts, frozen completion - selections, in-flight writers and reader protection. A lost part-publication + descriptors, committed Dataset/Split manifests and still-eligible completed + parts. Unpublished parts have an explicit commit deadline. Final publication + must be fenced against expiry reclamation, including a request admitted + before expiry whose publication arrives later. A lost part-publication response or missing intermediate MPU record is not proof that a chunk is - unused. Refuse deletion when reference or reader state cannot be confirmed. + unused. Refuse deletion when references or the publication fence cannot be confirmed. 4. Keep S3 MPU session, current-part and immutable part-generation keys under a common upload prefix in Chunk-KV so the scanner can enumerate one upload's references with a bounded prefix scan. The S3 multipart authority owns that - key layout and the metadata-only Abort/expiry transition; old part - generations remain available until the scanner proves their bytes are - unreachable. An aborted or expired upload becomes a candidate only after + key layout and Abort/expiry semantics. Only completed data resources and + published references are required by this contract; it does not authorize + durable upload workflows. Old part generations remain available until the + scanner proves their bytes are unreachable. An aborted or expired upload becomes a candidate only after the age gate and reference checks. 5. Recheck chunk identity, layout generation and exact range against current authority immediately before reclaim. Treat lost delete replies idempotently and keep an in-memory scan cursor and bounded work budget. Restart may rescan old chunks. Report examined, - eligible, deferred and reclaimed bytes without per-upload metric labels. + eligible, retired, deferred and physically reclaimed bytes without per-upload + metric labels. Snapshot deletion denies new requests and waits its persisted + grace period; readers are not registered or renewed. Layout retirement uses + its bounded validity interval; overdue readers may fail explicitly. ## Dependencies - R92 supplies in-chunk strip reclamation after R95 qualifies dead ranges. -- The S3 multipart authority supplies grouped MPU keys and durable session, - part and completion references. The scanner also recognizes Iceberg's frozen +- The S3 multipart authority supplies grouped MPU keys and completed part + and final object references. The scanner also recognizes Iceberg's frozen MPU selection. -- Reader protection and published generation references must be queryable - before physical deletion is enabled; R168 may use the qualified range-delete +- Published references, part deadlines and deletion/layout grace must be + verifiable before physical deletion is enabled; R168 may use the qualified range-delete interface for ordinary S3 object deletion. ## Acceptance @@ -64,15 +75,24 @@ add metadata writes to the upload path and still miss those pre-record crashes. scan runs, assert it defers the chunk until the age gate and in-flight writer proof settle, then discovers the orphan without a part record. Invariant: missing upload metadata alone cannot reclaim data. Integration test. -- Given a current reader, changed layout generation or uncertain authority, +- Given an unexpired deletion/layout grace, changed layout generation or uncertain authority, when range deletion is attempted, assert no bytes are reused. After the - reader releases and identity is confirmed, repeat the same delete through a + grace expires and identity is confirmed, repeat the same delete through a lost reply and assert one idempotent result. Invariant: reclamation is fenced - by current chunk and reader authority. Integration test. + by current chunk and publication authority, without reader sessions. Integration test. - Given a large backlog and foreground load, when scanning runs, assert it respects item, byte and time budgets and reports deferred/reclaimed progress. Invariant: cleanup cannot monopolize the data path. Integration test. +- Given adjacent, overlapping and unaligned byte ranges, when the server records + their retirement and restarts, assert exact union, unchanged live gaps and + idempotent replay. A new stale-layout range fails without mutation; an already + retired range can be queried again. Invariant: monotonic exact retirement. + Integration test. +- Given retirement success with blocks still allocated, when the client receives + that response, assert Deferred rather than Reclaimed. Invariant: logical + retirement never reports physical capacity recovery. Unit test. + Required gates: - `pixi run -- cargo test -p crowdb-chunkdb --all-targets` diff --git a/doc/design/chunkdb/design-crowdb-chunkdb.md b/doc/design/chunkdb/design-crowdb-chunkdb.md index f6e1a931a..03e610d6a 100644 --- a/doc/design/chunkdb/design-crowdb-chunkdb.md +++ b/doc/design/chunkdb/design-crowdb-chunkdb.md @@ -948,10 +948,25 @@ mutating RPC acquires the per-chunk lock before its RMW cycle: with segments as cleanup intent → free segments → clear the segment list and persist the tombstone → `guard.refresh(chunk)`. - `delete_chunk_range`: offset and length are independent u32 byte values, - describing an exact half-open object range without KiB rounding. The RPC - returns `Unimplemented` without changing storage until shared-object range - reclamation is supported. An intersecting strip is not permission to free - its blocks while neighbouring objects remain live. + describing an exact half-open object range without KiB rounding. A qualified + request includes the observed layout revision and targets a sealed shared + user-data chunk. The caller must already establish absence of live and future + references. The server atomically records merged exact dead ranges in a + co-located resource record, guarded by the chunk's KV revision; the unchanged + chunk participates in that conditional batch without changing its layout. + Resource identity includes the writer epoch and sealed extent. Existing + retirement replays without another mutation; a new stale-layout range fails. + Records are bounded to 4096 intervals and decoded within 128 KiB. Foreground + QueryChunk does not load or return this resource metadata. Conflict or unknown + write results do not trigger confirmation GETs inside the retirement handler. + `reclaimed=false` means the exact range is retired but physical space remains + allocated, so callers keep it Deferred. Only a completed physical deletion + reports `reclaimed=true`. Missing qualification revisions are rejected, and + old responses default to unreclaimed. An intersecting strip is never permission + to free blocks containing live neighbours. Mixed-strip physical compaction and + independent orphan qualification are not implemented by range retirement. +- `update_chunk_strip`: compatibility wrapper over the one-strip form of + `replace_chunk_strip_range`. - `replace_chunk_strip_range`: `check_range` → `acquire` → validate Active or Sealed state, expected revision, exact old range, contiguous capacity, monotonic sequences, segment ownership, and mirror/EC geometry → commit only diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 857320df2..90b3280ba 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -165,8 +165,9 @@ Dataset expiry 依赖 upload/part 记录，Iceberg discovery/write sweep 依赖 都不是“封口 chunk 范围减去全部存活引用”的独立扫描器。删除写前 intent 后，写出 payload 而尚未保存结果就宕机会产生不可发现的范围。tree 内部 orphan 回收不能代替用户 payload GC。 -**继续核查（2026-10-24）**：生产 `chunkdb_rpc_service/mutations.rs::handle_delete_range` -明确返回 `Unimplemented`，没有更新 used-range/bitmap；`reclaim_location` 将其当作 Deferred。 +**初始核查（2026-10-24）**：生产 `chunkdb_rpc_service/mutations.rs::handle_delete_range` +当时返回 `Unimplemented`，没有更新 used-range/bitmap。R4.3 已补精确退休， +但物理回收仍由 R4.4 跟踪，`reclaim_location` 在未物理释放时仍为 Deferred。 因此测试替身返回 Reclaimed 不能证明实际范围已释放。底层依赖是 [R95](../backlog/R95-chunkdb-chunk-range-delete.md)（范围资格与孤儿发现）及 [R92](../backlog/R92-chunkdb-in-chunk-gc.md)（混存 strip 的物理回收，当前仅占位设计）。 @@ -208,6 +209,15 @@ S3 普通对象是直接 PUT，不能未经设计就套用 Dataset head 机制 - [ ] **R4.2 实现提交期限和回收隔离**：part 结果携带期限，提交权威边界验证；覆盖期限前准入 但期限后落地的 RPC、GC 与最终引用并发、成功发布后的长期存活。若需新增持久化资源事实， 先列出最小字段和每请求 RPC 成本供 review。Files: Dataset publication/upload、Iceberg file、GC。 +- [ ] **R4.4 补齐物理释放和资格扫描**：在 R4.3 基础上实现死亡 strip 释放/混存搬迁、布局 + grace 与迟到 IO 拒绝，再接 R4.1/R4.2 的全引用资格检查。不得仅因 RPC Success 计入物理 reclaimed。 + 后续补按 chunk 批量提交候选，避免逐 part 单独退休；4096 区间达到上限时需有界分块/归并， + 不能把超额候选静默丢弃。初始退休接口允许单范围调用，不声称批量扫描已完成。 + 已核实 reader 的 strip 布局检查允许非重叠空洞，但现有 replacement 要求非空、等 capacity， + 不能直接拿通用 replacement 删除 strip。`reconcile_pending_chunks` 目前仅 startup 调用， + 不能据此承诺运行中的 grace 到期清理；需独立有界周期扫描，不为每个上传新增任务。 + 底层 `crowdb-kv-client::put_cas` 目前冲突只返回 revision，未返回 value；本批不做冲突后 + GET 重试，返回冲突由下一轮扫描重新评估。补全 raw KV observed 协议的缺口需单独跟踪。 ### D4 — 全局硬配额，还是每台 Access 的资源准入？仅在发现活跃需求后讨论 @@ -425,6 +435,18 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 ## 已完成与验证记录 +- **2026-10-24，R4.3**：生产 DeleteChunkRange 支持 expected layout revision、封口用户共享 + chunk 的精确区间；在相同 KV group 以 chunk KV revision 条件批量更新 + `/chunk-ranges/` 和原样 chunk。资源包含 writer/封口身份、布局代际、有界合并区间； + 不改变 QueryChunk 内容，不新增锁或上传 journal。重复退休零写，新范围遇 stale revision + 返回冲突，unknown 不确认 GET。RPC 资源退休返回 `reclaimed=false`，客户端仍 Deferred； + 旧请求无 qualification revision 被拒绝，旧响应缺物理完成字段按 false 解释。 + `range_retirement_test` 7 项（真实 KV 与生产 RPC）、`reclamation_test` 6 项、 + `chunkdb_wrappers_test` 10 项通过；覆盖精确相邻/重叠、live gap、重启重放、并发、stale、 + 物理身份复用、4096 上限/合并、Active/system/非法范围和未释放不误报完成。 + 均经 pixi，真实服务测试先 clean-env；四 crate all-target/all-feature clippy 和 fmt 通过。 + 尚未完成混存物理释放、无过程状态孤儿扫描和提交期限隔离；R4/R95/R92 未完成。 + - **2026-10-24，F3.1**：同一 next 多窗口使用请求局部的 selection 准备结果，绑定 authority/Dataset/Snapshot/规则，复用准入、summary、schema 和 explicit-ID 校验。 81 个候选、五个窗口、两个 explicit ID：独立逐页调用实测 25 bounded GET + 10 scan， @@ -476,7 +498,8 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 workspace fmt 和 `git diff --check`。无新锁、无 unsafe 例外、无 HTTP wire 协议变化。 测试辅助代码的依赖/格式 lint 已修正；HTTP 测试释放客户端后再关闭 listener。 - **R4 未完成**：已定位物理孤儿发现缺口，并记录用户确认的未发布 part 有效期。 - 尚未新增范围 GC 或删除上传/Composition/Split 的过程状态，不能将本批读取优化算作该清理完成。 + 已补精确范围退休，尚未完成物理释放/独立孤儿扫描，也尚未删除上传/Composition/Split + 的过程状态；不能将读取优化或退休记录算作无过程状态 GC 已完成。 尚无当前源码的 HCA/TPS 对比，不声称吞吐提升比例。 diff --git a/lib/crowdb-chunk-client/src/reclamation.rs b/lib/crowdb-chunk-client/src/reclamation.rs index b3a7ff4db..448ac0cca 100644 --- a/lib/crowdb-chunk-client/src/reclamation.rs +++ b/lib/crowdb-chunk-client/src/reclamation.rs @@ -39,8 +39,9 @@ pub async fn reclaim_location(allocator: &dyn ChunkAllocator, location: &Locatio } if chunk.writer_epoch != 0 { u32::try_from(end).map_err(|_| IoError::MetadataConflict("range end exceeds protocol".into()))?; - if chunk.acknowledged_cursor < end - && !matches!( + if chunk.modify_ts == 0 + || chunk.acknowledged_cursor < end + || !matches!( ChunkState::try_from(chunk.state), Ok(ChunkState::Sealed | ChunkState::Deleted) ) @@ -53,9 +54,15 @@ pub async fn reclaim_location(allocator: &dyn ChunkAllocator, location: &Locatio .map_err(|_| IoError::MetadataConflict("range offset exceeds protocol".into()))?, chunk_size: u32::try_from(location.length) .map_err(|_| IoError::MetadataConflict("range length exceeds protocol".into()))?, + expected_modify_ts: chunk.modify_ts, }; return match allocator.delete_chunk_range(request).await { - Ok(_) | Err(IoError::ChunkNotFound(_)) => Ok(ReclaimOutcome::Reclaimed), + Ok(response) => Ok(if response.reclaimed { + ReclaimOutcome::Reclaimed + } else { + ReclaimOutcome::Deferred + }), + Err(IoError::ChunkNotFound(_)) => Ok(ReclaimOutcome::Reclaimed), Err(IoError::Unsupported(_)) => Ok(ReclaimOutcome::Deferred), Err(error) => Err(error), }; diff --git a/lib/crowdb-chunk-client/tests/reclamation_test.rs b/lib/crowdb-chunk-client/tests/reclamation_test.rs index ae047e53f..3601b530a 100644 --- a/lib/crowdb-chunk-client/tests/reclamation_test.rs +++ b/lib/crowdb-chunk-client/tests/reclamation_test.rs @@ -14,6 +14,7 @@ struct TestAllocator { range_calls: AtomicUsize, chunk_calls: AtomicUsize, range_supported: bool, + range_reclaimed: bool, delete_failed: bool, expected_range: (u32, u32), } @@ -24,6 +25,8 @@ impl TestAllocator { chunk: Chunk { id: Some(ChunkId { high: 1, low: 2 }), state: ChunkState::Sealed as i32, + modify_ts: 4, + acknowledged_cursor: 4096, sealed_length: 4, writer_epoch: u64::from(shared), ..Chunk::default() @@ -31,6 +34,7 @@ impl TestAllocator { range_calls: AtomicUsize::new(0), chunk_calls: AtomicUsize::new(0), range_supported: false, + range_reclaimed: true, delete_failed: false, expected_range: (1024, 2048), } @@ -86,11 +90,14 @@ impl ChunkAllocator for TestAllocator { async fn delete_chunk_range(&self, request: DeleteChunkRangeRequest) -> Result { assert_eq!(request.chunk_id, self.chunk.id); assert_eq!((request.chunk_offset, request.chunk_size), self.expected_range); + assert_eq!(request.expected_modify_ts, self.chunk.modify_ts); self.range_calls.fetch_add(1, Ordering::Relaxed); if !self.range_supported { return Err(IoError::Unsupported("shared ranges pending".into())); } - Ok(DeleteChunkRangeResponse::default()) + Ok(DeleteChunkRangeResponse { + reclaimed: self.range_reclaimed, + }) } } @@ -125,16 +132,22 @@ async fn shared_range_remains_deferred_until_storage_supports_it() { ReclaimOutcome::Deferred ); allocator.range_supported = true; + allocator.range_reclaimed = false; + assert_eq!( + reclaim_location(&allocator, &location).await.unwrap(), + ReclaimOutcome::Deferred + ); + allocator.range_reclaimed = true; assert_eq!( reclaim_location(&allocator, &location).await.unwrap(), ReclaimOutcome::Reclaimed ); - assert_eq!(allocator.range_calls.load(Ordering::Relaxed), 2); + assert_eq!(allocator.range_calls.load(Ordering::Relaxed), 3); assert_eq!(allocator.chunk_calls.load(Ordering::Relaxed), 0); } #[tokio::test] -async fn uncertain_shared_write_waits_for_readable_cursor_or_terminal_chunk() { +async fn shared_retirement_requires_a_sealed_writer_and_readable_range() { let mut allocator = TestAllocator::new(true); allocator.range_supported = true; allocator.expected_range = (0, 4096); @@ -149,15 +162,20 @@ async fn uncertain_shared_write_waits_for_readable_cursor_or_terminal_chunk() { allocator.chunk.acknowledged_cursor = 4096; assert_eq!( reclaim_location(&allocator, &location).await.unwrap(), - ReclaimOutcome::Reclaimed + ReclaimOutcome::Deferred ); allocator.chunk.acknowledged_cursor = 0; allocator.chunk.state = ChunkState::Sealed as i32; + assert_eq!( + reclaim_location(&allocator, &location).await.unwrap(), + ReclaimOutcome::Deferred + ); + allocator.chunk.acknowledged_cursor = 4096; assert_eq!( reclaim_location(&allocator, &location).await.unwrap(), ReclaimOutcome::Reclaimed ); - assert_eq!(allocator.range_calls.load(Ordering::Relaxed), 2); + assert_eq!(allocator.range_calls.load(Ordering::Relaxed), 1); } #[tokio::test] @@ -187,6 +205,7 @@ async fn shared_byte_ranges_are_never_rounded_into_neighbouring_objects() { (1024 * 1024 * 1024 - 1, 1), ] { allocator.expected_range = (offset, length); + allocator.chunk.acknowledged_cursor = u64::from(offset) + u64::from(length); let mut location = allocator.location(); location.offset = u64::from(offset); location.length = u64::from(length); diff --git a/lib/crowdb-chunkdb-client/src/rpc_transport.rs b/lib/crowdb-chunkdb-client/src/rpc_transport.rs index 999850a00..b70a7da74 100644 --- a/lib/crowdb-chunkdb-client/src/rpc_transport.rs +++ b/lib/crowdb-chunkdb-client/src/rpc_transport.rs @@ -586,6 +586,7 @@ impl ChunkdbRpcTransport { chunk_id: chunk_id_off.as_ref(), chunk_offset: req.chunk_offset, chunk_size: req.chunk_size, + expected_modify_ts: req.expected_modify_ts, }; let fb_req = FBDeleteChunkRangeRequest::create(&mut builder, &args); builder.finish(fb_req, None); @@ -599,7 +600,9 @@ impl ChunkdbRpcTransport { )); } check_ret_code(r.ret_code(), r.error_msg())?; - Ok(DeleteChunkRangeResponse {}) + Ok(DeleteChunkRangeResponse { + reclaimed: r.reclaimed(), + }) } pub async fn send_allocate_replacement_segment( diff --git a/lib/crowdb-protocol/src/fb_wrappers/chunkdb.rs b/lib/crowdb-protocol/src/fb_wrappers/chunkdb.rs index b0af161da..75135c75b 100644 --- a/lib/crowdb-protocol/src/fb_wrappers/chunkdb.rs +++ b/lib/crowdb-protocol/src/fb_wrappers/chunkdb.rs @@ -280,6 +280,9 @@ pub struct FBDeleteChunkRangeResponseRef<'a> { } impl<'a> FBDeleteChunkRangeResponseRef<'a> { + pub fn reclaimed(&self) -> bool { + self.root.is_some_and(|r| r.reclaimed()) + } pub fn new(buf: &'a [u8]) -> Self { Self { root: parse_root::(buf), diff --git a/lib/crowdb-protocol/src/fbs/chunkdb.fbs b/lib/crowdb-protocol/src/fbs/chunkdb.fbs index 7bd4ae731..c8c504196 100644 --- a/lib/crowdb-protocol/src/fbs/chunkdb.fbs +++ b/lib/crowdb-protocol/src/fbs/chunkdb.fbs @@ -429,6 +429,7 @@ table FBDeleteChunkRangeRequest { chunk_id:crowdb.rpc.proto.FBInt128; chunk_offset:uint32; chunk_size:uint32; + expected_modify_ts:uint64; } table FBDeleteChunkRangeResponse { @@ -438,6 +439,7 @@ table FBDeleteChunkRangeResponse { error_msg:string; range_start:uint32; range_end:uint32; + reclaimed:bool = false; } // ── Replacement Allocation + Fenced Range Swap ───────────────── diff --git a/lib/crowdb-protocol/src/types/chunkdb.rs b/lib/crowdb-protocol/src/types/chunkdb.rs index e94ee9a0f..86d2927ab 100644 --- a/lib/crowdb-protocol/src/types/chunkdb.rs +++ b/lib/crowdb-protocol/src/types/chunkdb.rs @@ -472,10 +472,15 @@ pub struct DeleteChunkRangeRequest { pub chunk_id: Option, pub chunk_offset: u32, pub chunk_size: u32, + /// Chunk layout revision used to qualify the range. Zero is not qualified. + pub expected_modify_ts: u64, } #[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] -pub struct DeleteChunkRangeResponse {} +pub struct DeleteChunkRangeResponse { + /// True only after physical resources for the requested range are released. + pub reclaimed: bool, +} #[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] pub struct AllocateReplacementSegmentRequest { diff --git a/lib/crowdb-protocol/tests/chunkdb_wrappers_test.rs b/lib/crowdb-protocol/tests/chunkdb_wrappers_test.rs index bcb6cd8fc..9b5519d2c 100644 --- a/lib/crowdb-protocol/tests/chunkdb_wrappers_test.rs +++ b/lib/crowdb-protocol/tests/chunkdb_wrappers_test.rs @@ -341,6 +341,7 @@ fn delete_chunk_range_response_success() { error_msg: None, range_start: 0, range_end: 0, + reclaimed: true, }, ); fbb.finish(resp, None); @@ -350,9 +351,26 @@ fn delete_chunk_range_response_success() { assert!(view.valid()); assert_eq!(view.request_id(), Some(42)); assert!(view.ok()); + assert!(view.reclaimed()); assert_eq!(view.ret_code(), FBChunkdbRetCode::Success); } +#[test] +fn range_response_without_physical_completion_is_not_reclaimed() { + let mut builder = FlatBufferBuilder::new(); + let response = FBDeleteChunkRangeResponse::create( + &mut builder, + &FBDeleteChunkRangeResponseArgs { + ret_code: FBChunkdbRetCode::Success, + ..Default::default() + }, + ); + builder.finish(response, None); + let view = FBDeleteChunkRangeResponseRef::new(builder.finished_data()); + assert!(view.ok()); + assert!(!view.reclaimed()); +} + #[test] fn list_chunks_response_with_next_token() { let mut fbb = FlatBufferBuilder::new(); From 6dd3baac1b017a6ac0499f0fc132ff8aea76911b Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 10:48:30 +0800 Subject: [PATCH 096/258] Complete access cleanup when chunk deletion is accepted --- .../tests/iceberg_gc_budget_test.rs | 4 +- doc/backlog/R95-chunkdb-chunk-range-delete.md | 9 ++-- .../design-crowdb-access-server.md | 6 +++ doc/design/chunkdb/design-crowdb-chunkdb.md | 8 ++-- doc/working/plan-access-simplification.md | 42 ++++++++++++------- lib/crowdb-access-dataset/src/payload.rs | 2 +- lib/crowdb-access-iceberg/src/gc/task.rs | 1 + .../tests/common/gc_blocks.rs | 4 +- lib/crowdb-chunk-client/src/reclamation.rs | 24 ++++------- .../tests/reclamation_test.rs | 34 ++++++++++++--- 10 files changed, 85 insertions(+), 49 deletions(-) diff --git a/app/crowdb-access-server/tests/iceberg_gc_budget_test.rs b/app/crowdb-access-server/tests/iceberg_gc_budget_test.rs index 379018af5..7a04ebe52 100644 --- a/app/crowdb-access-server/tests/iceberg_gc_budget_test.rs +++ b/app/crowdb-access-server/tests/iceberg_gc_budget_test.rs @@ -85,7 +85,7 @@ impl FileBlockStore for TestBlocks { async fn reclaim(&self, _root: &ChunkRoot) -> Result { self.deletes.fetch_add(1, Ordering::Relaxed); - Ok(ReclaimOutcome::Reclaimed) + Ok(ReclaimOutcome::Accepted) } } @@ -109,7 +109,7 @@ async fn chunk_io_budget_rejects_work_before_dispatch_and_resets_per_step() { assert!(matches!(blocks.reclaim(&root).await, Err(FileIoError::Bounds))); assert_eq!(inner.deletes.load(Ordering::Relaxed), 0); budget.reset(); - assert_eq!(blocks.reclaim(&root).await.unwrap(), ReclaimOutcome::Reclaimed); + assert_eq!(blocks.reclaim(&root).await.unwrap(), ReclaimOutcome::Accepted); assert_eq!(inner.deletes.load(Ordering::Relaxed), 1); } diff --git a/doc/backlog/R95-chunkdb-chunk-range-delete.md b/doc/backlog/R95-chunkdb-chunk-range-delete.md index a961230a3..005576f0a 100644 --- a/doc/backlog/R95-chunkdb-chunk-range-delete.md +++ b/doc/backlog/R95-chunkdb-chunk-range-delete.md @@ -19,8 +19,9 @@ add metadata writes to the upload path and still miss those pre-record crashes. writers are rejected. Persist merged dead intervals and the sealed physical identity as resource metadata, atomically fenced by the chunk revision. This background resource state is approved; per-upload process records are - not. A successful retirement is distinct from physical reclamation: return - an explicit unreclaimed result until blocks have actually been released. + not. Successful retirement completes the caller's notification responsibility; + Access must not wait for physical reclamation or poll the returned physical + status. ChunkDB owns subsequent release and restart recovery independently. Never round a dead frame outward to the usage bitmap's coarser granularity. 2. Use a bounded, chunk-centered scanner to find old chunks and ranges whose bytes have no live owner. Do not create per-MPU cleanup intents, candidate @@ -90,8 +91,8 @@ add metadata writes to the upload path and still miss those pre-record crashes. retired range can be queried again. Invariant: monotonic exact retirement. Integration test. - Given retirement success with blocks still allocated, when the client receives - that response, assert Deferred rather than Reclaimed. Invariant: logical - retirement never reports physical capacity recovery. Unit test. + that response, assert Accepted without further polling. Invariant: notification + completion is independent of physical capacity recovery. Unit test. Required gates: diff --git a/doc/design/access-server/design-crowdb-access-server.md b/doc/design/access-server/design-crowdb-access-server.md index 5995d8950..c6e1f57a8 100644 --- a/doc/design/access-server/design-crowdb-access-server.md +++ b/doc/design/access-server/design-crowdb-access-server.md @@ -133,6 +133,12 @@ Iceberg file-record construction also lives in the Iceberg library, including the format rule shared by ordinary uploads and multipart completion. Iceberg GC uses a separate chunk client with Iceberg's small-write policy. The Iceberg library owns its GC storage budget and file-block adapters. +After proving a payload range unreachable, GC hands its exact byte range to +ChunkDB. A successful deletion notification completes Access's responsibility; +Access does not poll physical reclamation or retain progress for DiskDB capacity +release. `ReclaimOutcome::Accepted` means the notification was accepted (or the +chunk is already absent), not that disk space is reusable. Physical range +retirement, layout grace and compaction belong to ChunkDB and DiskDB. Each small-write policy also chooses its chunk capacity; each protocol's large-write policy chooses its own maximum chunk size. The deployment profile sets RPC workers and DiskIO connections independently of these data limits. diff --git a/doc/design/chunkdb/design-crowdb-chunkdb.md b/doc/design/chunkdb/design-crowdb-chunkdb.md index 03e610d6a..645d02a7e 100644 --- a/doc/design/chunkdb/design-crowdb-chunkdb.md +++ b/doc/design/chunkdb/design-crowdb-chunkdb.md @@ -959,10 +959,10 @@ mutating RPC acquires the per-chunk lock before its RMW cycle: Records are bounded to 4096 intervals and decoded within 128 KiB. Foreground QueryChunk does not load or return this resource metadata. Conflict or unknown write results do not trigger confirmation GETs inside the retirement handler. - `reclaimed=false` means the exact range is retired but physical space remains - allocated, so callers keep it Deferred. Only a completed physical deletion - reports `reclaimed=true`. Missing qualification revisions are rejected, and - old responses default to unreclaimed. An intersecting strip is never permission + A successful response means ChunkDB has accepted cleanup ownership. Access + finishes its notification without waiting for the physical `reclaimed` flag; + physical capacity release is a separate ChunkDB/DiskDB responsibility. + Missing qualification revisions are rejected. An intersecting strip is never permission to free blocks containing live neighbours. Mixed-strip physical compaction and independent orphan qualification are not implemented by range retirement. - `update_chunk_strip`: compatibility wrapper over the one-strip form of diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 90b3280ba..a78cc6a5e 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -47,6 +47,9 @@ Upstream: [Dataset SDK](../backlog/R220-dataset-fat-client-plan.md)、 - **用户决定（2026-10-24）**：未发布 MPU part 有明确可提交期限；过期拒绝新提交， 客户端重传。GC 在期限后再等待安全期回收；已发布资源不能按该临时期限回收。 期限必须随 part 结果返回，最终提交验证期限；本地时钟检查不能单独证明迟到发布安全。 +- **用户澄清（2026-10-24）**：Access GC 只需通知 ChunkDB 的 DeleteChunkRange 接口。 + 接口成功即完成本层责任，不等待物理释放、不轮询 reclaimed、不保存物理回收进度。 + ChunkDB 内部范围回收尚属单独工作，不作为本轮 Access 清理的实施内容。 - 幂等按操作效果判断：同目标同内容的普通覆盖可直接重试，不要求原响应重放、 operation ledger 或多次查询来“证明幂等”。有反例按 R1–R4 单独 review； 不把“最小提交回执 + 查询结果”设成所有写 API 的默认成本。 @@ -167,20 +170,21 @@ Dataset expiry 依赖 upload/part 记录，Iceberg discovery/write sweep 依赖 **初始核查（2026-10-24）**：生产 `chunkdb_rpc_service/mutations.rs::handle_delete_range` 当时返回 `Unimplemented`，没有更新 used-range/bitmap。R4.3 已补精确退休， -但物理回收仍由 R4.4 跟踪，`reclaim_location` 在未物理释放时仍为 Deferred。 -因此测试替身返回 Reclaimed 不能证明实际范围已释放。底层依赖是 +物理回收独立于本计划；`reclaim_location` 收到成功后返回 Accepted，表示通知已接受， +不表示容量已释放。底层独立工作是 [R95](../backlog/R95-chunkdb-chunk-range-delete.md)（范围资格与孤儿发现）及 [R92](../backlog/R92-chunkdb-in-chunk-gc.md)（混存 strip 的物理回收，当前仅占位设计）。 不能用“把 RPC 改为成功”或把共享 chunk 整块删除替代实现；R95 旧文档的 reader/session 约束需按本计划已确认的删除 grace、part 期限重新整理。R168 的逐对象 cleanup record 也不是本计划自动批准的新状态。涉及新持久化资源字段先列出用途、写入频率和替代方案。 -**已确认的物理资源事实（2026-10-24，待实施）**：R95 若按范围逐步回收，需要在 chunk/strip +**已确认的物理资源事实（2026-10-24，ChunkDB 独立工作）**：R95 若按范围逐步回收，需要在 chunk/strip 元数据中保留已经判定死亡的精确区间及布局代际。它表示“哪些物理字节已经不可再引用”， 不是 upload 的步骤、进度或 owner session；由后台 GC 按 chunk 合并写，前台上传不追加 cleanup/journal RPC。现有 `usage_bitmap` 描述的粒度不能直接用于任意字节范围，不能把 一个 frame 的删除向邻居取整。部分死亡 strip 暂时只记录死亡事实，释放整块分配或搬迁 -存活部分后才能报告物理 reclaimed；R92 负责后者及逻辑地址到新物理布局的映射。 +存活部分后才增加实际可用容量；R92 负责后者及逻辑地址到新物理布局的映射。 +Access 不等待这些步骤，不消费物理 reclaimed 字段。 失败流程：GC 先完成发布隔离与活引用检查 → 以 chunk revision 条件更新死亡范围/布局 → 物理释放。进程在二者间退出时，从资源事实重做释放；丢回复时重复操作不得伤及新代际。 @@ -209,15 +213,15 @@ S3 普通对象是直接 PUT，不能未经设计就套用 Dataset head 机制 - [ ] **R4.2 实现提交期限和回收隔离**：part 结果携带期限，提交权威边界验证；覆盖期限前准入 但期限后落地的 RPC、GC 与最终引用并发、成功发布后的长期存活。若需新增持久化资源事实， 先列出最小字段和每请求 RPC 成本供 review。Files: Dataset publication/upload、Iceberg file、GC。 -- [ ] **R4.4 补齐物理释放和资格扫描**：在 R4.3 基础上实现死亡 strip 释放/混存搬迁、布局 - grace 与迟到 IO 拒绝，再接 R4.1/R4.2 的全引用资格检查。不得仅因 RPC Success 计入物理 reclaimed。 - 后续补按 chunk 批量提交候选，避免逐 part 单独退休；4096 区间达到上限时需有界分块/归并， - 不能把超额候选静默丢弃。初始退休接口允许单范围调用，不声称批量扫描已完成。 - 已核实 reader 的 strip 布局检查允许非重叠空洞，但现有 replacement 要求非空、等 capacity， - 不能直接拿通用 replacement 删除 strip。`reconcile_pending_chunks` 目前仅 startup 调用， - 不能据此承诺运行中的 grace 到期清理；需独立有界周期扫描，不为每个上传新增任务。 - 底层 `crowdb-kv-client::put_cas` 目前冲突只返回 revision，未返回 value；本批不做冲突后 - GET 重试，返回冲突由下一轮扫描重新评估。补全 raw KV observed 协议的缺口需单独跟踪。 +- [ ] **R4.5 简化通知入口的前置查询**：当前接口要求 expected_modify_ts，reclaim_location + 因此先 QueryChunk，并兼容旧的 dedicated DeleteChunk 分支。与后续 ChunkDB 接口实现一起 + 收敛为 chunk_id/offset/length 的单次 DeleteChunkRange 通知；布局、封口与物理回收判断归 + ChunkDB。删除此查询需要同步调整服务端的 revision 契约，另行实施。 + +**已移出 Access 实施范围**：死亡 strip 释放、混存搬迁、布局 grace、迟到 IO、后台物理扫描 +属于 R92/R95 的 ChunkDB 工作。未提交的实验保存于 git stash +`e28ee7be`（`WIP whole-strip reclamation saved separately from access cleanup`），未并入本次修改。 +已有精确退休接口不代表完整物理回收已实现；本计划不以它的完成作为 Access 通知成功条件。 ### D4 — 全局硬配额，还是每台 Access 的资源准入？仅在发现活跃需求后讨论 @@ -435,12 +439,20 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 ## 已完成与验证记录 +- **2026-10-24，R4.4**：按用户明确的职责边界，DeleteChunkRange 成功即返回 Accepted， + 不等待 `reclaimed`；已接受的 dedicated 删除也不再检查 strips/cleanup intents 是否排空。 + Dataset/Iceberg 同步使用 Accepted，Iceberg reclaimed_bytes 注明为逻辑通知完成量， + 不代表磁盘可复用容量。无新增持久化、轮询或确认 RPC；不支持/未准入仍 Deferred， + 错误仍传播。物理回收实验另存 stash，未提交；R4.5 保留前置 QueryChunk 的简化任务。 + pixi 下 `reclamation_test` 7、`split_reclaim_test` 2、`gc_worker_test` 21、 + `iceberg_gc_budget_test` 2 项通过；四 crate all-target/all-feature clippy 与 fmt 通过。 + - **2026-10-24，R4.3**：生产 DeleteChunkRange 支持 expected layout revision、封口用户共享 chunk 的精确区间；在相同 KV group 以 chunk KV revision 条件批量更新 `/chunk-ranges/` 和原样 chunk。资源包含 writer/封口身份、布局代际、有界合并区间； 不改变 QueryChunk 内容，不新增锁或上传 journal。重复退休零写，新范围遇 stale revision - 返回冲突，unknown 不确认 GET。RPC 资源退休返回 `reclaimed=false`，客户端仍 Deferred； - 旧请求无 qualification revision 被拒绝，旧响应缺物理完成字段按 false 解释。 + 返回冲突，unknown 不确认 GET。该批曾让客户端等待 `reclaimed`；用户随后明确此边界过重， + R4.4 改为成功通知即结束。当前服务仍拒绝无 qualification revision 的旧请求。 `range_retirement_test` 7 项（真实 KV 与生产 RPC）、`reclamation_test` 6 项、 `chunkdb_wrappers_test` 10 项通过；覆盖精确相邻/重叠、live gap、重启重放、并发、stale、 物理身份复用、4096 上限/合并、Active/system/非法范围和未释放不误报完成。 diff --git a/lib/crowdb-access-dataset/src/payload.rs b/lib/crowdb-access-dataset/src/payload.rs index a3b85c5af..474110276 100644 --- a/lib/crowdb-access-dataset/src/payload.rs +++ b/lib/crowdb-access-dataset/src/payload.rs @@ -194,7 +194,7 @@ impl PayloadStorage for ChunkPayloadStorage { complete &= crowdb_chunk_client::reclaim_location(allocator.as_ref(), location) .await .map_err(|error| UploadError::Storage(error.to_string()))? - == ReclaimOutcome::Reclaimed; + == ReclaimOutcome::Accepted; } Ok(complete) } diff --git a/lib/crowdb-access-iceberg/src/gc/task.rs b/lib/crowdb-access-iceberg/src/gc/task.rs index f7fe28f48..8211af114 100644 --- a/lib/crowdb-access-iceberg/src/gc/task.rs +++ b/lib/crowdb-access-iceberg/src/gc/task.rs @@ -76,6 +76,7 @@ pub struct GcTask { pub marked: u64, pub deleted: u64, /// Logical bytes of completed file records, not freed disk allocation or parity bytes. + /// Logical bytes whose cleanup notification was accepted, not reusable disk capacity. pub reclaimed_bytes: u64, } diff --git a/lib/crowdb-access-iceberg/tests/common/gc_blocks.rs b/lib/crowdb-access-iceberg/tests/common/gc_blocks.rs index 4984afadb..24e10b0c0 100644 --- a/lib/crowdb-access-iceberg/tests/common/gc_blocks.rs +++ b/lib/crowdb-access-iceberg/tests/common/gc_blocks.rs @@ -42,7 +42,7 @@ impl FileBlockStore for TestReclaimBlocks { if self.reply_loss.swap(false, Ordering::Relaxed) { return Err(crowdb_chunk_client::IoError::Internal("lost deletion response".into()).into()); } - Ok(ReclaimOutcome::Reclaimed) + Ok(ReclaimOutcome::Accepted) } async fn reclaim_location(&self, location: &Location) -> Result { @@ -57,6 +57,6 @@ impl FileBlockStore for TestReclaimBlocks { }); self.deletes.fetch_add(1, Ordering::Relaxed); self.location_deletes.fetch_add(1, Ordering::Relaxed); - Ok(ReclaimOutcome::Reclaimed) + Ok(ReclaimOutcome::Accepted) } } diff --git a/lib/crowdb-chunk-client/src/reclamation.rs b/lib/crowdb-chunk-client/src/reclamation.rs index 448ac0cca..022009282 100644 --- a/lib/crowdb-chunk-client/src/reclamation.rs +++ b/lib/crowdb-chunk-client/src/reclamation.rs @@ -6,7 +6,10 @@ use crate::{ChunkAllocator, IoError, Result}; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum ReclaimOutcome { - Reclaimed, + /// ChunkDB accepted durable cleanup ownership, or the chunk is already gone. + /// This does not report physical capacity reclaimed by DiskDB. + Accepted, + /// ChunkDB has not accepted this notification yet. Deferred, } @@ -29,7 +32,7 @@ pub async fn reclaim_location(allocator: &dyn ChunkAllocator, location: &Locatio Ok(response) => response .chunk .ok_or_else(|| IoError::MetadataConflict("missing chunk response".into()))?, - Err(IoError::ChunkNotFound(_)) => return Ok(ReclaimOutcome::Reclaimed), + Err(IoError::ChunkNotFound(_)) => return Ok(ReclaimOutcome::Accepted), Err(error) => return Err(error), }; if chunk.id != Some(chunk_id) { @@ -57,12 +60,7 @@ pub async fn reclaim_location(allocator: &dyn ChunkAllocator, location: &Locatio expected_modify_ts: chunk.modify_ts, }; return match allocator.delete_chunk_range(request).await { - Ok(response) => Ok(if response.reclaimed { - ReclaimOutcome::Reclaimed - } else { - ReclaimOutcome::Deferred - }), - Err(IoError::ChunkNotFound(_)) => Ok(ReclaimOutcome::Reclaimed), + Ok(_) | Err(IoError::ChunkNotFound(_)) => Ok(ReclaimOutcome::Accepted), Err(IoError::Unsupported(_)) => Ok(ReclaimOutcome::Deferred), Err(error) => Err(error), }; @@ -87,15 +85,11 @@ pub async fn reclaim_location(allocator: &dyn ChunkAllocator, location: &Locatio Ok(response) => response .chunk .ok_or_else(|| IoError::MetadataConflict("missing deletion response".into()))?, - Err(IoError::ChunkNotFound(_)) => return Ok(ReclaimOutcome::Reclaimed), + Err(IoError::ChunkNotFound(_)) => return Ok(ReclaimOutcome::Accepted), Err(error) => return Err(error), }; - if deleted.id != Some(chunk_id) - || deleted.state != ChunkState::Deleted as i32 - || !deleted.strips.is_empty() - || !deleted.cleanup_intents.is_empty() - { + if deleted.id != Some(chunk_id) || deleted.state != ChunkState::Deleted as i32 { return Ok(ReclaimOutcome::Deferred); } - Ok(ReclaimOutcome::Reclaimed) + Ok(ReclaimOutcome::Accepted) } diff --git a/lib/crowdb-chunk-client/tests/reclamation_test.rs b/lib/crowdb-chunk-client/tests/reclamation_test.rs index 3601b530a..25128a04f 100644 --- a/lib/crowdb-chunk-client/tests/reclamation_test.rs +++ b/lib/crowdb-chunk-client/tests/reclamation_test.rs @@ -12,6 +12,7 @@ use crowdb_protocol::common::ChunkId; struct TestAllocator { chunk: Chunk, range_calls: AtomicUsize, + query_calls: AtomicUsize, chunk_calls: AtomicUsize, range_supported: bool, range_reclaimed: bool, @@ -32,6 +33,7 @@ impl TestAllocator { ..Chunk::default() }, range_calls: AtomicUsize::new(0), + query_calls: AtomicUsize::new(0), chunk_calls: AtomicUsize::new(0), range_supported: false, range_reclaimed: true, @@ -67,6 +69,7 @@ impl ChunkAllocator for TestAllocator { async fn query_chunk(&self, request: QueryChunkRequest) -> Result { assert_eq!(request.chunk_id, self.chunk.id); + self.query_calls.fetch_add(1, Ordering::Relaxed); Ok(QueryChunkResponse { chunk: Some(self.chunk.clone()), ..QueryChunkResponse::default() @@ -114,14 +117,14 @@ async fn dedicated_chunk_deletion_checks_ownership_and_retries_failures() { allocator.delete_failed = false; assert_eq!( reclaim_location(&allocator, &location).await.unwrap(), - ReclaimOutcome::Reclaimed + ReclaimOutcome::Accepted ); assert_eq!(allocator.chunk_calls.load(Ordering::Relaxed), 2); assert_eq!(allocator.range_calls.load(Ordering::Relaxed), 0); } #[tokio::test] -async fn shared_range_remains_deferred_until_storage_supports_it() { +async fn shared_range_finishes_when_chunkdb_accepts_notification_without_physical_completion() { let mut allocator = TestAllocator::new(true); allocator.expected_range = (134, 234); let mut location = allocator.location(); @@ -135,17 +138,36 @@ async fn shared_range_remains_deferred_until_storage_supports_it() { allocator.range_reclaimed = false; assert_eq!( reclaim_location(&allocator, &location).await.unwrap(), - ReclaimOutcome::Deferred + ReclaimOutcome::Accepted ); + assert_eq!(allocator.range_calls.load(Ordering::Relaxed), 2); + assert_eq!(allocator.query_calls.load(Ordering::Relaxed), 2); allocator.range_reclaimed = true; assert_eq!( reclaim_location(&allocator, &location).await.unwrap(), - ReclaimOutcome::Reclaimed + ReclaimOutcome::Accepted ); assert_eq!(allocator.range_calls.load(Ordering::Relaxed), 3); assert_eq!(allocator.chunk_calls.load(Ordering::Relaxed), 0); } +#[tokio::test] +async fn accepted_dedicated_deletion_does_not_wait_for_chunkdb_cleanup() { + use crowdb_protocol::chunkdb::rpc::{ChunkStrip, StripCleanupIntent}; + let mut allocator = TestAllocator::new(false); + allocator.chunk.strips.push(ChunkStrip::default()); + allocator + .chunk + .cleanup_intents + .push(StripCleanupIntent::default()); + assert_eq!( + reclaim_location(&allocator, &allocator.location()).await.unwrap(), + ReclaimOutcome::Accepted + ); + assert_eq!(allocator.chunk_calls.load(Ordering::Relaxed), 1); + assert_eq!(allocator.query_calls.load(Ordering::Relaxed), 1); +} + #[tokio::test] async fn shared_retirement_requires_a_sealed_writer_and_readable_range() { let mut allocator = TestAllocator::new(true); @@ -173,7 +195,7 @@ async fn shared_retirement_requires_a_sealed_writer_and_readable_range() { allocator.chunk.acknowledged_cursor = 4096; assert_eq!( reclaim_location(&allocator, &location).await.unwrap(), - ReclaimOutcome::Reclaimed + ReclaimOutcome::Accepted ); assert_eq!(allocator.range_calls.load(Ordering::Relaxed), 1); } @@ -211,7 +233,7 @@ async fn shared_byte_ranges_are_never_rounded_into_neighbouring_objects() { location.length = u64::from(length); assert_eq!( reclaim_location(&allocator, &location).await.unwrap(), - ReclaimOutcome::Reclaimed + ReclaimOutcome::Accepted ); } assert_eq!(allocator.range_calls.load(Ordering::Relaxed), 5); From 0d2eee57502eb967cca88f8f2bc822f94ae73ac1 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 10:59:18 +0800 Subject: [PATCH 097/258] Reuse upload mutation results without redundant metadata reads --- doc/working/plan-access-simplification.md | 11 +- lib/crowdb-access-dataset/src/upload.rs | 15 +- lib/crowdb-access-dataset/src/upload/index.rs | 24 +- .../src/upload/ownership.rs | 105 +++++---- lib/crowdb-access-dataset/src/upload/parts.rs | 24 +- .../tests/common/upload_fault.rs | 79 +++++++ .../tests/upload_mutation_cost_test.rs | 209 ++++++++++++++++++ 7 files changed, 389 insertions(+), 78 deletions(-) create mode 100644 lib/crowdb-access-dataset/tests/common/upload_fault.rs create mode 100644 lib/crowdb-access-dataset/tests/upload_mutation_cost_test.rs diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index a78cc6a5e..52666cb0b 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -293,8 +293,6 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 - [ ] **P4.1 缩减 upload/part 状态与回读**：列出 create/intent/write/accept/index/complete/abort 的必要资源事实与过程状态；删除后者，改成客户端控制完整 part/最终清单； 成功结果直接返回，不为续传保存进度。需要保留 intent 的具体反例先按 R4 review。Files: Dataset `upload.rs`、`upload/{parts,index,ownership}.rs`。 -- [ ] **P4.2 修复 CAS 冲突重读**：`index_part`、`abort/cleanup/cleanup_part` 直接使用 observation； - 合并不同 part 的索引更新时保留双方结果。claim/release 已正确使用 observation，不重复改写。Files: `upload/{index,ownership,retirement}.rs`。 - [ ] **P4.3 缩小 owner/part 争用范围**：测 128-part 索引页、增长的 retired owners 的争用和编码； 比较更小记录，证明 claim-vs-reclaim 和迟到 claim 不破坏共享引用。Files: `upload/{index,ownership,retirement,reclaim}.rs`、相关并发/丢回复测试。 @@ -439,6 +437,15 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 ## 已完成与验证记录 +- **2026-10-24，P4.2**：part 索引、abort、cleanup 完成标记、part 清理均直接沿 CAS 的 + observed value 重试；初始 part 枚举的结果也直接交给清理，不重复 GET。 + 索引连续四次冲突保留其他四个 part；abort 观察到新 owner 立即停止；cleanup 在通知前后 + 的连续冲突不追加 GET。create 成功 0G + 1C，冲突复用 descriptor；单页 complete + 正常/重复均 2G，复用本次已校验 parts，不回读 descriptor/index。unknown 直接传播， + 客户端可重新请求；不把 timeout 当作 definite conflict。没有改变上传生命周期或删除 intent。 + 新增 `upload_mutation_cost_test` 六项均通过；Dataset `--all-targets`、all-target/all-feature + clippy、workspace fmt 和 diff 检查通过，均经 pixi。P4.1/P4.3 与 R4 继续跟踪状态删除和争用。 + - **2026-10-24，R4.4**：按用户明确的职责边界，DeleteChunkRange 成功即返回 Accepted， 不等待 `reclaimed`；已接受的 dedicated 删除也不再检查 strips/cleanup intents 是否排空。 Dataset/Iceberg 同步使用 Accepted，Iceberg reclaimed_bytes 注明为逻辑通知完成量， diff --git a/lib/crowdb-access-dataset/src/upload.rs b/lib/crowdb-access-dataset/src/upload.rs index 127ee1a48..bee70b765 100644 --- a/lib/crowdb-access-dataset/src/upload.rs +++ b/lib/crowdb-access-dataset/src/upload.rs @@ -143,25 +143,22 @@ impl DatasetUploads { retired_owners: Vec::new(), cleanup_complete: false, }; - let created = match self + match self .store .compare_exchange(location, None, &encode(&record)?) .await? { - CasOutcome::Applied(_) => true, + CasOutcome::Applied(_) => Ok(record), CasOutcome::Conflict(Some(value)) => { let existing = decode_session(&value.bytes)?; if existing.length != length || existing.part_size != part_size || existing.md5 != md5 { return Err(UploadError::Conflict); } - false + let mut existing = existing; + existing.parts = self.accepted_parts(location, &existing).await?; + Ok(existing) } - CasOutcome::Conflict(None) => return Err(UploadError::Conflict), - }; - if created { - self.session(location).await - } else { - self.get(location).await + CasOutcome::Conflict(None) => Err(UploadError::Conflict), } } diff --git a/lib/crowdb-access-dataset/src/upload/index.rs b/lib/crowdb-access-dataset/src/upload/index.rs index d42ac3844..dfb130fb2 100644 --- a/lib/crowdb-access-dataset/src/upload/index.rs +++ b/lib/crowdb-access-dataset/src/upload/index.rs @@ -33,8 +33,8 @@ impl DatasetUploads { } pub(super) async fn index_part(&self, location: &[u8], part: &UploadPart) -> Result<(), UploadError> { let key = index_key(location, part.index / PAGE_PARTS); + let mut stored = self.store.get(&key).await?; for _ in 0..16 { - let stored = self.store.get(&key).await?; let mut parts = stored .as_ref() .map(|v| decode::>(&v.bytes)) @@ -52,17 +52,17 @@ impl DatasetUploads { } parts.push(part.clone()); parts.sort_by_key(|v| v.index); - if matches!( - self.store - .compare_exchange( - &key, - stored.as_ref().map(|v| v.bytes.as_slice()), - &encode(&parts)? - ) - .await?, - CasOutcome::Applied(_) - ) { - return Ok(()); + match self + .store + .compare_exchange( + &key, + stored.as_ref().map(|v| v.bytes.as_slice()), + &encode(&parts)?, + ) + .await? + { + CasOutcome::Applied(_) => return Ok(()), + CasOutcome::Conflict(observed) => stored = observed, } } Err(UploadError::Pending) diff --git a/lib/crowdb-access-dataset/src/upload/ownership.rs b/lib/crowdb-access-dataset/src/upload/ownership.rs index 1345a98fd..bce1a069d 100644 --- a/lib/crowdb-access-dataset/src/upload/ownership.rs +++ b/lib/crowdb-access-dataset/src/upload/ownership.rs @@ -78,23 +78,27 @@ impl DatasetUploads { } /// # Errors - /// Claimed data conflicts. Deferred physical reclamation remains observable. + /// Claimed data conflicts. Deferred cleanup notifications remain observable. pub async fn abort(&self, location: &[u8]) -> Result { + let mut stored = self.store.get(location).await?.ok_or(UploadError::NotFound)?; for _ in 0..16 { - let stored = self.store.get(location).await?.ok_or(UploadError::NotFound)?; let mut record = decode_session(&stored.bytes)?; if !record.owners.is_empty() { return Err(UploadError::Conflict); } if record.state != UploadState::Aborted { record.state = UploadState::Aborted; - if !matches!( - self.store - .compare_exchange(location, Some(&stored.bytes), &encode(&record)?) - .await?, - CasOutcome::Applied(_) - ) { - continue; + match self + .store + .compare_exchange(location, Some(&stored.bytes), &encode(&record)?) + .await? + { + CasOutcome::Applied(_) => {} + CasOutcome::Conflict(Some(observed)) => { + stored = observed; + continue; + } + CasOutcome::Conflict(None) => return Err(UploadError::NotFound), } } return self.cleanup(location, record).await; @@ -102,54 +106,63 @@ impl DatasetUploads { Err(UploadError::Pending) } - async fn cleanup(&self, location: &[u8], record: UploadRecord) -> Result { + async fn cleanup(&self, location: &[u8], mut record: UploadRecord) -> Result { if record.cleanup_complete { - return self.get(location).await; + record.parts = self.accepted_parts(location, &record).await?; + return Ok(record); } let entries = self.entries(location, &record).await?; let mut complete = true; for entry in entries { - complete &= self.cleanup_part(&part_key(location, entry.part.index)).await?; + complete &= self + .cleanup_part(&part_key(location, entry.part.index), entry) + .await?; } if complete { + let mut expected = encode(&record)?; for _ in 0..16 { - let stored = self.store.get(location).await?.ok_or(UploadError::NotFound)?; - let mut current = decode_session(&stored.bytes)?; - if current.state != UploadState::Aborted { + if record.state != UploadState::Aborted { return Err(UploadError::Conflict); } - if current.cleanup_complete { + if record.cleanup_complete { break; } - current.cleanup_complete = true; - if matches!( - self.store - .compare_exchange(location, Some(&stored.bytes), &encode(¤t)?) - .await?, - CasOutcome::Applied(_) - ) { - break; + record.cleanup_complete = true; + match self + .store + .compare_exchange(location, Some(&expected), &encode(&record)?) + .await? + { + CasOutcome::Applied(_) => break, + CasOutcome::Conflict(Some(observed)) => { + record = decode_session(&observed.bytes)?; + expected = observed.bytes; + } + CasOutcome::Conflict(None) => return Err(UploadError::NotFound), } } + if !record.cleanup_complete { + return Err(UploadError::Pending); + } } - self.get(location).await + record.parts = self.accepted_parts(location, &record).await?; + Ok(record) } - async fn cleanup_part(&self, key: &[u8]) -> Result { + async fn cleanup_part(&self, key: &[u8], mut record: super::PartEntry) -> Result { + let mut expected = encode(&record)?; for _ in 0..16 { - let Some(stored) = self.store.get(key).await? else { - return Ok(true); - }; - let mut record = decode_part(&stored.bytes)?; if record.state != PartState::Reclaimed { record.state = PartState::Reclaimed; - if !matches!( - self.store - .compare_exchange(key, Some(&stored.bytes), &encode(&record)?) - .await?, - CasOutcome::Applied(_) - ) { - continue; + let value = encode(&record)?; + match self.store.compare_exchange(key, Some(&expected), &value).await? { + CasOutcome::Applied(_) => expected = value, + CasOutcome::Conflict(Some(observed)) => { + record = decode_part(&observed.bytes)?; + expected = observed.bytes; + continue; + } + CasOutcome::Conflict(None) => return Ok(true), } } if record.part.locations.is_empty() { @@ -158,15 +171,17 @@ impl DatasetUploads { if !self.payloads.discard(&record.part.locations).await? { return Ok(false); } - let expected = encode(&record)?; record.part.locations.clear(); - if matches!( - self.store - .compare_exchange(key, Some(&expected), &encode(&record)?) - .await?, - CasOutcome::Applied(_) - ) { - return Ok(true); + match self + .store + .compare_exchange(key, Some(&expected), &encode(&record)?) + .await? + { + CasOutcome::Applied(_) | CasOutcome::Conflict(None) => return Ok(true), + CasOutcome::Conflict(Some(observed)) => { + record = decode_part(&observed.bytes)?; + expected = observed.bytes; + } } } Err(UploadError::Pending) diff --git a/lib/crowdb-access-dataset/src/upload/parts.rs b/lib/crowdb-access-dataset/src/upload/parts.rs index 67a413be3..c7e29b3de 100644 --- a/lib/crowdb-access-dataset/src/upload/parts.rs +++ b/lib/crowdb-access-dataset/src/upload/parts.rs @@ -177,7 +177,8 @@ impl DatasetUploads { let stored = self.store.get(location).await?.ok_or(UploadError::NotFound)?; let mut session = decode_session(&stored.bytes)?; if matches!(session.state, UploadState::Complete | UploadState::Claimed) { - return self.get(location).await; + session.parts = self.accepted_parts(location, &session).await?; + return Ok(session); } if session.state != UploadState::Open || now >= session.expires_at { return Err(UploadError::Conflict); @@ -202,16 +203,19 @@ impl DatasetUploads { .compare_exchange(location, Some(&stored.bytes), &encode(&session)?) .await? { - CasOutcome::Applied(_) => self.get(location).await, - CasOutcome::Conflict(Some(value)) - if matches!( - decode_session(&value.bytes)?.state, - UploadState::Complete | UploadState::Claimed - ) => - { - self.get(location).await + CasOutcome::Applied(_) => { + session.parts = parts; + Ok(session) } - CasOutcome::Conflict(_) => Err(UploadError::Conflict), + CasOutcome::Conflict(Some(value)) => { + let mut observed = decode_session(&value.bytes)?; + if !matches!(observed.state, UploadState::Complete | UploadState::Claimed) { + return Err(UploadError::Conflict); + } + observed.parts = parts; + Ok(observed) + } + CasOutcome::Conflict(None) => Err(UploadError::Conflict), } } } diff --git a/lib/crowdb-access-dataset/tests/common/upload_fault.rs b/lib/crowdb-access-dataset/tests/common/upload_fault.rs new file mode 100644 index 000000000..2613a2d61 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/common/upload_fault.rs @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use crowdb_access_dataset::{CasOutcome, DatasetStore, StoreError, StoredValue}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +use super::upload::TestStore; + +pub enum TestMutation { + Conflict(Vec), + Apply, + LoseReply, +} + +pub struct TestUploadFaultStore { + pub inner: Arc, + key: Vec, + mutations: Vec, + pub reads: AtomicUsize, + pub writes: AtomicUsize, +} + +impl TestUploadFaultStore { + pub fn new(inner: Arc, key: Vec, mutations: Vec) -> Arc { + Arc::new(Self { + inner, + key, + mutations, + reads: AtomicUsize::new(0), + writes: AtomicUsize::new(0), + }) + } +} + +#[async_trait] +impl DatasetStore for TestUploadFaultStore { + async fn get(&self, key: &[u8]) -> Result, StoreError> { + if key == self.key { + self.reads.fetch_add(1, Ordering::Relaxed); + } + self.inner.get(key).await + } + + async fn compare_exchange( + &self, + key: &[u8], + expected: Option<&[u8]>, + value: &[u8], + ) -> Result { + if key != self.key { + return self.inner.compare_exchange(key, expected, value).await; + } + let index = self.writes.fetch_add(1, Ordering::Relaxed); + let current = self.inner.get(key).await?; + assert_eq!( + current.as_ref().map(|stored| stored.bytes.as_slice()), + expected, + "retry must use the last returned observation" + ); + match self.mutations.get(index) { + Some(TestMutation::Conflict(observed)) => { + assert!(matches!( + self.inner.compare_exchange(key, expected, observed).await?, + CasOutcome::Applied(_) + )); + Ok(CasOutcome::Conflict(self.inner.get(key).await?)) + } + Some(TestMutation::LoseReply) => { + self.inner.compare_exchange(key, expected, value).await?; + Err(StoreError::Rejected) + } + Some(TestMutation::Apply) | None => self.inner.compare_exchange(key, expected, value).await, + } + } +} diff --git a/lib/crowdb-access-dataset/tests/upload_mutation_cost_test.rs b/lib/crowdb-access-dataset/tests/upload_mutation_cost_test.rs new file mode 100644 index 000000000..c47cbc482 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/upload_mutation_cost_test.rs @@ -0,0 +1,209 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/upload.rs"] +mod upload; +#[path = "common/upload_fault.rs"] +mod upload_fault; + +use bytes::Bytes; +use crowdb_access_dataset::{ + DatasetIdentity, DatasetStore, DatasetUploads, NamespacePath, OperationId, UploadError, UploadState, +}; +use md5::{Digest, Md5}; +use serde_json::{json, Value}; +use std::sync::{atomic::Ordering, Arc}; +use upload::{TestPayload, TestStore}; +use upload_fault::{TestMutation, TestUploadFaultStore}; + +fn location() -> Vec { + DatasetUploads::location( + &DatasetIdentity::new(NamespacePath::root(), "mutation-cost").unwrap(), + OperationId::random(), + ) + .unwrap() +} + +async fn complete_upload(store: Arc, payloads: Arc, key: &[u8]) { + let uploads = DatasetUploads::new(store, payloads); + uploads + .create(key, 4, 4, Md5::digest(b"data").into(), 100) + .await + .unwrap(); + uploads + .put_part(key, 0, Bytes::from_static(b"data"), 1) + .await + .unwrap(); + uploads.complete(key, 2).await.unwrap(); +} + +async fn changed_records(store: &TestStore, key: &[u8], field: &str) -> Vec { + let mut value: Value = serde_json::from_slice(&store.get(key).await.unwrap().unwrap().bytes).unwrap(); + (101..105) + .map(|number| { + value[field] = json!(number); + TestMutation::Conflict(serde_json::to_vec(&value).unwrap()) + }) + .collect() +} + +#[tokio::test] +async fn create_and_complete_return_known_results_without_confirmation_reads() { + let store = Arc::new(TestStore::default()); + let uploads = DatasetUploads::new(store.clone(), Arc::new(TestPayload::default())); + let key = location(); + let declared = uploads + .create(&key, 4, 4, Md5::digest(b"data").into(), 100) + .await + .unwrap(); + assert_eq!(store.reads.load(Ordering::Relaxed), 0); + assert_eq!( + uploads.create(&key, 4, 4, declared.md5, 100).await.unwrap(), + declared + ); + assert_eq!( + store.reads.load(Ordering::Relaxed), + 1, + "only the part index is read" + ); + uploads + .put_part(&key, 0, Bytes::from_static(b"data"), 1) + .await + .unwrap(); + for _ in 0..2 { + let reads = store.reads.load(Ordering::Relaxed); + let complete = uploads.complete(&key, 2).await.unwrap(); + assert_eq!(complete.state, UploadState::Complete); + assert_eq!(complete.parts.len(), 1); + assert_eq!(store.reads.load(Ordering::Relaxed) - reads, 2); + } +} + +#[tokio::test] +async fn four_index_conflicts_merge_other_parts_using_each_observation() { + let store = Arc::new(TestStore::default()); + let payloads = Arc::new(TestPayload::default()); + let uploads = DatasetUploads::new(store.clone(), payloads.clone()); + let key = location(); + uploads.create(&key, 20, 4, [0; 16], 100).await.unwrap(); + let mut parts = Vec::new(); + let mut mutations = Vec::new(); + for index in 1..5 { + parts.push( + uploads + .put_part(&key, index, Bytes::from_static(b"data"), 1) + .await + .unwrap(), + ); + mutations.push(TestMutation::Conflict(serde_json::to_vec(&parts).unwrap())); + } + let mut index_key = key.clone(); + index_key.extend_from_slice(b"\0I\0\0\0\0"); + store.delete(&index_key).await.unwrap(); + let fault = TestUploadFaultStore::new(store, index_key, mutations); + let uploads = DatasetUploads::new(fault.clone(), payloads); + let first = uploads + .put_part(&key, 0, Bytes::from_static(b"data"), 1) + .await + .unwrap(); + assert_eq!(fault.reads.load(Ordering::Relaxed), 1); + assert_eq!(fault.writes.load(Ordering::Relaxed), 5); + parts.insert(0, first); + assert_eq!(uploads.get(&key).await.unwrap().parts, parts); +} + +#[tokio::test] +async fn abort_reuses_conflicts_and_stops_when_an_owner_wins() { + let store = Arc::new(TestStore::default()); + let payloads = Arc::new(TestPayload::default()); + let key = location(); + complete_upload(store.clone(), payloads.clone(), &key).await; + let mut mutations = changed_records(&store, &key, "expires_at").await; + let mut claimed: Value = serde_json::from_slice(&store.get(&key).await.unwrap().unwrap().bytes).unwrap(); + claimed["state"] = json!("claimed"); + claimed["owners"] = json!([OperationId::random()]); + mutations.push(TestMutation::Conflict(serde_json::to_vec(&claimed).unwrap())); + let fault = TestUploadFaultStore::new(store, key.clone(), mutations); + let uploads = DatasetUploads::new(fault.clone(), payloads); + assert!(matches!(uploads.abort(&key).await, Err(UploadError::Conflict))); + assert_eq!(fault.reads.load(Ordering::Relaxed), 1); + assert_eq!(fault.writes.load(Ordering::Relaxed), 5); + assert_eq!(uploads.read_part(&key, 0).await.unwrap(), b"data".as_slice()); +} + +#[tokio::test] +async fn cleanup_part_reuses_observations_before_and_after_notification() { + let store = Arc::new(TestStore::default()); + let payloads = Arc::new(TestPayload::default()); + let key = location(); + complete_upload(store.clone(), payloads.clone(), &key).await; + let mut part_key = key.clone(); + part_key.extend_from_slice(b"\0P\0\0\0\0"); + let mut mutations = changed_records(&store, &part_key, "deadline").await; + mutations.push(TestMutation::Apply); + let mut retired: Value = + serde_json::from_slice(&store.get(&part_key).await.unwrap().unwrap().bytes).unwrap(); + retired["state"] = json!("Reclaimed"); + for number in 105..109 { + retired["deadline"] = json!(number); + mutations.push(TestMutation::Conflict(serde_json::to_vec(&retired).unwrap())); + } + let fault = TestUploadFaultStore::new(store, part_key, mutations); + let uploads = DatasetUploads::new(fault.clone(), payloads); + assert!(uploads.abort(&key).await.unwrap().cleanup_complete); + assert_eq!( + fault.reads.load(Ordering::Relaxed), + 1, + "the initial entry enumeration only" + ); + assert_eq!(fault.writes.load(Ordering::Relaxed), 10); +} + +#[tokio::test] +async fn cleanup_completion_conflicts_do_not_reload_the_upload() { + let store = Arc::new(TestStore::default()); + let payloads = Arc::new(TestPayload::default()); + let key = location(); + complete_upload(store.clone(), payloads.clone(), &key).await; + let mut mutations = vec![TestMutation::Apply]; + for mutation in changed_records(&store, &key, "expires_at").await { + let TestMutation::Conflict(bytes) = mutation else { + unreachable!() + }; + let mut value: Value = serde_json::from_slice(&bytes).unwrap(); + value["state"] = json!("aborted"); + mutations.push(TestMutation::Conflict(serde_json::to_vec(&value).unwrap())); + } + let fault = TestUploadFaultStore::new(store, key.clone(), mutations); + let uploads = DatasetUploads::new(fault.clone(), payloads); + assert!(uploads.abort(&key).await.unwrap().cleanup_complete); + assert_eq!(fault.reads.load(Ordering::Relaxed), 1); + assert_eq!(fault.writes.load(Ordering::Relaxed), 6); +} + +#[tokio::test] +async fn unknown_completion_is_propagated_without_confirmation_get() { + let store = Arc::new(TestStore::default()); + let payloads = Arc::new(TestPayload::default()); + let key = location(); + let uploads = DatasetUploads::new(store.clone(), payloads.clone()); + uploads + .create(&key, 4, 4, Md5::digest(b"data").into(), 100) + .await + .unwrap(); + uploads + .put_part(&key, 0, Bytes::from_static(b"data"), 1) + .await + .unwrap(); + let fault = TestUploadFaultStore::new(store, key.clone(), vec![TestMutation::LoseReply]); + let uploads = DatasetUploads::new(fault.clone(), payloads); + assert!(matches!( + uploads.complete(&key, 2).await, + Err(UploadError::Store(_)) + )); + assert_eq!(fault.reads.load(Ordering::Relaxed), 1); + assert_eq!( + uploads.complete(&key, 3).await.unwrap().state, + UploadState::Complete + ); +} From e38252a2309ab476851475280aff33d8e46b8708 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 11:08:28 +0800 Subject: [PATCH 098/258] Carry multipart observations through publication and cleanup --- .../s3/design-crowdb-access-s3.md | 9 + doc/working/plan-access-simplification.md | 19 +- .../src/metadata/multipart_repository.rs | 55 ++---- .../multipart_repository/completion.rs | 17 +- .../metadata/multipart_repository/exchange.rs | 70 ++++++++ .../metadata/multipart_repository/parts.rs | 127 +++++++++---- .../multipart_repository/publication.rs | 30 ++-- .../metadata/multipart_repository/terminal.rs | 22 +-- .../tests/common/multipart_cost.rs | 170 ++++++++++++++++++ .../tests/common/multipart_progress.rs | 30 ++++ .../tests/multipart_repository_test.rs | 106 ++++++++++- 11 files changed, 541 insertions(+), 114 deletions(-) create mode 100644 lib/crowdb-access-s3/src/metadata/multipart_repository/exchange.rs create mode 100644 lib/crowdb-access-s3/tests/common/multipart_cost.rs diff --git a/doc/design/access-server/s3/design-crowdb-access-s3.md b/doc/design/access-server/s3/design-crowdb-access-s3.md index a488f9130..2d057c6be 100644 --- a/doc/design/access-server/s3/design-crowdb-access-s3.md +++ b/doc/design/access-server/s3/design-crowdb-access-s3.md @@ -199,6 +199,15 @@ iteration instead of discarding it and reading again. This coordination is scope to one upload and preserves exact staged-byte accounting and completion selection; independent uploads share no reservation record. +After a successful reservation, the writer reuses its known part pointer and +immutable generation to publish and settle it. Recovery loads that evidence +when a later request helps the reservation. A bound upload is loaded directly +through its authoritative key and validated against its bucket, object, and +upload identity; the listing index is not reread at every stage. Definite +conflicts retain the typed upload record for the next operation. Part progress +has a five-second work deadline so continuous contention cannot run forever; +expiration of this local work budget returns a retryable busy result. + Completion validates the ordered selected part numbers, raw MD5 values, minimum nonfinal size, and current generations. It freezes the selection under the session compare-and-swap, composes chunk locations with adjusted logical diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 52666cb0b..3064172ad 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -299,10 +299,10 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 ### P5 — S3 multipart（客户端控制 part；协议资源与过程状态分开审查） - [ ] **P5.1 精简 MPU 持久化与重复装载**：先说明 UploadId/ListParts/Complete 所需资源事实， - 删除无用过程阶段；复核旧实现新 part `7G + 4C` 路径，把已知 session/generation/ - pointer 传给 settle；验证 identity index 是否只是列表元数据，不能未经证明删除权威检查。Files: S3 `metadata/multipart_repository{.rs,/parts.rs}`。 -- [ ] **P5.2 保留 typed CAS 结果**：exchange 返回 applied/observed，消除 complete/abort/settle - 对同 key 的 conflict 后 GET；同时覆盖同 part 替换、不同 part 并发、complete/abort 和不确定写入。Files: S3 `metadata/multipart_repository/{completion,publication,terminal}.rs`。 + 删除无用过程阶段。已完成正常 part 的 session/generation/pointer 直传，repository 成本 + 从旧 `7G + 4C` 降至实测 `2G + 4C`；独立 identity 查询仍验证 listing index。 + 剩余任务是客户端最终清单与 pending/accounting/generation 的必要性审查及删除， + ListParts、完成数据和最终对象引用不能丢失。Files: S3 `metadata/multipart_repository{.rs,/parts.rs}`。 ### P6 — Iceberg 旧 multipart / 全局 credits @@ -437,6 +437,17 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 ## 已完成与验证记录 +- **2026-10-24，P5.2 与 P5.1 装载部分**：S3 MPU 的内部 exchange 保留并校验 typed + observation；freeze、abort、settle、发布完成标记和 object predecessor 冲突不再 GET + 同一 key。绑定 upload 后直读权威记录，校验 bucket/object/upload，阶段间不重读列表索引； + reserve 成功复用本次 generation 与 pointer，后续帮助恢复仍加载其证据。 + routed actor 实测新 part `2G + 4C`、相同内容重试 `2G + 0C`、单 part freeze `3G + 1C`、 + publish `4G + 2C`；不含外层 bucket/auth/payload IO。unknown 保留现有核对语义。 + 固定次数会把帮助其他 part 的有效进展算作失败，改用五秒工作期限，不放大重试次数。 + MPU repository 20 项含同 part 十个覆盖者、不同 part 十个写者、两类 object 冲突、 + 终态 observation、丢回复与 exact generation；S3 `--all-targets` 及 native MPU 五项通过。 + all-target/all-feature clippy、fmt 和 diff 检查通过；没有新增过程记录、锁或 wire 字段。 + - **2026-10-24，P4.2**：part 索引、abort、cleanup 完成标记、part 清理均直接沿 CAS 的 observed value 重试；初始 part 枚举的结果也直接交给清理，不重复 GET。 索引连续四次冲突保留其他四个 part；abort 观察到新 owner 立即停止；cleanup 在通知前后 diff --git a/lib/crowdb-access-s3/src/metadata/multipart_repository.rs b/lib/crowdb-access-s3/src/metadata/multipart_repository.rs index 5fd608f41..ba7d0bece 100644 --- a/lib/crowdb-access-s3/src/metadata/multipart_repository.rs +++ b/lib/crowdb-access-s3/src/metadata/multipart_repository.rs @@ -3,7 +3,6 @@ //! CAS-backed multipart authority in the S3 Chunk-KV namespace. -use crowdb_access_multipart::next_revision; use std::sync::Arc; use super::{ @@ -12,12 +11,14 @@ use super::{ }; mod completion; +mod exchange; mod listing; mod parts; mod publication; mod terminal; pub use completion::CompletionPart; +use exchange::MultipartExchange; pub use listing::{MultipartPartPage, MultipartUploadPage}; pub use terminal::MultipartExpiryPage; @@ -110,8 +111,19 @@ impl MultipartRepository { &self, identity: &MultipartSessionRecord, ) -> Result, MultipartRepositoryError> { - self.load_identity(identity.bucket_id, &identity.object_key, &identity.upload_id) - .await + self.store + .get(self.session_key(identity)) + .await? + .map(|value| { + MultipartSessionRecord::decode( + &value.value, + identity.bucket_id, + &identity.object_key, + &identity.upload_id, + ) + .map_err(Into::into) + }) + .transpose() } /// Reads one session through its bucket, object and upload identity. @@ -141,43 +153,6 @@ impl MultipartRepository { .transpose() } - /// Moves one session phase under an exact-value CAS fence. - /// - /// # Errors - /// Rejects invalid revisions, changed identity and uncertain storage - /// writes that cannot be confirmed by a matching read. - pub async fn exchange( - &self, - previous: &MultipartSessionRecord, - next: &MultipartSessionRecord, - ) -> Result { - if previous.bucket_id != next.bucket_id - || previous.object_key != next.object_key - || previous.upload_id != next.upload_id - || next_revision(previous.revision) != Some(next.revision) - { - return Err(MultipartRepositoryError::Conflict); - } - let key = self.session_key(previous); - let expected = previous.encode()?; - let value = next.encode()?; - match self - .store - .compare_exchange_observed(key, expected, value.clone()) - .await - { - Ok((true, _)) => Ok(true), - Ok((false, observed)) => Ok(observed.is_some_and(|observed| observed.value == value)), - Err(error) => { - if self.load(next).await?.as_ref() == Some(next) { - Ok(true) - } else { - Err(error.into()) - } - } - } - } - /// Reads one current part generation. /// /// # Errors diff --git a/lib/crowdb-access-s3/src/metadata/multipart_repository/completion.rs b/lib/crowdb-access-s3/src/metadata/multipart_repository/completion.rs index f288a5cc2..2ac71cd76 100644 --- a/lib/crowdb-access-s3/src/metadata/multipart_repository/completion.rs +++ b/lib/crowdb-access-s3/src/metadata/multipart_repository/completion.rs @@ -8,7 +8,9 @@ use crowdb_access_multipart::{ }; use sha2::{Digest as _, Sha256}; -use super::{MultipartPhase, MultipartRepository, MultipartRepositoryError, MultipartSessionRecord}; +use super::{ + MultipartExchange, MultipartPhase, MultipartRepository, MultipartRepositoryError, MultipartSessionRecord, +}; const MIN_NONFINAL_PART_BYTES: u64 = 5 * 1024 * 1024; @@ -106,7 +108,18 @@ impl MultipartRepository { .map(|value| Sha256::digest(value.value).into()), ); next.etag = Some(assembled.etag); - Ok(self.exchange(¤t, &next).await?.then_some(next)) + Ok(match self.exchange_observed(¤t, &next).await? { + MultipartExchange::Applied => Some(next), + MultipartExchange::Conflict(Some(observed)) + if matches!( + observed.phase, + MultipartPhase::Publishing | MultipartPhase::Published + ) && observed.completion_request_digest == Some(request_digest) => + { + Some(*observed) + } + MultipartExchange::Conflict(_) => None, + }) } } diff --git a/lib/crowdb-access-s3/src/metadata/multipart_repository/exchange.rs b/lib/crowdb-access-s3/src/metadata/multipart_repository/exchange.rs new file mode 100644 index 000000000..854a31a29 --- /dev/null +++ b/lib/crowdb-access-s3/src/metadata/multipart_repository/exchange.rs @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_access_multipart::next_revision; + +use super::{MultipartRepository, MultipartRepositoryError, MultipartSessionRecord}; + +pub(super) enum MultipartExchange { + Applied, + Conflict(Option>), +} + +impl MultipartRepository { + /// Moves one upload under an exact-value CAS fence. + /// # Errors + /// Rejects invalid revisions, changed identity and unconfirmed storage writes. + pub async fn exchange( + &self, + previous: &MultipartSessionRecord, + next: &MultipartSessionRecord, + ) -> Result { + Ok(match self.exchange_observed(previous, next).await? { + MultipartExchange::Applied => true, + MultipartExchange::Conflict(observed) => observed.as_deref() == Some(next), + }) + } + + pub(super) async fn exchange_observed( + &self, + previous: &MultipartSessionRecord, + next: &MultipartSessionRecord, + ) -> Result { + if previous.bucket_id != next.bucket_id + || previous.object_key != next.object_key + || previous.upload_id != next.upload_id + || next_revision(previous.revision) != Some(next.revision) + { + return Err(MultipartRepositoryError::Conflict); + } + let expected = previous.encode()?; + let value = next.encode()?; + match self + .store + .compare_exchange_observed(self.session_key(previous), expected, value) + .await + { + Ok((true, _)) => Ok(MultipartExchange::Applied), + Ok((false, observed)) => Ok(MultipartExchange::Conflict( + observed + .map(|observed| { + MultipartSessionRecord::decode( + &observed.value, + previous.bucket_id, + &previous.object_key, + &previous.upload_id, + ) + }) + .transpose()? + .map(Box::new), + )), + Err(error) => { + if self.load(next).await?.as_ref() == Some(next) { + Ok(MultipartExchange::Applied) + } else { + Err(error.into()) + } + } + } + } +} diff --git a/lib/crowdb-access-s3/src/metadata/multipart_repository/parts.rs b/lib/crowdb-access-s3/src/metadata/multipart_repository/parts.rs index 9366a93a8..ccfa91882 100644 --- a/lib/crowdb-access-s3/src/metadata/multipart_repository/parts.rs +++ b/lib/crowdb-access-s3/src/metadata/multipart_repository/parts.rs @@ -10,10 +10,15 @@ use crowdb_access_multipart::{ use sha2::{Digest as _, Sha256}; use super::{ - MetadataKey, MultipartPartRecord, MultipartPhase, MultipartRepository, MultipartRepositoryError, - MultipartSessionRecord, PutIfAbsentOutcome, + MetadataKey, MultipartExchange, MultipartPartRecord, MultipartPhase, MultipartRepository, + MultipartRepositoryError, MultipartSessionRecord, PutIfAbsentOutcome, }; +pub(super) struct PartProgress { + pub(super) accepted: Option, + pub(super) current: MultipartSessionRecord, +} + impl MultipartRepository { /// Publishes a streamed part only while the upload remains open. /// @@ -32,27 +37,42 @@ impl MultipartRepository { session: &MultipartSessionRecord, part: &MultipartPartRecord, now_ms: u64, + ) -> Result, MultipartRepositoryError> { + tokio::time::timeout( + std::time::Duration::from_secs(5), + self.put_part_until_settled(session, part, now_ms), + ) + .await + .map_err(|_| MultipartRepositoryError::Busy)? + } + + async fn put_part_until_settled( + &self, + session: &MultipartSessionRecord, + part: &MultipartPartRecord, + now_ms: u64, ) -> Result, MultipartRepositoryError> { let mut current = self .load(session) .await? .ok_or(MultipartRepositoryError::Conflict)?; loop { - if current.pending.is_some() { - self.settle_pending_part(¤t).await?; - } else if let Some(saved) = self.reserve_stream_part(¤t, part, now_ms).await? { - return Ok(Some(saved)); - } - let latest = self - .load(session) - .await? - .ok_or(MultipartRepositoryError::Conflict)?; - if latest.revision == current.revision { - return Ok(None); + let helping = current.pending.is_some(); + let progress = if helping { + self.settle_current_part(¤t).await? + } else { + self.reserve_stream_part(¤t, part, now_ms).await? + }; + if let Some(saved) = progress.accepted { + if !helping + || (saved.number == part.number + && saved.length == part.length + && saved.raw_md5 == part.raw_md5) + { + return Ok(Some(saved)); + } } - current = latest; - // Another publication advanced the durable fence. Re-evaluate our - // part against that state without uploading its bytes again. + current = progress.current; } } @@ -61,7 +81,7 @@ impl MultipartRepository { current: &MultipartSessionRecord, part: &MultipartPartRecord, now_ms: u64, - ) -> Result, MultipartRepositoryError> { + ) -> Result { if current.phase != MultipartPhase::Open || !live_at(current.created_ms, current.expires_ms, now_ms) || part.bucket_id != current.bucket_id @@ -75,7 +95,10 @@ impl MultipartRepository { let before = self.part(current, part.number).await?; if let Some(existing) = &before { if existing.length == part.length && existing.raw_md5 == part.raw_md5 { - return Ok(Some(existing.clone())); + return Ok(PartProgress { + accepted: Some(existing.clone()), + current: current.clone(), + }); } } let mut after = part.clone(); @@ -110,10 +133,14 @@ impl MultipartRepository { after_digest: Sha256::digest(after.encode()?).into(), after_length: after.length, }); - if !self.exchange(current, &reserved).await? { - return Ok(None); + match self.exchange_observed(current, &reserved).await? { + MultipartExchange::Applied => self.finish_reserved_part(&reserved, after, before).await, + MultipartExchange::Conflict(Some(observed)) => Ok(PartProgress { + accepted: None, + current: *observed, + }), + MultipartExchange::Conflict(None) => Err(MultipartRepositoryError::Conflict), } - self.settle_pending_part(&reserved).await } async fn persist_generation( @@ -173,12 +200,22 @@ impl MultipartRepository { &self, session: &MultipartSessionRecord, ) -> Result, MultipartRepositoryError> { + Ok(self.settle_current_part(session).await?.accepted) + } + + pub(super) async fn settle_current_part( + &self, + session: &MultipartSessionRecord, + ) -> Result { let current = self .load(session) .await? .ok_or(MultipartRepositoryError::Conflict)?; if current != *session { - return Ok(None); + return Ok(PartProgress { + accepted: None, + current, + }); } let pending = current .pending @@ -188,25 +225,45 @@ impl MultipartRepository { .part_generation(¤t, pending.number, pending.after_revision) .await? .ok_or(MultipartRepositoryError::InvalidPart)?; + let before = self.part(¤t, pending.number).await?; + self.finish_reserved_part(¤t, after, before).await + } + + async fn finish_reserved_part( + &self, + current: &MultipartSessionRecord, + after: MultipartPartRecord, + before: Option, + ) -> Result { + let pending = current + .pending + .as_ref() + .ok_or(MultipartRepositoryError::Conflict)?; let digest: [u8; 32] = Sha256::digest(after.encode()?).into(); if after.length != pending.after_length || digest != pending.after_digest { return Err(MultipartRepositoryError::InvalidPart); } - self.publish_reserved_pointer(¤t, pending, &after).await?; + self.publish_reserved_pointer(current, pending, &after, before) + .await?; let mut settled = current.clone(); settled.revision = next_revision(current.revision).ok_or(MultipartRepositoryError::Conflict)?; settled.pending = None; - if self.exchange(¤t, &settled).await? { - return Ok(Some(after)); - } - let latest = self - .load(¤t) - .await? - .ok_or(MultipartRepositoryError::Conflict)?; - if latest.pending.is_none() && self.part(&latest, after.number).await?.as_ref() == Some(&after) { - Ok(Some(after)) - } else { - Ok(None) + match self.exchange_observed(current, &settled).await? { + MultipartExchange::Applied => Ok(PartProgress { + accepted: Some(after), + current: settled, + }), + MultipartExchange::Conflict(Some(observed)) if observed.revision > current.revision => { + Ok(PartProgress { + accepted: Some(after), + current: *observed, + }) + } + MultipartExchange::Conflict(Some(observed)) => Ok(PartProgress { + accepted: None, + current: *observed, + }), + MultipartExchange::Conflict(None) => Err(MultipartRepositoryError::Conflict), } } @@ -215,8 +272,8 @@ impl MultipartRepository { session: &MultipartSessionRecord, pending: &PendingPartMutation, after: &MultipartPartRecord, + current: Option, ) -> Result<(), MultipartRepositoryError> { - let current = self.part(session, pending.number).await?; if current.as_ref() == Some(after) { return Ok(()); } diff --git a/lib/crowdb-access-s3/src/metadata/multipart_repository/publication.rs b/lib/crowdb-access-s3/src/metadata/multipart_repository/publication.rs index 90599c27f..ae3ad729e 100644 --- a/lib/crowdb-access-s3/src/metadata/multipart_repository/publication.rs +++ b/lib/crowdb-access-s3/src/metadata/multipart_repository/publication.rs @@ -7,8 +7,8 @@ use crowdb_access_multipart::{next_revision, MultipartComposer}; use sha2::{Digest as _, Sha256}; use super::{ - MetadataKey, MultipartPhase, MultipartRepository, MultipartRepositoryError, MultipartSessionRecord, - PutIfAbsentOutcome, + MetadataKey, MultipartExchange, MultipartPhase, MultipartRepository, MultipartRepositoryError, + MultipartSessionRecord, PutIfAbsentOutcome, }; use crate::integrity::multipart_checksum_marker; use crate::metadata::ObjectRecord; @@ -95,18 +95,21 @@ impl MultipartRepository { } let mutation = if let Some(before) = observed { self.store - .compare_exchange(key.clone(), before.value, value.clone()) + .compare_exchange_observed(key.clone(), before.value, value.clone()) .await } else { self.store .put_if_absent(key.clone(), value.clone()) .await - .map(|outcome| matches!(outcome, PutIfAbsentOutcome::Inserted { .. })) + .map(|outcome| match outcome { + PutIfAbsentOutcome::Inserted { .. } => (true, None), + PutIfAbsentOutcome::Existing(observed) => (false, Some(observed)), + }) }; match mutation { - Ok(true) => {} - Ok(false) => { - if self.store.get(key).await?.as_ref().map(|entry| &entry.value) != Some(&value) { + Ok((true, _)) => {} + Ok((false, observed)) => { + if observed.as_ref().map(|entry| &entry.value) != Some(&value) { return Err(MultipartRepositoryError::Conflict); } } @@ -126,13 +129,12 @@ impl MultipartRepository { let mut next = current.clone(); next.revision = next_revision(current.revision).ok_or(MultipartRepositoryError::Conflict)?; next.phase = MultipartPhase::Published; - if self.exchange(current, &next).await? { - Ok(next) - } else { - self.load(current) - .await? - .filter(|record| record.phase == MultipartPhase::Published) - .ok_or(MultipartRepositoryError::Conflict) + match self.exchange_observed(current, &next).await? { + MultipartExchange::Applied => Ok(next), + MultipartExchange::Conflict(Some(observed)) if observed.phase == MultipartPhase::Published => { + Ok(*observed) + } + MultipartExchange::Conflict(_) => Err(MultipartRepositoryError::Conflict), } } } diff --git a/lib/crowdb-access-s3/src/metadata/multipart_repository/terminal.rs b/lib/crowdb-access-s3/src/metadata/multipart_repository/terminal.rs index 1d09f075c..7b778e843 100644 --- a/lib/crowdb-access-s3/src/metadata/multipart_repository/terminal.rs +++ b/lib/crowdb-access-s3/src/metadata/multipart_repository/terminal.rs @@ -4,7 +4,8 @@ //! Terminal multipart session transitions. use super::{ - MetadataKey, MultipartPhase, MultipartRepository, MultipartRepositoryError, MultipartSessionRecord, + MetadataKey, MultipartExchange, MultipartPhase, MultipartRepository, MultipartRepositoryError, + MultipartSessionRecord, }; use crate::metadata::BucketId; use crowdb_access_multipart::next_revision; @@ -105,11 +106,7 @@ impl MultipartRepository { .await? .ok_or(MultipartRepositoryError::Conflict)?; if current.pending.is_some() { - self.settle_pending_part(¤t).await?; - current = self - .load(session) - .await? - .ok_or(MultipartRepositoryError::Conflict)?; + current = self.settle_current_part(¤t).await?.current; if current.pending.is_some() { return Err(MultipartRepositoryError::Busy); } @@ -123,13 +120,12 @@ impl MultipartRepository { let mut next = current.clone(); next.revision = next_revision(current.revision).ok_or(MultipartRepositoryError::Conflict)?; next.phase = MultipartPhase::Aborted; - if self.exchange(¤t, &next).await? { - Ok(next) - } else { - self.load(session) - .await? - .filter(|record| record.phase == MultipartPhase::Aborted) - .ok_or(MultipartRepositoryError::Conflict) + match self.exchange_observed(¤t, &next).await? { + MultipartExchange::Applied => Ok(next), + MultipartExchange::Conflict(Some(observed)) if observed.phase == MultipartPhase::Aborted => { + Ok(*observed) + } + MultipartExchange::Conflict(_) => Err(MultipartRepositoryError::Conflict), } } } diff --git a/lib/crowdb-access-s3/tests/common/multipart_cost.rs b/lib/crowdb-access-s3/tests/common/multipart_cost.rs new file mode 100644 index 000000000..1d591528e --- /dev/null +++ b/lib/crowdb-access-s3/tests/common/multipart_cost.rs @@ -0,0 +1,170 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + part, repository_with_probe, session, CompletionPart, MetadataKey, MultipartPhase, + MultipartRepositoryError, TenantId, +}; + +fn tenant() -> TenantId { + TenantId::new(b"tenant".to_vec()).unwrap() +} + +fn completion() -> [CompletionPart; 1] { + [CompletionPart { + number: 1, + etag: "09".repeat(16), + }] +} + +#[tokio::test] +async fn normal_part_and_completion_have_no_post_write_metadata_reload() { + let (repository, probe, _) = repository_with_probe().await; + let session = session(); + repository.begin(&session).await.unwrap(); + let before = probe.counts(); + repository + .put_stream_part(&session, &part(), 110) + .await + .unwrap() + .unwrap(); + let after = probe.counts(); + assert_eq!((after.0 - before.0, after.1 - before.1), (2, 4)); + repository + .put_stream_part(&session, &part(), 111) + .await + .unwrap() + .unwrap(); + let replay = probe.counts(); + assert_eq!((replay.0 - after.0, replay.1 - after.1), (2, 0)); + let frozen = repository + .freeze_completion(&session, &completion(), 120) + .await + .unwrap() + .unwrap(); + let freeze = probe.counts(); + assert_eq!((freeze.0 - replay.0, freeze.1 - replay.1), (3, 1)); + assert_eq!( + repository.publish_completion(&frozen).await.unwrap().phase, + MultipartPhase::Published + ); + let published = probe.counts(); + assert_eq!((published.0 - freeze.0, published.1 - freeze.1), (4, 2)); +} + +#[tokio::test] +async fn abort_accepts_the_returned_terminal_record_without_get() { + let (repository, probe, _) = repository_with_probe().await; + let session = session(); + repository.begin(&session).await.unwrap(); + let mut aborted = session.clone(); + aborted.revision += 1; + aborted.phase = MultipartPhase::Aborted; + probe.conflict( + MetadataKey::multipart_session(&tenant(), session.bucket_id, &session.upload_id), + aborted.encode().unwrap(), + 0, + ); + let before = probe.counts(); + assert_eq!(repository.abort(&session).await.unwrap(), aborted); + let after = probe.counts(); + assert_eq!((after.0 - before.0, after.1 - before.1), (1, 1)); +} + +#[tokio::test] +async fn completion_reuses_a_concurrent_identical_freeze_observation() { + let (repository, probe, store) = repository_with_probe().await; + let session = session(); + repository.begin(&session).await.unwrap(); + repository.put_stream_part(&session, &part(), 110).await.unwrap(); + let open = repository.load(&session).await.unwrap().unwrap(); + let frozen = repository + .freeze_completion(&session, &completion(), 120) + .await + .unwrap() + .unwrap(); + let key = MetadataKey::multipart_session(&tenant(), session.bucket_id, &session.upload_id); + assert!(store + .compare_exchange(key.clone(), frozen.encode().unwrap(), open.encode().unwrap()) + .await + .unwrap()); + probe.conflict(key, frozen.encode().unwrap(), 0); + assert_eq!( + repository + .freeze_completion(&session, &completion(), 120) + .await + .unwrap(), + Some(frozen) + ); +} + +#[tokio::test] +async fn publication_finish_reuses_the_published_observation() { + let (repository, probe, _) = repository_with_probe().await; + let session = session(); + repository.begin(&session).await.unwrap(); + repository.put_stream_part(&session, &part(), 110).await.unwrap(); + let frozen = repository + .freeze_completion(&session, &completion(), 120) + .await + .unwrap() + .unwrap(); + let mut published = frozen.clone(); + published.revision += 1; + published.phase = MultipartPhase::Published; + probe.conflict( + MetadataKey::multipart_session(&tenant(), session.bucket_id, &session.upload_id), + published.encode().unwrap(), + 0, + ); + assert_eq!(repository.publish_completion(&frozen).await.unwrap(), published); +} + +#[tokio::test] +async fn object_conflict_returns_without_get_or_overwriting_a_competitor() { + for predecessor in [None, Some(b"old".to_vec())] { + let (repository, probe, store) = repository_with_probe().await; + let session = session(); + repository.begin(&session).await.unwrap(); + repository.put_stream_part(&session, &part(), 110).await.unwrap(); + let key = MetadataKey::object(&tenant(), session.bucket_id, &session.object_key).unwrap(); + if let Some(value) = predecessor { + store.put_if_absent(key.clone(), value).await.unwrap(); + } + let frozen = repository + .freeze_completion(&session, &completion(), 120) + .await + .unwrap() + .unwrap(); + probe.conflict(key, b"concurrent replacement".to_vec(), 0); + assert!(matches!( + repository.publish_completion(&frozen).await, + Err(MultipartRepositoryError::Conflict) + )); + } +} + +#[tokio::test] +async fn clearing_reservation_reuses_the_returned_upload_without_get() { + let (repository, probe, _) = repository_with_probe().await; + let session = session(); + repository.begin(&session).await.unwrap(); + let mut settled = session.clone(); + settled.revision = 3; + settled.part_count = 1; + settled.staged_bytes = part().length; + probe.conflict( + MetadataKey::multipart_session(&tenant(), session.bucket_id, &session.upload_id), + settled.encode().unwrap(), + 1, + ); + let before = probe.counts(); + let saved = repository + .put_stream_part(&session, &part(), 110) + .await + .unwrap() + .unwrap(); + assert_eq!(saved.number, 1); + let after = probe.counts(); + assert_eq!((after.0 - before.0, after.1 - before.1), (2, 4)); +} diff --git a/lib/crowdb-access-s3/tests/common/multipart_progress.rs b/lib/crowdb-access-s3/tests/common/multipart_progress.rs index 0eacaf673..9ff4c734c 100644 --- a/lib/crowdb-access-s3/tests/common/multipart_progress.rs +++ b/lib/crowdb-access-s3/tests/common/multipart_progress.rs @@ -39,6 +39,36 @@ async fn concurrent_distinct_parts_publish_without_reuploading() { assert!(current.pending.is_none()); } +#[tokio::test] +async fn concurrent_replacements_return_their_own_part_content() { + let (repository, _, _) = repository().await; + let repository = Arc::new(repository); + let session = session(); + repository.begin(&session).await.unwrap(); + let mut tasks = Vec::new(); + for digest in 1..=10 { + let repository = repository.clone(); + let session = session.clone(); + tasks.push(tokio::spawn(async move { + let mut value = part(); + value.raw_md5 = [digest; 16]; + let saved = repository + .put_stream_part(&session, &value, 110) + .await + .unwrap() + .unwrap(); + assert_eq!(saved.raw_md5, value.raw_md5); + })); + } + for task in tasks { + task.await.unwrap(); + } + let current = repository.load(&session).await.unwrap().unwrap(); + assert_eq!(current.part_count, 1); + assert_eq!(current.staged_bytes, 5); + assert!(current.pending.is_none()); +} + #[tokio::test] async fn orphan_generation_does_not_poison_retry_or_changed_part() { for identical in [true, false] { diff --git a/lib/crowdb-access-s3/tests/multipart_repository_test.rs b/lib/crowdb-access-s3/tests/multipart_repository_test.rs index b6dd8ec9c..9b9da897e 100644 --- a/lib/crowdb-access-s3/tests/multipart_repository_test.rs +++ b/lib/crowdb-access-s3/tests/multipart_repository_test.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -28,6 +28,8 @@ use crowdb_protocol::common::ChunkId; use sha2::{Digest as _, Sha256}; use tokio::sync::{mpsc, oneshot}; +#[path = "common/multipart_cost.rs"] +mod cost; #[path = "common/multipart_progress.rs"] mod progress; @@ -43,6 +45,35 @@ impl ChunkKvRangeCatalogSource for Catalog { enum ActorRequest { Point(PointOperation, oneshot::Sender>), Scan(ScanRequest, oneshot::Sender>), + Inject(TestConflict), +} + +struct TestConflict { + key: Vec, + value: Vec, + skip: usize, +} + +struct TestProbe { + reads: Arc, + writes: Arc, + lose_reply: Arc, + sender: mpsc::UnboundedSender, +} + +impl TestProbe { + fn counts(&self) -> (usize, usize) { + ( + self.reads.load(Ordering::Relaxed), + self.writes.load(Ordering::Relaxed), + ) + } + + fn conflict(&self, key: Vec, value: Vec, skip: usize) { + self.sender + .send(ActorRequest::Inject(TestConflict { key, value, skip })) + .unwrap(); + } } struct ActorTransport(mpsc::UnboundedSender); @@ -159,17 +190,60 @@ fn scan_reply(request: &ScanRequest, values: &HashMap, RpcValue>) -> Chu } async fn repository() -> (MultipartRepository, Arc, Arc) { - let (sender, mut receiver) = mpsc::unbounded_channel::(); - let lose_reply = Arc::new(AtomicBool::new(false)); - let actor_lose_reply = Arc::clone(&lose_reply); + let (repository, probe, store) = repository_with_probe().await; + (repository, probe.lose_reply.clone(), store) +} + +fn spawn_actor( + mut receiver: mpsc::UnboundedReceiver, + reads: Arc, + writes: Arc, + lose_reply: Arc, +) { tokio::spawn(async move { let mut values = HashMap::new(); + let mut conflict: Option = None; + let mut forbidden_get: Option> = None; while let Some(request) = receiver.recv().await { match request { ActorRequest::Point(operation, response) => { let is_mutation = !matches!(operation, PointOperation::Get { .. }); + if let PointOperation::Get { key } = &operation { + reads.fetch_add(1, Ordering::Relaxed); + assert_ne!( + Some(key), + forbidden_get.as_ref(), + "GET after definite CAS conflict" + ); + } else { + writes.fetch_add(1, Ordering::Relaxed); + } + let mutation_key = match &operation { + PointOperation::CompareExchange { key, .. } + | PointOperation::PutIfAbsent { key, .. } => Some(key), + _ => None, + }; + if let Some(fault) = conflict.as_mut().filter(|fault| Some(&fault.key) == mutation_key) { + if fault.skip == 0 { + let fault = conflict.take().unwrap(); + let revision = values + .get(&fault.key) + .map_or(1, |value: &RpcValue| value.revision + 1); + forbidden_get = Some(fault.key.clone()); + values.insert( + fault.key.clone(), + RpcValue { + key: fault.key, + value: fault.value, + revision, + }, + ); + } else { + fault.skip -= 1; + } + } let result = reply(operation, &mut values); - if is_mutation && actor_lose_reply.swap(false, Ordering::SeqCst) { + if is_mutation && lose_reply.swap(false, Ordering::SeqCst) { let _ = response.send(Err(ClientError::Transport("committed reply lost".into()))); } else { let _ = response.send(Ok(result)); @@ -178,9 +252,29 @@ async fn repository() -> (MultipartRepository, Arc, Arc { let _ = response.send(Ok(scan_reply(&request, &values))); } + ActorRequest::Inject(fault) => { + conflict = Some(fault); + forbidden_get = None; + } } } }); +} + +async fn repository_with_probe() -> (MultipartRepository, Arc, Arc) { + let (sender, receiver) = mpsc::unbounded_channel::(); + let probe = Arc::new(TestProbe { + reads: Arc::new(AtomicUsize::new(0)), + writes: Arc::new(AtomicUsize::new(0)), + lose_reply: Arc::new(AtomicBool::new(false)), + sender: sender.clone(), + }); + spawn_actor( + receiver, + probe.reads.clone(), + probe.writes.clone(), + probe.lose_reply.clone(), + ); let mut page = ChunkKvRangeCatalogPage { generation: 1, page_index: 0, @@ -234,7 +328,7 @@ async fn repository() -> (MultipartRepository, Arc, Arc Date: Sat, 24 Oct 2026 11:17:25 +0800 Subject: [PATCH 099/258] Apply namespace property changes without workflow or response records --- app/crowdb-access-server/src/iceberg/http.rs | 3 - .../src/iceberg/namespace_write.rs | 50 ++- .../tests/common/iceberg_client.py | 2 +- .../iceberg_namespace_write_http_test.rs | 28 +- .../iceberge/design-crowdb-iceberg.md | 33 +- doc/working/plan-access-simplification.md | 13 + .../src/namespace/update.rs | 125 +++---- .../tests/common/namespace_property_race.rs | 92 ++++++ .../tests/common/store.rs | 21 +- .../tests/namespace_recovery_test.rs | 4 +- .../tests/namespace_update_test.rs | 311 ++++++++++++------ 11 files changed, 476 insertions(+), 206 deletions(-) create mode 100644 lib/crowdb-access-iceberg/tests/common/namespace_property_race.rs diff --git a/app/crowdb-access-server/src/iceberg/http.rs b/app/crowdb-access-server/src/iceberg/http.rs index d855dcd58..f58ecfb79 100644 --- a/app/crowdb-access-server/src/iceberg/http.rs +++ b/app/crowdb-access-server/src/iceberg/http.rs @@ -387,9 +387,6 @@ impl IcebergHttpService { let warehouse = warehouse(query)?; let mut config = CatalogConfig::for_capabilities(warehouse.as_deref(), capabilities)?; config.endpoints = Route::endpoints(&self.installed_routes(), capabilities); - if self.namespaces.is_some() { - config.idempotency_key_lifetime = Some("PT24H".into()); - } Ok(response( 200, serde_json::to_vec(&config).map_err(|_| service_unavailable())?, diff --git a/app/crowdb-access-server/src/iceberg/namespace_write.rs b/app/crowdb-access-server/src/iceberg/namespace_write.rs index 26fee4a8e..74c846046 100644 --- a/app/crowdb-access-server/src/iceberg/namespace_write.rs +++ b/app/crowdb-access-server/src/iceberg/namespace_write.rs @@ -70,6 +70,15 @@ impl NamespaceWrites { let request_key = RequestKey::parse(header, now).map_err(|_| bad_request())?; let uri = request.uri().clone(); let bytes = read_body(request.into_body()).await?; + if route == "POST /namespaces/{namespace}/properties" { + let NamespaceMutation::Update(identifier, changes) = + namespace_request::parse(route, &uri, &bytes)? + else { + return Err(bad_request()); + }; + let (status, body) = self.update_properties(context, identifier, changes).await?; + return Ok(response(status, body)); + } let mut digest = Sha256::new(); for value in [route.as_bytes(), uri.to_string().as_bytes(), bytes.as_slice()] { digest.update((value.len() as u64).to_be_bytes()); @@ -155,17 +164,9 @@ impl NamespaceWrites { .await .map_err(|error| mutation_error(&error))?, ), - NamespaceMutation::Update(identifier, changes) => self - .repository - .update_properties(&NamespacePropertyRequest { - context: retry.context, - identity: retry.identity, - principal: retry.principal.clone(), - identifier, - changes, - }) - .await - .map_err(|error| mutation_error(&error))?, + NamespaceMutation::Update(identifier, changes) => { + return self.update_properties(retry.context, identifier, changes).await; + } NamespaceMutation::Drop(identifier) => self .dropper .drop_namespace(&NamespaceDropRequest { @@ -187,6 +188,33 @@ impl NamespaceWrites { .map_err(|_| service_unavailable())?; Ok((outcome.status, bytes)) } + + async fn update_properties( + &self, + context: CatalogContext, + identifier: crowdb_access_iceberg::namespace::NamespaceIdentifier, + changes: crowdb_access_iceberg::namespace::PropertyChanges, + ) -> Result<(u16, Vec), IcebergErrorResponse> { + let outcome = self + .repository + .update_properties(&NamespacePropertyRequest { + context, + identifier, + changes, + }) + .await + .map_err(|error| mutation_error(&error))? + .ok_or_else(|| { + IcebergErrorResponse::new(404, "NoSuchNamespaceException", "Namespace does not exist") + })?; + let body = serde_json::to_vec(&serde_json::json!({ + "removed": outcome.removed, + "updated": outcome.updated, + "missing": outcome.missing, + })) + .map_err(|_| service_unavailable())?; + Ok((200, body)) + } } pub(super) async fn read_body(mut body: Incoming) -> Result, IcebergErrorResponse> { diff --git a/app/crowdb-access-server/tests/common/iceberg_client.py b/app/crowdb-access-server/tests/common/iceberg_client.py index f0745c3b8..684f5eb5d 100644 --- a/app/crowdb-access-server/tests/common/iceberg_client.py +++ b/app/crowdb-access-server/tests/common/iceberg_client.py @@ -38,7 +38,7 @@ def main(): "POST /v1/{prefix}/namespaces/{namespace}/properties", "DELETE /v1/{prefix}/namespaces/{namespace}", } - assert response.json()["idempotency-key-lifetime"] == "PT24H" + assert "idempotency-key-lifetime" not in response.json() catalog = load_catalog("crowdb", **properties) namespaces = catalog.list_namespaces() assert isinstance(namespaces, list) diff --git a/app/crowdb-access-server/tests/iceberg_namespace_write_http_test.rs b/app/crowdb-access-server/tests/iceberg_namespace_write_http_test.rs index e73cd62a0..7c785f456 100644 --- a/app/crowdb-access-server/tests/iceberg_namespace_write_http_test.rs +++ b/app/crowdb-access-server/tests/iceberg_namespace_write_http_test.rs @@ -163,6 +163,7 @@ async fn namespace_writes_use_distinct_credentials_and_replay_success_and_termin async fn verify_property_replay(store: &common::TestStore, address: std::net::SocketAddr) { let update_key = fresh_key(store).await; + let records = store.values.load().len(); let changes = r#"{"removals":["owner","missing"],"updates":{"value":"kept"}}"#; let updated = send( address, @@ -174,17 +175,24 @@ async fn verify_property_replay(store: &common::TestStore, address: std::net::So ) .await; assert_eq!(updated.0, 200); + let repeated = send( + address, + "POST", + "/v1/namespaces/parent/properties", + "w", + Some(&update_key), + changes, + ) + .await; + assert_eq!(repeated.0, 200); assert_eq!( - send( - address, - "POST", - "/v1/namespaces/parent/properties", - "w", - Some(&update_key), - changes - ) - .await, - updated + serde_json::from_str::(&repeated.1).unwrap(), + serde_json::json!({"removed": [], "updated": ["value"], "missing": ["missing", "owner"]}) + ); + assert_eq!( + store.values.load().len(), + records, + "property updates must not create journals or response records" ); assert_eq!( send( diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg.md b/doc/design/access-server/iceberge/design-crowdb-iceberg.md index 7177f0a79..25ef873e5 100644 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg.md +++ b/doc/design/access-server/iceberge/design-crowdb-iceberg.md @@ -234,20 +234,25 @@ epochs, missing targets and tombstones are not visible; corruption is an error. Active-context checks bracket resolution so retirement cannot turn an old-domain lookup into a response from the replacement catalog. -Property updates persist their input and immutable before/after snapshots, then -CAS the whole namespace authority. Publication advances the property and mutation -revisions but preserves the name epoch and admission fence. An operation marker -protects uncertain publication evidence until the terminal result is durable; -another writer can finish that operation before replacing its marker. Cleanup -uses a conditional write and advances the mutation revision again. After a -definitive CAS conflict proves the input revision is obsolete, a property update -may return to preparation with fresh snapshots; unknown outcomes never take that -path. Work is bounded and exhaustion remains retryable, not a terminal conflict. -Property preparation uses the same holder-bound marker dispatcher as creation -and drop. It can finish interrupted child admission or a nonempty drop before -publishing properties; recursive helpers consume the caller's phase budget. -Namespace mutations use the shared HTTP retry ledger before executing their -durable operation driver; terminal client errors are retained alongside success. +Property updates apply the request's assignments and removals directly to the +current namespace authority through one conditional write. This advances the +property and mutation revisions once, preserving the name epoch and admission +fence. A definite conflict supplies the next authority; the request reapplies its +changes while preserving other properties. Deletion, changed identity or a +pending lifecycle operation blocks that update. Active-context validation +brackets resolution and checks completion. Retry work is bounded. + +Property requests persist no input, operation journal, marker, response payload +or HTTP retry binding. A request that leaves the properties unchanged performs +no write. A lost reply is returned as an error; the client can submit its changes +again. Each response describes that request's actual removed and missing keys, +so a repeated removal can move from removed to missing. Repeating assignments +can overwrite a concurrent assignment to the same property. Other properties +are retained. Old property journals remain readable by recovery, including the +holder-bound marker dispatcher used before new updates. Creation and drop +currently use their durable drivers and the shared HTTP retry ledger. +The catalog configuration does not advertise a universal idempotency-key +lifetime, since independent property updates do not retain operation responses. Namespace creation installs a recoverable parent/name reservation before a parent authority CAS. Nested admission leaves a pending-operation marker and advances only diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 3064172ad..d91c229f6 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -437,6 +437,19 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 ## 已完成与验证记录 +- **2026-10-24，P9.3，R1 的 namespace 属性场景**：属性请求直接修改现有 authority， + 删除新请求的 NamespaceOperation、输入/响应 payload、pending marker 和 HTTP RetryLedger。 + 请求不再携带 operation/principal 作为存储层属性参数；入口仍逐次鉴权。CAS 用于保留未提及 + 的属性及删除/名称代际；连续四次真实冲突直用 observed authority，零额外 GET。 + 单层 namespace 实测正常 5G + 1C（含三次 context 检查），无变化重试 5G + 0C； + 元数据 key 总数不增加。unknown 无确认 GET，客户端重试得到当次 removed/missing， + 可重写同属性，并不保证重放原响应。旧 property journal 仍可由恢复器结清，正常请求不创建。 + 配置移除全局 PT24H 响应重放广告，避免把部分旧路由的能力宣称为所有 mutation 的合同。 + namespace update 11、recovery 4、HTTP write 4、route 2 项通过；create/drop/admission/model + 回归通过。两 crate all-target/all-feature clippy、workspace fmt 与 diff 检查通过。 + HTTP 服务测试已在允许本地监听的环境验证；初次沙箱 bind 拒绝不属于代码失败。 + + - **2026-10-24，P5.2 与 P5.1 装载部分**：S3 MPU 的内部 exchange 保留并校验 typed observation；freeze、abort、settle、发布完成标记和 object predecessor 冲突不再 GET 同一 key。绑定 upload 后直读权威记录，校验 bucket/object/upload，阶段间不重读列表索引； diff --git a/lib/crowdb-access-iceberg/src/namespace/update.rs b/lib/crowdb-access-iceberg/src/namespace/update.rs index e6f840807..452a076f8 100644 --- a/lib/crowdb-access-iceberg/src/namespace/update.rs +++ b/lib/crowdb-access-iceberg/src/namespace/update.rs @@ -1,86 +1,95 @@ -use crate::catalog::{CatalogContext, CatalogError}; +use crate::catalog::{CasOutcome, CatalogContext, CatalogError}; use crate::error::ValidationError; use crate::operation::{PayloadStore, RequestIdentity}; use crate::record::StorageRecord; use super::{ - NamespaceAction, NamespaceAuthority, NamespaceIdentifier, NamespaceJournal, NamespaceLifecycle, - NamespaceMutation, NamespaceOperation, NamespaceOutcome, NamespacePhase, NamespaceRepository, - PropertyChanges, + NamespaceAuthority, NamespaceIdentifier, NamespaceJournal, NamespaceLifecycle, NamespaceMutation, + NamespaceOperation, NamespacePhase, NamespaceRepository, PropertyChanges, }; #[derive(Clone, Debug)] pub struct NamespacePropertyRequest { pub context: CatalogContext, - pub identity: RequestIdentity, - pub principal: String, pub identifier: NamespaceIdentifier, pub changes: PropertyChanges, } impl NamespaceRepository { + /// Applies one independent set of property changes to the current authority. + /// Repeating a removal reports its current missing state rather than replaying + /// an earlier response. No operation or response record is persisted. /// # Errors - /// Rejects invalid changes, changed retry input, retired contexts and storage failures. - /// Returns no outcome if the namespace is absent before an operation is installed. + /// Rejects invalid changes, retired contexts, deletion races and storage failures. pub async fn update_properties( &self, request: &NamespacePropertyRequest, - ) -> Result, CatalogError> { + ) -> Result, CatalogError> { request.changes.validate()?; - if request.principal.is_empty() || request.principal.len() > 256 || request.principal.contains('\0') { - return Err(ValidationError::Text.into()); - } - let input = serde_json::to_vec(&request.changes).map_err(|_| ValidationError::Record)?; - let journal = NamespaceJournal::new(self.store.clone()); - if let Some(existing) = journal.load(request.context, request.identity.operation).await? { - if existing.action != NamespaceAction::Update - || existing.identity != request.identity - || existing.principal != request.principal - || existing.identifier != request.identifier - || PayloadStore::new(self.store.clone()).get(&existing.input).await? != input - { - return Err(CatalogError::Conflict); - } - if existing.phase == NamespacePhase::Prepared { - self.settle_property_target(request.context, &request.identifier) - .await?; - } - return self - .resume_property_update(request.context, request.identity.operation) - .await - .map(Some); - } - let Some(authority) = self + let Some(mut authority) = self .settle_property_target(request.context, &request.identifier) .await? else { return Ok(None); }; - Self::prepare_property_bytes(&authority, &request.changes, request.identity)?; - let input = PayloadStore::new(self.store.clone()) - .put(request.context.catalog, request.identity.operation, &input) - .await?; - journal - .begin(NamespaceOperation { - context: request.context, - identity: request.identity, - principal: request.principal.clone(), - action: NamespaceAction::Update, - identifier: request.identifier.clone(), - namespace: authority.namespace, - parent: authority.parent, - phase: NamespacePhase::Prepared, - revision: 1, - input, - mutation: None, - scan_after: Vec::new(), - scan_generation: 0, - outcome: None, - }) - .await?; - self.resume_property_update(request.context, request.identity.operation) - .await - .map(Some) + let key = super::authority_key(authority.catalog, authority.namespace); + let key_bytes = key.encode()?; + for _ in 0..16 { + if authority.lifecycle != NamespaceLifecycle::Ready { + return Err(CatalogError::Conflict); + } + if authority.pending_operation.is_some() { + return Err(CatalogError::Busy); + } + let update = authority.properties.apply(&request.changes)?; + if update.properties == authority.properties { + self.check_context(request.context).await?; + return Ok(Some(update)); + } + let expected = StorageRecord::NamespaceAuthority(Box::new(authority.clone())).encode()?; + let mut next = authority.clone(); + next.properties = update.properties.clone(); + next.property_revision = next + .property_revision + .checked_add(1) + .ok_or(ValidationError::GenerationExhausted)?; + next.mutation_revision = next + .mutation_revision + .checked_add(1) + .ok_or(ValidationError::GenerationExhausted)?; + let value = StorageRecord::NamespaceAuthority(Box::new(next)).encode()?; + match self + .store + .compare_exchange( + &key_bytes, + Some(&expected), + &value, + crate::operation::mutation_identity(&key_bytes, Some(&expected), &value), + ) + .await? + { + CasOutcome::Applied(_) => { + self.check_context(request.context).await?; + return Ok(Some(update)); + } + CasOutcome::Conflict(Some(observed)) => { + let StorageRecord::NamespaceAuthority(observed) = + StorageRecord::decode(&key, &observed.bytes)? + else { + return Err(ValidationError::Record.into()); + }; + if observed.identifier != authority.identifier + || observed.name_epoch != authority.name_epoch + || observed.parent != authority.parent + { + return Err(CatalogError::Conflict); + } + authority = *observed; + } + CasOutcome::Conflict(None) => return Ok(None), + } + } + Err(CatalogError::Busy) } pub(super) fn prepare_property_bytes( diff --git a/lib/crowdb-access-iceberg/tests/common/namespace_property_race.rs b/lib/crowdb-access-iceberg/tests/common/namespace_property_race.rs new file mode 100644 index 000000000..c47bc89c3 --- /dev/null +++ b/lib/crowdb-access-iceberg/tests/common/namespace_property_race.rs @@ -0,0 +1,92 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use crowdb_access_iceberg::{ + catalog::{CasOutcome, CatalogStore, StoreError, StoredValue}, + namespace::{ChildScan, NamespaceAuthority, NamespaceStore}, + operation::mutation_identity, + record::StorageRecord, +}; +use crowdb_chunk_kv_client::MultiScanPage; +use crowdb_protocol::chunk_kv::ClientRequestId; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +use crate::common::TestStore; + +pub struct TestPropertyRaceStore { + pub inner: Arc, + pub reads: AtomicUsize, + pub attempts: AtomicUsize, + key: Vec, + updates: Vec, +} + +impl TestPropertyRaceStore { + pub fn new(inner: Arc, key: Vec, updates: Vec) -> Arc { + Arc::new(Self { + inner, + key, + updates, + reads: AtomicUsize::new(0), + attempts: AtomicUsize::new(0), + }) + } +} + +#[async_trait] +impl CatalogStore for TestPropertyRaceStore { + async fn get(&self, key: &[u8]) -> Result, StoreError> { + self.reads.fetch_add(1, Ordering::Relaxed); + self.inner.get(key).await + } + + async fn compare_exchange( + &self, + key: &[u8], + expected: Option<&[u8]>, + value: &[u8], + identity: ClientRequestId, + ) -> Result { + if key != self.key { + return self.inner.compare_exchange(key, expected, value, identity).await; + } + let index = self.attempts.fetch_add(1, Ordering::Relaxed); + let current = self.inner.values.load().get(key).cloned(); + assert_eq!( + current.as_ref().map(|current| current.bytes.as_slice()), + expected, + "the next CAS must use the returned authority" + ); + if let Some(update) = self.updates.get(index) { + let bytes = StorageRecord::NamespaceAuthority(Box::new(update.clone())).encode()?; + assert!(matches!( + self.inner + .compare_exchange(key, expected, &bytes, mutation_identity(key, expected, &bytes)) + .await?, + CasOutcome::Applied(_) + )); + return Ok(CasOutcome::Conflict(self.inner.values.load().get(key).cloned())); + } + self.inner.compare_exchange(key, expected, value, identity).await + } +} + +#[async_trait] +impl NamespaceStore for TestPropertyRaceStore { + async fn scan_children(&self, request: ChildScan) -> Result { + self.inner.scan_children(request).await + } + + async fn delete_mapping( + &self, + key: &[u8], + expected: &[u8], + identity: ClientRequestId, + ) -> Result { + self.inner.delete_mapping(key, expected, identity).await + } +} diff --git a/lib/crowdb-access-iceberg/tests/common/store.rs b/lib/crowdb-access-iceberg/tests/common/store.rs index b9c680bd2..516157591 100644 --- a/lib/crowdb-access-iceberg/tests/common/store.rs +++ b/lib/crowdb-access-iceberg/tests/common/store.rs @@ -129,12 +129,27 @@ impl TestStore { } } - async fn pause_namespace_authority(&self, key: &[u8], expected: Option<&[u8]>, value: &[u8]) { + async fn pause_namespace_update(&self, key: &[u8], expected: Option<&[u8]>, value: &[u8]) { if let Ok(crowdb_access_iceberg::record::StorageRecord::NamespaceAuthority(authority)) = crowdb_access_iceberg::key::IcebergKey::decode(key) .and_then(|key| crowdb_access_iceberg::record::StorageRecord::decode(&key, value)) { - if authority.pending_operation.is_some() && expected.is_some() { + if expected.is_some() + && (authority.pending_operation.is_some() + || expected + .and_then(|bytes| { + crowdb_access_iceberg::key::IcebergKey::decode(key) + .ok() + .and_then(|key| { + crowdb_access_iceberg::record::StorageRecord::decode(&key, bytes).ok() + }) + }) + .is_some_and(|record| { + matches!(record, + crowdb_access_iceberg::record::StorageRecord::NamespaceAuthority(before) + if before.property_revision < authority.property_revision) + })) + { if let Some(barrier) = &self.namespace_update_barrier { if self.namespace_update_visits.fetch_add(1, Ordering::SeqCst) < 2 { barrier.wait().await; @@ -267,7 +282,7 @@ impl CatalogStore for TestStore { } } } - self.pause_namespace_authority(key, expected, value).await; + self.pause_namespace_update(key, expected, value).await; if let Ok(crowdb_access_iceberg::record::StorageRecord::Active(root)) = crowdb_access_iceberg::key::IcebergKey::decode(key) .and_then(|key| crowdb_access_iceberg::record::StorageRecord::decode(&key, value)) diff --git a/lib/crowdb-access-iceberg/tests/namespace_recovery_test.rs b/lib/crowdb-access-iceberg/tests/namespace_recovery_test.rs index 7bbc78e7d..551a4d199 100644 --- a/lib/crowdb-access-iceberg/tests/namespace_recovery_test.rs +++ b/lib/crowdb-access-iceberg/tests/namespace_recovery_test.rs @@ -135,8 +135,6 @@ async fn setup() -> (TestNamespace, NamespaceIdentifier) { async fn update(fixture: &TestNamespace, identifier: NamespaceIdentifier) { let request = NamespacePropertyRequest { context: fixture.context, - identity: identity(), - principal: "writer".into(), identifier, changes: PropertyChanges { removals: Vec::new(), @@ -147,7 +145,7 @@ async fn update(fixture: &TestNamespace, identifier: NamespaceIdentifier) { for _ in 0..4 { match repository.update_properties(&request).await { Ok(Some(outcome)) => { - assert_eq!(outcome.status, 200); + assert_eq!(outcome.updated, ["updated"]); let authority = repository .load(fixture.context, &request.identifier) .await diff --git a/lib/crowdb-access-iceberg/tests/namespace_update_test.rs b/lib/crowdb-access-iceberg/tests/namespace_update_test.rs index 12f430a68..d64111c00 100644 --- a/lib/crowdb-access-iceberg/tests/namespace_update_test.rs +++ b/lib/crowdb-access-iceberg/tests/namespace_update_test.rs @@ -4,6 +4,10 @@ mod common; mod fixture; #[path = "common/namespace_store.rs"] mod namespace_store; +#[path = "common/namespace_property_race.rs"] +mod race; +#[path = "common/namespace_recovery_store.rs"] +mod recovery_store; use std::collections::BTreeMap; use std::sync::{atomic::Ordering, Arc}; @@ -15,17 +19,11 @@ use crowdb_access_iceberg::namespace::{ NamespaceAuthority, NamespaceLifecycle, NamespaceProperties, NamespacePropertyRequest, NamespaceRepository, PropertyChanges, }; -use crowdb_access_iceberg::operation::{PayloadStore, RequestIdentity}; use fixture::TestNamespace; fn request(fixture: &TestNamespace, authority: &NamespaceAuthority) -> NamespacePropertyRequest { NamespacePropertyRequest { context: fixture.context, - identity: RequestIdentity { - operation: OperationId::random(), - issued_ms: 100, - }, - principal: "writer".into(), identifier: authority.identifier.clone(), changes: PropertyChanges { removals: vec!["old".into(), "absent".into()], @@ -52,17 +50,9 @@ async fn property_publication_is_atomic_and_does_not_change_identity_or_admissio let request = request(&fixture, &original); let repository = NamespaceRepository::new(fixture.store.clone()); let result = repository.update_properties(&request).await.unwrap().unwrap(); - assert_eq!(result.status, 200); - let body = PayloadStore::new(fixture.store.clone()) - .get(&result.body) - .await - .unwrap(); - assert_eq!( - serde_json::from_slice::(&body).unwrap(), - serde_json::json!({ - "removed": ["old"], "updated": ["owner"], "missing": ["absent"], - }) - ); + assert_eq!(result.removed, ["old"]); + assert_eq!(result.updated, ["owner"]); + assert_eq!(result.missing, ["absent"]); let selected = repository .load(fixture.context, &original.identifier) .await @@ -72,16 +62,17 @@ async fn property_publication_is_atomic_and_does_not_change_identity_or_admissio assert_eq!(selected.name_epoch, original.name_epoch); assert_eq!(selected.admission_fence, original.admission_fence); assert_eq!(selected.property_revision, original.property_revision + 1); - assert_eq!(selected.mutation_revision, original.mutation_revision + 2); + assert_eq!(selected.mutation_revision, original.mutation_revision + 1); assert_eq!(selected.pending_operation, None); assert_eq!( selected.properties.entries(), &BTreeMap::from([("keep".into(), "value".into()), ("owner".into(), "数据库".into()),]) ); - assert_eq!( - repository.update_properties(&request).await.unwrap(), - Some(result) - ); + let before = fixture.store.writes.load(Ordering::Relaxed); + let replay = repository.update_properties(&request).await.unwrap().unwrap(); + assert!(replay.removed.is_empty()); + assert_eq!(replay.missing, ["absent", "old"]); + assert_eq!(fixture.store.writes.load(Ordering::Relaxed), before); assert_eq!( repository .load(fixture.context, &original.identifier) @@ -92,70 +83,42 @@ async fn property_publication_is_atomic_and_does_not_change_identity_or_admissio } #[tokio::test] -async fn every_lost_write_reply_resumes_one_property_revision_and_original_response() { - let (baseline, authority) = setup().await; - let before = baseline.store.writes.load(Ordering::SeqCst); - NamespaceRepository::new(baseline.store.clone()) - .update_properties(&request(&baseline, &authority)) - .await - .unwrap(); - let writes = baseline.store.writes.load(Ordering::SeqCst) - before; - assert!(writes >= 10); - for offset in 1..=writes { - let (fixture, authority) = setup().await; - let request = request(&fixture, &authority); - fixture.store.fail_after.store( - fixture.store.writes.load(Ordering::SeqCst) + offset, - Ordering::SeqCst, - ); - assert!( - NamespaceRepository::new(fixture.store.clone()) - .update_properties(&request) - .await - .is_err(), - "offset {offset}" - ); - let restarted = NamespaceRepository::new(fixture.store.clone()); - let outcome = restarted.update_properties(&request).await.unwrap().unwrap(); - assert_eq!(outcome.status, 200, "offset {offset}"); - assert_eq!( - restarted.update_properties(&request).await.unwrap(), - Some(outcome) - ); - let selected = restarted - .load(fixture.context, &authority.identifier) - .await - .unwrap() - .unwrap(); - assert_eq!( - selected.property_revision, - authority.property_revision + 1, - "offset {offset}" - ); - assert_eq!(selected.pending_operation, None, "offset {offset}"); - } +async fn lost_write_reply_is_retried_from_resource_facts_without_a_response_record() { + let (fixture, authority) = setup().await; + let request = request(&fixture, &authority); + let repository = NamespaceRepository::new(fixture.store.clone()); + let writes = fixture.store.writes.load(Ordering::Relaxed); + let reads = fixture.store.reads.load(Ordering::Relaxed); + let records = fixture.store.values.load().len(); + fixture.store.fail_after.store(writes + 1, Ordering::Relaxed); + assert!(repository.update_properties(&request).await.is_err()); + assert_eq!(fixture.store.reads.load(Ordering::Relaxed) - reads, 4); + let restarted = NamespaceRepository::new(fixture.store.clone()); + let result = restarted.update_properties(&request).await.unwrap().unwrap(); + assert!(result.removed.is_empty()); + assert_eq!(result.missing, ["absent", "old"]); + assert_eq!(fixture.store.writes.load(Ordering::Relaxed), writes + 1); + assert_eq!(fixture.store.values.load().len(), records); } #[tokio::test] -async fn changed_request_input_or_principal_never_reuses_a_completed_result() { +async fn independent_requests_apply_changes_to_current_properties_without_a_ledger() { let (fixture, authority) = setup().await; - let original = request(&fixture, &authority); + let mut request = request(&fixture, &authority); let repository = NamespaceRepository::new(fixture.store.clone()); - repository.update_properties(&original).await.unwrap(); - let writes = fixture.store.writes.load(Ordering::SeqCst); - let mut changed = original.clone(); - changed.changes.updates.insert("different".into(), "value".into()); - assert!(matches!( - repository.update_properties(&changed).await, - Err(CatalogError::Conflict) - )); - changed = original; - changed.principal = "another-writer".into(); - assert!(matches!( - repository.update_properties(&changed).await, - Err(CatalogError::Conflict) - )); - assert_eq!(fixture.store.writes.load(Ordering::SeqCst), writes); + repository.update_properties(&request).await.unwrap(); + request.changes.updates.insert("owner".into(), "second".into()); + repository.update_properties(&request).await.unwrap(); + request.changes.updates.insert("owner".into(), "数据库".into()); + repository.update_properties(&request).await.unwrap(); + let selected = repository + .load(fixture.context, &authority.identifier) + .await + .unwrap() + .unwrap(); + assert_eq!(selected.properties.entries().get("owner").unwrap(), "数据库"); + assert_eq!(selected.property_revision, authority.property_revision + 3); + assert!(selected.pending_operation.is_none()); } #[tokio::test] @@ -179,7 +142,7 @@ async fn invalid_changes_and_exhausted_revisions_fail_before_any_durable_write() Err(CatalogError::Invalid(ValidationError::RecordTooLarge)) )); assert_eq!(fixture.store.writes.load(Ordering::SeqCst), writes); - authority.mutation_revision = u64::MAX - 1; + authority.mutation_revision = u64::MAX; fixture.publish(&authority).await; let writes = fixture.store.writes.load(Ordering::SeqCst); assert!(matches!( @@ -213,22 +176,7 @@ async fn concurrent_property_writers_rebase_only_after_a_definitive_cas_loss() { .await .unwrap(); for result in [results.0, results.1] { - match result { - Ok(Some(outcome)) => assert_eq!(outcome.status, 200), - Err(CatalogError::Busy) => {} - other => panic!("unexpected concurrent result: {other:?}"), - } - } - for request in [&first, &second] { - assert_eq!( - repository - .update_properties(request) - .await - .unwrap() - .unwrap() - .status, - 200 - ); + result.unwrap().unwrap(); } let selected = repository .load(fixture.context, &authority.identifier) @@ -254,6 +202,13 @@ async fn absent_dropping_and_retired_namespaces_do_not_accept_property_writes() fixture.publish(&authority).await; assert!(repository.update_properties(&request).await.is_err()); fixture.root(fixture.context, RootState::Fencing).await; + assert!(matches!( + repository.update_properties(&request).await, + Err(CatalogError::Busy) + )); + let mut replacement = fixture.context; + replacement.activation_epoch += 1; + fixture.root(replacement, RootState::Ready).await; assert!(matches!( repository.update_properties(&request).await, Err(CatalogError::Conflict) @@ -261,18 +216,168 @@ async fn absent_dropping_and_retired_namespaces_do_not_accept_property_writes() } #[tokio::test] -async fn foreign_property_marker_is_rejected_without_helping_its_owner() { +async fn normal_property_write_is_one_resource_cas_and_no_new_records() { + let (fixture, authority) = setup().await; + let repository = NamespaceRepository::new(fixture.store.clone()); + let records = fixture.store.values.load().len(); + let reads = fixture.store.reads.load(Ordering::Relaxed); + let attempts = fixture.store.cas_attempts.load(Ordering::Relaxed); + let writes = fixture.store.writes.load(Ordering::Relaxed); + let request = request(&fixture, &authority); + repository.update_properties(&request).await.unwrap().unwrap(); + assert_eq!(fixture.store.reads.load(Ordering::Relaxed) - reads, 5); + assert_eq!(fixture.store.cas_attempts.load(Ordering::Relaxed) - attempts, 1); + assert_eq!(fixture.store.writes.load(Ordering::Relaxed) - writes, 1); + assert_eq!(fixture.store.values.load().len(), records); + repository.update_properties(&request).await.unwrap().unwrap(); + assert_eq!(fixture.store.reads.load(Ordering::Relaxed) - reads, 10); + assert_eq!(fixture.store.cas_attempts.load(Ordering::Relaxed) - attempts, 1); +} + +#[tokio::test] +async fn legacy_property_journal_can_finish_before_a_new_stateless_update() { + use crowdb_access_iceberg::{ + namespace::{ + NamespaceAction, NamespaceJournal, NamespaceOperation, NamespacePhase, NamespaceRecovery, + }, + operation::{PayloadStore, RequestIdentity}, + }; let (fixture, original) = setup().await; + let request = request(&fixture, &original); + let operation = OperationId::random(); + let input = PayloadStore::new(fixture.store.clone()) + .put( + fixture.context.catalog, + operation, + &serde_json::to_vec(&request.changes).unwrap(), + ) + .await + .unwrap(); + NamespaceJournal::new(fixture.store.clone()) + .begin(NamespaceOperation { + context: fixture.context, + identity: RequestIdentity { + operation, + issued_ms: 100, + }, + principal: "legacy-writer".into(), + action: NamespaceAction::Update, + identifier: original.identifier.clone(), + namespace: original.namespace, + parent: original.parent, + phase: NamespacePhase::Prepared, + revision: 1, + input, + mutation: None, + scan_after: Vec::new(), + scan_generation: 0, + outcome: None, + }) + .await + .unwrap(); + let recovery = NamespaceRecovery::new(fixture.store.clone()); + for _ in 0..4 { + let page = recovery.recover_page(fixture.context, None).await.unwrap(); + assert!(page.failures.is_empty(), "{:?}", page.failures); + if NamespaceJournal::new(fixture.store.clone()) + .load(fixture.context, operation) + .await + .unwrap() + .unwrap() + .phase + == NamespacePhase::Complete + { + break; + } + } + let records = fixture.store.values.load().len(); + let writes = fixture.store.writes.load(Ordering::Relaxed); + let result = NamespaceRepository::new(fixture.store.clone()) + .update_properties(&request) + .await + .unwrap() + .unwrap(); + assert!(result.removed.is_empty()); + assert_eq!(fixture.store.values.load().len(), records); + assert_eq!(fixture.store.writes.load(Ordering::Relaxed), writes); +} + +#[tokio::test] +async fn four_real_property_conflicts_merge_the_observed_authority_without_get() { + let (fixture, original) = setup().await; + let mut current = original.clone(); + let mut observations = Vec::new(); + for index in 0..4 { + let mut properties = current.properties.entries().clone(); + properties.insert(format!("parallel-{index}"), "retained".into()); + current.properties = NamespaceProperties::new(properties).unwrap(); + current.property_revision += 1; + current.mutation_revision += 1; + observations.push(current.clone()); + } + let key = crowdb_access_iceberg::namespace::authority_key(original.catalog, original.namespace) + .encode() + .unwrap(); + let store = race::TestPropertyRaceStore::new(fixture.store.clone(), key, observations); + let repository = NamespaceRepository::new(store.clone()); + repository + .update_properties(&request(&fixture, &original)) + .await + .unwrap() + .unwrap(); + assert_eq!(store.attempts.load(Ordering::Relaxed), 5); + assert_eq!(store.reads.load(Ordering::Relaxed), 5); + let selected = repository + .load(fixture.context, &original.identifier) + .await + .unwrap() + .unwrap(); + for index in 0..4 { + assert_eq!( + selected + .properties + .entries() + .get(&format!("parallel-{index}")) + .unwrap(), + "retained" + ); + } + assert_eq!(selected.properties.entries().get("owner").unwrap(), "数据库"); + assert_eq!(selected.properties.entries().get("keep").unwrap(), "value"); +} + +#[tokio::test] +async fn observed_deletion_stops_property_writes_without_helping_or_reloading() { + let (fixture, original) = setup().await; + let mut dropping = original.clone(); + dropping.lifecycle = NamespaceLifecycle::Dropping; + dropping.pending_operation = Some(OperationId::random()); + dropping.mutation_revision += 1; + let key = crowdb_access_iceberg::namespace::authority_key(original.catalog, original.namespace) + .encode() + .unwrap(); + let store = race::TestPropertyRaceStore::new(fixture.store.clone(), key, vec![dropping]); + assert!(matches!( + NamespaceRepository::new(store.clone()) + .update_properties(&request(&fixture, &original)) + .await, + Err(CatalogError::Conflict) + )); + assert_eq!(store.reads.load(Ordering::Relaxed), 4); + assert_eq!(store.attempts.load(Ordering::Relaxed), 1); +} + +#[tokio::test] +async fn foreign_property_marker_is_rejected_without_helping_its_owner() { + let (fixture, _) = setup().await; let repository = NamespaceRepository::new(fixture.store.clone()); - let owner = request(&fixture, &original); - repository.update_properties(&owner).await.unwrap(); let mut foreign = fixture.authority(None, &["foreign"]); - foreign.pending_operation = Some(owner.identity.operation); + foreign.pending_operation = Some(OperationId::random()); fixture.publish(&foreign).await; let writes = fixture.store.writes.load(Ordering::SeqCst); assert!(matches!( repository.update_properties(&request(&fixture, &foreign)).await, - Err(CatalogError::Invalid(ValidationError::IdentityMismatch)) + Err(CatalogError::Invalid(_)) )); assert_eq!(fixture.store.writes.load(Ordering::SeqCst), writes); assert_eq!( From 2e459e2c4eb26cd9c56bb021cd47232a973bb97c Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 11:23:17 +0800 Subject: [PATCH 100/258] Remove shared HTTP response ledgers from Iceberg writes --- .../src/iceberg/metrics.rs | 18 -- .../src/iceberg/namespace_write.rs | 62 +--- .../src/iceberg/table_write.rs | 79 +---- .../src/iceberg/table_write/lifecycle.rs | 25 +- .../src/iceberg/table_write/mutation.rs | 7 +- .../tests/common/iceberg_store.rs | 7 +- .../iceberg_namespace_write_http_test.rs | 34 ++- .../tests/iceberg_route_test.rs | 1 - .../iceberge/design-crowdb-iceberg.md | 36 ++- doc/working/plan-access-simplification.md | 19 +- lib/crowdb-access-iceberg/src/operation.rs | 4 +- .../src/operation/result.rs | 37 +-- .../src/operation/retry.rs | 215 +------------- .../tests/gc_worker_test.rs | 7 - .../tests/request_identity_test.rs | 13 + lib/crowdb-access-iceberg/tests/retry_test.rs | 278 ------------------ 16 files changed, 126 insertions(+), 716 deletions(-) create mode 100644 lib/crowdb-access-iceberg/tests/request_identity_test.rs delete mode 100644 lib/crowdb-access-iceberg/tests/retry_test.rs diff --git a/app/crowdb-access-server/src/iceberg/metrics.rs b/app/crowdb-access-server/src/iceberg/metrics.rs index a863d90db..10188c09e 100644 --- a/app/crowdb-access-server/src/iceberg/metrics.rs +++ b/app/crowdb-access-server/src/iceberg/metrics.rs @@ -47,9 +47,6 @@ pub struct MetricCounts { #[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)] pub struct IcebergMetricsSnapshot { pub routes: [[MetricCounts; OUTCOME_COUNT]; ROUTE_COUNT], - pub retry_new: u64, - pub retry_resume: u64, - pub retry_replay: u64, pub selected_versions: [u64; 3], pub chunk_read: Option, pub chunk_small_write: Option, @@ -104,7 +101,6 @@ impl Counters { pub(super) struct IcebergMetrics { routes: [[Counters; OUTCOME_COUNT]; ROUTE_COUNT], - retry: [AtomicU64; 3], selected_versions: [AtomicU64; 3], } @@ -112,7 +108,6 @@ impl Default for IcebergMetrics { fn default() -> Self { Self { routes: array::from_fn(|_| array::from_fn(|_| Counters::new())), - retry: array::from_fn(|_| AtomicU64::new(0)), selected_versions: array::from_fn(|_| AtomicU64::new(0)), } } @@ -122,9 +117,6 @@ impl IcebergMetrics { pub(super) fn snapshot(&self) -> IcebergMetricsSnapshot { IcebergMetricsSnapshot { routes: array::from_fn(|route| array::from_fn(|outcome| self.routes[route][outcome].snapshot())), - retry_new: self.retry[0].load(Ordering::Relaxed), - retry_resume: self.retry[1].load(Ordering::Relaxed), - retry_replay: self.retry[2].load(Ordering::Relaxed), selected_versions: array::from_fn(|index| self.selected_versions[index].load(Ordering::Relaxed)), chunk_read: None, chunk_small_write: None, @@ -143,7 +135,6 @@ pub(super) struct RequestObservation { response_bytes: AtomicU64, dispatch_ns: AtomicU64, status: AtomicU16, - retry: AtomicU8, version: AtomicU8, catalog: CatalogStoreOperationMeter, } @@ -158,7 +149,6 @@ impl RequestObservation { response_bytes: AtomicU64::new(0), dispatch_ns: AtomicU64::new(0), status: AtomicU16::new(0), - retry: AtomicU8::new(0), version: AtomicU8::new(0), catalog: CatalogStoreOperationMeter::default(), }) @@ -214,10 +204,6 @@ impl Drop for RequestObservation { counters .catalog_conditional_delete .fetch_add(catalog.conditional_delete, Ordering::Relaxed); - let retry = self.retry.load(Ordering::Relaxed); - if retry != 0 { - self.metrics.retry[usize::from(retry - 1)].fetch_add(1, Ordering::Relaxed); - } let version = self.version.load(Ordering::Relaxed); if (1..=3).contains(&version) { self.metrics.selected_versions[usize::from(version - 1)].fetch_add(1, Ordering::Relaxed); @@ -244,10 +230,6 @@ pub(super) fn record_request_bytes(length: usize) { }); } -pub(super) fn record_retry(kind: u8) { - let _ = REQUEST_OBSERVATION.try_with(|span| span.retry.store(kind, Ordering::Relaxed)); -} - pub(super) fn record_selected_version(version: u8) { let _ = REQUEST_OBSERVATION.try_with(|span| span.version.store(version, Ordering::Relaxed)); } diff --git a/app/crowdb-access-server/src/iceberg/namespace_write.rs b/app/crowdb-access-server/src/iceberg/namespace_write.rs index 74c846046..52598712b 100644 --- a/app/crowdb-access-server/src/iceberg/namespace_write.rs +++ b/app/crowdb-access-server/src/iceberg/namespace_write.rs @@ -7,22 +7,26 @@ use crowdb_access_iceberg::namespace::{ NamespaceCreateRequest, NamespaceCreator, NamespaceDropRequest, NamespaceDropper, NamespacePropertyRequest, NamespaceRepository, NamespaceStore, }; -use crowdb_access_iceberg::operation::{PayloadStore, RetryAdmission, RetryLedger, RetryRecord}; +use crowdb_access_iceberg::operation::{PayloadStore, RequestIdentity}; use crowdb_access_iceberg::wire::{IcebergErrorResponse, Principal, RequestKey}; use http_body_util::BodyExt; use hyper::body::Incoming; use hyper::{Request, Response}; -use sha2::{Digest, Sha256}; use super::body::IcebergBody; use super::http::{bad_request, response, service_unavailable}; use super::namespace_request::{self, NamespaceMutation}; +struct NamespaceMutationRequest { + context: CatalogContext, + identity: RequestIdentity, + principal: String, +} + pub(super) struct NamespaceWrites { creator: NamespaceCreator, dropper: NamespaceDropper, repository: NamespaceRepository, - ledger: RetryLedger, payloads: PayloadStore, } @@ -32,7 +36,6 @@ impl NamespaceWrites { creator: NamespaceCreator::new(store.clone()), dropper: NamespaceDropper::new(store.clone()), repository: NamespaceRepository::new(store.clone()), - ledger: RetryLedger::new(store.clone()), payloads: PayloadStore::new(store), } } @@ -79,34 +82,10 @@ impl NamespaceWrites { let (status, body) = self.update_properties(context, identifier, changes).await?; return Ok(response(status, body)); } - let mut digest = Sha256::new(); - for value in [route.as_bytes(), uri.to_string().as_bytes(), bytes.as_slice()] { - digest.update((value.len() as u64).to_be_bytes()); - digest.update(value); - } - let retry = RetryRecord { + let retry = NamespaceMutationRequest { identity: request_key.identity(), principal: principal.name.into(), - route: route.into(), - digest: digest.finalize().into(), context, - retained_until_ms: 0, - status: 0, - body: Vec::new(), - }; - let retry = match self.admit(retry, request_key, now).await? { - RetryAdmission::Replay(record) => { - super::metrics::record_retry(3); - return Ok(response(record.status, record.body)); - } - RetryAdmission::New(record) => { - super::metrics::record_retry(1); - record - } - RetryAdmission::Resume(record) => { - super::metrics::record_retry(2); - record - } }; let result = match namespace_request::parse(route, &uri, &bytes) { Ok(mutation) => self.mutate(&retry, mutation).await, @@ -120,35 +99,12 @@ impl NamespaceWrites { ), Err(error) => return Err(error), }; - self.ledger - .finish(retry, status, body.clone(), now_ms()?) - .await - .map_err(|error| mutation_error(&error))?; Ok(response(status, body)) } - async fn admit( - &self, - mut retry: RetryRecord, - request_key: RequestKey, - now: u64, - ) -> Result { - for _ in 0..8 { - match self.ledger.begin(retry.clone(), now).await { - Err(CatalogError::Busy) if matches!(request_key, RequestKey::Internal(_)) => { - retry.identity = RequestKey::parse(None, now) - .map_err(|_| bad_request())? - .identity(); - } - result => return result.map_err(|error| mutation_error(&error)), - } - } - Err(service_unavailable()) - } - async fn mutate( &self, - retry: &RetryRecord, + retry: &NamespaceMutationRequest, mutation: NamespaceMutation, ) -> Result<(u16, Vec), IcebergErrorResponse> { let outcome = match mutation { diff --git a/app/crowdb-access-server/src/iceberg/table_write.rs b/app/crowdb-access-server/src/iceberg/table_write.rs index 717c50cc4..050156825 100644 --- a/app/crowdb-access-server/src/iceberg/table_write.rs +++ b/app/crowdb-access-server/src/iceberg/table_write.rs @@ -5,7 +5,7 @@ use crowdb_access_iceberg::{ commit::{CommitProofLimits, StagedCommitLimits, TableCreator}, file::FileBlockStore, namespace::{NamespaceRepository, NamespaceStore}, - operation::{PayloadStore, RetryAdmission, RetryLedger, RetryRecord}, + operation::{PayloadStore, RequestIdentity}, table::TableRepository, wire::{IcebergErrorResponse, Principal, RequestKey}, }; @@ -15,7 +15,7 @@ use sha2::{Digest, Sha256}; use super::{ body::{IcebergBody, SpoolPermit}, http::{bad_request, response, service_unavailable}, - namespace_write::{mutation_error, now_ms, read_body}, + namespace_write::{now_ms, read_body}, table_limits, }; @@ -23,6 +23,13 @@ mod lifecycle; mod mutation; pub(super) mod request; +pub(super) struct TableMutation { + context: CatalogContext, + identity: RequestIdentity, + principal: String, + digest: [u8; 32], +} + pub(super) struct TableWrites { store: Arc, blocks: Arc, @@ -30,7 +37,6 @@ pub(super) struct TableWrites { namespaces: NamespaceRepository, tables: TableRepository, lifecycles: crowdb_access_iceberg::table::TableLifecycles, - ledger: RetryLedger, payloads: PayloadStore, limits: CommitProofLimits, active: Arc, @@ -56,7 +62,6 @@ impl TableWrites { }), namespaces: NamespaceRepository::new(store.clone()), tables: TableRepository::new(store.clone()), - ledger: RetryLedger::new(store.clone()), payloads: PayloadStore::new(store), limits, active: Arc::new(AtomicUsize::new(0)), @@ -114,15 +119,11 @@ impl TableWrites { digest.update((value.len() as u64).to_be_bytes()); digest.update(value); } - let mut record = RetryRecord { + let record = TableMutation { identity: key.identity(), principal: principal.name.into(), - route: route.into(), digest: digest.finalize().into(), context, - retained_until_ms: 0, - status: 0, - body: Vec::new(), }; if method == Method::POST && uri.path() != "/v1/tables/rename" { if let Some(body) = self @@ -141,39 +142,11 @@ impl TableWrites { return Ok(response(200, body)); } } - let admission = self.admit(&mut record, key, now).await?; - let (record, resuming) = match admission { - RetryAdmission::Replay(record) => { - super::metrics::record_retry(3); - let body = self.decorate_response( - record.status, - record.body, - configuration_target, - context, - authority, - principal, - request_host.as_deref(), - )?; - return Ok(response(record.status, body)); - } - RetryAdmission::New(record) => { - super::metrics::record_retry(1); - (record, false) - } - RetryAdmission::Resume(record) => { - super::metrics::record_retry(2); - (record, true) - } - }; if method == Method::DELETE || uri.path() == "/v1/tables/rename" { let result = self - .mutate_lifecycle(&record, capabilities, resuming, &method, &uri, &bytes) + .mutate_lifecycle(&record, capabilities, &method, &uri, &bytes) .await; let (status, body) = self.outcome_response(result).await?; - self.ledger - .finish(record, status, body.clone(), now_ms()?) - .await - .map_err(|error| mutation_error(&error))?; return Ok(response(status, body)); } let target = request::parse(&uri); @@ -182,10 +155,6 @@ impl TableWrites { Err(error) => Err(error), }; let (status, body) = self.outcome_response(result).await?; - self.ledger - .finish(record, status, body.clone(), now_ms()?) - .await - .map_err(|error| mutation_error(&error))?; let body = self.decorate_response( status, body, @@ -303,30 +272,4 @@ impl TableWrites { }; Some((target.namespace, name, staged)) } - - async fn admit( - &self, - record: &mut RetryRecord, - key: RequestKey, - now: u64, - ) -> Result { - for _ in 0..8 { - match self.ledger.begin(record.clone(), now).await { - Err(crowdb_access_iceberg::catalog::CatalogError::Busy) - if matches!(key, RequestKey::Internal(_)) => - { - record.identity = RequestKey::parse(None, now) - .map_err(|_| bad_request())? - .identity(); - } - result => { - return result.map_err(|error| { - tracing::warn!(%error, "table retry ledger admission failed"); - mutation_error(&error) - }) - } - } - } - Err(service_unavailable()) - } } diff --git a/app/crowdb-access-server/src/iceberg/table_write/lifecycle.rs b/app/crowdb-access-server/src/iceberg/table_write/lifecycle.rs index 1a45c62df..6c803e388 100644 --- a/app/crowdb-access-server/src/iceberg/table_write/lifecycle.rs +++ b/app/crowdb-access-server/src/iceberg/table_write/lifecycle.rs @@ -7,7 +7,6 @@ use crowdb_access_iceberg::{ commit::TableCommitOutcome, key::NameSuffix, namespace::NamespaceIdentifier, - operation::RetryRecord, table::{TableLifecycleAction, TableLifecycleRequest}, wire::IcebergErrorResponse, }; @@ -43,9 +42,8 @@ impl Identifier { impl TableWrites { pub(super) async fn mutate_lifecycle( &self, - record: &RetryRecord, + record: &super::TableMutation, capabilities: Capabilities, - resuming: bool, method: &Method, uri: &Uri, bytes: &[u8], @@ -81,14 +79,11 @@ impl TableWrites { TableLifecycleAction::Drop { purge_requested }, ) }; - let existing = if resuming { - self.lifecycles - .load(record.context, record.identity.operation) - .await - .map_err(|_| service_unavailable())? - } else { - None - }; + let existing = self + .lifecycles + .load(record.context, record.identity.operation) + .await + .map_err(|_| service_unavailable())?; let version = if let Some(operation) = existing { operation.before.format_version } else { @@ -111,8 +106,12 @@ impl TableWrites { } self.lifecycles.execute(&TableLifecycleRequest { context: record.context, identity: record.identity, principal: record.principal.clone(), namespace, name, action }).await.map_err(|error| { - tracing::error!(%error, "table lifecycle remains recoverable; retry with the same request key"); - service_unavailable() + if matches!(error, crowdb_access_iceberg::catalog::CatalogError::Conflict) { + IcebergErrorResponse::new(409, "CommitFailedException", "Table lifecycle request conflicts") + } else { + tracing::error!(%error, "table lifecycle remains recoverable; retry with the same request key"); + service_unavailable() + } }) } } diff --git a/app/crowdb-access-server/src/iceberg/table_write/mutation.rs b/app/crowdb-access-server/src/iceberg/table_write/mutation.rs index 7786c1fe8..75a0a2a38 100644 --- a/app/crowdb-access-server/src/iceberg/table_write/mutation.rs +++ b/app/crowdb-access-server/src/iceberg/table_write/mutation.rs @@ -10,7 +10,6 @@ use crowdb_access_iceberg::{ CreateTableRequest, DirectCommitError, StagedCommitRequest, TableCommitJournal, TableCommitOperation, TableCommitOutcome, TableCommitPhase, TableCreationRequest, TableRequirement, }, - operation::RetryRecord, wire::IcebergErrorResponse, }; @@ -22,7 +21,7 @@ struct CommitInput<'input> { impl TableWrites { pub(super) async fn try_direct_update( &self, - record: &RetryRecord, + record: &super::TableMutation, capabilities: Capabilities, uri: &hyper::Uri, body: &[u8], @@ -90,7 +89,7 @@ impl TableWrites { pub(super) async fn mutate( &self, - record: &RetryRecord, + record: &super::TableMutation, capabilities: Capabilities, target: Target, body: Vec, @@ -170,7 +169,7 @@ impl TableWrites { async fn update( &self, - record: &RetryRecord, + record: &super::TableMutation, capabilities: Capabilities, namespace: &crowdb_access_iceberg::namespace::NamespaceIdentifier, name: &str, diff --git a/app/crowdb-access-server/tests/common/iceberg_store.rs b/app/crowdb-access-server/tests/common/iceberg_store.rs index 1d69a0691..cca72d599 100644 --- a/app/crowdb-access-server/tests/common/iceberg_store.rs +++ b/app/crowdb-access-server/tests/common/iceberg_store.rs @@ -261,7 +261,12 @@ impl CatalogStore for TestStore { )) => { (mode == 2 && scope == crowdb_access_iceberg::key::CatalogScope::NamespaceAuthority) - || (mode == 3 && scope == crowdb_access_iceberg::key::CatalogScope::Operation) + || (mode == 3 + && scope == crowdb_access_iceberg::key::CatalogScope::NamespaceOperation + && matches!(crowdb_access_iceberg::key::IcebergKey::decode(key).and_then(|key| + crowdb_access_iceberg::record::StorageRecord::decode(&key, value)), + Ok(crowdb_access_iceberg::record::StorageRecord::NamespaceOperation(operation)) + if operation.phase == crowdb_access_iceberg::namespace::NamespacePhase::Complete)) || (mode == 5 && scope == crowdb_access_iceberg::key::CatalogScope::TableHead) || (mode == 4 && scope diff --git a/app/crowdb-access-server/tests/iceberg_namespace_write_http_test.rs b/app/crowdb-access-server/tests/iceberg_namespace_write_http_test.rs index 7c785f456..d91e538d7 100644 --- a/app/crowdb-access-server/tests/iceberg_namespace_write_http_test.rs +++ b/app/crowdb-access-server/tests/iceberg_namespace_write_http_test.rs @@ -260,7 +260,7 @@ async fn verify_drop_replay( } #[tokio::test] -async fn malformed_and_missing_parent_results_are_retained_before_any_later_retry() { +async fn failed_requests_without_a_resource_operation_can_retry_after_the_cause_changes() { let (store, address, stop, server) = setup().await; let key = fresh_key(&store).await; let body = r#"{"namespace":["missing","child"]}"#; @@ -280,8 +280,10 @@ async fn malformed_and_missing_parent_results_are_retained_before_any_later_retr 200 ); assert_eq!( - send(address, "POST", "/v1/namespaces", "w", Some(&key), body).await, - failed + send(address, "POST", "/v1/namespaces", "w", Some(&key), body) + .await + .0, + 200 ); let invalid_key = fresh_key(&store).await; let malformed = send(address, "POST", "/v1/namespaces", "w", Some(&invalid_key), "{").await; @@ -290,6 +292,19 @@ async fn malformed_and_missing_parent_results_are_retained_before_any_later_retr send(address, "POST", "/v1/namespaces", "w", Some(&invalid_key), "{").await, malformed ); + assert_eq!( + send( + address, + "POST", + "/v1/namespaces", + "w", + Some(&invalid_key), + r#"{"namespace":["corrected"]}"# + ) + .await + .0, + 200 + ); assert_eq!( send(address, "POST", "/v1/namespaces", "w", Some("invalid"), "{}") .await @@ -298,7 +313,7 @@ async fn malformed_and_missing_parent_results_are_retained_before_any_later_retr ); assert_eq!( send(address, "POST", "/v1/namespaces", "w", None, body).await.0, - 200 + 409 ); assert_eq!( send(address, "DELETE", "/v1/namespaces/missing", "w", None, "") @@ -311,7 +326,7 @@ async fn malformed_and_missing_parent_results_are_retained_before_any_later_retr } #[tokio::test] -async fn colliding_uuidv7_headers_have_independent_durable_http_replay() { +async fn colliding_uuidv7_headers_do_not_create_shared_http_binding_state() { let (store, address, stop, server) = setup().await; let now = u64::try_from(SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis()).unwrap(); let mut seen = std::collections::BTreeMap::new(); @@ -385,7 +400,14 @@ async fn colliding_uuidv7_headers_have_independent_durable_http_replay() { scope: SystemScope::RetryOverflow, suffix: second.as_bytes().to_vec(), }; - assert!(store.values.load().contains_key(&overflow.encode().unwrap())); + assert!(!store.values.load().contains_key(&overflow.encode().unwrap())); + assert!(store.values.load().keys().all(|key| !matches!( + IcebergKey::decode(key), + Ok(IcebergKey::System { + scope: SystemScope::RetryBinding | SystemScope::RetryOverflow, + .. + }) + ))); stop.send(()).unwrap(); server.await.unwrap(); } diff --git a/app/crowdb-access-server/tests/iceberg_route_test.rs b/app/crowdb-access-server/tests/iceberg_route_test.rs index fd1d17247..133be899c 100644 --- a/app/crowdb-access-server/tests/iceberg_route_test.rs +++ b/app/crowdb-access-server/tests/iceberg_route_test.rs @@ -286,7 +286,6 @@ async fn discovery_uses_installed_routes_and_unsupported_paths_leave_no_record() assert_eq!(snapshot.routes[2][0].requests, 1); assert_eq!(snapshot.routes[2][0].request_bytes, bytes); assert!(snapshot.routes[2][0].response_bytes > 0); - assert_eq!(snapshot.retry_new, 1); assert_eq!(snapshot.routes[1][0].requests, 1); assert_eq!(snapshot.routes[1][0].response_bytes, 0); } diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg.md b/doc/design/access-server/iceberge/design-crowdb-iceberg.md index 25ef873e5..ccfb1d39c 100644 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg.md +++ b/doc/design/access-server/iceberge/design-crowdb-iceberg.md @@ -202,24 +202,27 @@ neither clear nor smaller restart settings can shorten existing bounds. Listeners stop admission before bounded draining; startup and periodic reconciliation resume interrupted management operations. -Management, audit and shared REST retry ledgers each use 4096 deterministic -fast-hash slots with exact-identity overflow keys. An occupied slot does not -reject a different identity: it routes that identity to its own durable key. -Neither slot nor overflow records are evicted inside their retention window; -overflow storage is subject to normal disk capacity and physical reclamation. -Client identities use UUIDv7 issuance time with a 24-hour admission window and -30-second future-clock allowance. Retention starts at first admission and includes -grace. Principal, digest and catalog context must match before REST replay; -catalog replacement prevents old-body replay or rebinding. Terminal results are -immutable, while transient failures retain recoverable state. Requests without -client keys receive distinct internal identities, not cross-request deduplication. +Management and audit ledgers use 4096 deterministic fast-hash slots with +exact-identity overflow keys. An occupied slot routes a different identity to +its own durable key. Their retention and authority checks belong to the +management operation contract. Client write identities use UUIDv7 issuance +time with a 30-second future-clock allowance; requests without a client key +receive independent internal identities. + +REST namespace and table writes do not allocate a shared HTTP retry binding +or cache terminal HTTP results. Each request validates its current authority +and input. Existing business drivers bind operation identities to their stable +resources and validate their own request preconditions; direct expected-base +table updates retain their head CAS contract. A failed request without a +business operation can be submitted again after the resource state changes. +Legacy HTTP bindings, overflow keys and response payloads remain decodable +for retention cleanup, with no producer on new request paths. Immutable operation payloads use 32-KiB pages with a 2-MiB aggregate limit. Each reference binds catalog, operation, content digest and total size; readers validate -every page and the complete digest. Small retry responses remain inline. Larger -responses publish one immutable manifest only after all pages are durable, then -complete the system retry binding. A lost reply resumes page writes or replays the -published manifest without changing the original response. +every page and the complete digest. Business operation drivers use these +references for their existing inputs and results. Legacy large HTTP response +manifests remain readable by cleanup; independent requests do not create them. Namespace operation journals occupy a separate key scope from HTTP responses. They preserve request identity, principal, stable target and parent IDs, immutable @@ -250,7 +253,8 @@ so a repeated removal can move from removed to missing. Repeating assignments can overwrite a concurrent assignment to the same property. Other properties are retained. Old property journals remain readable by recovery, including the holder-bound marker dispatcher used before new updates. Creation and drop -currently use their durable drivers and the shared HTTP retry ledger. +currently use their durable drivers, whose simplification is independent of +the removed outer HTTP response ledger. The catalog configuration does not advertise a universal idempotency-key lifetime, since independent property updates do not retain operation responses. diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index d91c229f6..bddacadce 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -320,10 +320,8 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 ### P8 — Iceberg RetryLedger（执行 D3；按 route 判定必要性） -- [ ] **P8.1 清掉确定冲突后的查询**：begin 使用返回 binding，仅真实碰撞才查 overflow；finish - 使用已返回 result，不重新 GET。Files: Iceberg `operation/{retry,ledger,result}.rs`。 -- [ ] **P8.2 按实际用途删除重试账本**：按 route 对照业务 journal 与 HTTP binding/input/result，判定实际重试效果； - 普通覆盖删除 ledger/body/状态，条件或多资源操作按 R2/R3 处理；不默认保留“最小回执”。迁移测试和 SDK。Files: `operation/{retry,result,payload}.rs`、Access `iceberg/{table_write,namespace_write}`。 +HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。旧格式仅供解码/GC； +内部业务 driver 的清理继续由 R2/R3/P9/P10 跟踪，不再优化已删除的账本路径。 ### P9 — Iceberg namespace / table 多阶段操作 @@ -437,6 +435,19 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 ## 已完成与验证记录 +- **2026-10-24，P8.2（P8.1 随路径删除）**：namespace/table 写入口不再 begin/finish + 通用 HTTP RetryLedger，删除 admission 重试、New/Resume/Replay 状态、response 编码/装载 + driver 及对应指标。表生命周期从真实业务操作读取恢复所需版本，不依赖外层 Resume 提示； + changed request 冲突映射保留。写者鉴权、context、direct commit expected base 均保留。 + 旧 Retry/RetryResult 格式仍由 codec 和 GC 识别；没有新写入口，不批量删除未过期记录。 + namespace create 的缺失父目录/无业务记录错误在父目录建好后可用原 key 成功， + malformed 输入可修正重试；已有业务操作仍验证其身份，不靠 HTTP 缓存遮住新状态。 + 相同 hash slot 的两个 UUID 请求不创建 retry binding/overflow，仍独立成功。 + namespace HTTP 4、table write 8、lifecycle 5、route 2 项通过；组件 GC 21、namespace + update 11/create 4/drop 6、request identity 1 项通过。两 crate all-target/all-feature clippy、 + fmt 与 diff 检查通过。内部 journal 尚未删除，不能把本项当作 R2/R3 已完成。 + + - **2026-10-24，P9.3，R1 的 namespace 属性场景**：属性请求直接修改现有 authority， 删除新请求的 NamespaceOperation、输入/响应 payload、pending marker 和 HTTP RetryLedger。 请求不再携带 operation/principal 作为存储层属性参数；入口仍逐次鉴权。CAS 用于保留未提及 diff --git a/lib/crowdb-access-iceberg/src/operation.rs b/lib/crowdb-access-iceberg/src/operation.rs index 1acfffd79..d4d669dc8 100644 --- a/lib/crowdb-access-iceberg/src/operation.rs +++ b/lib/crowdb-access-iceberg/src/operation.rs @@ -1,4 +1,4 @@ -//! Durable management requests and HTTP retry identities. +//! Management operations, request identities and legacy retry record decoding. mod identity; mod ledger; @@ -13,4 +13,4 @@ pub(crate) use ledger::{ledger_locate, LedgerLocation}; pub use management::{ManagementAction, ManagementOperation, ManagementPhase, ManagementRequest}; pub use payload::{PayloadPage, PayloadReference, PayloadStore, MAX_PAYLOAD_BYTES, PAYLOAD_PAGE_BYTES}; pub use result::RetryResult; -pub use retry::{RetryAdmission, RetryLedger, RetryRecord}; +pub use retry::RetryRecord; diff --git a/lib/crowdb-access-iceberg/src/operation/result.rs b/lib/crowdb-access-iceberg/src/operation/result.rs index d2283d1f0..35690e142 100644 --- a/lib/crowdb-access-iceberg/src/operation/result.rs +++ b/lib/crowdb-access-iceberg/src/operation/result.rs @@ -1,8 +1,8 @@ -use crate::catalog::CatalogError; +//! Legacy paged HTTP response format; new requests do not create these records. + use crate::error::ValidationError; -use crate::record::StorageRecord; -use super::{PayloadReference, PayloadStore, RetryRecord}; +use super::{PayloadReference, RetryRecord}; pub const INLINE_RETRY_BYTES: usize = 16 * 1024; @@ -29,34 +29,3 @@ impl RetryResult { Ok(()) } } - -pub(super) async fn encode_result( - payloads: &PayloadStore, - request: &RetryRecord, -) -> Result, CatalogError> { - if request.body.len() <= INLINE_RETRY_BYTES { - return Ok(StorageRecord::Retry(Box::new(request.clone())).encode()?); - } - let body = payloads - .put(request.context.catalog, request.identity.operation, &request.body) - .await?; - let mut binding = request.clone(); - binding.body.clear(); - Ok(StorageRecord::RetryResult(Box::new(RetryResult { binding, body })).encode()?) -} - -pub(super) async fn decode_result( - payloads: &PayloadStore, - record: StorageRecord, -) -> Result { - match record { - StorageRecord::Retry(record) => Ok(*record), - StorageRecord::RetryResult(result) => { - let mut record = result.binding; - record.body = payloads.get(&result.body).await?; - record.validate()?; - Ok(record) - } - _ => Err(ValidationError::Record.into()), - } -} diff --git a/lib/crowdb-access-iceberg/src/operation/retry.rs b/lib/crowdb-access-iceberg/src/operation/retry.rs index 918e99c25..ea181eb52 100644 --- a/lib/crowdb-access-iceberg/src/operation/retry.rs +++ b/lib/crowdb-access-iceberg/src/operation/retry.rs @@ -1,14 +1,10 @@ -use std::sync::Arc; +//! Legacy HTTP response records retained for decoding and garbage collection. -use crate::catalog::{CasOutcome, CatalogContext, CatalogError, CatalogStore, RootState}; +use crate::catalog::CatalogContext; use crate::error::ValidationError; -use crate::key::{CatalogScope, IcebergKey, SystemScope}; -use crate::record::StorageRecord; +use crate::key::{CatalogScope, IcebergKey}; -use super::{ - ledger_locate, mutation_identity, LedgerLocation, PayloadStore, RequestIdentity, MAX_PAYLOAD_BYTES, - RETRY_WINDOW_MS, -}; +use super::{RequestIdentity, MAX_PAYLOAD_BYTES}; #[derive(Clone, Debug, Eq, PartialEq)] pub struct RetryRecord { @@ -51,209 +47,6 @@ impl RetryRecord { suffix: self.identity.operation.as_bytes().to_vec(), } } - - fn same_request(&self, other: &Self) -> bool { - self.identity == other.identity - && self.principal == other.principal - && self.route == other.route - && self.digest == other.digest - && self.context == other.context - } -} - -#[derive(Debug)] -pub enum RetryAdmission { - New(RetryRecord), - Resume(RetryRecord), - Replay(RetryRecord), -} - -pub struct RetryLedger { - store: Arc, -} - -impl RetryLedger { - #[must_use] - pub fn new(store: Arc) -> Self { - Self { store } - } - - /// # Errors - /// Rejects changed input/principal/domain, expired keys, collisions and storage failures. - pub async fn begin(&self, mut request: RetryRecord, now_ms: u64) -> Result { - request.status = 0; - request.body.clear(); - request.retained_until_ms = now_ms - .checked_add(RETRY_WINDOW_MS) - .and_then(|time| time.checked_add(30_000)) - .ok_or(ValidationError::Deadline)?; - request.validate()?; - self.check_context(request.context).await?; - let bytes = StorageRecord::Retry(Box::new(request.clone())).encode()?; - for _ in 0..4 { - let key = match ledger_locate( - self.store.as_ref(), - SystemScope::RetryBinding, - request.identity.operation, - ) - .await? - { - LedgerLocation::Existing(key, value) => { - let StorageRecord::Retry(existing) = StorageRecord::decode(&key, &value.bytes)? else { - return Err(ValidationError::Record.into()); - }; - if !existing.same_request(&request) || now_ms > existing.retained_until_ms { - return Err(CatalogError::Conflict); - } - return self.existing(*existing).await; - } - LedgerLocation::Vacant(key) => { - request.identity.validate(now_ms)?; - key - } - }; - if self.cas(&key, None, &bytes).await? { - return Ok(RetryAdmission::New(request)); - } - } - Err(CatalogError::Busy) - } - - /// # Errors - /// Rejects conflicting final results, domain changes, expired bindings and storage failures. - pub async fn finish( - &self, - mut request: RetryRecord, - status: u16, - body: Vec, - now_ms: u64, - ) -> Result { - if !terminal_status(status) { - return Ok(false); - } - self.check_context(request.context).await?; - let LedgerLocation::Existing(key, previous) = ledger_locate( - self.store.as_ref(), - SystemScope::RetryBinding, - request.identity.operation, - ) - .await? - else { - return Err(CatalogError::Conflict); - }; - let StorageRecord::Retry(binding) = StorageRecord::decode(&key, &previous.bytes)? else { - return Err(ValidationError::Record.into()); - }; - if !binding.same_request(&request) || now_ms > binding.retained_until_ms { - return Err(CatalogError::Conflict); - } - request.retained_until_ms = binding.retained_until_ms; - request.status = status; - request.body = body; - request.validate()?; - let result_key = request.result_key(); - let payloads = PayloadStore::new(self.store.clone()); - let result = if let Some(existing) = self.store.get(&result_key.encode()?).await? { - let record = - super::result::decode_result(&payloads, StorageRecord::decode(&result_key, &existing.bytes)?) - .await?; - if record != request { - return Err(CatalogError::Conflict); - } - existing.bytes - } else { - super::result::encode_result(&payloads, &request).await? - }; - if !self.cas(&result_key, None, &result).await? { - let existing = self - .store - .get(&result_key.encode()?) - .await? - .ok_or(CatalogError::Conflict)?; - if existing.bytes != result { - return Err(CatalogError::Conflict); - } - } - request.body.clear(); - let binding = StorageRecord::Retry(Box::new(request)).encode()?; - self.cas(&key, Some(&previous.bytes), &binding).await?; - Ok(true) - } - - async fn existing(&self, binding: RetryRecord) -> Result { - let key = binding.result_key(); - let Some(value) = self.store.get(&key.encode()?).await? else { - if binding.status != 0 { - return Err(ValidationError::Record.into()); - } - return Ok(RetryAdmission::Resume(binding)); - }; - let payloads = PayloadStore::new(self.store.clone()); - let result = - super::result::decode_result(&payloads, StorageRecord::decode(&key, &value.bytes)?).await?; - if !binding.same_request(&result) - || result.status == 0 - || binding.retained_until_ms != result.retained_until_ms - || (binding.status != 0 && binding.status != result.status) - { - return Err(ValidationError::Record.into()); - } - if binding.status == 0 { - let LedgerLocation::Existing(binding_key, _) = ledger_locate( - self.store.as_ref(), - SystemScope::RetryBinding, - binding.identity.operation, - ) - .await? - else { - return Err(CatalogError::Conflict); - }; - let previous = StorageRecord::Retry(Box::new(binding.clone())).encode()?; - let mut completed = binding; - completed.status = result.status; - self.cas( - &binding_key, - Some(&previous), - &StorageRecord::Retry(Box::new(completed)).encode()?, - ) - .await?; - } - Ok(RetryAdmission::Replay(result)) - } - - async fn check_context(&self, context: CatalogContext) -> Result<(), CatalogError> { - let key = IcebergKey::System { - scope: SystemScope::ActiveRoot, - suffix: Vec::new(), - }; - let value = self - .store - .get(&key.encode()?) - .await? - .ok_or(CatalogError::Uninitialized)?; - let StorageRecord::Active(root) = StorageRecord::decode(&key, &value.bytes)? else { - return Err(ValidationError::Record.into()); - }; - if root.state != RootState::Ready || root.context != context { - return Err(CatalogError::Conflict); - } - Ok(()) - } - - async fn cas( - &self, - key: &IcebergKey, - expected: Option<&[u8]>, - value: &[u8], - ) -> Result { - let key = key.encode()?; - Ok(matches!( - self.store - .compare_exchange(&key, expected, value, mutation_identity(&key, expected, value)) - .await?, - CasOutcome::Applied(_) - )) - } } fn terminal_status(status: u16) -> bool { diff --git a/lib/crowdb-access-iceberg/tests/gc_worker_test.rs b/lib/crowdb-access-iceberg/tests/gc_worker_test.rs index 1f8e869ed..13233f8b0 100644 --- a/lib/crowdb-access-iceberg/tests/gc_worker_test.rs +++ b/lib/crowdb-access-iceberg/tests/gc_worker_test.rs @@ -516,13 +516,6 @@ async fn retired_cleanup_removes_expired_primary_binding_and_projection_without_ StorageRecord::decode(&overflow, &value.bytes).unwrap(), StorageRecord::Retry(Box::new(new.clone())) ); - assert!(matches!( - crowdb_access_iceberg::operation::RetryLedger::new(fixture.store.clone()) - .begin(new, 1_000_000) - .await - .unwrap(), - crowdb_access_iceberg::operation::RetryAdmission::Resume(_) - )); } #[tokio::test] diff --git a/lib/crowdb-access-iceberg/tests/request_identity_test.rs b/lib/crowdb-access-iceberg/tests/request_identity_test.rs new file mode 100644 index 000000000..1a70885b9 --- /dev/null +++ b/lib/crowdb-access-iceberg/tests/request_identity_test.rs @@ -0,0 +1,13 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_access_iceberg::operation::RequestIdentity; + +#[test] +fn uuidv7_wire_keys_validate_version_and_clock() { + let valid = "00000000-0064-7000-8000-000000000001"; + assert_eq!(RequestIdentity::parse(valid, 100).unwrap().issued_ms, 100); + assert!(RequestIdentity::parse("00000000-0064-4000-8000-000000000001", 100).is_err()); + assert!(RequestIdentity::parse("00000001-0064-7000-8000-000000000001", 100).is_err()); + assert!(RequestIdentity::parse("not-a-uuid", 100).is_err()); +} diff --git a/lib/crowdb-access-iceberg/tests/retry_test.rs b/lib/crowdb-access-iceberg/tests/retry_test.rs deleted file mode 100644 index 10c43898b..000000000 --- a/lib/crowdb-access-iceberg/tests/retry_test.rs +++ /dev/null @@ -1,278 +0,0 @@ -#[path = "common/store.rs"] -mod common; - -use common::TestStore; -use crowdb_access_iceberg::catalog::{CatalogError, CatalogRepository, ClearBounds, ManagementPrivilege}; -use crowdb_access_iceberg::key::OperationId; -use crowdb_access_iceberg::operation::{ - ManagementAction, ManagementRequest, RequestIdentity, RetryAdmission, RetryLedger, RetryRecord, -}; -use std::sync::atomic::Ordering; -use std::sync::Arc; - -#[tokio::test] -async fn lost_result_reply_repairs_binding_before_replay() { - let (store, _, request) = setup().await; - let ledger = RetryLedger::new(store.clone()); - ledger.begin(request.clone(), 100).await.unwrap(); - store - .fail_after - .store(store.writes.load(Ordering::SeqCst) + 1, Ordering::SeqCst); - assert!(ledger - .finish(request.clone(), 409, b"conflict".to_vec(), 101) - .await - .is_err()); - let ledger = RetryLedger::new(store.clone()); - assert!(matches!( - ledger.begin(request.clone(), 102).await.unwrap(), - RetryAdmission::Replay(_) - )); - let key = crowdb_access_iceberg::operation::ledger_key( - crowdb_access_iceberg::key::SystemScope::RetryBinding, - request.identity.operation, - ) - .unwrap(); - let values = store.values.load(); - let record = crowdb_access_iceberg::record::StorageRecord::decode( - &key, - &values.get(&key.encode().unwrap()).unwrap().bytes, - ) - .unwrap(); - let crowdb_access_iceberg::record::StorageRecord::Retry(binding) = record else { - panic!("retry binding") - }; - assert_eq!(binding.status, 409); - assert!(binding.body.is_empty()); -} - -async fn setup() -> (Arc, CatalogRepository, RetryRecord) { - let store = Arc::new(TestStore::default()); - let repository = CatalogRepository::new( - store.clone(), - ClearBounds { - request_ms: 1, - root_lease_ms: 0, - delegated_access_ms: 0, - clock_skew_ms: 1, - }, - ) - .unwrap(); - repository - .execute( - ManagementRequest { - identity: RequestIdentity { - operation: OperationId::random(), - issued_ms: 100, - }, - principal: "manager".into(), - action: ManagementAction::Initialize, - expected_epoch: 0, - display_name: "catalog".into(), - confirmation: None, - capabilities: None, - }, - ManagementPrivilege::Manage, - 100, - ) - .await - .unwrap(); - let record = RetryRecord { - identity: RequestIdentity { - operation: OperationId::random(), - issued_ms: 100, - }, - principal: "reader".into(), - route: "POST /namespaces".into(), - digest: [1; 32], - context: repository.status().await.unwrap().0.context, - retained_until_ms: 0, - status: 0, - body: Vec::new(), - }; - (store, repository, record) -} - -#[tokio::test] -async fn final_conflicts_replay_but_server_errors_resume() { - let (store, _, request) = setup().await; - let ledger = RetryLedger::new(store.clone()); - assert!(matches!( - ledger.begin(request.clone(), 100).await.unwrap(), - RetryAdmission::New(_) - )); - assert!(!ledger - .finish(request.clone(), 503, b"busy".to_vec(), 101) - .await - .unwrap()); - assert!(matches!( - RetryLedger::new(store.clone()) - .begin(request.clone(), 102) - .await - .unwrap(), - RetryAdmission::Resume(_) - )); - assert!(ledger - .finish(request.clone(), 409, b"conflict".to_vec(), 103) - .await - .unwrap()); - let RetryAdmission::Replay(result) = RetryLedger::new(store).begin(request, 104).await.unwrap() else { - panic!("expected replay") - }; - assert_eq!(result.status, 409); - assert_eq!(result.body, b"conflict"); -} - -#[tokio::test] -async fn changed_principal_digest_and_retired_domain_never_replay() { - let (store, repository, request) = setup().await; - let ledger = RetryLedger::new(store); - ledger.begin(request.clone(), 100).await.unwrap(); - ledger - .finish(request.clone(), 200, b"old".to_vec(), 101) - .await - .unwrap(); - let mut changed = request.clone(); - changed.principal = "another".into(); - assert!(matches!( - ledger.begin(changed, 102).await, - Err(CatalogError::Conflict) - )); - let mut changed = request.clone(); - changed.digest = [2; 32]; - assert!(matches!( - ledger.begin(changed, 102).await, - Err(CatalogError::Conflict) - )); - let clear = ManagementRequest { - identity: RequestIdentity { - operation: OperationId::random(), - issued_ms: 100, - }, - principal: "clearer".into(), - action: ManagementAction::Clear, - expected_epoch: 1, - display_name: "empty".into(), - confirmation: Some(request.context.catalog), - capabilities: None, - }; - let _ = repository - .execute(clear.clone(), ManagementPrivilege::Clear, 103) - .await; - repository - .execute(clear, ManagementPrivilege::Clear, 200) - .await - .unwrap(); - assert!(matches!( - ledger.begin(request.clone(), 201).await, - Err(CatalogError::Conflict) - )); - let mut rebound = request; - rebound.context = repository.status().await.unwrap().0.context; - assert!(matches!( - ledger.begin(rebound, 201).await, - Err(CatalogError::Conflict) - )); -} - -#[test] -fn uuidv7_wire_keys_validate_version_and_clock() { - let valid = "00000000-0064-7000-8000-000000000001"; - assert_eq!(RequestIdentity::parse(valid, 100).unwrap().issued_ms, 100); - assert!(RequestIdentity::parse("00000000-0064-4000-8000-000000000001", 100).is_err()); - assert!(RequestIdentity::parse("00000001-0064-7000-8000-000000000001", 100).is_err()); - assert!(RequestIdentity::parse("not-a-uuid", 100).is_err()); -} - -#[tokio::test] -async fn colliding_keys_use_exact_overflow_and_replay_independently() { - use crowdb_access_iceberg::key::SystemScope; - use crowdb_access_iceberg::operation::ledger_key; - let (store, _, request) = setup().await; - let ledger = RetryLedger::new(store.clone()); - let target = ledger_key(SystemScope::RetryBinding, request.identity.operation).unwrap(); - let mut collision = request.clone(); - collision.identity.operation = (1_u128..1_000_000) - .find_map(|number| { - let candidate = OperationId::from_bytes(&number.to_be_bytes()).unwrap(); - (candidate != request.identity.operation - && ledger_key(SystemScope::RetryBinding, candidate).unwrap() == target) - .then_some(candidate) - }) - .expect("colliding bounded slot"); - assert!(matches!( - ledger.begin(request.clone(), 100).await.unwrap(), - RetryAdmission::New(_) - )); - assert!(matches!( - ledger.begin(collision.clone(), 101).await.unwrap(), - RetryAdmission::New(_) - )); - let overflow = crowdb_access_iceberg::key::IcebergKey::System { - scope: SystemScope::RetryOverflow, - suffix: collision.identity.operation.as_bytes().to_vec(), - }; - assert!(store.values.load().contains_key(&overflow.encode().unwrap())); - ledger - .finish(request.clone(), 204, Vec::new(), 102) - .await - .unwrap(); - ledger - .finish(collision.clone(), 409, b"collision".to_vec(), 102) - .await - .unwrap(); - assert!(matches!( - ledger.begin(request, 103).await.unwrap(), - RetryAdmission::Replay(record) if record.status == 204 - )); - assert!(matches!( - ledger.begin(collision, 103).await.unwrap(), - RetryAdmission::Replay(record) if record.status == 409 && record.body == b"collision" - )); -} - -#[tokio::test] -async fn large_retry_bodies_recover_after_every_page_manifest_and_binding_reply_loss() { - use crowdb_access_iceberg::operation::PAYLOAD_PAGE_BYTES; - let body = vec![31; PAYLOAD_PAGE_BYTES * 2 + 1]; - for lost_write in 1..=5 { - let (store, _, mut request) = setup().await; - request.principal = "writer".into(); - let ledger = RetryLedger::new(store.clone()); - ledger.begin(request.clone(), 100).await.unwrap(); - store - .fail_after - .store(store.writes.load(Ordering::SeqCst) + lost_write, Ordering::SeqCst); - assert!(ledger - .finish(request.clone(), 200, body.clone(), 101) - .await - .is_err()); - let recovered = RetryLedger::new(store.clone()); - match recovered.begin(request.clone(), 102).await.unwrap() { - RetryAdmission::Resume(_) => { - recovered - .finish(request.clone(), 200, body.clone(), 103) - .await - .unwrap(); - } - RetryAdmission::Replay(result) => assert_eq!(result.body, body), - RetryAdmission::New(_) => panic!("existing identity must not be readmitted"), - } - let RetryAdmission::Replay(result) = recovered.begin(request.clone(), 104).await.unwrap() else { - panic!("replay") - }; - assert_eq!(result.body, body); - assert_eq!(result.principal, "writer"); - let mut changed = request.clone(); - changed.principal = "reader".into(); - assert!(matches!( - recovered.begin(changed, 104).await, - Err(CatalogError::Conflict) - )); - let writes = store.writes.load(Ordering::SeqCst); - assert!(matches!( - recovered.finish(request, 200, vec![0; body.len()], 104).await, - Err(CatalogError::Conflict) - )); - assert_eq!(store.writes.load(Ordering::SeqCst), writes); - } -} From 0705eae69a7ea453ccca51553c9c8c4da203233a Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 11:32:24 +0800 Subject: [PATCH 101/258] Bound concurrent index preparation and remove a payload copy --- .../dataset/design-dataset-control.md | 8 ++ doc/working/plan-access-simplification.md | 20 ++- .../src/inspection/index.rs | 18 ++- .../src/payload/reader.rs | 7 +- .../tests/common/inspection_writes.rs | 98 ++++++++++++++ .../tests/inspection_write_cost_test.rs | 128 ++++++++++++++++++ 6 files changed, 271 insertions(+), 8 deletions(-) create mode 100644 lib/crowdb-access-dataset/tests/common/inspection_writes.rs create mode 100644 lib/crowdb-access-dataset/tests/inspection_write_cost_test.rs diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index 3c40f0433..a26a1ea27 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -97,6 +97,14 @@ metadata and separate value bindings. Ordinary inspection and indexed read plann do not load a complete Manifest or chunk payload. Inventory and parent/child indexes filter visibility through commit evidence, not merely candidate record state. +Preparing an index overlaps at most sixteen immutable row writes, with at most +2 MiB of encoded keys and values queued. Encoding holds at most one additional +row under the existing per-row bound. No row is copied into another batch buffer. +All row writes must succeed before inventory, parent/child and ready summaries +are installed. Failure leaves the index unavailable; repeating the request checks +the same immutable rows. This bounds parallel work while preserving the existing +number of conditional mutations and each row's identity. + List pages default to twenty items and cap at one hundred with a 1 MiB metadata window. Exact matrices admit twenty IDs by twelve fields. IDs preserve up to 16 KiB of binary data; Schema admits up to 256 fields with names up to 1024 bytes. Projection diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index bddacadce..853b315fc 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -345,9 +345,11 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ### P12 — Dataset inspection 索引 -- [ ] **P12.1 降低索引写入粒度**：当前 F 个 schema field、N 个 sample、E 个非空 sample-field - 产生 `F + N + 2E` 行写和 3 个 summary/index 写，不含 Manifest/publication。 - 比较批量与有界并发，保留顺序、预算和未完整索引不可见性。Files: Dataset `inspection/{index,preparation}.rs`。 +- [ ] **P12.1 比较 routed batch 与已完成的有界并发**：当前 F 个 schema field、N 个 sample、 + E 个非空 sample-field 仍有 `F + N + 2E` 行写和 3 个 summary/index 写。 + 已完成 16 条/2 MiB 的并发准备，消除逐行串行等待；后续测 grouped RPC 能否进一步减少 + RTT 与消息数量，计入 batch 输入复制、分区分组和失败重试成本，不能把并发算成 RPC 数减少。 + Files: Dataset `inspection/{index,preparation}.rs`、`chunk_store.rs`、Chunk-KV batch 接口。 - [ ] **P12.2 精简索引准备/回收检查点**：保留服务必需的不可变索引，按 D1 删除辅助恢复进度，失败重做； 未发布索引/候选纳入 R4 孤儿清理。显式 legacy preparation 仍是 mutation，读 miss 不自动持久化构建。Files: `inspection/{legacy,retirement,reclaim}.rs` 及索引恢复测试。 @@ -384,7 +386,7 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 - [ ] **F7 比较预读与 create-CAS 总成本**：新 immutable 页的 GET+CAS 可能比直接 CAS 更贵， replay 则可能相反；比较新写/重放比例，复用本请求已知值，不能无证明改普通 PUT。Files: Dataset Composition/Split receipts、Iceberg `operation/payload.rs`、journal validation。 - [ ] **F8 减少 payload 复制**：审查 HTTP 收整 part、`BytesMut` 拼接、inline copy、兼容 - UploadedChunkReader 的拼接后 `to_vec` 和 S3 coalescing；先查活跃调用，再移交 owner 或流式 + reader 之外的 S3 coalescing；UploadedChunkReader 已改为最终 Vec 一次拼接。先查活跃调用，再移交 owner 或流式 发送。Files: Dataset payload/upload、Access HTTP batch、S3/native 接收路径。验证部分帧、非整齐 size、held buffer 和实际复制字节。 - [ ] **F9 分解摘要/编码/分配 CPU**：complete 重读 part 并做 part/whole MD5，native part 与 field 还各有摘要；MD5 不能直接拼合，只有同范围可信校验证据才可复用。测大 owner/part-index @@ -435,6 +437,16 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，P12.1a / F8.1**：新建索引同时推进最多 16 条 immutable row CAS、 + 队列最多 2 MiB 编码 key/value，另有一条正在编码的有界 row；没有额外拼接或复制批次。 + 所有 row 成功后才写 inventory/child/ready。延迟 store 证明存在并发、条数与字节界限 + 都生效；12 个 sample 的 37 条 row mutation 保持不变，不能据此宣称 RPC 数减少。 + 中间行失败保持索引不可见，重建 authority 后可验证并重做 immutable rows。 + UploadedChunkReader 直接填最终预分配 Vec，删除 BytesMut→Vec 的整段复制和扩容。 + 新增 inspection write 两项及 Dataset `--all-targets` 全部通过；包含既有跨 part range、 + digest、cancel、Composition/Split 及不可见性回归。Dataset all-target/all-feature clippy、 + workspace fmt 与 diff 检查通过。grouped RPC 比较及其他活跃 payload 路径继续在待办中。 + - **2026-10-24，P8.2（P8.1 随路径删除）**：namespace/table 写入口不再 begin/finish 通用 HTTP RetryLedger，删除 admission 重试、New/Resume/Replay 状态、response 编码/装载 driver 及对应指标。表生命周期从真实业务操作读取恢复所需版本，不依赖外层 Resume 提示； diff --git a/lib/crowdb-access-dataset/src/inspection/index.rs b/lib/crowdb-access-dataset/src/inspection/index.rs index 36afdc415..097584608 100644 --- a/lib/crowdb-access-dataset/src/inspection/index.rs +++ b/lib/crowdb-access-dataset/src/inspection/index.rs @@ -5,10 +5,14 @@ use super::{key, InspectionState, SnapshotSummary}; use crate::{ AuthorityError, CasOutcome, DatasetAuthority, DatasetIdentity, FieldLocator, ManifestRecord, SnapshotId, }; +use futures::{stream::FuturesUnordered, StreamExt}; use sha2::{Digest, Sha256}; use super::rows::{encode, next_row, EncodedRow, IndexPosition}; +const MAX_PENDING_ROWS: usize = 16; +const MAX_PENDING_BYTES: usize = 2 * 1024 * 1024; + impl DatasetAuthority { pub(crate) async fn persist_inspection_index( &self, @@ -42,8 +46,20 @@ impl DatasetAuthority { manifest: &ManifestRecord, ) -> Result<(), AuthorityError> { let mut position = IndexPosition::default(); + let mut pending = FuturesUnordered::new(); + let mut pending_bytes = 0; while let Some(row) = next_row(identity, manifest, &mut position)? { - self.put_inspection_bytes(row).await?; + let size = row.key.len() + row.bytes.len(); + while pending.len() >= MAX_PENDING_ROWS || pending_bytes + size > MAX_PENDING_BYTES { + let (size, result) = pending.next().await.ok_or(AuthorityError::Corrupt)?; + pending_bytes -= size; + result?; + } + pending_bytes += size; + pending.push(async move { (size, self.put_inspection_bytes(row).await) }); + } + while let Some((_, result)) = pending.next().await { + result?; } Ok(()) } diff --git a/lib/crowdb-access-dataset/src/payload/reader.rs b/lib/crowdb-access-dataset/src/payload/reader.rs index c0d125aa2..c883c9d1c 100644 --- a/lib/crowdb-access-dataset/src/payload/reader.rs +++ b/lib/crowdb-access-dataset/src/payload/reader.rs @@ -3,7 +3,6 @@ use crate::{ChunkReadError, ChunkReader, DatasetUploads, ReadCancellation, UploadError}; use async_trait::async_trait; -use bytes::BytesMut; use std::ops::Range; /// Compatibility reader for bounded native/legacy reads. Large fields use streams. @@ -34,7 +33,9 @@ impl ChunkReader for UploadedChunkReader { { return Err(ChunkReadError::Failed("bounded field range required".into())); } - let mut output = BytesMut::new(); + let mut output = Vec::with_capacity( + usize::try_from(range.end - range.start).map_err(|_| ChunkReadError::Truncated)?, + ); let first = range.start / u64::from(record.part_size()); let end = range.end.div_ceil(u64::from(record.part_size())); for index in first..end { @@ -55,7 +56,7 @@ impl ChunkReader for UploadedChunkReader { if cancel.is_cancelled() { return Err(ChunkReadError::Cancelled); } - Ok(output.to_vec()) + Ok(output) } } fn map_error(error: &UploadError) -> ChunkReadError { diff --git a/lib/crowdb-access-dataset/tests/common/inspection_writes.rs b/lib/crowdb-access-dataset/tests/common/inspection_writes.rs new file mode 100644 index 000000000..e799d1e34 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/common/inspection_writes.rs @@ -0,0 +1,98 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use crowdb_access_dataset::{CasOutcome, DatasetStore, StoreError, StoredValue}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +use super::dataset_store::TestDatasetStore; + +pub struct TestIndexWrites { + pub inner: TestDatasetStore, + scope_offset: usize, + pending: AtomicUsize, + bytes: AtomicUsize, + values: AtomicUsize, + pub peak_pending: AtomicUsize, + pub peak_bytes: AtomicUsize, + pub peak_values: AtomicUsize, + pub rows: AtomicUsize, + pub fail_value: AtomicBool, +} + +impl TestIndexWrites { + pub fn new(identity_bytes: usize) -> Self { + Self { + inner: TestDatasetStore::default(), + scope_offset: b"DS\0I\0".len() + identity_bytes + 16, + pending: AtomicUsize::new(0), + bytes: AtomicUsize::new(0), + values: AtomicUsize::new(0), + peak_pending: AtomicUsize::new(0), + peak_bytes: AtomicUsize::new(0), + peak_values: AtomicUsize::new(0), + rows: AtomicUsize::new(0), + fail_value: AtomicBool::new(false), + } + } +} + +struct TestWriteGuard<'store> { + store: &'store TestIndexWrites, + bytes: usize, + value: bool, +} + +impl Drop for TestWriteGuard<'_> { + fn drop(&mut self) { + self.store.pending.fetch_sub(1, Ordering::Relaxed); + self.store.bytes.fetch_sub(self.bytes, Ordering::Relaxed); + if self.value { + self.store.values.fetch_sub(1, Ordering::Relaxed); + } + } +} + +#[async_trait] +impl DatasetStore for TestIndexWrites { + async fn get(&self, key: &[u8]) -> Result, StoreError> { + self.inner.get(key).await + } + + async fn compare_exchange( + &self, + key: &[u8], + expected: Option<&[u8]>, + value: &[u8], + ) -> Result { + let kind = key.get(self.scope_offset).copied(); + if !key.starts_with(b"DS\0I\0") || !matches!(kind, Some(b'S' | b'M' | b'F' | b'V')) { + return self.inner.compare_exchange(key, expected, value).await; + } + let guard = TestWriteGuard { + store: self, + bytes: key.len() + value.len(), + value: kind == Some(b'V'), + }; + self.rows.fetch_add(1, Ordering::Relaxed); + self.peak_pending.fetch_max( + self.pending.fetch_add(1, Ordering::Relaxed) + 1, + Ordering::Relaxed, + ); + self.peak_bytes.fetch_max( + self.bytes.fetch_add(guard.bytes, Ordering::Relaxed) + guard.bytes, + Ordering::Relaxed, + ); + if guard.value { + self.peak_values + .fetch_max(self.values.fetch_add(1, Ordering::Relaxed) + 1, Ordering::Relaxed); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + if self.fail_value.swap(false, Ordering::Relaxed) { + return Err(StoreError::Rejected); + } + } else { + tokio::task::yield_now().await; + } + self.inner.compare_exchange(key, expected, value).await + } +} diff --git a/lib/crowdb-access-dataset/tests/inspection_write_cost_test.rs b/lib/crowdb-access-dataset/tests/inspection_write_cost_test.rs new file mode 100644 index 000000000..cc9162a24 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/inspection_write_cost_test.rs @@ -0,0 +1,128 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/dataset_store.rs"] +#[allow(dead_code)] +mod dataset_store; +#[path = "common/inspection_writes.rs"] +mod inspection_writes; + +use crowdb_access_dataset::{ + DatasetAuthority, DatasetIdentity, DatasetStore, FieldDefinition, FieldLocator, FieldRecord, + ManifestRecord, NamespacePath, OperationId, SampleRecord, SchemaRecord, SnapshotId, +}; +use inspection_writes::TestIndexWrites; +use md5::{Digest, Md5}; +use std::sync::{atomic::Ordering, Arc}; + +fn manifest(samples: u8, bytes: usize) -> ManifestRecord { + let value = vec![255; bytes]; + ManifestRecord { + version: 1, + snapshot: SnapshotId::random(), + schema: SchemaRecord { + version: 1, + fields: vec![FieldDefinition { + name: "body".into(), + required: true, + }], + }, + samples: (0..samples) + .map(|id| SampleRecord { + sample_id: vec![id], + fields: vec![FieldRecord { + name: "body".into(), + value: FieldLocator::Inline { + md5: Md5::digest(&value).into(), + value: value.clone(), + }, + }], + }) + .collect(), + } +} + +fn ready_key(identity: &DatasetIdentity, snapshot: SnapshotId) -> Vec { + let mut key = b"DS\0I\0".to_vec(); + key.extend(identity.key().unwrap()); + key.extend(snapshot.as_bytes()); + key.push(b'H'); + key +} + +#[tokio::test] +async fn index_writes_overlap_with_both_count_and_encoded_byte_limits() { + for bytes in [1, 64 * 1024] { + let identity = DatasetIdentity::new(NamespacePath::root(), "write-budget").unwrap(); + let store = Arc::new(TestIndexWrites::new(identity.key().unwrap().len())); + let authority = DatasetAuthority::new(store.clone()); + authority.create_dataset(identity.clone()).await.unwrap(); + let samples = 12; + let snapshot = authority + .prepare_snapshot_with_operation( + &identity, + None, + bincode::serialize(&manifest(samples, bytes)).unwrap(), + OperationId::random(), + ) + .await + .unwrap(); + assert_eq!(store.rows.load(Ordering::Relaxed), 1 + 3 * usize::from(samples)); + let pending = store.peak_pending.load(Ordering::Relaxed); + assert!((2..=16).contains(&pending), "peak requests: {pending}"); + assert!(store.peak_bytes.load(Ordering::Relaxed) <= 2 * 1024 * 1024); + if bytes > 1 { + assert!( + store.peak_values.load(Ordering::Relaxed) <= 7, + "encoded values must consume the byte budget" + ); + } + assert!(store + .get(&ready_key(&identity, snapshot)) + .await + .unwrap() + .is_some()); + assert!(authority.latest(&identity).await.unwrap().is_none()); + } +} + +#[tokio::test] +async fn failed_parallel_row_keeps_index_hidden_and_retries_immutable_rows() { + let identity = DatasetIdentity::new(NamespacePath::root(), "write-failure").unwrap(); + let store = Arc::new(TestIndexWrites::new(identity.key().unwrap().len())); + let authority = DatasetAuthority::new(store.clone()); + authority.create_dataset(identity.clone()).await.unwrap(); + let operation = OperationId::random(); + let bytes = bincode::serialize(&manifest(12, 1)).unwrap(); + store.fail_value.store(true, Ordering::Relaxed); + assert!(authority + .prepare_snapshot_with_operation(&identity, None, bytes.clone(), operation) + .await + .is_err()); + let snapshot = authority + .publication_status(&identity, operation) + .await + .unwrap() + .publication + .snapshot; + assert!(store + .get(&ready_key(&identity, snapshot)) + .await + .unwrap() + .is_none()); + assert!(authority.latest(&identity).await.unwrap().is_none()); + let restarted = DatasetAuthority::new(store.clone()); + assert_eq!( + restarted + .prepare_snapshot_with_operation(&identity, None, bytes, operation) + .await + .unwrap(), + snapshot + ); + assert!(store + .get(&ready_key(&identity, snapshot)) + .await + .unwrap() + .is_some()); + assert!(restarted.latest(&identity).await.unwrap().is_none()); +} From e6a5c6cc08dc6a12c78528f77cc61ab66266adb6 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 11:46:59 +0800 Subject: [PATCH 102/258] Share read admission and locators within continuation requests --- .../src/dataset/api/batch.rs | 65 +++++----- .../src/dataset/api/query.rs | 19 +-- .../tests/dataset_read_query_contract_test.rs | 10 +- .../dataset/design-dataset-control.md | 8 ++ doc/working/plan-access-simplification.md | 46 ++++++- lib/crowdb-access-dataset/src/inspection.rs | 1 + .../src/inspection/evaluate.rs | 24 +++- .../src/inspection/read.rs | 77 +++++++++--- lib/crowdb-access-dataset/src/lib.rs | 4 +- lib/crowdb-access-dataset/src/read_query.rs | 6 + .../src/read_query/page.rs | 114 ++++++++++++++---- lib/crowdb-access-dataset/src/split/read.rs | 17 ++- .../tests/read_query_planning_test.rs | 40 ++++++ lib/crowdb-sdk-native/src/dataset/read.rs | 31 +++-- 14 files changed, 351 insertions(+), 111 deletions(-) diff --git a/app/crowdb-access-server/src/dataset/api/batch.rs b/app/crowdb-access-server/src/dataset/api/batch.rs index d782118c4..2da8130fe 100644 --- a/app/crowdb-access-server/src/dataset/api/batch.rs +++ b/app/crowdb-access-server/src/dataset/api/batch.rs @@ -4,7 +4,9 @@ use super::{error::ApiError, read_json, snapshot, ApiResult}; use crate::dataset::{DatasetBody, DatasetHttpService}; use base64::{engine::general_purpose::STANDARD, Engine}; -use crowdb_access_dataset::{DatasetIdentity, FieldLocator, SampleView, SnapshotId, UploadReadDescriptor}; +use crowdb_access_dataset::{ + DatasetIdentity, FieldLocator, ReadPage, ResolvedReadPage, SampleView, SnapshotId, UploadReadDescriptor, +}; use futures::stream; use http_body_util::{BodyExt, StreamBody}; use hyper::{ @@ -19,8 +21,6 @@ pub(super) struct BatchRequest { pub snapshot: String, pub sample_ids: Vec, pub fields: Vec, - #[serde(skip)] - pub sample_id_limit: Option, } impl BatchRequest { fn ids(&self) -> Result>, ApiError> { @@ -28,10 +28,7 @@ impl BatchRequest { .iter() .map(|value| { let bytes = STANDARD.decode(value).map_err(|_| ApiError::invalid())?; - if bytes.is_empty() - || bytes.len() > self.sample_id_limit.unwrap_or(1024) - || STANDARD.encode(&bytes) != *value - { + if bytes.is_empty() || bytes.len() > 1024 || STANDARD.encode(&bytes) != *value { return Err(ApiError::invalid()); } Ok(bytes) @@ -69,40 +66,52 @@ impl DatasetHttpService { .0 .clone(); let request: BatchRequest = read_json(request).await?; - self.batch(identity, request, None, true, request_id).await + self.batch(identity, request, request_id).await } pub(super) async fn batch( &self, identity: &DatasetIdentity, request: BatchRequest, - next: Option, - end: bool, request_id: String, ) -> ApiResult { let id = snapshot(&request.snapshot)?; let ids = request.ids()?; let authority = self.reads.authority(); - if ids.is_empty() || request.sample_id_limit.is_none() { - authority.check_snapshot_readable(identity, id).await?; - } + authority.check_snapshot_readable(identity, id).await?; let samples = if ids.is_empty() { Vec::new() - } else if request.sample_id_limit.is_some() { - authority - .resolve_inspection_batch_locators( - identity, - id, - &ids, - &request.fields, - &crowdb_access_dataset::ReadCancellation::new(), - ) - .await? } else { authority .resolve_batch_locators(identity, id, &ids, &request.fields) .await? }; - let header = make_header(id, &request.sample_ids, &request.fields, &samples, next, end); + self.render_batch( + ResolvedReadPage { + page: ReadPage { + snapshot: id, + ids, + fields: request.fields, + next: None, + }, + samples, + }, + request_id, + ) + } + + pub(super) fn render_batch(&self, resolved: ResolvedReadPage, request_id: String) -> ApiResult { + let request = resolved.page; + let samples = resolved.samples; + let ids: Vec<_> = request.ids.iter().map(|id| STANDARD.encode(id)).collect(); + let end = request.next.is_none(); + let header = make_header( + request.snapshot, + &ids, + &request.fields, + &samples, + request.next, + end, + ); let encoded = serde_json::to_vec(&header).map_err(|_| ApiError::integrity())?; if encoded.len() > 8 * 1024 * 1024 { return Err(ApiError::too_large()); @@ -111,11 +120,11 @@ impl DatasetHttpService { prefix.extend_from_slice(&frame(1, &Bytes::from(encoded))?); let locators = samples .into_iter() - .flat_map(|sample| { + .flat_map(|mut sample| { request .fields .iter() - .filter_map(move |name| sample.fields.get(name).cloned()) + .filter_map(move |name| sample.fields.remove(name)) .filter(|v| !matches!(v, FieldLocator::Tombstone)) }) .collect(); @@ -204,7 +213,7 @@ impl BatchState { if let Some(bytes) = self.pending.take() { return Ok(bytes); } - let Some(locator) = self.locators.get(self.index) else { + let Some(locator) = self.locators.get_mut(self.index) else { self.finished = true; return frame(0, &Bytes::new()); }; @@ -216,7 +225,7 @@ impl BatchState { if value.is_empty() { return self.end_field(); } - let bytes = Bytes::copy_from_slice(value); + let bytes = Bytes::from(std::mem::take(value)); let digest = *md5; self.part = 1; self.data_frame(bytes, digest) diff --git a/app/crowdb-access-server/src/dataset/api/query.rs b/app/crowdb-access-server/src/dataset/api/query.rs index ed59ea4dc..5860d0e20 100644 --- a/app/crowdb-access-server/src/dataset/api/query.rs +++ b/app/crowdb-access-server/src/dataset/api/query.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{batch::BatchRequest, json, read_json, ApiError, ApiResult, DatasetHttpService}; +use super::{json, read_json, ApiError, ApiResult, DatasetHttpService}; use base64::{engine::general_purpose::STANDARD, Engine}; use crowdb_access_dataset::{ DatasetIdentity, DatasetReads, ReadCancellation, ReadPartition, ReadShuffle, ReadSpec, Selection, @@ -129,21 +129,8 @@ impl DatasetHttpService { .clone(); let input: NextRead = read_json(request).await?; let page = DatasetReads::new(self.reads.authority()) - .next(identity, &input.cursor, &ReadCancellation::new()) + .next_with_locators(identity, &input.cursor, &ReadCancellation::new()) .await?; - let end = page.next.is_none(); - self.batch( - identity, - BatchRequest { - snapshot: page.snapshot.to_string(), - sample_ids: page.ids.iter().map(|id| STANDARD.encode(id)).collect(), - fields: page.fields, - sample_id_limit: Some(16 * 1024), - }, - page.next, - end, - request_id, - ) - .await + self.render_batch(page, request_id) } } diff --git a/app/crowdb-access-server/tests/dataset_read_query_contract_test.rs b/app/crowdb-access-server/tests/dataset_read_query_contract_test.rs index ed4ea8eb9..9761732ee 100644 --- a/app/crowdb-access-server/tests/dataset_read_query_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_read_query_contract_test.rs @@ -12,14 +12,18 @@ use std::sync::Arc; #[tokio::test] async fn split_read_continuation_replays_without_remote_progress_or_cleanup() { - let test = TestDatasetHttp::start(Arc::new(TestStore::default())).await; + use std::sync::atomic::Ordering; + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start(store.clone()).await; let client = Client::new(); let (root, snapshot) = common::fixture(&client, &test.origin).await; let revision = published_split(&client, &root, &snapshot).await; let binding = response(client.post(format!("{root}/reads/bind")).json(&plan()), 200).await; assert_eq!(binding["snapshot"], snapshot); assert_eq!(binding["split"]["revision"], revision); + let before = store.admission_reads.load(Ordering::Relaxed); let first = batch(&client, &root, &binding["cursor"]).await; + assert_eq!(store.admission_reads.load(Ordering::Relaxed) - before, 1); assert!(!first["sample_ids"].as_array().unwrap().is_empty()); let independent = Client::new(); assert_eq!(batch(&independent, &root, &binding["cursor"]).await, first); @@ -52,8 +56,8 @@ async fn unsplit_delivery_has_no_outer_duplicate_admission_reads() { let writes = store.mutations.load(Ordering::Relaxed); let page = batch(&client, &root, &binding["cursor"]).await; assert_eq!(page["sample_ids"].as_array().unwrap().len(), 3); - // Selection and delivery still each own one bounded authority check. - assert_eq!(store.admission_reads.load(Ordering::Relaxed) - before, 2); + // Selection and delivery share one request-local authority check. + assert_eq!(store.admission_reads.load(Ordering::Relaxed) - before, 1); assert_eq!(store.mutations.load(Ordering::Relaxed), writes); drop(client); test.stop().await; diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index a26a1ea27..802f9dfe0 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -111,6 +111,14 @@ of binary data; Schema admits up to 256 fields with names up to 1024 bytes. Proj and selection use aggregate byte/deadline/cancellation budgets. Native bounded reads reject oversized external values before materializing their contents. +Continuation delivery resolves selection and field locators within one request. +It reuses that request's Snapshot admission, schema and already scanned membership; +Split visibility checks reuse admission for the same Snapshot. No admission handle +is returned to callers. A separate request checks deletion again, including empty +results. Explicit batch reads remain independent admitted requests. HTTP delivery +moves inline payload owners from resolved locators into output buffers without +cloning the payload. + Old snapshots without projections require explicit resumable inspection preparation. Unsupported or oversized legacy state is reported, never represented as an empty Snapshot. Field preview is explicit and capped at 64 KiB after read-only admission; it diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 853b315fc..66a5eb0b2 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -149,6 +149,33 @@ CAS 不再满足只证明“本次条件不成立”，不能谎称“第一次 新请求如何处理和无 journal 替代方案；有保留中间记录的需求时单独确认。 Files: Iceberg `table/lifecycle/{operation,completion,reservation}.rs`、`namespace/`。 +**rename 的具体持久化例外，待用户 review（2026-10-24）**：不能仅删除 journal 后依次改 +三个 key。设父 namespace P 中已有 A，rename A→B 先预放 B mapping，再 CAS TableHead。 +如果 P 的删除已经标记 Dropping，空目录探测先经过尚未可见的 B，随后 head 切到 B， +探测再经过已失效的 A，就可能误判为空；改名线程此时退出，P 被删，已有表丢失可达名称。 +跨 namespace rename 的目标父目录被删也有相同问题。读取前后各 GET 一次父目录不能弥补 +线程在 head CAS 后退出的窗口；增加等待时间也不能消除它。 + +最小候选方案是在参与 rename 的父 namespace 中保留有界准入占位，内容仅为稳定 TableId、 +受影响的目标名称与预期 head 代际，持有期间删除返回可重试 busy。目标 mapping 仍通过 +CAS 保证名称唯一；TableHead CAS 是名称切换点；mapping 只在与 head 的 ID/name epoch +一致时可见。完成后条件清除占位；进程退出后从 head/mapping 的最终事实完成名称发布， +或先推进 head fence 使迟到 CAS 失效，再取消占位。两边父目录按稳定 ID 顺序占用， +拿不到第二个时释放第一个并返回 busy，不能等待形成环。 +这仍是服务端持久化协调状态，不能把它改名后声称“没有状态”。它只服务 rename 与严格 +非空删除，不保存输入/输出 body、进度页或 HTTP 响应；预计每个参与父目录增加占用/清除 +两次 CAS，替代当前完整 journal/payload 阶段，实际成本与竞争须回归计数。同父目录改名 +存在共享 CAS 竞争，普通读取不登记也不争用该记录。不新增进程锁。 + +替代方案是实现底层跨 key 原子条件提交，或暂不提供 rename。当前 CatalogStore 只暴露 +单 key CAS；Chunk-KV 的 batch_mutate 明确不提供跨 key/partition 原子性，不能冒充事务。 +新建 child 可以在发布后校验父身份并把失败的新资源交给 GC；rename 移动的是既有表， +不能接受线程退出后把原表变成无名称孤儿。准入占位是否允许必须先确认，再改资源格式。 + +- [ ] **R3.1 rename 最小协调合同**：用户 review 上述持久化例外后，实现 namespace 准入 + 占位、head/name 的可见性与取消 fence；删除生命周期输入/输出 journal。验证同父目录 + 扫描跨过旧/新名称、跨 namespace drop、占位后各失败点、反向并发 rename 和迟到 CAS。 + #### R4 — 去掉过程状态后，GC 是否还能发现所有孤儿？这是实现前置检查 **具体流程**：物理文件已写出 → 尚未保存 part/最终引用时进程退出。若 GC 只扫描 upload @@ -372,9 +399,10 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 - [ ] **F1 请求级成本计数**：记录 GET/bounded GET/scan/CAS/PUT/delete、重复 key、冲突与 unknown、真实 routed RPC、串行阶段和复制字节；用测试 wrapper/已有 metrics，不加生产 每 RPC 日志或 profiling 锁。Files: Dataset/S3/Iceberg 测试 store、现有指标、`tools/sdk/`。 -- [ ] **F2 合并一次请求的删除准入**：`DatasetReads::next`、HTTP/native batch 和 inspection - 仍可重复查询删除/提交状态。建立 snapshot-bound 请求内结果，独立入口仍需准入；不跨请求 - 缓存删除许可。Files: Dataset `read_query/`、`inspection/`、Access `dataset/api/batch.rs`、native `dataset/read.rs`。 +- [ ] **F2 复核 legacy/多源读取的重复准入**：现代 continuation 的 selection→HTTP/native + delivery 已共用请求局部的 Snapshot/schema/membership，Split 同 Snapshot 也复用准入。 + 剩余复核 legacy `DatasetReadService`、多源/祖先元数据，独立入口和不同 Snapshot 仍各自准入； + 不跨请求缓存删除许可。Files: Dataset `transport.rs`、`inspection/`、`split/`。 - [ ] **F3 避免无状态读取重复规划**：大 shuffle group + 小 batch 会反复重建同一 group； selection 窗口还重复加载 schema/membership。比较有界客户端 group 缓存、请求内规划复用 与 token 大小；不恢复持久化 session。Files: Dataset `read_query/page.rs`、`inspection/evaluate.rs`、SDK iterable。 @@ -437,6 +465,18 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，F2.2 / F8.2**：增加 `next_with_locators`，HTTP/native continuation 在同一 + 调用内复用 Snapshot 准入、schema 和已扫描成员。私有准备结果绑定 authority/identity/ + Snapshot，不向调用者暴露可复用 admission；普通 next/显式 batch 仍独立检查。 + 81 个候选、shuffle 取 7 个 sample：两次独立操作实测 27 bounded GET + 2 schema scan， + 组合操作为 17 bounded GET + 1 schema scan，member scan 都是五次，零 mutation。 + HTTP 无 Split/有 Split 每 next 均仅一次删除准入；旧 continuation 在后续请求仍被删除拒绝。 + Split 初始 revision 复用已完成的 Snapshot 可见性证明，没有降低 revision 绑定检查。 + HTTP 不再把 ID encode→decode，locator→body 使用移交 Vec，删除两次 inline payload 复制。 + Dataset `--all-targets`、HTTP batch/read-query 四项、native payload/read 七项和三 crate + all-target/all-feature clippy、fmt、diff 检查通过。保留短尾 part、digest、held buffer、 + cancel/drop、空 Split、二进制 ID、未知字段/成员和跨实例重放回归。没有新增持久化/缓存。 + - **2026-10-24，P12.1a / F8.1**：新建索引同时推进最多 16 条 immutable row CAS、 队列最多 2 MiB 编码 key/value，另有一条正在编码的有界 row；没有额外拼接或复制批次。 所有 row 成功后才写 inventory/child/ready。延迟 store 证明存在并发、条数与字节界限 diff --git a/lib/crowdb-access-dataset/src/inspection.rs b/lib/crowdb-access-dataset/src/inspection.rs index 1baf35408..5116b35df 100644 --- a/lib/crowdb-access-dataset/src/inspection.rs +++ b/lib/crowdb-access-dataset/src/inspection.rs @@ -27,6 +27,7 @@ mod exact; mod materialize; mod matrix; mod read; +pub(crate) use read::PreparedInspectionRead; mod uploads; pub use matrix::InspectionSample; diff --git a/lib/crowdb-access-dataset/src/inspection/evaluate.rs b/lib/crowdb-access-dataset/src/inspection/evaluate.rs index 447ec0400..106ec3aef 100644 --- a/lib/crowdb-access-dataset/src/inspection/evaluate.rs +++ b/lib/crowdb-access-dataset/src/inspection/evaluate.rs @@ -34,13 +34,13 @@ impl DatasetAuthority { .await } - pub(crate) async fn prepare_inspection_selection<'a>( + pub(crate) async fn prepare_inspection_selection<'a, 'selection>( &'a self, identity: &'a DatasetIdentity, snapshot: SnapshotId, - selection: &'a Selection, + selection: &'selection Selection, work: &mut InspectionWork<'_>, - ) -> Result, InspectionError> { + ) -> Result, InspectionError> { let summary = self.ready_inspection(identity, snapshot, work).await?; let schema = self .inspection_schema_definitions(identity, &summary, work) @@ -69,6 +69,7 @@ impl DatasetAuthority { compiled, cursor, summaries: BTreeMap::from([(snapshot, summary)]), + schema, }) } @@ -103,15 +104,26 @@ impl DatasetAuthority { } /// Local to one bounded request; its authority and immutable query cannot be replaced. -pub(crate) struct PreparedInspectionSelection<'a> { +pub(crate) struct PreparedInspectionSelection<'a, 'selection> { authority: &'a DatasetAuthority, identity: &'a DatasetIdentity, snapshot: SnapshotId, - compiled: EffectiveSelection<'a>, + compiled: EffectiveSelection<'selection>, cursor: CursorScope, summaries: BTreeMap, + schema: Vec, } -impl PreparedInspectionSelection<'_> { +impl<'a> PreparedInspectionSelection<'a, '_> { + pub(crate) fn into_delivery(self) -> super::read::PreparedInspectionRead<'a> { + super::read::PreparedInspectionRead { + authority: self.authority, + identity: self.identity, + snapshot: self.snapshot, + summaries: self.summaries, + schema: self.schema, + } + } + pub(crate) async fn page( &mut self, after: Option>, diff --git a/lib/crowdb-access-dataset/src/inspection/read.rs b/lib/crowdb-access-dataset/src/inspection/read.rs index 4313c7d09..5317953c5 100644 --- a/lib/crowdb-access-dataset/src/inspection/read.rs +++ b/lib/crowdb-access-dataset/src/inspection/read.rs @@ -45,28 +45,68 @@ impl DatasetAuthority { let schema = self .inspection_schema_definitions(identity, &summary, &mut work) .await?; + PreparedInspectionRead { + authority: self, + identity, + snapshot, + schema, + summaries: BTreeMap::from([(snapshot, summary)]), + } + .resolve(ids, fields, &mut work, false) + .await + } +} + +pub(crate) struct PreparedInspectionRead<'a> { + pub(super) authority: &'a DatasetAuthority, + pub(super) identity: &'a DatasetIdentity, + pub(super) snapshot: SnapshotId, + pub(super) schema: Vec, + pub(super) summaries: BTreeMap, +} + +impl PreparedInspectionRead<'_> { + pub(crate) async fn resolve( + &mut self, + ids: &[Vec], + fields: &[String], + work: &mut InspectionWork<'_>, + known_members: bool, + ) -> Result, InspectionError> { + if ids.is_empty() { + return Ok(Vec::new()); + } + if ids.len() > crate::MAX_BATCH_SIZE + || fields.len() > 256 + || ids.iter().collect::>().len() != ids.len() + || fields.iter().collect::>().len() != fields.len() + { + return Err(AuthorityError::Invalid(DatasetError::InvalidManifest).into()); + } if fields .iter() - .any(|field| !schema.iter().any(|known| known.name == *field)) + .any(|field| !self.schema.iter().any(|known| known.name == *field)) { return Err(AuthorityError::Invalid(DatasetError::InvalidManifest).into()); } - let mut summaries = BTreeMap::from([(snapshot, summary)]); let mut output = Vec::new(); for id in ids { if id.is_empty() || id.len() > 16 * 1024 { return Err(InspectionError::Limit); } - let mut member = key::scope(identity, snapshot, b'M').map_err(AuthorityError::from)?; - key::component(&mut member, id); - let value = work - .get(&member, 128 * 1024) - .await? - .ok_or(AuthorityError::Invalid(DatasetError::SampleNotFound))?; - let stored: Vec = - serde_json::from_slice(&value.bytes).map_err(|_| InspectionError::Corrupt)?; - if stored != *id { - return Err(InspectionError::Corrupt); + if !known_members { + let mut member = + key::scope(self.identity, self.snapshot, b'M').map_err(AuthorityError::from)?; + key::component(&mut member, id); + let value = work + .get(&member, 128 * 1024) + .await? + .ok_or(AuthorityError::Invalid(DatasetError::SampleNotFound))?; + let stored: Vec = + serde_json::from_slice(&value.bytes).map_err(|_| InspectionError::Corrupt)?; + if stored != *id { + return Err(InspectionError::Corrupt); + } } let mut sample = SampleView { sample_id: id.clone(), @@ -75,11 +115,20 @@ impl DatasetAuthority { }; for field in fields { if let Some(metadata) = self - .effective_field_metadata(identity, snapshot, id, field, &mut summaries, &mut work) + .authority + .effective_field_metadata( + self.identity, + self.snapshot, + id, + field, + &mut self.summaries, + work, + ) .await? { let locator = self - .composition_field_locator(identity, id, &metadata, &mut work) + .authority + .composition_field_locator(self.identity, id, &metadata, work) .await?; sample.fields.insert(field.clone(), locator); } diff --git a/lib/crowdb-access-dataset/src/lib.rs b/lib/crowdb-access-dataset/src/lib.rs index 3f7ca1ea7..40c8d5569 100644 --- a/lib/crowdb-access-dataset/src/lib.rs +++ b/lib/crowdb-access-dataset/src/lib.rs @@ -83,4 +83,6 @@ pub use store::{ }; mod read_query; -pub use read_query::{DatasetReads, ReadBinding, ReadPage, ReadPartition, ReadShuffle, ReadSpec, ReadSplit}; +pub use read_query::{ + DatasetReads, ReadBinding, ReadPage, ReadPartition, ReadShuffle, ReadSpec, ReadSplit, ResolvedReadPage, +}; diff --git a/lib/crowdb-access-dataset/src/read_query.rs b/lib/crowdb-access-dataset/src/read_query.rs index 4376ae733..f57938296 100644 --- a/lib/crowdb-access-dataset/src/read_query.rs +++ b/lib/crowdb-access-dataset/src/read_query.rs @@ -53,6 +53,12 @@ pub struct ReadPage { pub fields: Vec, pub next: Option, } + +/// One request's selected IDs and delivery locators, resolved under the same admission. +pub struct ResolvedReadPage { + pub page: ReadPage, + pub samples: Vec, +} #[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct Continuation { diff --git a/lib/crowdb-access-dataset/src/read_query/page.rs b/lib/crowdb-access-dataset/src/read_query/page.rs index ef155b221..593d9bc21 100644 --- a/lib/crowdb-access-dataset/src/read_query/page.rs +++ b/lib/crowdb-access-dataset/src/read_query/page.rs @@ -3,13 +3,22 @@ use super::{ binding::{budget, inspection_error, split_error}, - AuthorityError, Continuation, DatasetError, DatasetIdentity, DatasetReads, ReadPage, + AuthorityError, Continuation, DatasetError, DatasetIdentity, DatasetReads, ReadPage, ResolvedReadPage, +}; +use crate::{ + inspection::{work::InspectionWork, PreparedInspectionRead}, + MetadataScanBudget, ReadCancellation, }; -use crate::{inspection::work::InspectionWork, ReadCancellation}; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; use md5::{Digest, Md5}; use std::time::{Duration, Instant}; +struct SelectedGroup<'a> { + ids: Vec>, + after: Option>, + delivery: PreparedInspectionRead<'a>, +} + impl DatasetReads { /// Replays the caller's position. No read record, heartbeat or acknowledgement is written. /// A partial shuffle group is reconstructed from its immutable start boundary. @@ -21,11 +30,37 @@ impl DatasetReads { token: &str, cancel: &ReadCancellation, ) -> Result { + Ok(self.next_page(identity, token, cancel, false).await?.page) + } + + /// Resolves selection and delivery metadata in the same bounded request. + /// No reusable admission permit or server-side read state escapes the call. + /// # Errors + /// Rejects invalid continuations, deletion, projection, corruption and work limits. + pub async fn next_with_locators( + &self, + identity: &DatasetIdentity, + token: &str, + cancel: &ReadCancellation, + ) -> Result { + self.next_page(identity, token, cancel, true).await + } + + async fn next_page( + &self, + identity: &DatasetIdentity, + token: &str, + cancel: &ReadCancellation, + resolve: bool, + ) -> Result { let mut cursor = Continuation::decode(identity, token)?; - let (mut ids, after) = - tokio::time::timeout(Duration::from_secs(2), self.group(identity, &cursor, cancel)) - .await - .map_err(|_| DatasetError::ReadLimitExceeded)??; + let SelectedGroup { + mut ids, + after, + mut delivery, + } = tokio::time::timeout(Duration::from_secs(2), self.group(identity, &cursor, cancel)) + .await + .map_err(|_| DatasetError::ReadLimitExceeded)??; let count = ids.len(); if let Some(shuffle) = &cursor.spec.shuffle { ids.sort_by_cached_key(|id| { @@ -52,6 +87,24 @@ impl DatasetReads { .saturating_add(cursor.spec.batch_size) .min(ids.len()); let batch = ids[cursor.offset..end].to_vec(); + let samples = if resolve { + let mut work = InspectionWork::new( + self.authority.store.as_ref(), + cancel, + MetadataScanBudget { + items: 100, + bytes: 16 * 1024 * 1024, + timeout: Duration::from_secs(2), + }, + ) + .map_err(inspection_error)?; + delivery + .resolve(&batch, &cursor.spec.fields, &mut work, true) + .await + .map_err(inspection_error)? + } else { + Vec::new() + }; let finished = end == ids.len(); let next = if finished && after.is_none() { None @@ -68,20 +121,23 @@ impl DatasetReads { } Some(cursor.encode()?) }; - Ok(ReadPage { - snapshot: cursor.snapshot, - ids: batch, - fields: cursor.spec.fields, - next, + Ok(ResolvedReadPage { + page: ReadPage { + snapshot: cursor.snapshot, + ids: batch, + fields: cursor.spec.fields, + next, + }, + samples, }) } - async fn group( - &self, - identity: &DatasetIdentity, + async fn group<'a>( + &'a self, + identity: &'a DatasetIdentity, cursor: &Continuation, cancel: &ReadCancellation, - ) -> Result<(Vec>, Option>), AuthorityError> { + ) -> Result, AuthorityError> { let p = &cursor.spec.partition; let size = cursor.spec.shuffle.as_ref().map_or_else( || { @@ -103,16 +159,6 @@ impl DatasetReads { .map_err(|_| DatasetError::CursorMismatch)?; let deadline = Instant::now() + Duration::from_secs(2); let mut bytes = 0usize; - let revision = if let Some(source) = &cursor.split { - Some( - self.authority - .get_split_revision(identity, cursor.snapshot, source.revision, budget(), cancel) - .await - .map_err(split_error)?, - ) - } else { - None - }; let mut selection_work = InspectionWork::new(self.authority.store.as_ref(), cancel, budget()).map_err(inspection_error)?; let mut selection = self @@ -125,6 +171,18 @@ impl DatasetReads { ) .await .map_err(inspection_error)?; + let revision = if let Some(source) = &cursor.split { + let mut split_work = InspectionWork::new(self.authority.store.as_ref(), cancel, budget()) + .map_err(inspection_error)?; + Some( + self.authority + .read_admitted_split_revision(identity, cursor.snapshot, source.revision, &mut split_work) + .await + .map_err(split_error)?, + ) + } else { + None + }; for _ in 0..5000 { if cancel.is_cancelled() { return Err(DatasetError::ReadCancelled.into()); @@ -162,7 +220,11 @@ impl DatasetReads { } after = page.after; if ids.len() == size || after.is_none() { - return Ok((ids, after)); + return Ok(SelectedGroup { + ids, + after, + delivery: selection.into_delivery(), + }); } selection_work = InspectionWork::new(self.authority.store.as_ref(), cancel, limits) .map_err(inspection_error)?; diff --git a/lib/crowdb-access-dataset/src/split/read.rs b/lib/crowdb-access-dataset/src/split/read.rs index 35744ad66..ad340a5a8 100644 --- a/lib/crowdb-access-dataset/src/split/read.rs +++ b/lib/crowdb-access-dataset/src/split/read.rs @@ -88,6 +88,18 @@ impl DatasetAuthority { work: &mut InspectionWork<'_>, ) -> Result { self.require_split_reader(identity, snapshot, work).await?; + self.read_admitted_split_revision(identity, snapshot, revision, work) + .await + } + + /// The same request must already have admitted this exact Snapshot. + pub(crate) async fn read_admitted_split_revision( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + revision: SplitRevisionId, + work: &mut InspectionWork<'_>, + ) -> Result { if !self .split_revision_visible(identity, snapshot, revision, work) .await? @@ -129,10 +141,7 @@ impl DatasetAuthority { work: &mut InspectionWork<'_>, ) -> Result { if self.inspection_initial_split(identity, snapshot, work).await? == Some(revision) { - return self - .inspection_snapshot_committed(identity, snapshot, work) - .await - .map_err(Into::into); + return Ok(true); } let target = CommitTarget::SplitRevision { snapshot, revision }; if let Some(value) = work diff --git a/lib/crowdb-access-dataset/tests/read_query_planning_test.rs b/lib/crowdb-access-dataset/tests/read_query_planning_test.rs index 60e8b8868..a54250c51 100644 --- a/lib/crowdb-access-dataset/tests/read_query_planning_test.rs +++ b/lib/crowdb-access-dataset/tests/read_query_planning_test.rs @@ -67,6 +67,46 @@ fn delta(store: &access_store::TestAccessStore, before: [u64; 5]) -> [u64; 5] { std::array::from_fn(|i| after[i] - before[i]) } +#[tokio::test] +async fn selection_and_delivery_share_one_admission_schema_and_membership_scan() { + let (store, authority, identity) = fixture().await; + let reads = DatasetReads::new(authority.clone()); + let bound = reads + .bind(&identity, spec(Selection::Prefix(b"sample-".to_vec()))) + .await + .unwrap(); + let cancel = ReadCancellation::new(); + let before = costs(&store); + let page = reads.next(&identity, &bound.cursor, &cancel).await.unwrap(); + let reference = authority + .resolve_inspection_batch_locators(&identity, page.snapshot, &page.ids, &page.fields, &cancel) + .await + .unwrap(); + let independent = delta(&store, before); + assert_eq!(independent, [2, 27, 2, 5, 0]); + let before = costs(&store); + let combined = reads + .next_with_locators(&identity, &bound.cursor, &cancel) + .await + .unwrap(); + assert_eq!(combined.page.ids, page.ids); + assert_eq!(combined.page.next, page.next); + assert_eq!(combined.samples, reference); + assert_eq!(delta(&store, before), [1, 17, 1, 5, 0]); + // A separate request must read the deletion mark again. + let before = costs(&store); + DatasetReads::new(authority) + .next_with_locators(&identity, &bound.cursor, &cancel) + .await + .unwrap(); + assert_eq!(delta(&store, before), [1, 17, 1, 5, 0]); + cancel.cancel(); + assert!(reads + .next_with_locators(&identity, &bound.cursor, &cancel) + .await + .is_err()); +} + #[tokio::test] async fn multi_window_shuffle_prepares_once_and_new_requests_recheck() { let (store, authority, identity) = fixture().await; diff --git a/lib/crowdb-sdk-native/src/dataset/read.rs b/lib/crowdb-sdk-native/src/dataset/read.rs index a3d225bcd..0b95312a3 100644 --- a/lib/crowdb-sdk-native/src/dataset/read.rs +++ b/lib/crowdb-sdk-native/src/dataset/read.rs @@ -65,22 +65,22 @@ impl DatasetClient { let batch = request .run(async { Ok(reads - .next(&self.identity, cursor, &ReadCancellation::new()) + .next_with_locators(&self.identity, cursor, &ReadCancellation::new()) .await?) }) .await?; - let end = batch.next.is_none(); - self.batch( + let end = batch.page.next.is_none(); + Ok(self.finish_batch( request, BatchRequest { - snapshot: batch.snapshot, - sample_ids: batch.ids, - fields: batch.fields, + snapshot: batch.page.snapshot, + sample_ids: batch.page.ids, + fields: batch.page.fields, }, - batch.next, + batch.samples, + batch.page.next, end, - ) - .await + )) } async fn batch( @@ -116,6 +116,17 @@ impl DatasetClient { }) .await? }; + Ok(self.finish_batch(request, batch, samples, next, end)) + } + + fn finish_batch( + &self, + request: RequestLease, + batch: BatchRequest, + samples: Vec, + next: Option, + end: bool, + ) -> ReadBatch { let (fields, locators) = descriptors(&batch, samples); let header = Arc::new(BatchHeader { snapshot: batch.snapshot, @@ -131,7 +142,7 @@ impl DatasetClient { Arc::clone(&self.runtime.resources), Arc::clone(&self.uploads), ); - Ok(ReadBatch { header, body }) + ReadBatch { header, body } } } From 68d5ce475b157aef7ce555f0fb5437994877ac31 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 11:52:02 +0800 Subject: [PATCH 103/258] Reuse verified single-part digests for identical field ranges --- .../dataset/design-crowdb-access-dataset.md | 7 + doc/working/plan-access-simplification.md | 13 +- lib/crowdb-access-dataset/src/upload.rs | 1 + lib/crowdb-access-dataset/src/upload/parts.rs | 12 +- .../src/upload/stream.rs | 8 ++ .../src/upload/verification.rs | 33 +++++ .../tests/common/upload.rs | 13 +- .../tests/upload_stream_test.rs | 4 + .../tests/upload_test.rs | 26 ++++ .../src/dataset/read/field.rs | 37 ++++-- .../tests/common/dataset_digest_payload.rs | 51 +++++++ .../tests/dataset_digest_test.rs | 124 ++++++++++++++++++ 12 files changed, 306 insertions(+), 23 deletions(-) create mode 100644 lib/crowdb-access-dataset/src/upload/verification.rs create mode 100644 lib/crowdb-sdk-native/tests/common/dataset_digest_payload.rs create mode 100644 lib/crowdb-sdk-native/tests/dataset_digest_test.rs diff --git a/doc/design/access-server/dataset/design-crowdb-access-dataset.md b/doc/design/access-server/dataset/design-crowdb-access-dataset.md index 1cba2546d..743fa423a 100644 --- a/doc/design/access-server/dataset/design-crowdb-access-dataset.md +++ b/doc/design/access-server/dataset/design-crowdb-access-dataset.md @@ -227,6 +227,9 @@ only after the chunk's acknowledged cursor covers the location and the bytes pass integrity checks. Readable but unconfirmed bytes remain pending. A fenced abandoned allocation is reclaimed before replacing it under the same part ID. Completion verifies every part and the whole declared MD5 with bounded buffers. +For one part whose declared length and MD5 match the whole upload, verification +hashes the stored bytes once and reuses that exact-range result. Multiple part +digests cannot be concatenated into the whole-upload MD5. Expiry scans bounded pages of upload authorities directly. Each caller retains its scan cursor locally; restart repeats the scan and loses no cleanup obligation. @@ -294,6 +297,10 @@ description per request, then fetch each part record once. Part indices, lengths and payload digests remain checked; EOF does not reload metadata. The description is request-owned and registers no remote reader or retention. New requests still perform admission, and reclaimed parts fail instead of producing successful EOF. +Native delivery reuses a single part's successful-EOF length and MD5 proof only +when the part is the complete field range and matches the field's expected digest. +Partial data and failed streams provide no proof. Multiple parts still require +the independent whole-field digest, and inline fields remain checked directly. The internal legacy scan adapter also returns a caller-owned cursor without loading or persisting progress. Requests have no shared inactivity TTL. diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 66a5eb0b2..e8e4ae00f 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -416,8 +416,8 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 - [ ] **F8 减少 payload 复制**：审查 HTTP 收整 part、`BytesMut` 拼接、inline copy、兼容 reader 之外的 S3 coalescing；UploadedChunkReader 已改为最终 Vec 一次拼接。先查活跃调用，再移交 owner 或流式 发送。Files: Dataset payload/upload、Access HTTP batch、S3/native 接收路径。验证部分帧、非整齐 size、held buffer 和实际复制字节。 -- [ ] **F9 分解摘要/编码/分配 CPU**：complete 重读 part 并做 part/whole MD5，native part 与 - field 还各有摘要；MD5 不能直接拼合，只有同范围可信校验证据才可复用。测大 owner/part-index +- [ ] **F9 分解剩余摘要/编码/分配 CPU**：单 part complete/native field 已复用同范围的已验证 + MD5，multipart 仍校验各 part 和整体，不能拼合摘要。继续测大 owner/part-index 编码、JSON/base64、header clone 和 allocations。Files: Dataset upload、native `dataset/read/field.rs`、inspection response。 - [ ] **F10 混合大小准入与调度**：测 count/bytes 限制、慢消费者占用、单线程多个 in-flight、 有界 prefetch、batch 等待、1 ms arena 重试和 10 ms cancellation 轮询。区分 Access、CQ/WR、 @@ -465,6 +465,15 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，F9.1**：单 part 完成仍读取并验证实际存储内容，但其长度/MD5 与整个 upload + 相同时不再重复哈希同一范围。VerifiedUploadPart 仅在成功 EOF 后给出长度＋MD5 证据； + native 单 part 字段消费该证据并核对字段期望值，不再次计算整体 MD5。两条路径的逻辑 + 摘要输入由 2L 变为 L，IO/元数据 RPC 未减少；multipart 与 inline 继续原有整体摘要。 + 失败/部分读取没有证据；同长度替换内容且伪造 part MD5 仍不能冒充原字段的整体证明， + native EOF 报 sticky integrity error，已交付 owner 保持有效。没有放宽完整性合同或新增状态。 + upload stream 2/upload 7、native digest 1/payload 3/read 4 项及两 crate + all-target/all-feature clippy、fmt、diff 检查通过。尚无本项的同源 CPU/TPS 实测。 + - **2026-10-24，F2.2 / F8.2**：增加 `next_with_locators`，HTTP/native continuation 在同一 调用内复用 Snapshot 准入、schema 和已扫描成员。私有准备结果绑定 authority/identity/ Snapshot，不向调用者暴露可复用 admission；普通 next/显式 batch 仍独立检查。 diff --git a/lib/crowdb-access-dataset/src/upload.rs b/lib/crowdb-access-dataset/src/upload.rs index bee70b765..9f7f6df34 100644 --- a/lib/crowdb-access-dataset/src/upload.rs +++ b/lib/crowdb-access-dataset/src/upload.rs @@ -11,6 +11,7 @@ mod read; pub use read::UploadReadDescriptor; mod retirement; mod stream; +mod verification; pub use stream::VerifiedUploadPart; use bytes::Bytes; diff --git a/lib/crowdb-access-dataset/src/upload/parts.rs b/lib/crowdb-access-dataset/src/upload/parts.rs index c7e29b3de..d32c5ac3f 100644 --- a/lib/crowdb-access-dataset/src/upload/parts.rs +++ b/lib/crowdb-access-dataset/src/upload/parts.rs @@ -10,7 +10,6 @@ use async_trait::async_trait; use bytes::Bytes; use crowdb_chunk_client::{FramedWriteBuffer, IoError, SmallWriteIntent}; use crowdb_protocol::chunkdb::rpc::Location; -use md5::{Digest, Md5}; use std::sync::Arc; impl DatasetUploads { @@ -187,16 +186,7 @@ impl DatasetUploads { if parts.len() != part_count(&session) as usize { return Err(UploadError::Conflict); } - let mut hash = Md5::new(); - let mut length = 0; - for part in &parts { - let bytes = self.verified_part(part).await?; - hash.update(&bytes); - length += part.length; - } - if length != session.length || <[u8; 16]>::from(hash.finalize()) != session.md5 { - return Err(UploadError::Integrity); - } + self.verify_completion(&session, &parts).await?; session.state = UploadState::Complete; match self .store diff --git a/lib/crowdb-access-dataset/src/upload/stream.rs b/lib/crowdb-access-dataset/src/upload/stream.rs index 3593c68d0..16afe671e 100644 --- a/lib/crowdb-access-dataset/src/upload/stream.rs +++ b/lib/crowdb-access-dataset/src/upload/stream.rs @@ -48,6 +48,14 @@ impl UploadReadDescriptor { } impl VerifiedUploadPart { + /// Returns the exact verified byte range and digest only after successful EOF. + /// Partial data, cancellation and failed integrity checks do not certify a part. + #[must_use] + pub fn verified_digest(&self) -> Option<(u64, [u8; 16])> { + (!self.failed && self.stream.is_none() && self.delivered == self.expected.length) + .then_some((self.expected.length, self.expected.md5)) + } + /// Failure is terminal; subsequent pulls cannot report successful EOF. /// # Errors /// Returns storage failures, truncated/oversized payloads or digest mismatch. diff --git a/lib/crowdb-access-dataset/src/upload/verification.rs b/lib/crowdb-access-dataset/src/upload/verification.rs new file mode 100644 index 000000000..bf1fe5811 --- /dev/null +++ b/lib/crowdb-access-dataset/src/upload/verification.rs @@ -0,0 +1,33 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use md5::{Digest, Md5}; + +use super::{DatasetUploads, UploadError, UploadPart, UploadRecord}; + +impl DatasetUploads { + pub(super) async fn verify_completion( + &self, + upload: &UploadRecord, + parts: &[UploadPart], + ) -> Result<(), UploadError> { + if let [part] = parts { + if part.length != upload.length || part.md5 != upload.md5 { + return Err(UploadError::Integrity); + } + self.verified_part(part).await?; + return Ok(()); + } + let mut hash = Md5::new(); + let mut length = 0_u64; + for part in parts { + let bytes = self.verified_part(part).await?; + hash.update(&bytes); + length = length.checked_add(part.length).ok_or(UploadError::Integrity)?; + } + if length != upload.length || <[u8; 16]>::from(hash.finalize()) != upload.md5 { + return Err(UploadError::Integrity); + } + Ok(()) + } +} diff --git a/lib/crowdb-access-dataset/tests/common/upload.rs b/lib/crowdb-access-dataset/tests/common/upload.rs index 23b26dabd..d3e724da7 100644 --- a/lib/crowdb-access-dataset/tests/common/upload.rs +++ b/lib/crowdb-access-dataset/tests/common/upload.rs @@ -147,6 +147,7 @@ pub struct TestPayload { pub defer_reclaim: AtomicBool, pub unconfirmed: AtomicBool, pub abandoned: AtomicBool, + pub corrupt_read: AtomicBool, } #[async_trait] impl PayloadStorage for TestPayload { @@ -187,11 +188,19 @@ impl PayloadStorage for TestPayload { .and_then(|l| l.chunk_id) .ok_or(UploadError::Integrity)? .low; - self.data + let bytes = self + .data .load() .get(&index) .cloned() - .ok_or(UploadError::Storage("missing bytes".into())) + .ok_or(UploadError::Storage("missing bytes".into()))?; + if self.corrupt_read.load(Ordering::Relaxed) { + let mut bytes = bytes.to_vec(); + bytes[0] ^= 1; + Ok(Bytes::from(bytes)) + } else { + Ok(bytes) + } } async fn durability( &self, diff --git a/lib/crowdb-access-dataset/tests/upload_stream_test.rs b/lib/crowdb-access-dataset/tests/upload_stream_test.rs index 37a481899..8257a8f0b 100644 --- a/lib/crowdb-access-dataset/tests/upload_stream_test.rs +++ b/lib/crowdb-access-dataset/tests/upload_stream_test.rs @@ -32,9 +32,11 @@ async fn stream_retains_allocation_without_collecting_and_verifies_at_eof() { assert!(uploads.read_part_stream(&location, 0).await.is_err()); uploads.complete(&location, 2).await.unwrap(); let mut stream = uploads.read_part_stream(&location, 0).await.unwrap(); + assert!(stream.verified_digest().is_none()); assert_eq!(payloads.pulls.load(Ordering::Relaxed), 0); let first = stream.next().await.unwrap().unwrap(); let second = stream.next().await.unwrap().unwrap(); + assert!(stream.verified_digest().is_none()); assert!(Arc::ptr_eq( first.allocation().unwrap(), second.allocation().unwrap() @@ -43,6 +45,7 @@ async fn stream_retains_allocation_without_collecting_and_verifies_at_eof() { assert_eq!(first.as_ref(), b"da"); assert_eq!(second.as_ref(), b"ta"); assert!(stream.next().await.unwrap().is_none()); + assert_eq!(stream.verified_digest(), Some((4, Md5::digest(b"data").into()))); assert!(stream.next().await.unwrap().is_none()); drop(stream); drop(uploads); @@ -65,6 +68,7 @@ async fn corruption_truncation_overrun_and_partial_transport_failure_stay_termin } let pulls = payloads.pulls.load(Ordering::Relaxed); assert!(stream.next().await.is_err()); + assert!(stream.verified_digest().is_none()); assert_eq!(payloads.pulls.load(Ordering::Relaxed), pulls); } } diff --git a/lib/crowdb-access-dataset/tests/upload_test.rs b/lib/crowdb-access-dataset/tests/upload_test.rs index 230b6bc89..a136f0d5a 100644 --- a/lib/crowdb-access-dataset/tests/upload_test.rs +++ b/lib/crowdb-access-dataset/tests/upload_test.rs @@ -18,6 +18,32 @@ fn location(name: &str) -> Vec { ) .unwrap() } + +#[tokio::test] +async fn single_part_completion_still_verifies_actual_stored_bytes() { + let payloads = Arc::new(TestPayload::default()); + let uploads = DatasetUploads::new(Arc::new(TestStore::default()), payloads.clone()); + let key = location("single-corruption"); + uploads + .create(&key, 4, 4, Md5::digest(b"data").into(), 100) + .await + .unwrap(); + uploads + .put_part(&key, 0, Bytes::from_static(b"data"), 1) + .await + .unwrap(); + payloads.corrupt_read.store(true, Ordering::Relaxed); + assert!(matches!( + uploads.complete(&key, 2).await, + Err(UploadError::Integrity) + )); + assert_eq!(uploads.metadata(&key).await.unwrap().state, UploadState::Open); + payloads.corrupt_read.store(false, Ordering::Relaxed); + assert_eq!( + uploads.complete(&key, 3).await.unwrap().state, + UploadState::Complete + ); +} #[tokio::test] async fn lost_storage_reply_recovers_exact_intent_without_another_allocation() { let store = Arc::new(TestStore::default()); diff --git a/lib/crowdb-sdk-native/src/dataset/read/field.rs b/lib/crowdb-sdk-native/src/dataset/read/field.rs index 05df5b937..4f936af80 100644 --- a/lib/crowdb-sdk-native/src/dataset/read/field.rs +++ b/lib/crowdb-sdk-native/src/dataset/read/field.rs @@ -22,7 +22,12 @@ pub(super) struct FieldReader { source: Source, length: u64, expected: [u8; 16], - digest: Md5, + digest: FieldDigest, +} + +enum FieldDigest { + Incremental(Md5), + SinglePart { verified: bool }, } impl FieldReader { @@ -58,7 +63,7 @@ impl FieldReader { source, length, expected, - digest: Md5::new(), + digest: FieldDigest::Incremental(Md5::new()), }) } @@ -82,6 +87,9 @@ impl FieldReader { if record.length() != self.length || record.md5() != self.expected { return Err(Error::Integrity); } + if record.part_count() == 1 { + self.digest = FieldDigest::SinglePart { verified: false }; + } *metadata = Some(record); } let record = metadata.as_ref().ok_or(Error::Integrity)?; @@ -100,16 +108,27 @@ impl FieldReader { if let Some(payload) = request.run(async { Ok(reader.next().await?) }).await? { break Some(payload); } + if record.part_count() == 1 { + if reader.verified_digest() != Some((self.length, self.expected)) { + return Err(Error::Integrity); + } + self.digest = FieldDigest::SinglePart { verified: true }; + } *stream = None; *part += 1; } } }; - if payload.is_none() - && (self.offset != self.length - || <[u8; 16]>::from(self.digest.clone().finalize()) != self.expected) - { - return Err(Error::Integrity); + if payload.is_none() { + let verified = match &self.digest { + FieldDigest::Incremental(digest) => { + <[u8; 16]>::from(digest.clone().finalize()) == self.expected + } + FieldDigest::SinglePart { verified } => *verified, + }; + if self.offset != self.length || !verified { + return Err(Error::Integrity); + } } Ok(payload) } @@ -122,7 +141,9 @@ impl FieldReader { if self.offset > self.length { return Err(Error::Integrity); } - self.digest.update(bytes); + if let FieldDigest::Incremental(digest) = &mut self.digest { + digest.update(bytes); + } Ok(()) } } diff --git a/lib/crowdb-sdk-native/tests/common/dataset_digest_payload.rs b/lib/crowdb-sdk-native/tests/common/dataset_digest_payload.rs new file mode 100644 index 000000000..5bf139f2d --- /dev/null +++ b/lib/crowdb-sdk-native/tests/common/dataset_digest_payload.rs @@ -0,0 +1,51 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use arc_swap::ArcSwap; +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_access_dataset::{PayloadDurability, PayloadStorage, UploadError}; +use crowdb_chunk_client::SmallWriteIntent; +use crowdb_protocol::{chunkdb::rpc::Location, common::ChunkId}; +use std::sync::Arc; + +#[derive(Default)] +pub struct TestDigestPayload { + pub bytes: ArcSwap>, +} + +#[async_trait] +impl PayloadStorage for TestDigestPayload { + async fn write( + &self, + _: &[u8], + data: Bytes, + intent: Arc, + ) -> Result, UploadError> { + let location = Location { + chunk_id: Some(ChunkId { high: 1, low: 1 }), + offset: 0, + length: data.len() as u64, + logical_offset: 0, + logical_length: data.len() as u64, + }; + intent + .before_write(&location) + .await + .map_err(|error| UploadError::Storage(error.to_string()))?; + self.bytes.store(Arc::new(data.to_vec())); + Ok(vec![location]) + } + + async fn read(&self, _: &[Location]) -> Result { + Ok(Bytes::copy_from_slice(self.bytes.load().as_slice())) + } + + async fn durability(&self, _: &[Location]) -> Result { + Ok(PayloadDurability::Durable) + } + + async fn discard(&self, _: &[Location]) -> Result { + Ok(true) + } +} diff --git a/lib/crowdb-sdk-native/tests/dataset_digest_test.rs b/lib/crowdb-sdk-native/tests/dataset_digest_test.rs new file mode 100644 index 000000000..f1bdda2e6 --- /dev/null +++ b/lib/crowdb-sdk-native/tests/dataset_digest_test.rs @@ -0,0 +1,124 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/metadata.rs"] +mod metadata; +#[path = "common/dataset_digest_payload.rs"] +mod payload; +#[path = "common/storage.rs"] +mod storage; + +use crowdb_access_dataset::{ + ChunkKvDatasetStore, DatasetAuthority, DatasetIdentity, DatasetStore, DatasetUploads, FieldDefinition, + FieldLocator, FieldRecord, ManifestRecord, NamespacePath, OperationId, SampleRecord, SchemaRecord, + SnapshotId, +}; +use crowdb_sdk_native::{ + dataset::{BatchRequest, DatasetClient}, + Error, OperationContext, +}; +use md5::{Digest, Md5}; +use std::{ + num::NonZeroU64, + sync::Arc, + time::{Duration, Instant}, +}; + +fn context(id: u64) -> OperationContext { + OperationContext::new( + NonZeroU64::new(id).unwrap(), + Instant::now() + Duration::from_secs(5), + ) +} + +#[tokio::test] +async fn completed_part_digest_only_certifies_the_identical_field_range() { + for replace_part in [false, true] { + let metadata = metadata::client().await; + let runtime = storage::runtime_with_metadata(metadata.clone()); + let store = Arc::new(ChunkKvDatasetStore::new(metadata)); + let authority = Arc::new(DatasetAuthority::new(store.clone())); + let payloads = Arc::new(payload::TestDigestPayload::default()); + let uploads = Arc::new(DatasetUploads::new(store.clone(), payloads.clone())); + let identity = DatasetIdentity::new(NamespacePath::root(), "digest").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + let location = DatasetUploads::location(&identity, OperationId::random()).unwrap(); + let expected: [u8; 16] = Md5::digest(b"original").into(); + uploads.create(&location, 8, 8, expected, u64::MAX).await.unwrap(); + uploads + .put_part(&location, 0, bytes::Bytes::from_static(b"original"), 1) + .await + .unwrap(); + uploads.complete(&location, 2).await.unwrap(); + let snapshot = publish(&authority, &identity, location.clone(), expected).await; + if replace_part { + payloads.bytes.store(Arc::new(b"replaced".to_vec())); + let mut key = location; + key.extend_from_slice(b"\0P\0\0\0\0"); + let old = store.get(&key).await.unwrap().unwrap(); + let mut value: serde_json::Value = serde_json::from_slice(&old.bytes).unwrap(); + value["part"]["md5"] = serde_json::json!(<[u8; 16]>::from(Md5::digest(b"replaced"))); + store + .compare_exchange(&key, Some(&old.bytes), &serde_json::to_vec(&value).unwrap()) + .await + .unwrap(); + } + let client = DatasetClient::from_services_for_tests(runtime.clone(), identity, authority, uploads); + let mut batch = client + .read_batch( + context(1), + BatchRequest { + snapshot, + sample_ids: vec![b"one".to_vec()], + fields: vec!["value".into()], + }, + ) + .await + .unwrap(); + let held = batch.body.next().await.unwrap().unwrap().buffer; + if replace_part { + assert_eq!(held.as_slice(), b"replaced"); + assert!(matches!(batch.body.next().await, Err(Error::Integrity))); + assert!(matches!(batch.body.next().await, Err(Error::Integrity))); + } else { + assert_eq!(held.as_slice(), b"original"); + assert!(batch.body.next().await.unwrap().is_none()); + } + runtime.shutdown().await.unwrap(); + assert_eq!(held.as_slice().len(), 8); + } +} + +async fn publish( + authority: &DatasetAuthority, + identity: &DatasetIdentity, + location: Vec, + md5: [u8; 16], +) -> SnapshotId { + let manifest = ManifestRecord { + version: 1, + snapshot: SnapshotId::random(), + schema: SchemaRecord { + version: 1, + fields: vec![FieldDefinition { + name: "value".into(), + required: true, + }], + }, + samples: vec![SampleRecord { + sample_id: b"one".to_vec(), + fields: vec![FieldRecord { + name: "value".into(), + value: FieldLocator::Chunk { + location, + length: 8, + md5, + }, + }], + }], + }; + authority + .publish_snapshot(identity, None, bincode::serialize(&manifest).unwrap()) + .await + .unwrap() +} From 825a420e24037baff0af2aa45b7cd3341b3145a8 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 12:12:34 +0800 Subject: [PATCH 104/258] Replace rename request journals with bounded namespace occupancy --- .../src/iceberg/table_write.rs | 7 +- .../src/iceberg/table_write/lifecycle.rs | 5 +- .../tests/iceberg_namespace_http_test.rs | 1 + .../tests/iceberg_namespace_sdk_test.rs | 1 + .../tests/iceberg_table_lifecycle_test.rs | 2 +- .../iceberge/design-crowdb-iceberg.md | 69 ++-- doc/working/plan-access-simplification.md | 26 +- lib/crowdb-access-iceberg/src/namespace.rs | 2 + .../src/namespace/admission.rs | 3 + .../src/namespace/authority.rs | 12 + .../src/namespace/create.rs | 1 + .../src/namespace/drop_fence.rs | 9 + .../src/namespace/rename.rs | 59 ++++ .../src/namespace/repair.rs | 8 + .../src/record/namespace.rs | 46 ++- lib/crowdb-access-iceberg/src/table.rs | 6 +- .../src/table/lifecycle.rs | 46 ++- .../src/table/lifecycle/admission.rs | 3 + .../src/table/lifecycle/preparation.rs | 18 +- .../src/table/lifecycle/reservation.rs | 9 +- lib/crowdb-access-iceberg/src/table/rename.rs | 162 +++++++++ .../src/table/rename/admission.rs | 125 +++++++ .../src/table/rename/recovery.rs | 88 +++++ .../src/table/rename/reservation.rs | 122 +++++++ .../tests/common/namespace.rs | 1 + .../tests/common/rename_conflict.rs | 102 ++++++ .../tests/common/store.rs | 7 +- .../tests/common/table_rename.rs | 102 ++++++ .../tests/namespace_drop_test.rs | 5 +- .../tests/namespace_journal_test.rs | 1 + .../tests/namespace_record_test.rs | 1 + .../tests/table_commit_publication_test.rs | 4 +- .../tests/table_lifecycle_race_test.rs | 312 ++++++++---------- .../tests/table_lifecycle_test.rs | 131 +------- .../tests/table_rename_legacy_test.rs | 95 ++++++ .../tests/table_rename_test.rs | 255 ++++++++++++++ lib/crowdb-protocol/src/fbs/iceberg.fbs | 10 + 37 files changed, 1495 insertions(+), 361 deletions(-) create mode 100644 lib/crowdb-access-iceberg/src/namespace/rename.rs create mode 100644 lib/crowdb-access-iceberg/src/table/rename.rs create mode 100644 lib/crowdb-access-iceberg/src/table/rename/admission.rs create mode 100644 lib/crowdb-access-iceberg/src/table/rename/recovery.rs create mode 100644 lib/crowdb-access-iceberg/src/table/rename/reservation.rs create mode 100644 lib/crowdb-access-iceberg/tests/common/rename_conflict.rs create mode 100644 lib/crowdb-access-iceberg/tests/common/table_rename.rs create mode 100644 lib/crowdb-access-iceberg/tests/table_rename_legacy_test.rs create mode 100644 lib/crowdb-access-iceberg/tests/table_rename_test.rs diff --git a/app/crowdb-access-server/src/iceberg/table_write.rs b/app/crowdb-access-server/src/iceberg/table_write.rs index 050156825..35bc9a4f6 100644 --- a/app/crowdb-access-server/src/iceberg/table_write.rs +++ b/app/crowdb-access-server/src/iceberg/table_write.rs @@ -143,11 +143,10 @@ impl TableWrites { } } if method == Method::DELETE || uri.path() == "/v1/tables/rename" { - let result = self + let outcome = self .mutate_lifecycle(&record, capabilities, &method, &uri, &bytes) - .await; - let (status, body) = self.outcome_response(result).await?; - return Ok(response(status, body)); + .await?; + return Ok(response(outcome.status, outcome.body)); } let target = request::parse(&uri); let result = match target { diff --git a/app/crowdb-access-server/src/iceberg/table_write/lifecycle.rs b/app/crowdb-access-server/src/iceberg/table_write/lifecycle.rs index 6c803e388..a349151f0 100644 --- a/app/crowdb-access-server/src/iceberg/table_write/lifecycle.rs +++ b/app/crowdb-access-server/src/iceberg/table_write/lifecycle.rs @@ -4,10 +4,9 @@ use super::{ }; use crowdb_access_iceberg::{ catalog::{Capabilities, FormatAction}, - commit::TableCommitOutcome, key::NameSuffix, namespace::NamespaceIdentifier, - table::{TableLifecycleAction, TableLifecycleRequest}, + table::{TableLifecycleAction, TableLifecycleOutcome, TableLifecycleRequest}, wire::IcebergErrorResponse, }; use hyper::{Method, Uri}; @@ -47,7 +46,7 @@ impl TableWrites { method: &Method, uri: &Uri, bytes: &[u8], - ) -> Result { + ) -> Result { let (namespace, name, action) = if uri.path() == "/v1/tables/rename" { if *method != Method::POST { return Err(super::super::table_read::unsupported()); diff --git a/app/crowdb-access-server/tests/iceberg_namespace_http_test.rs b/app/crowdb-access-server/tests/iceberg_namespace_http_test.rs index 43eef58fa..01584c7a7 100644 --- a/app/crowdb-access-server/tests/iceberg_namespace_http_test.rs +++ b/app/crowdb-access-server/tests/iceberg_namespace_http_test.rs @@ -242,6 +242,7 @@ async fn complete_list_exhaustion_never_returns_a_truncated_success() { mutation_revision: 1, lifecycle: NamespaceLifecycle::Ready, pending_operation: None, + rename: None, properties: NamespaceProperties::default(), }; values.insert( diff --git a/app/crowdb-access-server/tests/iceberg_namespace_sdk_test.rs b/app/crowdb-access-server/tests/iceberg_namespace_sdk_test.rs index be4c017a4..4e84d92a8 100644 --- a/app/crowdb-access-server/tests/iceberg_namespace_sdk_test.rs +++ b/app/crowdb-access-server/tests/iceberg_namespace_sdk_test.rs @@ -58,6 +58,7 @@ fn install(test: &TestTableHttp, count: usize, padding: usize, stale: bool) { mutation_revision: 1, lifecycle: NamespaceLifecycle::Ready, pending_operation: None, + rename: None, properties: NamespaceProperties::default(), }; values.insert( diff --git a/app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs b/app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs index 1fd30c9d6..97c75c7d1 100644 --- a/app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs +++ b/app/crowdb-access-server/tests/iceberg_table_lifecycle_test.rs @@ -117,7 +117,7 @@ async fn rename_preserves_metadata_supports_cross_namespace_and_never_aliases_ol ) .await; value(test.post(TABLES, "w", None, &create()).await, 200).await; - empty(test.post(RENAME, "w", Some(&identity), &rename).await).await; + value(test.post(RENAME, "w", Some(&identity), &rename).await, 409).await; value( test.post("/v1/namespaces", "w", None, &json!({"namespace":["destination"]})) .await, diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg.md b/doc/design/access-server/iceberge/design-crowdb-iceberg.md index ccfb1d39c..eca8e4dfd 100644 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg.md +++ b/doc/design/access-server/iceberge/design-crowdb-iceberg.md @@ -104,9 +104,10 @@ native catalog. They are not file lookup indexes and cannot produce an object listing. The service must discover abandoned work before a table head exists and after a table has disappeared, without a table-ID reverse index: -- Create, commit and lifecycle journals and payload pages are keyed by operation - identity; creation can precede name/head publication and rename spans two live - namespaces. Their selected heads carry the frozen storage identity. +- Create, legacy commit and drop journals and payload pages are keyed by operation + identity; creation can precede name/head publication. Their selected heads carry + the frozen storage identity. Name moves use bounded namespace occupancy instead + of request journals; existing rename journals remain recoverable. - Multipart sessions and parts are keyed by upload identity/part number. Catalog admission records apply only to legacy credited sessions; new streamed sessions use independent per-session bounds. @@ -282,12 +283,14 @@ the proof. A live child restores Ready without changing the name epoch or proper revision. Only completion of both ranges permits the fenced tombstone CAS. Terminal replay and conditional cleanup cannot delete a recreated NamespaceId. Table-child probes resolve published mappings against the selected table head. -Unpublished table reservations are helped through their creation or lifecycle -journal. Ordinary table creation does not modify the namespace authority. A +Unpublished table reservations are helped through their creation journal, +namespace name-move occupancy or a legacy lifecycle journal. Ordinary table creation does not modify the namespace authority. A concurrent drop scan may miss a creator; later mapping reconciliation resolves the leftover state. Stable namespace identity prevents that state from becoming -visible under a recreated namespace. Rename retains its lifecycle admission -protocol. Corrupt table authority blocks the emptiness proof. +visible under a recreated namespace. Name moves occupy both participating parent +authorities before reserving the destination; deletion helps settle or fence the +move and returns retryable busy before taking a fresh emptiness proof. Corrupt +table authority blocks the emptiness proof. Each listener runs a namespace-journal sweep with bounded pages, per-operation phase budgets and a wall-clock deadline. The sweep resumes abandoned operations and their conditional mapping cleanup without requiring a client retry. Catalog @@ -794,18 +797,41 @@ markers and retains uncertain storage errors. Known semantic validation failures become durable client outcomes before any candidate is published. Recovery deadlines preserve journal evidence rather than canceling the logical operation. -Logical drop and same/cross-namespace rename use a bounded `TableLifecycleOperation` -journal. It fixes the original head and exact source mapping, request identity, -principal, input and candidate before publication. A single head CAS arbitrates -against metadata commits and other lifecycle operations. A losing operation keeps -its terminal conflict instead of rebasing onto a new generation or recreated name. -Rename changes the canonical identifier and name epoch, not table identity, UUID, -metadata generation, digest or file location. Destination reservation precedes a -namespace admission CAS. The admission marker remains until the head outcome and -destination mapping are durable; namespace-drop helpers finish or abort that exact -operation with a shared bounded work budget. The source stays head-qualified until -the move publishes, and the old name never becomes an alias. Cleanup conditionally -removes only the captured mapping, preserving names recreated with another identity. +Logical drop uses a bounded `TableLifecycleOperation` journal. It fixes the original +head, exact source mapping, request identity, principal, input and candidate before +publication. A losing operation keeps its terminal conflict instead of rebasing +onto a new generation or recreated name. + +Same/cross-namespace rename stores no new input, phase, response or lifecycle +journal. A single bounded `NamespaceRename` occupancy in each affected parent +contains the source mapping, target name, expected head generation/fence and a +fresh per-attempt reservation identity. Parents are occupied in stable ID order; +failure to acquire the second cancels and releases the first, without waiting on +another rename. Definite parent CAS conflicts use the observed authority directly, +preserving concurrent property changes. Ordinary readers do not acquire occupancy. + +The target name is reserved before one exact-head CAS switches the canonical +identifier and name epoch. Table identity, UUID, metadata generation, digest and +file location stay fixed. The destination is published before the captured source +mapping and parent occupancies are conditionally cleared. A namespace cannot enter +Dropping while occupied, including a same-parent move where a scan could otherwise +miss both names across the switch. Mapping visibility remains qualified by the +head, and the old name never becomes an alias. + +Recovery reads the head and name facts: a switched head completes the destination; +an unchanged head is first conditionally fenced to invalidate delayed publication, +then the reservation and occupancy can be removed. Conflicts use their returned +head. Namespace mapping sweeps and name-reservation/deletion helpers recover +abandoned occupancy. Every new attempt has a distinct reservation identity, even +when an HTTP request key is reused, preventing delayed cleanup of a canceled +attempt from deleting a later reservation. Cleanup never removes a recreated name +with a different identity. Existing lifecycle journals still finish through their +legacy recovery driver. + +Rename replies are generated directly from current resource state. After a +completed move the source is missing; repeating that request can return 404. +A recreated source with an occupied target conflicts. There is no historical +response replay or implicit rebasing of an in-flight head CAS. Drop tombstones the selected head without traversing snapshots or deleting files. A purge request persists a `TablePurgeTask` containing the tombstoned head, @@ -814,9 +840,8 @@ The purge task becomes eligible 20 minutes after that marker time; a delayed scheduler scan or restart does not restart the delay. A drop without an explicit purge request leaves the underlying files in place. Purge reclaims table-scoped files after the delay and owner checks, without snapshot reachability scans -inside the dropped table. Success is -retained before releasing rename head/namespace markers. Retrying after response -loss returns the original result without mutating a replacement table. The REST +inside the dropped table. Drop success is retained before releasing its markers. +Drop retries return the retained result without mutating a replacement table. The REST drop/rename routes require independent writer credentials and return empty success responses; stale table names fail normal load, exists, commit and credential refresh. diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index e8e4ae00f..70de57807 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -29,6 +29,8 @@ Upstream: [Dataset SDK](../backlog/R220-dataset-fat-client-plan.md)、 重试/重启不重置期限；已准入流可继续，过久的流允许失败，不能截断后报成功。 Access Server 不登记读者、不互相确认删除、不异步写释放记录。 - **非空 Iceberg namespace 删除必须失败**，已由用户决定，不再提出放宽选项。 +- **用户决定（2026-10-24）**：允许 rename 在参与父目录中保存最小有界占位，用于严格非空 + 删除和迟到 CAS 隔离；删除新 rename 的完整 journal、输入和响应。普通读取不登记。 - definite CAS conflict 立即使用返回的 observed value 计算合法重试，无“三次后才更新”。 返回值可能随后又变旧，因此仍需期限/重试上限；transport timeout 不等于 CAS 失败。 明确 expected revision 的业务操作不能擅自改 base 重放。 @@ -149,7 +151,7 @@ CAS 不再满足只证明“本次条件不成立”，不能谎称“第一次 新请求如何处理和无 journal 替代方案；有保留中间记录的需求时单独确认。 Files: Iceberg `table/lifecycle/{operation,completion,reservation}.rs`、`namespace/`。 -**rename 的具体持久化例外，待用户 review（2026-10-24）**：不能仅删除 journal 后依次改 +**rename 的具体持久化例外，用户已允许（2026-10-24）**：不能仅删除 journal 后依次改 三个 key。设父 namespace P 中已有 A，rename A→B 先预放 B mapping，再 CAS TableHead。 如果 P 的删除已经标记 Dropping，空目录探测先经过尚未可见的 B，随后 head 切到 B， 探测再经过已失效的 A，就可能误判为空；改名线程此时退出，P 被删，已有表丢失可达名称。 @@ -170,11 +172,7 @@ CAS 保证名称唯一；TableHead CAS 是名称切换点；mapping 只在与 he 替代方案是实现底层跨 key 原子条件提交，或暂不提供 rename。当前 CatalogStore 只暴露 单 key CAS；Chunk-KV 的 batch_mutate 明确不提供跨 key/partition 原子性，不能冒充事务。 新建 child 可以在发布后校验父身份并把失败的新资源交给 GC；rename 移动的是既有表， -不能接受线程退出后把原表变成无名称孤儿。准入占位是否允许必须先确认，再改资源格式。 - -- [ ] **R3.1 rename 最小协调合同**：用户 review 上述持久化例外后，实现 namespace 准入 - 占位、head/name 的可见性与取消 fence；删除生命周期输入/输出 journal。验证同父目录 - 扫描跨过旧/新名称、跨 namespace drop、占位后各失败点、反向并发 rename 和迟到 CAS。 +不能接受线程退出后把原表变成无名称孤儿。用户已确认允许最小占位，删除完整 journal。 #### R4 — 去掉过程状态后，GC 是否还能发现所有孤儿？这是实现前置检查 @@ -465,6 +463,22 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，R3.1 / P9.2（rename 部分）**：新 rename 只在参与父 authority 中保存一个 + `NamespaceRename`：原精确 mapping、目标 namespace/name、head generation/fence 和本次 + reservation ID。父目录按稳定 ID 占用，不等待第二个；冲突直接使用 observed，四次真实 + 冲突中禁止额外 GET 的回归通过。名字 reservation 后一次原 head CAS 切换，发布目标名， + 条件删除原 mapping，清父占位。同父目录为 6 次写入，跨目录 8 次（含 conditional delete）； + 成功后 key 总数不变，只有一个新目标名 key，没有新 journal/input/output/response key。 + 未切换 head 的恢复先 CAS 推进 fence，再撤占位；已切换则补目标名。普通读不登记。 + 同请求重试使用新的 reservation ID，阻止旧取消请求迟到清理新 reservation 的 ABA。 + namespace drop 遇占位先恢复再返回 Busy；下一次独立探测保持严格非空失败。反向跨目录 + 并发通过 barrier 同时占首父目录，不形成等待环。每个成功 mutation 的丢回复/进程重建、 + head CAS 前后暂停、并发 commit/drop、重建源名及旧 journal 恢复均通过。 + rename 不保留历史响应：源已移动可返回 404，重建源与已占目标冲突；进行中的旧 CAS + 失败不隐式换 base，新独立请求可选择当前资源。drop/create 的旧 driver 仍在 P9.2。 + Iceberg `--all-targets`、HTTP lifecycle/namespace 9 项、两 crate all-target/all-feature clippy、 + workspace fmt 和 diff 检查通过。没有 TPS 提升实测；ChunkDB 物理回收仍独立。 + - **2026-10-24，F9.1**：单 part 完成仍读取并验证实际存储内容，但其长度/MD5 与整个 upload 相同时不再重复哈希同一范围。VerifiedUploadPart 仅在成功 EOF 后给出长度＋MD5 证据； native 单 part 字段消费该证据并核对字段期望值，不再次计算整体 MD5。两条路径的逻辑 diff --git a/lib/crowdb-access-iceberg/src/namespace.rs b/lib/crowdb-access-iceberg/src/namespace.rs index 54c10d6b1..e4425a910 100644 --- a/lib/crowdb-access-iceberg/src/namespace.rs +++ b/lib/crowdb-access-iceberg/src/namespace.rs @@ -18,6 +18,7 @@ mod properties; mod publication; mod recovery; mod recovery_scan; +mod rename; mod repair; mod repository; mod reservation; @@ -38,6 +39,7 @@ pub use operation::{ pub use properties::{NamespaceProperties, PropertyChanges, PropertyUpdate, MAX_PROPERTIES}; pub use recovery::{NamespaceRecovery, NamespaceRecoveryPage}; pub use recovery_scan::{NamespaceRecoveryScan, NamespaceRecoveryStore}; +pub use rename::NamespaceRename; pub use repository::NamespaceRepository; pub use storage::{ChildScan, NamespaceStore}; pub use update::NamespacePropertyRequest; diff --git a/lib/crowdb-access-iceberg/src/namespace/admission.rs b/lib/crowdb-access-iceberg/src/namespace/admission.rs index 369ec483b..ac1229490 100644 --- a/lib/crowdb-access-iceberg/src/namespace/admission.rs +++ b/lib/crowdb-access-iceberg/src/namespace/admission.rs @@ -28,6 +28,9 @@ impl NamespaceCreator { }; let after = match StorageRecord::decode(&key, &value.bytes)? { StorageRecord::NamespaceAuthority(mut parent) => { + if parent.rename.is_some() { + return Err(CatalogError::Busy); + } if parent.lifecycle != NamespaceLifecycle::Ready || Some(parent.identifier.clone()) != operation.identifier.parent() { diff --git a/lib/crowdb-access-iceberg/src/namespace/authority.rs b/lib/crowdb-access-iceberg/src/namespace/authority.rs index 2a14ac9fb..fbbf173ee 100644 --- a/lib/crowdb-access-iceberg/src/namespace/authority.rs +++ b/lib/crowdb-access-iceberg/src/namespace/authority.rs @@ -22,6 +22,7 @@ pub struct NamespaceAuthority { pub mutation_revision: u64, pub lifecycle: NamespaceLifecycle, pub pending_operation: Option, + pub rename: Option>, pub properties: NamespaceProperties, } @@ -39,6 +40,17 @@ impl NamespaceAuthority { { return Err(ValidationError::Record); } + if let Some(rename) = &self.rename { + rename.validate()?; + if self.lifecycle != NamespaceLifecycle::Ready + || self.pending_operation.is_some() + || rename.source.catalog != self.catalog + || (rename.source.namespace != self.namespace + && rename.destination_namespace != self.namespace) + { + return Err(ValidationError::IdentityMismatch); + } + } Ok(()) } } diff --git a/lib/crowdb-access-iceberg/src/namespace/create.rs b/lib/crowdb-access-iceberg/src/namespace/create.rs index 6d1af80e2..f65d0fbd5 100644 --- a/lib/crowdb-access-iceberg/src/namespace/create.rs +++ b/lib/crowdb-access-iceberg/src/namespace/create.rs @@ -140,6 +140,7 @@ impl NamespaceCreator { mutation_revision: 1, lifecycle: NamespaceLifecycle::Ready, pending_operation: Some(operation.identity.operation), + rename: None, properties, })) } diff --git a/lib/crowdb-access-iceberg/src/namespace/drop_fence.rs b/lib/crowdb-access-iceberg/src/namespace/drop_fence.rs index b70eed35c..e28006754 100644 --- a/lib/crowdb-access-iceberg/src/namespace/drop_fence.rs +++ b/lib/crowdb-access-iceberg/src/namespace/drop_fence.rs @@ -26,6 +26,15 @@ impl NamespaceDropper { if authority.namespace != operation.namespace || authority.parent != operation.parent { return self.finish_outcome(operation, 404).await; } + if let Some(rename) = &authority.rename { + crate::table::TableRenames::from_parts( + self.creator.repository.store.clone(), + self.creator.names.clone(), + ) + .recover(operation.context, rename) + .await?; + return Err(CatalogError::Busy); + } if let Some(pending) = authority.pending_operation { let owner = NamespaceJournal::new(self.creator.repository.store.clone()) .load(operation.context, pending) diff --git a/lib/crowdb-access-iceberg/src/namespace/rename.rs b/lib/crowdb-access-iceberg/src/namespace/rename.rs new file mode 100644 index 000000000..b6f62e87a --- /dev/null +++ b/lib/crowdb-access-iceberg/src/namespace/rename.rs @@ -0,0 +1,59 @@ +use crate::{ + error::ValidationError, + key::{NamespaceId, OperationId}, + table::{name_key, TableMapping, TableMappingState}, +}; + +/// Bounded name-move occupancy shared by the affected namespace authorities. +/// Contains no request payload, response or execution phase. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NamespaceRename { + pub source: TableMapping, + pub destination_namespace: NamespaceId, + pub destination_name: String, + pub operation: OperationId, + pub head_generation: u64, + pub head_fence: u64, +} + +impl NamespaceRename { + /// # Errors + /// Rejects malformed names, unchanged destinations and exhausted head fences. + pub fn validate(&self) -> Result<(), ValidationError> { + self.source.validate()?; + name_key( + self.source.catalog, + self.destination_namespace, + &self.destination_name, + )?; + if self.source.state != TableMappingState::Published + || self.head_generation == 0 + || self.head_fence == 0 + || self.head_fence == u64::MAX + || self.source.name_epoch == u64::MAX + || (self.source.namespace == self.destination_namespace + && self.source.name == self.destination_name) + { + return Err(ValidationError::Record); + } + Ok(()) + } + + pub(crate) fn destination(&self, state: TableMappingState) -> TableMapping { + TableMapping { + namespace: self.destination_namespace, + name: self.destination_name.clone(), + name_epoch: self.source.name_epoch + 1, + operation: self.operation, + state, + ..self.source.clone() + } + } + + pub(crate) fn parents(&self) -> Vec { + let mut parents = vec![self.source.namespace, self.destination_namespace]; + parents.sort_unstable_by_key(|id| *id.as_bytes()); + parents.dedup(); + parents + } +} diff --git a/lib/crowdb-access-iceberg/src/namespace/repair.rs b/lib/crowdb-access-iceberg/src/namespace/repair.rs index e590c450c..453b0fe78 100644 --- a/lib/crowdb-access-iceberg/src/namespace/repair.rs +++ b/lib/crowdb-access-iceberg/src/namespace/repair.rs @@ -113,6 +113,14 @@ impl NamespaceRecovery { return Err(ValidationError::Record.into()); }; if mapping.resolves(&authority) { + if let Some(rename) = &authority.rename { + crate::table::TableRenames::from_parts( + self.creator.repository.store.clone(), + self.creator.names.clone(), + ) + .recover(context, rename) + .await?; + } return Ok(()); } } diff --git a/lib/crowdb-access-iceberg/src/record/namespace.rs b/lib/crowdb-access-iceberg/src/record/namespace.rs index fd2d8d882..d2420dcc9 100644 --- a/lib/crowdb-access-iceberg/src/record/namespace.rs +++ b/lib/crowdb-access-iceberg/src/record/namespace.rs @@ -1,8 +1,8 @@ use std::collections::BTreeMap; use crowdb_protocol::iceberg_fb::{ - FBNamespaceAuthority, FBNamespaceAuthorityArgs, FBNamespaceMapping, FBNamespaceMappingArgs, - FBNamespaceProperty, FBNamespacePropertyArgs, + FBNameMove, FBNameMoveArgs, FBNamespaceAuthority, FBNamespaceAuthorityArgs, FBNamespaceMapping, + FBNamespaceMappingArgs, FBNamespaceProperty, FBNamespacePropertyArgs, }; use flatbuffers::{FlatBufferBuilder, WIPOffset}; @@ -42,6 +42,11 @@ pub(super) fn encode_authority<'buffer>( )); } let properties = builder.create_vector(&properties); + let rename = authority + .rename + .as_ref() + .map(|rename| encode_rename(builder, rename)) + .transpose()?; Ok(FBNamespaceAuthority::create( builder, &FBNamespaceAuthorityArgs { @@ -60,6 +65,7 @@ pub(super) fn encode_authority<'buffer>( }, pending_operation, properties: Some(properties), + rename, }, )) } @@ -103,11 +109,47 @@ pub(super) fn decode_authority( Some(OperationId::from_bytes(value.pending_operation().bytes())?) }, properties: NamespaceProperties::new(properties)?, + rename: value.rename().map(decode_rename).transpose()?.map(Box::new), }; authority.validate()?; Ok(authority) } +fn encode_rename<'buffer>( + builder: &mut FlatBufferBuilder<'buffer>, + rename: &crate::namespace::NamespaceRename, +) -> Result>, ValidationError> { + rename.validate()?; + let source = super::table::encode_mapping(builder, &rename.source)?; + let destination_namespace = builder.create_vector(rename.destination_namespace.as_bytes()); + let destination_name = builder.create_string(&rename.destination_name); + let operation = builder.create_vector(rename.operation.as_bytes()); + Ok(FBNameMove::create( + builder, + &FBNameMoveArgs { + source: Some(source), + destination_namespace: Some(destination_namespace), + destination_name: Some(destination_name), + operation: Some(operation), + head_generation: rename.head_generation, + head_fence: rename.head_fence, + }, + )) +} + +fn decode_rename(value: FBNameMove<'_>) -> Result { + let rename = crate::namespace::NamespaceRename { + source: super::table::decode_mapping(value.source())?, + destination_namespace: NamespaceId::from_bytes(value.destination_namespace().bytes())?, + destination_name: value.destination_name().to_owned(), + operation: OperationId::from_bytes(value.operation().bytes())?, + head_generation: value.head_generation(), + head_fence: value.head_fence(), + }; + rename.validate()?; + Ok(rename) +} + pub(super) fn encode_mapping<'buffer>( builder: &mut FlatBufferBuilder<'buffer>, mapping: &NamespaceMapping, diff --git a/lib/crowdb-access-iceberg/src/table.rs b/lib/crowdb-access-iceberg/src/table.rs index 34641c18d..04a07acef 100644 --- a/lib/crowdb-access-iceberg/src/table.rs +++ b/lib/crowdb-access-iceberg/src/table.rs @@ -6,12 +6,13 @@ mod list; mod load; mod metadata; mod record; +mod rename; mod repository; pub use key::{head_key, name_key, storage_head_key}; pub use lifecycle::{ - TableLifecycleAction, TableLifecycleOperation, TableLifecyclePhase, TableLifecycleRequest, - TableLifecycles, TablePurgeTask, + TableLifecycleAction, TableLifecycleOperation, TableLifecycleOutcome, TableLifecyclePhase, + TableLifecycleRequest, TableLifecycles, TablePurgeTask, }; pub use list::{TableListLimits, TableListPage, TableLister}; pub use load::{SnapshotLoadingMode, TableLoad, TableLoadError, TableLoader}; @@ -24,4 +25,5 @@ pub use metadata::{ }; pub(crate) use metadata::{schema_default_identity, validate_layout_definitions, validate_schema_definition}; pub use record::{TableHead, TableLifecycle, TableMapping, TableMappingState}; +pub(crate) use rename::TableRenames; pub use repository::{SelectedTable, TableRepository}; diff --git a/lib/crowdb-access-iceberg/src/table/lifecycle.rs b/lib/crowdb-access-iceberg/src/table/lifecycle.rs index 1c61d9aad..25c6b08f4 100644 --- a/lib/crowdb-access-iceberg/src/table/lifecycle.rs +++ b/lib/crowdb-access-iceberg/src/table/lifecycle.rs @@ -38,6 +38,12 @@ pub struct TableLifecycleRequest { pub action: TableLifecycleAction, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TableLifecycleOutcome { + pub status: u16, + pub body: Vec, +} + pub struct TableLifecycles { store: Arc, names: Arc, @@ -62,14 +68,30 @@ impl TableLifecycles { /// # Errors /// Rejects changed request identities, retired catalogs and uncertain storage outcomes. - pub async fn execute(&self, request: &TableLifecycleRequest) -> Result { + pub async fn execute( + &self, + request: &TableLifecycleRequest, + ) -> Result { let input = preparation::input(request)?; if let Some(operation) = self.load(request.context, request.identity.operation).await? { self.match_request(request, &input, &operation).await?; + } else if matches!(request.action, TableLifecycleAction::Rename { .. }) { + let status = super::TableRenames::from_parts(self.store.clone(), self.names.clone()) + .execute(request) + .await?; + return TableLifecycleOutcome::new(status); } else if let Some(outcome) = self.prepare(request, &input).await? { - return Ok(outcome); + return self.inline(outcome).await; } - self.resume(request.context, request.identity.operation).await + self.inline(self.resume(request.context, request.identity.operation).await?) + .await + } + + async fn inline(&self, outcome: TableCommitOutcome) -> Result { + Ok(TableLifecycleOutcome { + status: outcome.status, + body: self.payloads.get(&outcome.body).await?, + }) } /// # Errors @@ -120,3 +142,21 @@ impl TableLifecycles { Err(CatalogError::Busy) } } + +impl TableLifecycleOutcome { + fn new(status: u16) -> Result { + let body = if status == 204 { + Vec::new() + } else { + let kind = if status == 404 { + "NoSuchTableException" + } else { + "CommitFailedException" + }; + serde_json::to_vec(&serde_json::json!({"error": {"code": status, "type": kind, + "message": "Table lifecycle target conflicts with current authority"}})) + .map_err(|_| crate::error::ValidationError::Record)? + }; + Ok(Self { status, body }) + } +} diff --git a/lib/crowdb-access-iceberg/src/table/lifecycle/admission.rs b/lib/crowdb-access-iceberg/src/table/lifecycle/admission.rs index 6a5290edd..cf375bf11 100644 --- a/lib/crowdb-access-iceberg/src/table/lifecycle/admission.rs +++ b/lib/crowdb-access-iceberg/src/table/lifecycle/admission.rs @@ -20,6 +20,9 @@ impl TableLifecycles { let StorageRecord::NamespaceAuthority(mut parent) = StorageRecord::decode(&key, &value.bytes)? else { return Err(ValidationError::Record.into()); }; + if parent.rename.is_some() { + return Err(CatalogError::Busy); + } if parent.lifecycle != NamespaceLifecycle::Ready || Some(&parent.identifier) != operation.destination_namespace.as_ref() { diff --git a/lib/crowdb-access-iceberg/src/table/lifecycle/preparation.rs b/lib/crowdb-access-iceberg/src/table/lifecycle/preparation.rs index c46664495..262cc2db8 100644 --- a/lib/crowdb-access-iceberg/src/table/lifecycle/preparation.rs +++ b/lib/crowdb-access-iceberg/src/table/lifecycle/preparation.rs @@ -71,23 +71,7 @@ impl TableLifecycles { candidate.lifecycle = TableLifecycle::Tombstone; (None, *purge_requested) } - TableLifecycleAction::Rename { namespace, name } => { - if namespace == &request.namespace && name == &request.name { - return self.initial_outcome(request, input, 204, "").await; - } - let Some(target) = self.namespaces.load(request.context, namespace).await? else { - return self - .initial_outcome(request, input, 404, "NoSuchNamespaceException") - .await; - }; - candidate.namespace = target.namespace; - candidate.name.clone_from(name); - candidate.name_epoch = candidate - .name_epoch - .checked_add(1) - .ok_or(ValidationError::GenerationExhausted)?; - (Some(namespace.clone()), false) - } + TableLifecycleAction::Rename { .. } => return Err(ValidationError::Record.into()), }; let operation = TableLifecycleOperation { context: request.context, diff --git a/lib/crowdb-access-iceberg/src/table/lifecycle/reservation.rs b/lib/crowdb-access-iceberg/src/table/lifecycle/reservation.rs index 8baca14e5..44998f1e7 100644 --- a/lib/crowdb-access-iceberg/src/table/lifecycle/reservation.rs +++ b/lib/crowdb-access-iceberg/src/table/lifecycle/reservation.rs @@ -78,10 +78,11 @@ impl TableLifecycles { mapping: &TableMapping, budget: &mut usize, ) -> Result<(), CatalogError> { - let operation = self - .load(context, mapping.operation) - .await? - .ok_or(ValidationError::Record)?; + let Some(operation) = self.load(context, mapping.operation).await? else { + return super::super::TableRenames::from_parts(self.store.clone(), self.names.clone()) + .help_reserved(context, mapping) + .await; + }; if !operation.is_rename() || operation.destination(TableMappingState::Reserved) != *mapping { return Err(ValidationError::IdentityMismatch.into()); } diff --git a/lib/crowdb-access-iceberg/src/table/rename.rs b/lib/crowdb-access-iceberg/src/table/rename.rs new file mode 100644 index 000000000..5fda6e9e9 --- /dev/null +++ b/lib/crowdb-access-iceberg/src/table/rename.rs @@ -0,0 +1,162 @@ +use std::sync::Arc; + +use crate::{ + catalog::{check_context, CasOutcome, CatalogContext, CatalogError, CatalogStore}, + error::ValidationError, + namespace::{ + NamespaceAuthority, NamespaceLifecycle, NamespaceRename, NamespaceRepository, NamespaceStore, + }, + operation::mutation_identity, + record::StorageRecord, +}; + +use super::{ + name_key, storage_head_key, TableHead, TableLifecycleAction, TableLifecycleRequest, TableMapping, + TableMappingState, TableRepository, +}; + +mod admission; +mod recovery; +mod reservation; + +pub(crate) struct TableRenames { + store: Arc, + names: Arc, + namespaces: NamespaceRepository, +} + +impl TableRenames { + pub(crate) fn from_parts(store: Arc, names: Arc) -> Self { + Self { + namespaces: NamespaceRepository::from_parts(store.clone(), names.clone()), + store, + names, + } + } + + pub(crate) async fn execute(&self, request: &TableLifecycleRequest) -> Result { + let TableLifecycleAction::Rename { namespace, name } = &request.action else { + return Err(ValidationError::Record.into()); + }; + let Some(source) = self.parent(request.context, &request.namespace).await? else { + return Ok(404); + }; + let target = if namespace == &request.namespace { + source.clone() + } else { + let Some(target) = self.parent(request.context, namespace).await? else { + return Ok(404); + }; + target + }; + let Some(selected) = TableRepository::new(self.store.clone()) + .select(request.context, source.namespace, &request.name) + .await? + else { + return Ok(404); + }; + if selected.head.pending_operation.is_some() { + return Err(CatalogError::Busy); + } + if namespace == &request.namespace && name == &request.name { + return Ok(204); + } + let key = name_key(request.context.catalog, source.namespace, &request.name)?; + let value = self + .store + .get(&key.encode()?) + .await? + .ok_or(CatalogError::Conflict)?; + let StorageRecord::TableMapping(mapping) = StorageRecord::decode(&key, &value.bytes)? else { + return Err(ValidationError::Record.into()); + }; + if !mapping.resolves(&selected.head) { + return Err(CatalogError::Conflict); + } + let rename = NamespaceRename { + source: mapping, + destination_namespace: target.namespace, + destination_name: name.clone(), + // A retried HTTP request must not reuse a canceled reservation's bytes. + operation: crate::key::OperationId::random(), + head_generation: selected.head.generation, + head_fence: selected.head.operation_fence, + }; + rename.validate()?; + let mut parents = vec![source, target]; + parents.sort_unstable_by_key(|parent| *parent.namespace.as_bytes()); + parents.dedup_by_key(|parent| parent.namespace); + if let Err(error) = self.admit(&rename, parents).await { + if matches!(error, CatalogError::Busy | CatalogError::Conflict) { + self.recover(request.context, &rename).await?; + } + return Err(error); + } + if let Err(error) = self.reserve(request.context, &rename).await { + if matches!(error, CatalogError::Busy | CatalogError::Conflict) { + self.recover(request.context, &rename).await?; + } + return Err(error); + } + self.publish(request.context, &rename, selected.head).await + } + + async fn parent( + &self, + context: CatalogContext, + name: &crate::namespace::NamespaceIdentifier, + ) -> Result, CatalogError> { + let parent = self.namespaces.load(context, name).await?; + if let Some(rename) = parent.as_ref().and_then(|parent| parent.rename.as_ref()) { + self.recover(context, rename).await?; + return Err(CatalogError::Busy); + } + if parent.as_ref().is_some_and(|parent| { + parent.lifecycle != NamespaceLifecycle::Ready || parent.pending_operation.is_some() + }) { + return Err(CatalogError::Busy); + } + Ok(parent) + } + + async fn publish( + &self, + context: CatalogContext, + rename: &NamespaceRename, + before: TableHead, + ) -> Result { + check_context(self.store.as_ref(), context).await?; + let mut candidate = before.clone(); + candidate.namespace = rename.destination_namespace; + candidate.name.clone_from(&rename.destination_name); + candidate.name_epoch += 1; + candidate.operation_fence += 1; + let key = storage_head_key(before.storage_location()); + let encoded = key.encode()?; + let expected = StorageRecord::TableHead(Box::new(before)).encode()?; + let bytes = StorageRecord::TableHead(Box::new(candidate.clone())).encode()?; + let result = self + .store + .compare_exchange( + &encoded, + Some(&expected), + &bytes, + mutation_identity(&encoded, Some(&expected), &bytes), + ) + .await?; + let (head, status) = match result { + CasOutcome::Applied(_) => (candidate, 204), + CasOutcome::Conflict(Some(value)) => { + let StorageRecord::TableHead(head) = StorageRecord::decode(&key, &value.bytes)? else { + return Err(ValidationError::Record.into()); + }; + let status = if value.bytes == bytes { 204 } else { 409 }; + (*head, status) + } + CasOutcome::Conflict(None) => return Err(CatalogError::Busy), + }; + self.settle(context, rename, head).await?; + check_context(self.store.as_ref(), context).await?; + Ok(status) + } +} diff --git a/lib/crowdb-access-iceberg/src/table/rename/admission.rs b/lib/crowdb-access-iceberg/src/table/rename/admission.rs new file mode 100644 index 000000000..1cfe08ba1 --- /dev/null +++ b/lib/crowdb-access-iceberg/src/table/rename/admission.rs @@ -0,0 +1,125 @@ +use super::{ + mutation_identity, CasOutcome, CatalogError, NamespaceAuthority, NamespaceLifecycle, NamespaceRename, + StorageRecord, TableRenames, ValidationError, +}; +use crate::namespace::authority_key; + +impl TableRenames { + pub(super) async fn admit( + &self, + rename: &NamespaceRename, + parents: Vec, + ) -> Result<(), CatalogError> { + for parent in parents { + self.occupy(rename, parent).await?; + } + Ok(()) + } + + async fn occupy( + &self, + rename: &NamespaceRename, + mut parent: NamespaceAuthority, + ) -> Result<(), CatalogError> { + let key = authority_key(parent.catalog, parent.namespace); + let encoded = key.encode()?; + let identifier = parent.identifier.clone(); + let name_epoch = parent.name_epoch; + for _ in 0..16 { + if parent.lifecycle != NamespaceLifecycle::Ready + || parent.pending_operation.is_some() + || parent.identifier != identifier + || parent.name_epoch != name_epoch + { + return Err(CatalogError::Conflict); + } + if let Some(existing) = &parent.rename { + return if existing.as_ref() == rename { + Ok(()) + } else { + Err(CatalogError::Busy) + }; + } + let expected = StorageRecord::NamespaceAuthority(Box::new(parent.clone())).encode()?; + let mut next = parent; + next.mutation_revision = next + .mutation_revision + .checked_add(1) + .ok_or(ValidationError::GenerationExhausted)?; + next.rename = Some(Box::new(rename.clone())); + let bytes = StorageRecord::NamespaceAuthority(Box::new(next)).encode()?; + match self + .store + .compare_exchange( + &encoded, + Some(&expected), + &bytes, + mutation_identity(&encoded, Some(&expected), &bytes), + ) + .await? + { + CasOutcome::Applied(_) => return Ok(()), + CasOutcome::Conflict(Some(value)) => { + let StorageRecord::NamespaceAuthority(observed) = + StorageRecord::decode(&key, &value.bytes)? + else { + return Err(ValidationError::Record.into()); + }; + parent = *observed; + } + CasOutcome::Conflict(None) => return Err(CatalogError::Conflict), + } + } + Err(CatalogError::Busy) + } + + pub(super) async fn release(&self, rename: &NamespaceRename) -> Result<(), CatalogError> { + for parent in rename.parents() { + let key = authority_key(rename.source.catalog, parent); + let encoded = key.encode()?; + let mut value = self.store.get(&encoded).await?; + let mut finished = false; + for _ in 0..16 { + let Some(current) = value else { + finished = true; + break; + }; + let StorageRecord::NamespaceAuthority(mut authority) = + StorageRecord::decode(&key, ¤t.bytes)? + else { + return Err(ValidationError::Record.into()); + }; + if authority.rename.as_deref() != Some(rename) { + finished = true; + break; + } + authority.rename = None; + authority.mutation_revision = authority + .mutation_revision + .checked_add(1) + .ok_or(ValidationError::GenerationExhausted)?; + let bytes = StorageRecord::NamespaceAuthority(authority).encode()?; + match self + .store + .compare_exchange( + &encoded, + Some(¤t.bytes), + &bytes, + mutation_identity(&encoded, Some(¤t.bytes), &bytes), + ) + .await? + { + CasOutcome::Applied(_) => { + finished = true; + break; + } + CasOutcome::Conflict(observed) => value = observed, + } + } + if !finished { + return Err(CatalogError::Busy); + } + } + Ok(()) + } +} diff --git a/lib/crowdb-access-iceberg/src/table/rename/recovery.rs b/lib/crowdb-access-iceberg/src/table/rename/recovery.rs new file mode 100644 index 000000000..65e872b5b --- /dev/null +++ b/lib/crowdb-access-iceberg/src/table/rename/recovery.rs @@ -0,0 +1,88 @@ +use super::{ + check_context, mutation_identity, storage_head_key, CasOutcome, CatalogContext, CatalogError, + NamespaceRename, StorageRecord, TableHead, TableMappingState, TableRenames, ValidationError, +}; + +impl TableRenames { + pub(crate) async fn recover( + &self, + context: CatalogContext, + rename: &NamespaceRename, + ) -> Result<(), CatalogError> { + rename.validate()?; + if rename.source.catalog != context.catalog { + return Err(ValidationError::IdentityMismatch.into()); + } + check_context(self.store.as_ref(), context).await?; + let key = storage_head_key(rename.source.storage_location()); + let value = self + .store + .get(&key.encode()?) + .await? + .ok_or(ValidationError::Record)?; + let StorageRecord::TableHead(head) = StorageRecord::decode(&key, &value.bytes)? else { + return Err(ValidationError::Record.into()); + }; + self.settle(context, rename, *head).await + } + + pub(super) async fn settle( + &self, + context: CatalogContext, + rename: &NamespaceRename, + mut head: TableHead, + ) -> Result<(), CatalogError> { + let key = storage_head_key(rename.source.storage_location()); + let encoded = key.encode()?; + for _ in 0..16 { + if head.operation_fence > rename.head_fence { + let destination = rename.destination(TableMappingState::Published); + if destination.resolves(&head) { + self.publish_name(&rename.destination(TableMappingState::Reserved)) + .await?; + } else { + self.remove(&rename.destination(TableMappingState::Reserved)) + .await?; + } + if !rename.source.resolves(&head) { + self.remove(&rename.source).await?; + } + self.release(rename).await?; + return check_context(self.store.as_ref(), context).await; + } + if head.operation_fence != rename.head_fence + || head.generation != rename.head_generation + || !rename.source.resolves(&head) + || head.pending_operation.is_some() + { + return Err(ValidationError::Record.into()); + } + // Win against any delayed name-switch CAS before removing occupancy. + let before = StorageRecord::TableHead(Box::new(head.clone())).encode()?; + let mut fenced = head.clone(); + fenced.operation_fence += 1; + let bytes = StorageRecord::TableHead(Box::new(fenced.clone())).encode()?; + match self + .store + .compare_exchange( + &encoded, + Some(&before), + &bytes, + mutation_identity(&encoded, Some(&before), &bytes), + ) + .await? + { + CasOutcome::Applied(_) => head = fenced, + CasOutcome::Conflict(Some(observed)) => { + let StorageRecord::TableHead(observed) = StorageRecord::decode(&key, &observed.bytes)? + else { + return Err(ValidationError::Record.into()); + }; + head = *observed; + } + CasOutcome::Conflict(None) => return Err(CatalogError::Busy), + } + } + Err(CatalogError::Busy) + } +} diff --git a/lib/crowdb-access-iceberg/src/table/rename/reservation.rs b/lib/crowdb-access-iceberg/src/table/rename/reservation.rs new file mode 100644 index 000000000..04d8879cd --- /dev/null +++ b/lib/crowdb-access-iceberg/src/table/rename/reservation.rs @@ -0,0 +1,122 @@ +use super::{ + mutation_identity, name_key, storage_head_key, CasOutcome, CatalogContext, CatalogError, NamespaceRename, + StorageRecord, TableMapping, TableMappingState, TableRenames, ValidationError, +}; + +impl TableRenames { + pub(super) async fn reserve( + &self, + context: CatalogContext, + rename: &NamespaceRename, + ) -> Result<(), CatalogError> { + let mapping = rename.destination(TableMappingState::Reserved); + let key = name_key(mapping.catalog, mapping.namespace, &mapping.name)?; + let encoded = key.encode()?; + let bytes = StorageRecord::TableMapping(mapping).encode()?; + for _ in 0..8 { + match self + .names + .compare_exchange(&encoded, None, &bytes, mutation_identity(&encoded, None, &bytes)) + .await? + { + CasOutcome::Applied(_) => return Ok(()), + CasOutcome::Conflict(Some(value)) if value.bytes == bytes => return Ok(()), + CasOutcome::Conflict(Some(value)) => { + let StorageRecord::TableMapping(existing) = StorageRecord::decode(&key, &value.bytes)? + else { + return Err(ValidationError::Record.into()); + }; + if existing.state == TableMappingState::Reserved { + Box::pin(crate::commit::TableCreator::help_reservation( + self.store.clone(), + self.names.clone(), + context, + &existing, + &mut 16, + )) + .await?; + return Err(CatalogError::Busy); + } + let head_key = storage_head_key(existing.storage_location()); + if let Some(head) = self.store.get(&head_key.encode()?).await? { + let StorageRecord::TableHead(head) = StorageRecord::decode(&head_key, &head.bytes)? + else { + return Err(ValidationError::Record.into()); + }; + if existing.resolves(&head) { + return Err(CatalogError::Conflict); + } + } + self.remove(&existing).await?; + } + CasOutcome::Conflict(None) => return Err(CatalogError::Busy), + } + } + Err(CatalogError::Busy) + } + + pub(super) async fn remove(&self, mapping: &TableMapping) -> Result<(), CatalogError> { + let key = name_key(mapping.catalog, mapping.namespace, &mapping.name)?.encode()?; + let bytes = StorageRecord::TableMapping(mapping.clone()).encode()?; + self.names + .delete_mapping(&key, &bytes, mutation_identity(&key, Some(&bytes), &[])) + .await?; + Ok(()) + } + + pub(crate) async fn help_reserved( + &self, + context: CatalogContext, + mapping: &TableMapping, + ) -> Result<(), CatalogError> { + let key = crate::namespace::authority_key(context.catalog, mapping.namespace); + if let Some(parent) = self.store.get(&key.encode()?).await? { + let StorageRecord::NamespaceAuthority(parent) = StorageRecord::decode(&key, &parent.bytes)? + else { + return Err(ValidationError::Record.into()); + }; + if let Some(rename) = parent + .rename + .as_ref() + .filter(|rename| rename.destination(TableMappingState::Reserved) == *mapping) + { + return self.recover(context, rename).await; + } + } + // Occupancy can be cleared only after the old head CAS is fenced or settled. + let key = storage_head_key(mapping.storage_location()); + if let Some(value) = self.store.get(&key.encode()?).await? { + let StorageRecord::TableHead(head) = StorageRecord::decode(&key, &value.bytes)? else { + return Err(ValidationError::Record.into()); + }; + let mut published = mapping.clone(); + published.state = TableMappingState::Published; + if published.resolves(&head) { + return self.publish_name(mapping).await; + } + } + self.remove(mapping).await + } + + pub(super) async fn publish_name(&self, reserved: &TableMapping) -> Result<(), CatalogError> { + let key = name_key(reserved.catalog, reserved.namespace, &reserved.name)?.encode()?; + let before = StorageRecord::TableMapping(reserved.clone()).encode()?; + let mut published = reserved.clone(); + published.state = TableMappingState::Published; + let bytes = StorageRecord::TableMapping(published).encode()?; + let result = self + .names + .compare_exchange( + &key, + Some(&before), + &bytes, + mutation_identity(&key, Some(&before), &bytes), + ) + .await?; + match result { + CasOutcome::Applied(_) => Ok(()), + CasOutcome::Conflict(Some(observed)) if observed.bytes == bytes => Ok(()), + CasOutcome::Conflict(_) => Err(CatalogError::Busy), + } + } +} diff --git a/lib/crowdb-access-iceberg/tests/common/namespace.rs b/lib/crowdb-access-iceberg/tests/common/namespace.rs index 03491f073..3dfc85463 100644 --- a/lib/crowdb-access-iceberg/tests/common/namespace.rs +++ b/lib/crowdb-access-iceberg/tests/common/namespace.rs @@ -91,6 +91,7 @@ impl TestNamespace { mutation_revision: 1, lifecycle: NamespaceLifecycle::Ready, pending_operation: None, + rename: None, properties: NamespaceProperties::default(), } } diff --git a/lib/crowdb-access-iceberg/tests/common/rename_conflict.rs b/lib/crowdb-access-iceberg/tests/common/rename_conflict.rs new file mode 100644 index 000000000..aa261bda3 --- /dev/null +++ b/lib/crowdb-access-iceberg/tests/common/rename_conflict.rs @@ -0,0 +1,102 @@ +use async_trait::async_trait; +use crowdb_access_iceberg::{ + catalog::{CasOutcome, CatalogStore, StoreError, StoredValue}, + key::IcebergKey, + namespace::{ChildScan, NamespaceStore}, + operation::mutation_identity, + record::StorageRecord, +}; +use crowdb_protocol::chunk_kv::ClientRequestId; +use std::sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, +}; + +pub struct TestRenameConflicts { + pub inner: Arc, + pub remaining: AtomicUsize, + pub injected: AtomicUsize, + awaiting_retry: AtomicBool, +} + +impl TestRenameConflicts { + pub fn new(inner: Arc) -> Self { + Self { + inner, + remaining: AtomicUsize::new(4), + injected: AtomicUsize::new(0), + awaiting_retry: AtomicBool::new(false), + } + } +} + +#[async_trait] +impl CatalogStore for TestRenameConflicts { + async fn get(&self, key: &[u8]) -> Result, StoreError> { + assert!( + !self.awaiting_retry.load(Ordering::SeqCst), + "CAS observed value must be used before another GET" + ); + self.inner.get(key).await + } + + async fn compare_exchange( + &self, + key: &[u8], + expected: Option<&[u8]>, + value: &[u8], + identity: ClientRequestId, + ) -> Result { + let decoded = IcebergKey::decode(key)?; + if let (Some(expected), StorageRecord::NamespaceAuthority(candidate)) = + (expected, StorageRecord::decode(&decoded, value)?) + { + if candidate.rename.is_some() && self.remaining.load(Ordering::SeqCst) > 0 { + let StorageRecord::NamespaceAuthority(mut competitor) = + StorageRecord::decode(&decoded, expected)? + else { + panic!() + }; + self.remaining.fetch_sub(1, Ordering::SeqCst); + self.injected.fetch_add(1, Ordering::SeqCst); + competitor.mutation_revision += 1; + let bytes = StorageRecord::NamespaceAuthority(competitor).encode()?; + assert!(matches!( + self.inner + .compare_exchange( + key, + Some(expected), + &bytes, + mutation_identity(key, Some(expected), &bytes) + ) + .await?, + CasOutcome::Applied(_) + )); + let observed = self.inner.get(key).await?; + self.awaiting_retry.store(true, Ordering::SeqCst); + return Ok(CasOutcome::Conflict(observed)); + } + } + let result = self.inner.compare_exchange(key, expected, value, identity).await; + self.awaiting_retry.store(false, Ordering::SeqCst); + result + } +} + +#[async_trait] +impl NamespaceStore for TestRenameConflicts { + async fn scan_children( + &self, + scan: ChildScan, + ) -> Result { + self.inner.scan_children(scan).await + } + async fn delete_mapping( + &self, + key: &[u8], + expected: &[u8], + identity: ClientRequestId, + ) -> Result { + self.inner.delete_mapping(key, expected, identity).await + } +} diff --git a/lib/crowdb-access-iceberg/tests/common/store.rs b/lib/crowdb-access-iceberg/tests/common/store.rs index 516157591..24f878ddb 100644 --- a/lib/crowdb-access-iceberg/tests/common/store.rs +++ b/lib/crowdb-access-iceberg/tests/common/store.rs @@ -123,7 +123,11 @@ impl TestStore { } else { &self.table_head_pause_before }; - if head.pending_operation.is_some() && enabled.swap(false, Ordering::SeqCst) { + if (head.pending_operation.is_some() || expected.is_some_and(|bytes| { + crowdb_access_iceberg::key::IcebergKey::decode(key).ok() + .and_then(|key| crowdb_access_iceberg::record::StorageRecord::decode(&key, bytes).ok()) + .is_some_and(|record| matches!(record, crowdb_access_iceberg::record::StorageRecord::TableHead(before) if before.name_epoch < head.name_epoch)) + })) && enabled.swap(false, Ordering::SeqCst) { self.table_head_entered.notify_one(); self.table_head_release.notified().await; } @@ -136,6 +140,7 @@ impl TestStore { { if expected.is_some() && (authority.pending_operation.is_some() + || authority.rename.is_some() || expected .and_then(|bytes| { crowdb_access_iceberg::key::IcebergKey::decode(key) diff --git a/lib/crowdb-access-iceberg/tests/common/table_rename.rs b/lib/crowdb-access-iceberg/tests/common/table_rename.rs new file mode 100644 index 000000000..8af71d16c --- /dev/null +++ b/lib/crowdb-access-iceberg/tests/common/table_rename.rs @@ -0,0 +1,102 @@ +use crate::creation::TestCreation; +use crowdb_access_iceberg::{ + catalog::CatalogStore, + key::OperationId, + namespace::{NamespaceIdentifier, NamespaceRecovery, NamespaceRepository}, + operation::RequestIdentity, + record::StorageRecord, + table::{TableHead, TableLifecycleAction, TableLifecycleRequest, TableRepository}, +}; + +pub fn request(test: &TestCreation, namespace: NamespaceIdentifier) -> TableLifecycleRequest { + TableLifecycleRequest { + context: test.fixture.context, + identity: RequestIdentity { + operation: OperationId::random(), + issued_ms: 100, + }, + principal: "writer".into(), + namespace: test.parent.identifier.clone(), + name: "events".into(), + action: TableLifecycleAction::Rename { + namespace, + name: "renamed".into(), + }, + } +} + +pub async fn setup(cross: bool) -> (TestCreation, TableLifecycleRequest, TableHead) { + let test = TestCreation::new().await; + test.creator().create(&test.request).await.unwrap(); + let target = if cross { + let parent = test.fixture.authority(None, &["destination"]); + test.fixture.publish(&parent).await; + parent.identifier + } else { + test.parent.identifier.clone() + }; + let before = TableRepository::new(test.fixture.store.clone()) + .select(test.fixture.context, test.parent.namespace, "events") + .await + .unwrap() + .unwrap() + .head; + let request = request(&test, target); + (test, request, before) +} + +pub async fn repair(test: &TestCreation) { + let recovery = NamespaceRecovery::new(test.fixture.store.clone()); + let mut cursor = None; + loop { + let page = recovery.repair_page(test.fixture.context, cursor).await.unwrap(); + assert!(page.failures.is_empty(), "{:?}", page.failures); + assert_eq!(page.deferred, 0); + cursor = page.continuation; + if cursor.is_none() { + break; + } + } +} + +pub async fn head(test: &TestCreation, before: &TableHead) -> TableHead { + let key = crowdb_access_iceberg::table::storage_head_key(before.storage_location()); + let value = test + .fixture + .store + .get(&key.encode().unwrap()) + .await + .unwrap() + .unwrap(); + let StorageRecord::TableHead(head) = StorageRecord::decode(&key, &value.bytes).unwrap() else { + panic!() + }; + *head +} + +pub async fn visible(test: &TestCreation, request: &TableLifecycleRequest) -> (bool, bool) { + let TableLifecycleAction::Rename { namespace, name } = &request.action else { + panic!() + }; + let parent = NamespaceRepository::new(test.fixture.store.clone()) + .load(request.context, namespace) + .await + .unwrap(); + let tables = TableRepository::new(test.fixture.store.clone()); + let old = tables + .select(request.context, test.parent.namespace, &request.name) + .await + .unwrap() + .is_some(); + let new = if let Some(parent) = parent { + tables + .select(request.context, parent.namespace, name) + .await + .unwrap() + .is_some() + } else { + false + }; + assert!(!(old && new), "one head must not resolve both names"); + (old, new) +} diff --git a/lib/crowdb-access-iceberg/tests/namespace_drop_test.rs b/lib/crowdb-access-iceberg/tests/namespace_drop_test.rs index 199c5c8a2..ce86a9049 100644 --- a/lib/crowdb-access-iceberg/tests/namespace_drop_test.rs +++ b/lib/crowdb-access-iceberg/tests/namespace_drop_test.rs @@ -243,7 +243,10 @@ async fn creator_and_empty_drop_share_one_admission_boundary() { let creator = NamespaceCreator::new(fixture.store.clone()); let dropper = NamespaceDropper::new(fixture.store.clone()); let results = tokio::time::timeout(std::time::Duration::from_secs(5), async { - tokio::join!(creator.create(&child), dropper.drop_namespace(&request)) + tokio::join!( + Box::pin(creator.create(&child)), + Box::pin(dropper.drop_namespace(&request)) + ) }) .await .unwrap(); diff --git a/lib/crowdb-access-iceberg/tests/namespace_journal_test.rs b/lib/crowdb-access-iceberg/tests/namespace_journal_test.rs index 80ee34e8b..f04f32180 100644 --- a/lib/crowdb-access-iceberg/tests/namespace_journal_test.rs +++ b/lib/crowdb-access-iceberg/tests/namespace_journal_test.rs @@ -276,6 +276,7 @@ async fn mutation_snapshots_are_key_bound_and_frozen_after_publication_starts() mutation_revision: 1, lifecycle: NamespaceLifecycle::Ready, pending_operation: None, + rename: None, properties: NamespaceProperties::default(), }; let before_bytes = StorageRecord::NamespaceAuthority(Box::new(authority.clone())) diff --git a/lib/crowdb-access-iceberg/tests/namespace_record_test.rs b/lib/crowdb-access-iceberg/tests/namespace_record_test.rs index e3775e730..8d3a2a14b 100644 --- a/lib/crowdb-access-iceberg/tests/namespace_record_test.rs +++ b/lib/crowdb-access-iceberg/tests/namespace_record_test.rs @@ -20,6 +20,7 @@ fn authority() -> NamespaceAuthority { mutation_revision: 1, lifecycle: NamespaceLifecycle::Ready, pending_operation: None, + rename: None, properties: NamespaceProperties::new(BTreeMap::from([("owner".into(), "数据库".into())])).unwrap(), } } diff --git a/lib/crowdb-access-iceberg/tests/table_commit_publication_test.rs b/lib/crowdb-access-iceberg/tests/table_commit_publication_test.rs index dda2fd5b2..fa5eaf066 100644 --- a/lib/crowdb-access-iceberg/tests/table_commit_publication_test.rs +++ b/lib/crowdb-access-iceberg/tests/table_commit_publication_test.rs @@ -276,8 +276,10 @@ async fn commit_winning_head_cas_forces_pending_lifecycle_to_abort_without_rebas .await .unwrap() .status, - 409 + if rename { 204 } else { 409 } ); + assert_eq!(head(&fixture).await.generation, committed.generation); + assert_eq!(head(&fixture).await.metadata_digest, committed.metadata_digest); } } diff --git a/lib/crowdb-access-iceberg/tests/table_lifecycle_race_test.rs b/lib/crowdb-access-iceberg/tests/table_lifecycle_race_test.rs index 474e37419..f1ead0b4b 100644 --- a/lib/crowdb-access-iceberg/tests/table_lifecycle_race_test.rs +++ b/lib/crowdb-access-iceberg/tests/table_lifecycle_race_test.rs @@ -8,46 +8,78 @@ mod creation; #[path = "common/namespace.rs"] #[allow(dead_code)] mod fixture; +#[path = "common/namespace_recovery_store.rs"] +mod namespace_recovery_store; #[path = "common/namespace_store.rs"] mod namespace_store; +#[path = "common/table_rename.rs"] +#[allow(dead_code)] +mod rename; -use creation::TestCreation; use crowdb_access_iceberg::{ catalog::CatalogError, key::OperationId, - namespace::{NamespaceDropRequest, NamespaceDropper, NamespaceIdentifier, NamespaceRepository}, + namespace::{NamespaceDropRequest, NamespaceDropper, NamespaceRepository}, operation::RequestIdentity, - table::{ - TableLifecycleAction, TableLifecyclePhase, TableLifecycleRequest, TableLifecycles, TableRepository, - }, + table::{TableLifecycleAction, TableLifecycleRequest, TableLifecycles, TableRepository}, }; use std::sync::atomic::Ordering; -fn request(test: &TestCreation, namespace: NamespaceIdentifier) -> TableLifecycleRequest { - TableLifecycleRequest { - context: test.fixture.context, - identity: RequestIdentity { - operation: OperationId::random(), - issued_ms: 100, - }, - principal: "writer".into(), - namespace: test.parent.identifier.clone(), - name: "events".into(), - action: TableLifecycleAction::Rename { - namespace, - name: "renamed".into(), - }, +#[tokio::test] +async fn canceled_request_retry_uses_a_fresh_reservation_against_delayed_cleanup() { + use crowdb_access_iceberg::{ + catalog::{CasOutcome, CatalogStore}, + namespace::NamespaceStore, + operation::mutation_identity, + table::name_key, + }; + let (test, request, _) = rename::setup(false).await; + let store = test.fixture.store.clone(); + let key = name_key(request.context.catalog, test.parent.namespace, "renamed") + .unwrap() + .encode() + .unwrap(); + let mut canceled = None; + for attempt in 0..2 { + store.table_head_pause_before.store(true, Ordering::SeqCst); + let task_store = store.clone(); + let task_request = request.clone(); + let pending = + tokio::spawn(async move { TableLifecycles::new(task_store).execute(&task_request).await }); + store.table_head_entered.notified().await; + let reserved = store.get(&key).await.unwrap().unwrap().bytes; + if attempt == 0 { + let drop = test.drop_request(); + assert!(matches!( + NamespaceDropper::new(store.clone()).drop_namespace(&drop).await, + Err(CatalogError::Busy) + )); + canceled = Some(reserved); + } else { + let old = canceled.as_ref().unwrap(); + assert_ne!(old, &reserved); + let result = store + .delete_mapping(&key, old, mutation_identity(&key, Some(old), &[])) + .await + .unwrap(); + assert!(matches!(result, CasOutcome::Conflict(Some(value)) if value.bytes == reserved)); + } + store.table_head_release.notify_one(); + assert_eq!( + pending.await.unwrap().unwrap().status, + if attempt == 0 { 409 } else { 204 } + ); } + assert_eq!(rename::visible(&test, &request).await, (false, true)); } #[tokio::test] -async fn namespace_drop_helps_rename_with_delayed_head_cas_response_and_cannot_remove_destination() { +async fn drop_fences_delayed_rename_or_preserves_the_committed_destination() { for after in [false, true] { - let test = TestCreation::new().await; - test.creator().create(&test.request).await.unwrap(); - let parent = test.fixture.authority(None, &["destination"]); - test.fixture.publish(&parent).await; - let request = request(&test, parent.identifier.clone()); + let (test, request, before) = rename::setup(true).await; + let TableLifecycleAction::Rename { namespace, .. } = &request.action else { + panic!() + }; let store = test.fixture.store.clone(); if after { store.table_head_pause_after.store(true, Ordering::SeqCst); @@ -59,60 +91,85 @@ async fn namespace_drop_helps_rename_with_delayed_head_cas_response_and_cannot_r let pending = tokio::spawn(async move { TableLifecycles::new(task_store).execute(&task_request).await }); store.table_head_entered.notified().await; - let result = NamespaceDropper::new(store.clone()) - .drop_namespace(&NamespaceDropRequest { - context: request.context, - identity: RequestIdentity { - operation: OperationId::random(), - issued_ms: 100, - }, - principal: "writer".into(), - identifier: parent.identifier.clone(), - }) - .await - .unwrap() - .unwrap(); - assert_eq!(result.status, 409); - let tables = TableRepository::new(store.clone()); - assert!(tables - .select(request.context, test.parent.namespace, "events") - .await - .unwrap() - .is_none()); - let selected = tables - .select(request.context, parent.namespace, "renamed") - .await - .unwrap() - .unwrap(); + let drop = NamespaceDropRequest { + context: request.context, + identity: RequestIdentity { + operation: OperationId::random(), + issued_ms: 100, + }, + principal: "writer".into(), + identifier: namespace.clone(), + }; + let dropper = NamespaceDropper::new(store.clone()); + assert!(matches!( + dropper.drop_namespace(&drop).await, + Err(CatalogError::Busy) + )); + let outcome = dropper.drop_namespace(&drop).await.unwrap().unwrap(); + assert_eq!(outcome.status, if after { 409 } else { 204 }); + assert_eq!(rename::visible(&test, &request).await, (!after, after)); store.table_head_release.notify_one(); - let delayed = pending.await.unwrap(); - assert!(delayed.is_ok() || matches!(delayed, Err(CatalogError::Busy))); assert_eq!( - TableLifecycles::new(store.clone()) - .execute(&request) - .await - .unwrap() - .status, - 204 + pending.await.unwrap().unwrap().status, + if after { 204 } else { 409 } ); - assert_eq!( - tables - .select(request.context, parent.namespace, "renamed") + assert_eq!(rename::visible(&test, &request).await, (!after, after)); + assert!(rename::head(&test, &before).await.operation_fence > before.operation_fence); + } +} + +#[tokio::test] +async fn interrupted_occupancies_recover_from_resources_at_every_mutation() { + for cross in [false, true] { + let (baseline, request, _) = rename::setup(cross).await; + let start = baseline.fixture.store.writes.load(Ordering::SeqCst); + TableLifecycles::new(baseline.fixture.store.clone()) + .execute(&request) + .await + .unwrap(); + let count = baseline.fixture.store.writes.load(Ordering::SeqCst) - start; + assert_eq!(count, if cross { 8 } else { 6 }); + for offset in 1..=count { + let (test, request, before) = rename::setup(cross).await; + let store = test.fixture.store.clone(); + let reads = test.blocks.reads.load(Ordering::SeqCst); + let writes = test.blocks.writes.load(Ordering::SeqCst); + store + .fail_after + .store(store.writes.load(Ordering::SeqCst) + offset, Ordering::SeqCst); + assert!( + TableLifecycles::new(store.clone()) + .execute(&request) + .await + .is_err(), + "cross={cross} offset={offset}" + ); + store.fail_after.store(0, Ordering::SeqCst); + rename::repair(&test).await; + let state = rename::visible(&test, &request).await; + assert!( + matches!(state, (true, false) | (false, true)), + "cross={cross} offset={offset}" + ); + let head = rename::head(&test, &before).await; + assert_eq!(head.metadata_location, before.metadata_location); + assert_eq!(head.generation, before.generation); + assert_eq!(head.metadata_digest, before.metadata_digest); + let outcome = TableLifecycles::new(store.clone()) + .execute(&request) .await - .unwrap() - .unwrap(), - selected - ); + .unwrap(); + assert_eq!(outcome.status, if state.0 { 204 } else { 404 }); + assert_eq!(rename::visible(&test, &request).await, (false, true)); + assert_eq!(test.blocks.reads.load(Ordering::SeqCst), reads); + assert_eq!(test.blocks.writes.load(Ordering::SeqCst), writes); + } } } #[tokio::test] -async fn losing_rename_cannot_publish_after_concurrent_drop_or_remove_recreated_source() { - let test = TestCreation::new().await; - test.creator().create(&test.request).await.unwrap(); - let parent = test.fixture.authority(None, &["destination"]); - test.fixture.publish(&parent).await; - let request = request(&test, parent.identifier.clone()); +async fn losing_rename_cannot_change_a_concurrent_drop_or_recreated_source() { + let (test, request, _) = rename::setup(true).await; let store = test.fixture.store.clone(); store.table_head_pause_before.store(true, Ordering::SeqCst); let task_request = request.clone(); @@ -140,112 +197,29 @@ async fn losing_rename_cannot_publish_after_concurrent_drop_or_remove_recreated_ let mut recreated = test.request.clone(); recreated.identity.operation = OperationId::random(); assert_eq!(test.creator().create(&recreated).await.unwrap().status, 200); - store.table_head_release.notify_one(); - assert_eq!(pending.await.unwrap().unwrap().status, 409); let tables = TableRepository::new(store.clone()); - assert!(tables - .select(request.context, parent.namespace, "renamed") - .await - .unwrap() - .is_none()); - assert!(tables + let replacement = tables .select(request.context, test.parent.namespace, "events") .await .unwrap() - .is_some()); + .unwrap(); + store.table_head_release.notify_one(); + assert_eq!(pending.await.unwrap().unwrap().status, 409); assert_eq!( - NamespaceDropper::new(store) - .drop_namespace(&NamespaceDropRequest { - context: request.context, - identity: RequestIdentity { - operation: OperationId::random(), - issued_ms: 100 - }, - principal: "writer".into(), - identifier: parent.identifier, - }) + tables + .select(request.context, test.parent.namespace, "events") .await .unwrap() - .unwrap() - .status, - 204 + .unwrap(), + replacement ); -} - -#[tokio::test] -async fn every_interrupted_rename_phase_composes_with_destination_namespace_drop() { - let baseline = TestCreation::new().await; - baseline.creator().create(&baseline.request).await.unwrap(); - let parent = baseline.fixture.authority(None, &["destination"]); - baseline.fixture.publish(&parent).await; - let start = baseline.fixture.store.writes.load(Ordering::SeqCst); - TableLifecycles::new(baseline.fixture.store.clone()) - .execute(&request(&baseline, parent.identifier)) + let TableLifecycleAction::Rename { namespace, .. } = &request.action else { + panic!() + }; + let target = NamespaceRepository::new(store) + .load(request.context, namespace) .await + .unwrap() .unwrap(); - let count = baseline.fixture.store.writes.load(Ordering::SeqCst) - start; - for offset in 1..=count { - let test = TestCreation::new().await; - test.creator().create(&test.request).await.unwrap(); - let parent = test.fixture.authority(None, &["destination"]); - test.fixture.publish(&parent).await; - let request = request(&test, parent.identifier.clone()); - let store = test.fixture.store.clone(); - store - .fail_after - .store(store.writes.load(Ordering::SeqCst) + offset, Ordering::SeqCst); - assert!(TableLifecycles::new(store.clone()) - .execute(&request) - .await - .is_err()); - store.fail_after.store(0, Ordering::SeqCst); - let result = NamespaceDropper::new(store.clone()) - .drop_namespace(&NamespaceDropRequest { - context: request.context, - identity: RequestIdentity { - operation: OperationId::random(), - issued_ms: 100, - }, - principal: "writer".into(), - identifier: parent.identifier.clone(), - }) - .await - .unwrap() - .unwrap(); - let outcome = TableLifecycles::new(store.clone()) - .execute(&request) - .await - .unwrap(); - let exists = NamespaceRepository::new(store.clone()) - .load(request.context, &parent.identifier) - .await - .unwrap() - .is_some(); - let moved = TableRepository::new(store.clone()) - .select(request.context, parent.namespace, "renamed") - .await - .unwrap() - .is_some(); - match result.status { - 204 => { - assert!(!exists && !moved); - assert_eq!(outcome.status, 404); - } - 409 => { - assert!(exists && moved); - assert_eq!(outcome.status, 204); - } - status => panic!("unexpected status {status} offset={offset}"), - } - if let Some(operation) = TableLifecycles::new(store) - .load(request.context, request.identity.operation) - .await - .unwrap() - { - assert!(matches!( - operation.phase, - TableLifecyclePhase::Complete | TableLifecyclePhase::Aborted - )); - } - } + assert!(target.rename.is_none()); } diff --git a/lib/crowdb-access-iceberg/tests/table_lifecycle_test.rs b/lib/crowdb-access-iceberg/tests/table_lifecycle_test.rs index b1275f2a0..1afed3acd 100644 --- a/lib/crowdb-access-iceberg/tests/table_lifecycle_test.rs +++ b/lib/crowdb-access-iceberg/tests/table_lifecycle_test.rs @@ -25,125 +25,6 @@ use crowdb_access_iceberg::{ }; use std::sync::atomic::Ordering; -async fn visible_during_rename( - test: &TestCreation, - request: &TableLifecycleRequest, - expected: &[u8], -) -> (bool, bool) { - use crowdb_access_iceberg::table::{ - SnapshotLoadingMode, TableListLimits, TableLister, TableLoad, TableLoader, TableMetadataLimits, - }; - let TableLifecycleAction::Rename { namespace, name } = &request.action else { - panic!("rename fixture required"); - }; - let loader = TableLoader::new( - test.fixture.store.clone(), - test.blocks.clone(), - TableMetadataLimits { - bytes: 2 * 1024 * 1024, - values: 200_000, - depth: 64, - string_bytes: 1024 * 1024, - collection_entries: 10_000, - }, - ); - let lister = TableLister::new(test.fixture.store.clone(), &[7; 32]).unwrap(); - let mut visible = Vec::new(); - for (parent, table) in [(&request.namespace, &request.name), (namespace, name)] { - let (response, page) = tokio::join!( - loader.load(request.context, parent, table, SnapshotLoadingMode::All, None), - lister.list( - request.context, - parent, - TableListLimits { - page_size: 1, - scanned: 10, - names_bytes: 1024, - }, - None - ) - ); - let page = page.unwrap().unwrap(); - let present = match response.unwrap() { - TableLoad::Missing => false, - TableLoad::Loaded { head, metadata, .. } => { - assert_eq!(metadata, expected); - assert_eq!(head.name, *table); - let original: serde_json::Value = serde_json::from_slice(expected).unwrap(); - assert_eq!( - head.table_uuid.unwrap().to_string(), - original["table-uuid"].as_str().unwrap() - ); - true - } - TableLoad::NotModified { .. } => panic!("unconditional load returned not modified"), - }; - assert_eq!(page.names.contains(table), present); - visible.push(present); - } - assert!( - !(visible[0] && visible[1]), - "old name must never alias the new name" - ); - (visible[0], visible[1]) -} - -#[tokio::test] -async fn interrupted_renames_keep_concurrent_list_and_load_head_qualified() { - for mode in [2, 3] { - let (baseline, request) = setup(mode).await; - let writes = baseline.fixture.store.writes.load(Ordering::SeqCst); - TableLifecycles::new(baseline.fixture.store.clone()) - .execute(&request) - .await - .unwrap(); - let count = baseline.fixture.store.writes.load(Ordering::SeqCst) - writes; - for offset in 1..=count { - let (test, request) = setup(mode).await; - let original = TableRepository::new(test.fixture.store.clone()) - .select(request.context, test.parent.namespace, &request.name) - .await - .unwrap() - .unwrap(); - let document = crowdb_access_iceberg::table::read_table_metadata_document( - test.blocks.clone(), - &original, - crowdb_access_iceberg::table::TableMetadataLimits { - bytes: 2 * 1024 * 1024, - values: 200_000, - depth: 64, - string_bytes: 1024 * 1024, - collection_entries: 10_000, - }, - ) - .await - .unwrap(); - let store = test.fixture.store.clone(); - store - .fail_after - .store(store.writes.load(Ordering::SeqCst) + offset, Ordering::SeqCst); - assert!(TableLifecycles::new(store.clone()) - .execute(&request) - .await - .is_err()); - store.fail_after.store(0, Ordering::SeqCst); - visible_during_rename(&test, &request, document.canonical()).await; - assert_eq!( - TableLifecycles::new(store) - .execute(&request) - .await - .unwrap() - .status, - 204 - ); - assert_eq!( - visible_during_rename(&test, &request, document.canonical()).await, - (false, true) - ); - } - } -} - async fn setup(mode: u8) -> (TestCreation, TableLifecycleRequest) { let test = TestCreation::new().await; assert_eq!(test.creator().create(&test.request).await.unwrap().status, 200); @@ -257,7 +138,7 @@ async fn assert_finished(test: &TestCreation, request: &TableLifecycleRequest) { #[tokio::test] async fn lifecycle_recovery_survives_every_durable_lost_response_without_file_io() { - for mode in 0..4 { + for mode in 0..2 { let (baseline, request) = setup(mode).await; let before = baseline.fixture.store.writes.load(Ordering::SeqCst); TableLifecycles::new(baseline.fixture.store.clone()) @@ -317,7 +198,7 @@ async fn lifecycle_recovery_survives_every_durable_lost_response_without_file_io #[tokio::test] async fn old_lifecycle_replay_cannot_remove_recreated_source_or_destination() { - for mode in 0..4 { + for mode in 0..2 { let (test, request) = setup(mode).await; let manager = TableLifecycles::new(test.fixture.store.clone()); assert_eq!(manager.execute(&request).await.unwrap().status, 204); @@ -412,7 +293,7 @@ async fn lifecycle_identity_reuse_rejects_changed_purge_or_principal() { #[tokio::test] async fn namespace_can_drop_after_table_drop_or_cross_namespace_move() { - for mode in [0, 1, 3] { + for mode in [0, 1] { let (test, request) = setup(mode).await; TableLifecycles::new(test.fixture.store.clone()) .execute(&request) @@ -429,7 +310,7 @@ async fn namespace_can_drop_after_table_drop_or_cross_namespace_move() { #[tokio::test] async fn interrupted_lifecycle_cannot_cleanup_a_source_recreated_before_recovery() { - for mode in [1, 3] { + for mode in [1] { let (baseline, request) = setup(mode).await; let start = baseline.fixture.store.writes.load(Ordering::SeqCst); TableLifecycles::new(baseline.fixture.store.clone()) @@ -490,7 +371,7 @@ async fn interrupted_lifecycle_cannot_cleanup_a_source_recreated_before_recovery #[tokio::test] async fn lifecycle_records_reject_metadata_mutation_foreign_keys_and_retired_contexts() { - let (test, request) = setup(3).await; + let (test, request) = setup(1).await; let manager = TableLifecycles::new(test.fixture.store.clone()); manager.execute(&request).await.unwrap(); let operation = manager @@ -504,7 +385,7 @@ async fn lifecycle_records_reject_metadata_mutation_foreign_keys_and_retired_con .encode() .is_err()); let mut corrupted = operation.clone(); - corrupted.purge_requested = true; + corrupted.candidate.namespace = crowdb_access_iceberg::key::NamespaceId::random(); assert!(StorageRecord::TableLifecycleOperation(Box::new(corrupted)) .encode() .is_err()); diff --git a/lib/crowdb-access-iceberg/tests/table_rename_legacy_test.rs b/lib/crowdb-access-iceberg/tests/table_rename_legacy_test.rs new file mode 100644 index 000000000..468651591 --- /dev/null +++ b/lib/crowdb-access-iceberg/tests/table_rename_legacy_test.rs @@ -0,0 +1,95 @@ +#[path = "common/file_blocks.rs"] +mod blocks; +#[path = "common/store.rs"] +mod common; +#[path = "common/table_creation.rs"] +#[allow(dead_code)] +mod creation; +#[path = "common/namespace.rs"] +#[allow(dead_code)] +mod fixture; +#[path = "common/namespace_recovery_store.rs"] +mod namespace_recovery_store; +#[path = "common/namespace_store.rs"] +mod namespace_store; +#[path = "common/table_rename.rs"] +#[allow(dead_code)] +mod rename; + +use crowdb_access_iceberg::{ + catalog::CatalogStore, + namespace::NamespaceRepository, + operation::PayloadStore, + record::StorageRecord, + table::{name_key, TableLifecycleAction, TableLifecycleOperation, TableLifecyclePhase, TableLifecycles}, +}; + +#[tokio::test] +async fn existing_rename_journals_remain_readable_and_can_finish() { + let (test, request, before) = rename::setup(true).await; + let TableLifecycleAction::Rename { namespace, name } = &request.action else { + panic!() + }; + let target = NamespaceRepository::new(test.fixture.store.clone()) + .load(request.context, namespace) + .await + .unwrap() + .unwrap(); + let key = name_key(before.catalog, before.namespace, &before.name).unwrap(); + let value = test + .fixture + .store + .get(&key.encode().unwrap()) + .await + .unwrap() + .unwrap(); + let StorageRecord::TableMapping(source) = StorageRecord::decode(&key, &value.bytes).unwrap() else { + panic!() + }; + let mut candidate = before.clone(); + candidate.namespace = target.namespace; + candidate.name.clone_from(name); + candidate.name_epoch += 1; + candidate.operation_fence += 1; + candidate.pending_operation = Some(request.identity.operation); + let input = serde_json::to_vec(&serde_json::json!({"namespace": request.namespace.components(), + "name": request.name, "action": {"rename": {"namespace": namespace.components(), "name": name}}})) + .unwrap(); + let input = PayloadStore::new(test.fixture.store.clone()) + .put(request.context.catalog, request.identity.operation, &input) + .await + .unwrap(); + let operation = TableLifecycleOperation { + context: request.context, + identity: request.identity, + principal: request.principal.clone(), + revision: 1, + phase: TableLifecyclePhase::Prepared, + input, + source, + before, + candidate, + destination_namespace: Some(namespace.clone()), + purge_requested: false, + admission: None, + outcome: None, + }; + test.fixture + .put( + operation.key(), + StorageRecord::TableLifecycleOperation(Box::new(operation)), + ) + .await; + let manager = TableLifecycles::new(test.fixture.store.clone()); + assert_eq!(manager.execute(&request).await.unwrap().status, 204); + assert_eq!(rename::visible(&test, &request).await, (false, true)); + assert_eq!( + manager + .load(request.context, request.identity.operation) + .await + .unwrap() + .unwrap() + .phase, + TableLifecyclePhase::Complete + ); +} diff --git a/lib/crowdb-access-iceberg/tests/table_rename_test.rs b/lib/crowdb-access-iceberg/tests/table_rename_test.rs new file mode 100644 index 000000000..92a56e51c --- /dev/null +++ b/lib/crowdb-access-iceberg/tests/table_rename_test.rs @@ -0,0 +1,255 @@ +#[path = "common/file_blocks.rs"] +mod blocks; +#[path = "common/store.rs"] +mod common; +#[path = "common/rename_conflict.rs"] +mod conflict; +#[path = "common/table_creation.rs"] +#[allow(dead_code)] +mod creation; +#[path = "common/namespace.rs"] +#[allow(dead_code)] +mod fixture; +#[path = "common/namespace_recovery_store.rs"] +mod namespace_recovery_store; +#[path = "common/namespace_store.rs"] +mod namespace_store; +#[path = "common/table_rename.rs"] +#[allow(dead_code)] +mod rename; + +use crowdb_access_iceberg::{ + catalog::CatalogError, + key::{IcebergKey, OperationId}, + namespace::{NamespaceAuthority, NamespaceLifecycle, NamespaceRepository}, + record::StorageRecord, + table::{TableLifecycleAction, TableLifecycles, TableRepository}, +}; +use std::sync::atomic::Ordering; + +#[tokio::test] +async fn repeated_namespace_conflicts_use_observed_values_without_gets() { + let (test, request, _) = rename::setup(false).await; + let store = std::sync::Arc::new(conflict::TestRenameConflicts::new(test.fixture.store.clone())); + assert_eq!( + TableLifecycles::new(store.clone()) + .execute(&request) + .await + .unwrap() + .status, + 204 + ); + assert_eq!(store.injected.load(Ordering::SeqCst), 4); + assert_no_occupancy(&test); + assert_eq!(rename::visible(&test, &request).await, (false, true)); +} + +fn assert_no_occupancy(test: &creation::TestCreation) { + for (key, value) in test.fixture.store.values.load().iter() { + let key = IcebergKey::decode(key).unwrap(); + if let StorageRecord::NamespaceAuthority(parent) = StorageRecord::decode(&key, &value.bytes).unwrap() + { + assert!(parent.rename.is_none()); + } + } +} + +#[tokio::test] +async fn rename_preserves_identity_and_creates_no_journal_or_response_records() { + for cross in [false, true] { + let (test, request, before) = rename::setup(cross).await; + let store = test.fixture.store.clone(); + let old_keys: std::collections::BTreeSet<_> = store.values.load().keys().cloned().collect(); + let writes = store.writes.load(Ordering::SeqCst); + let outcome = TableLifecycles::new(store.clone()) + .execute(&request) + .await + .unwrap(); + assert_eq!(outcome.status, 204); + assert!(outcome.body.is_empty()); + assert_eq!( + store.writes.load(Ordering::SeqCst) - writes, + if cross { 8 } else { 6 } + ); + assert_eq!(store.values.load().len(), old_keys.len()); + let new_keys: Vec<_> = store + .values + .load() + .keys() + .filter(|key| !old_keys.contains(*key)) + .cloned() + .collect(); + assert_eq!(new_keys.len(), 1, "only the destination name can be new"); + assert!(matches!( + IcebergKey::decode(&new_keys[0]).unwrap(), + IcebergKey::Catalog { + scope: crowdb_access_iceberg::key::CatalogScope::TableName, + .. + } + )); + let head = rename::head(&test, &before).await; + assert_eq!(head.table, before.table); + assert_eq!(head.table_uuid, before.table_uuid); + assert_eq!(head.generation, before.generation); + assert_eq!(head.metadata_location, before.metadata_location); + assert_eq!(head.metadata_digest, before.metadata_digest); + assert_eq!(head.name_epoch, before.name_epoch + 1); + assert_eq!(head.operation_fence, before.operation_fence + 1); + assert!(head.pending_operation.is_none()); + assert_no_occupancy(&test); + assert_eq!( + TableLifecycles::new(store.clone()) + .execute(&request) + .await + .unwrap() + .status, + 404 + ); + let mut recreated = test.request.clone(); + recreated.identity.operation = OperationId::random(); + test.creator().create(&recreated).await.unwrap(); + let tables = TableRepository::new(store.clone()); + let replacement = tables + .select(request.context, test.parent.namespace, "events") + .await + .unwrap() + .unwrap(); + assert!(matches!( + TableLifecycles::new(store.clone()).execute(&request).await, + Err(CatalogError::Conflict) + )); + assert_eq!( + tables + .select(request.context, test.parent.namespace, "events") + .await + .unwrap() + .unwrap() + .head + .table, + replacement.head.table + ); + assert_no_occupancy(&test); + } +} + +#[tokio::test] +async fn occupancy_codec_binds_both_parents_and_excludes_deletion_and_legacy_markers() { + let (test, request, before) = rename::setup(true).await; + let store = test.fixture.store.clone(); + store + .fail_after + .store(store.writes.load(Ordering::SeqCst) + 2, Ordering::SeqCst); + assert!(TableLifecycles::new(store.clone()) + .execute(&request) + .await + .is_err()); + store.fail_after.store(0, Ordering::SeqCst); + let parents: Vec = store + .values + .load() + .iter() + .filter_map(|(key, value)| { + match StorageRecord::decode(&IcebergKey::decode(key).unwrap(), &value.bytes).unwrap() { + StorageRecord::NamespaceAuthority(parent) if parent.rename.is_some() => Some(*parent), + _ => None, + } + }) + .collect(); + assert_eq!(parents.len(), 2); + assert_eq!(parents[0].rename, parents[1].rename); + for parent in parents { + let key = crowdb_access_iceberg::namespace::authority_key(parent.catalog, parent.namespace); + let record = StorageRecord::NamespaceAuthority(Box::new(parent.clone())); + assert_eq!( + StorageRecord::decode(&key, &record.encode().unwrap()).unwrap(), + record + ); + for mutation in 0..3 { + let mut bad = parent.clone(); + match mutation { + 0 => { + bad.lifecycle = NamespaceLifecycle::Dropping; + bad.pending_operation = Some(OperationId::random()); + } + 1 => bad.pending_operation = Some(OperationId::random()), + _ => { + bad.rename.as_mut().unwrap().source.catalog = + crowdb_access_iceberg::key::CatalogId::random(); + } + } + assert!(StorageRecord::NamespaceAuthority(Box::new(bad)).encode().is_err()); + } + } + rename::repair(&test).await; + assert_eq!(rename::head(&test, &before).await.name, before.name); + assert_no_occupancy(&test); +} + +#[tokio::test] +async fn opposing_moves_do_not_wait_on_each_other_or_lose_either_table() { + let (mut test, first, _) = rename::setup(true).await; + let TableLifecycleAction::Rename { namespace, .. } = &first.action else { + panic!() + }; + let mut create = test.request.clone(); + create.identity.operation = OperationId::random(); + create.namespace = namespace.clone(); + test.creator().create(&create).await.unwrap(); + std::sync::Arc::get_mut(&mut test.fixture.store) + .unwrap() + .namespace_update_barrier = Some(std::sync::Arc::new(tokio::sync::Barrier::new(2))); + let second = crowdb_access_iceberg::table::TableLifecycleRequest { + identity: crowdb_access_iceberg::operation::RequestIdentity { + operation: OperationId::random(), + issued_ms: 100, + }, + namespace: namespace.clone(), + action: TableLifecycleAction::Rename { + namespace: test.parent.identifier.clone(), + name: "other".into(), + }, + ..first.clone() + }; + let manager = TableLifecycles::new(test.fixture.store.clone()); + let (a, b) = tokio::time::timeout(std::time::Duration::from_secs(2), async { + tokio::join!( + Box::pin(manager.execute(&first)), + Box::pin(manager.execute(&second)) + ) + }) + .await + .unwrap(); + for result in [a, b] { + match result { + Ok(outcome) => assert_eq!(outcome.status, 204), + Err(CatalogError::Busy | CatalogError::Conflict) => (), + error => panic!("{error:?}"), + } + } + rename::repair(&test).await; + let tables = TableRepository::new(test.fixture.store.clone()); + let target = NamespaceRepository::new(test.fixture.store.clone()) + .load(first.context, namespace) + .await + .unwrap() + .unwrap(); + let a_old = tables + .select(first.context, test.parent.namespace, "events") + .await + .unwrap(); + let a_new = tables + .select(first.context, target.namespace, "renamed") + .await + .unwrap(); + let b_old = tables + .select(first.context, target.namespace, "events") + .await + .unwrap(); + let b_new = tables + .select(first.context, test.parent.namespace, "other") + .await + .unwrap(); + assert_ne!(a_old.is_some(), a_new.is_some()); + assert_ne!(b_old.is_some(), b_new.is_some()); + assert_no_occupancy(&test); +} diff --git a/lib/crowdb-protocol/src/fbs/iceberg.fbs b/lib/crowdb-protocol/src/fbs/iceberg.fbs index f5a4b2dc9..1b6853db1 100644 --- a/lib/crowdb-protocol/src/fbs/iceberg.fbs +++ b/lib/crowdb-protocol/src/fbs/iceberg.fbs @@ -93,6 +93,16 @@ table FBNamespaceAuthority { lifecycle:ubyte = 255; pending_operation:[ubyte] (required); properties:[FBNamespaceProperty] (required); + rename:FBNameMove; +} + +table FBNameMove { + source:FBTableMapping (required); + destination_namespace:[ubyte] (required); + destination_name:string (required); + operation:[ubyte] (required); + head_generation:ulong; + head_fence:ulong; } table FBNamespaceMapping { From a880c298faeebf5a66d333f7cf252aaf913be4a9 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 12:17:22 +0800 Subject: [PATCH 105/258] Reuse catalog ledger locations and conditional mutation observations --- doc/working/plan-access-simplification.md | 16 ++++++ .../src/catalog/recovery.rs | 49 +++++++------------ .../src/catalog/repository.rs | 41 ++++++++++------ .../tests/management_test.rs | 23 +++++++++ 4 files changed, 84 insertions(+), 45 deletions(-) diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 70de57807..c9d6f13f9 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -391,6 +391,14 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 - [ ] **P14.1 审查 root/context/初始化回读**：记录正常请求内重复 GET 和初始化写，复用已验证 immutable 事实，保留撤销和 catalog reset 的准入边界。凭据持久化不是 reader session。Files: Iceberg `catalog/{repository,storage}.rs`、Access `credentials.rs`、`credentials/ensure.rs`。 +- [ ] **P14.2 删除凭据 bootstrap 准备状态**：现状 `ensure_user` 先扫全部 credential，GET/创建 + `user-bootstrap/` 的加密候选，再 CAS credential，最后再全量 scan 取回 token。 + 准备记录不是认证来源，唯一用途是多个 bootstrap 共用随机 access key。改为 user 派生的 + 稳定 access-key 目标，secret 仍随机；在现有 credential scope 一次 create-CAS 保存最终 + 加密资源。冲突读取唯一该目标并核对 user/密文/启用状态，成功直接返回本次已知 token。 + 保留 legacy inventory 查找与显式 issue-user 的多 token 语义；旧准备记录不恢复/覆盖正式 + 凭据。Group-0 `put_cas` 当前只返回 revision，不能假装有 observed value；需单独补返回值 + 或保留一次确有必要的直接 GET，未知结果不能当作明确冲突。Files: Access credentials、S3 auth。 ## 其他性能任务 @@ -463,6 +471,14 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，P14.1a**：Catalog management install/audit 复用 ledger locator 的 absent/value， + authority publication 与退休 CAS 复用 observed，删除重复 GET。initialize 正常路径实测 + 19G + 7C（原 22G + 7C），重复已完成请求 2G + 0C（原 3G + 0C）；只是逻辑 store 成本， + 不是 routed TPS。没有改变 root/context、维护 grace 或旧 management 恢复/审计合同。 + management 12、admission 3、catalog contract 6 项、Iceberg all-target/all-feature clippy、 + fmt/diff 通过；包含并发初始化 100 轮、slot collision overflow 与各写入丢回复恢复。 + management journal 和凭据准备状态的必要性仍待各自审查，未据此宣称清理完成。 + - **2026-10-24，R3.1 / P9.2（rename 部分）**：新 rename 只在参与父 authority 中保存一个 `NamespaceRename`：原精确 mapping、目标 namespace/name、head generation/fence 和本次 reservation ID。父目录按稳定 ID 占用，不等待第二个；冲突直接使用 observed，四次真实 diff --git a/lib/crowdb-access-iceberg/src/catalog/recovery.rs b/lib/crowdb-access-iceberg/src/catalog/recovery.rs index b40433912..3131d917f 100644 --- a/lib/crowdb-access-iceberg/src/catalog/recovery.rs +++ b/lib/crowdb-access-iceberg/src/catalog/recovery.rs @@ -7,8 +7,8 @@ use crate::record::StorageRecord; use super::repository::{authority_key, decode_authority, elapsed_now, operation_bytes, root_key}; use super::{ - ActiveCatalogRecord, CatalogAuthority, CatalogContext, CatalogError, CatalogRepository, ClearTransition, - RootState, + ActiveCatalogRecord, CasOutcome, CatalogAuthority, CatalogContext, CatalogError, CatalogRepository, + ClearTransition, RootState, }; impl CatalogRepository { @@ -157,18 +157,13 @@ impl CatalogRepository { retired.lifecycle = super::CatalogLifecycle::Retired; let bytes = StorageRecord::Authority(retired).encode()?; let key = authority_key(transition.previous.catalog); - if !self - .cas(&key, Some(&operation.original_authority), &bytes) + match self + .exchange(&key, Some(&operation.original_authority), &bytes) .await? { - let actual = self - .store - .get(&key.encode()?) - .await? - .ok_or(ValidationError::Record)?; - if actual.bytes != bytes { - return Err(ValidationError::Record.into()); - } + CasOutcome::Applied(_) => (), + CasOutcome::Conflict(Some(actual)) if actual.bytes == bytes => (), + CasOutcome::Conflict(_) => return Err(ValidationError::Record.into()), } let mut published = operation.clone(); published.phase = ManagementPhase::Published; @@ -205,26 +200,19 @@ impl CatalogRepository { } else { None }; - if self.cas(&key, expected, &operation.result_authority).await? { - return Ok(()); + match self.exchange(&key, expected, &operation.result_authority).await? { + CasOutcome::Applied(_) => Ok(()), + CasOutcome::Conflict(Some(actual)) if actual.bytes == operation.result_authority => Ok(()), + CasOutcome::Conflict(_) => Err(ValidationError::Record.into()), } - let actual = self - .store - .get(&key.encode()?) - .await? - .ok_or(ValidationError::Record)?; - if actual.bytes != operation.result_authority { - return Err(ValidationError::Record.into()); - } - Ok(()) } async fn audit(&self, operation: &ManagementOperation, now_ms: u64) -> Result<(), CatalogError> { - let key = match ledger_locate(self.store.as_ref(), SystemScope::Audit, operation.id()).await? { - LedgerLocation::Existing(key, _) | LedgerLocation::Vacant(key) => key, + let (key, old) = match ledger_locate(self.store.as_ref(), SystemScope::Audit, operation.id()).await? { + LedgerLocation::Existing(key, value) => (key, Some(value)), + LedgerLocation::Vacant(key) => (key, None), }; let bytes = operation_bytes(operation)?; - let old = self.store.get(&key.encode()?).await?; if let Some(value) = &old { if value.bytes == bytes { return Ok(()); @@ -236,12 +224,13 @@ impl CatalogRepository { return Err(CatalogError::Busy); } } - if !self - .cas(&key, old.as_ref().map(|value| value.bytes.as_slice()), &bytes) + match self + .exchange(&key, old.as_ref().map(|value| value.bytes.as_slice()), &bytes) .await? { - return Err(CatalogError::Busy); + CasOutcome::Applied(_) => Ok(()), + CasOutcome::Conflict(Some(observed)) if observed.bytes == bytes => Ok(()), + CasOutcome::Conflict(_) => Err(CatalogError::Busy), } - Ok(()) } } diff --git a/lib/crowdb-access-iceberg/src/catalog/repository.rs b/lib/crowdb-access-iceberg/src/catalog/repository.rs index 72dcb8bbc..bd5364084 100644 --- a/lib/crowdb-access-iceberg/src/catalog/repository.rs +++ b/lib/crowdb-access-iceberg/src/catalog/repository.rs @@ -164,16 +164,25 @@ impl CatalogRepository { expected: Option<&[u8]>, value: &[u8], ) -> Result { - let key = key.encode()?; - let identity = mutation_identity(&key, expected, value); Ok(matches!( - self.store - .compare_exchange(&key, expected, value, identity) - .await?, + self.exchange(key, expected, value).await?, CasOutcome::Applied(_) )) } + pub(super) async fn exchange( + &self, + key: &IcebergKey, + expected: Option<&[u8]>, + value: &[u8], + ) -> Result { + let key = key.encode()?; + Ok(self + .store + .compare_exchange(&key, expected, value, mutation_identity(&key, expected, value)) + .await?) + } + pub(super) async fn phase( &self, operation: &ManagementOperation, @@ -309,18 +318,20 @@ impl CatalogRepository { else { return Ok(()); }; - let previous = self.store.get(&key.encode()?).await?; - if let Some(value) = &previous { - let StorageRecord::Management(old) = StorageRecord::decode(&key, &value.bytes)? else { - return Err(ValidationError::Record.into()); - }; - if old.id() == operation.id() { - return Ok(()); + match self.exchange(&key, None, &operation_bytes(operation)?).await? { + CasOutcome::Applied(_) => Ok(()), + CasOutcome::Conflict(Some(value)) => { + let StorageRecord::Management(existing) = StorageRecord::decode(&key, &value.bytes)? else { + return Err(ValidationError::Record.into()); + }; + if existing.id() == operation.id() { + Ok(()) + } else { + Err(CatalogError::Busy) + } } - return Err(CatalogError::Busy); + CasOutcome::Conflict(None) => Err(CatalogError::Busy), } - self.cas(&key, None, &operation_bytes(operation)?).await?; - Ok(()) } } diff --git a/lib/crowdb-access-iceberg/tests/management_test.rs b/lib/crowdb-access-iceberg/tests/management_test.rs index 3bdd06481..9f331672c 100644 --- a/lib/crowdb-access-iceberg/tests/management_test.rs +++ b/lib/crowdb-access-iceberg/tests/management_test.rs @@ -46,6 +46,29 @@ fn request(action: ManagementAction, epoch: u64, name: &str) -> ManagementReques } } +#[tokio::test] +async fn initialization_and_replay_do_not_reread_located_ledger_values() { + let store = Arc::new(TestStore::default()); + let catalog = repository(&store); + let initialize = request(ManagementAction::Initialize, 0, "cost"); + let result = catalog + .execute(initialize.clone(), ManagementPrivilege::Manage, 100) + .await + .unwrap(); + assert_eq!(store.reads.load(Ordering::SeqCst), 19); + assert_eq!(store.cas_attempts.load(Ordering::SeqCst), 7); + let reads = store.reads.load(Ordering::SeqCst); + assert_eq!( + catalog + .execute(initialize, ManagementPrivilege::Manage, 101) + .await + .unwrap(), + result + ); + assert_eq!(store.reads.load(Ordering::SeqCst) - reads, 2); + assert_eq!(store.cas_attempts.load(Ordering::SeqCst), 7); +} + #[tokio::test(start_paused = true)] async fn completed_clear_admits_one_retired_catalog_task() { let store = Arc::new(TestStore::default()); From 78119d90f15476f5014e22dc4b523cd1052abbf8 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 12:23:47 +0800 Subject: [PATCH 106/258] Create final bootstrap credentials without persisted preparation --- app/crowdb-access-server/src/credentials.rs | 6 +- .../src/credentials/ensure.rs | 131 +++++------------ .../tests/credential_authority_test.rs | 138 +++++++++++------- .../s3/design-crowdb-access-s3.md | 20 +++ doc/working/plan-access-simplification.md | 25 +++- lib/crowdb-access-s3/src/auth/secrets.rs | 23 ++- .../tests/credential_secret_test.rs | 19 +++ 7 files changed, 202 insertions(+), 160 deletions(-) diff --git a/app/crowdb-access-server/src/credentials.rs b/app/crowdb-access-server/src/credentials.rs index 493e9bb12..4e42c014a 100644 --- a/app/crowdb-access-server/src/credentials.rs +++ b/app/crowdb-access-server/src/credentials.rs @@ -34,8 +34,6 @@ pub enum CredentialAuthorityError { InvalidKey, #[error("S3 user has multiple credential records")] DuplicateUser, - #[error("S3 user credential preparation is malformed or belongs to another user")] - InvalidUserPreparation, #[error("S3 user credential is disabled or malformed")] InvalidUserCredential, } @@ -116,11 +114,11 @@ impl CredentialAuthority { if found.is_some() { return Err(CredentialAuthorityError::DuplicateUser); } - let credential = self.cipher.decrypt(user, access_key, &record.encrypted)?; + let mut credential = self.cipher.decrypt(user, access_key, &record.encrypted)?; if !credential.enabled { return Err(CredentialAuthorityError::InvalidUserCredential); } - let secret_key = String::from_utf8(credential.secret_key.clone()) + let secret_key = String::from_utf8(std::mem::take(&mut credential.secret_key)) .map_err(|_| CredentialAuthorityError::InvalidUserCredential)?; found = Some(IssuedUserToken { access_key_id: access_key.to_owned(), diff --git a/app/crowdb-access-server/src/credentials/ensure.rs b/app/crowdb-access-server/src/credentials/ensure.rs index 8d5929ac5..359f7e66c 100644 --- a/app/crowdb-access-server/src/credentials/ensure.rs +++ b/app/crowdb-access-server/src/credentials/ensure.rs @@ -1,122 +1,67 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! Concurrent bootstrap reserves one encrypted candidate before credential publication. +//! Bootstrap publishes one final encrypted credential without preparation state. -use super::{credential_key, CredentialAuthority, CredentialAuthorityError, CREATE_ATTEMPTS}; +use super::{credential_key, CredentialAuthority, CredentialAuthorityError}; use crowdb_access_s3::auth::{DurableCredentialRecord, IssuedUserToken}; use crowdb_kv_client::{Error as KvError, GetOutcome, ReadMode}; -use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -#[derive(Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct PreparedUser { - version: u32, - access_key_id: String, - record: Vec, -} - impl CredentialAuthority { - /// Reuse one credential across concurrent bootstrap callers and restarts. - /// The user-scoped preparation contains only the existing encrypted format; - /// publication never replaces another access key's durable record. - /// + /// Reuses an existing token or creates one final resource at a stable public + /// identifier. Concurrent bootstrap callers converge through create-if-absent; + /// each candidate secret is random. No upload/workflow recovery record exists. /// # Errors - /// Fails closed on duplicate/disabled users, malformed preparation, collisions - /// and uncertain Group-0 writes. A retry reconciles the same encrypted candidate. + /// Fails closed on disabled/duplicate users, conflicting user bindings, + /// malformed ciphertext and unavailable or uncertain Group-0 storage. pub async fn ensure_user(&self, user: &[u8]) -> Result { if let Some(token) = self.lookup_user(user).await? { return Ok(token); } - let prepared = self.prepare_user(user).await?; - let key = credential_key(&prepared.access_key_id); - match self.control.put_cas(0, 0, &key, &prepared.record, 0).await { - Ok(_) => {} - Err(KvError::CasFailed { .. } | KvError::CasBusy | KvError::OutcomeUnknown) => { - match self.control.get(0, 0, &key, ReadMode::Linearizable, None).await? { - GetOutcome::Found { value, .. } if value.as_ref() == prepared.record.as_slice() => {} - GetOutcome::Found { .. } => return Err(CredentialAuthorityError::Collision), - GetOutcome::NotFound => return Err(KvError::OutcomeUnknown.into()), - } - } - Err(error) => return Err(error.into()), - } - self.lookup_user(user) - .await? - .ok_or(CredentialAuthorityError::InvalidUserCredential) - } - - async fn prepare_user(&self, user: &[u8]) -> Result { - let key = format!("crowdb/s3/user-bootstrap/{}", hex::encode(Sha256::digest(user))); - let mut candidate = None; - for _ in 0..CREATE_ATTEMPTS { - match self - .control - .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) - .await? - { - GetOutcome::Found { value, .. } => { - if value.len() > 65536 { - return Err(CredentialAuthorityError::InvalidUserPreparation); - } - let prepared: PreparedUser = serde_json::from_slice(&value) - .map_err(|_| CredentialAuthorityError::InvalidUserPreparation)?; - self.validate_prepared(user, &prepared)?; - return Ok(prepared); - } - GetOutcome::NotFound => {} - } - let prepared = match candidate.as_ref() { - Some(prepared) => prepared, - None => candidate.insert(self.new_prepared(user)?), - }; - let value = - serde_json::to_vec(prepared).map_err(|_| CredentialAuthorityError::InvalidUserPreparation)?; - if value.len() > 65536 { - return Err(CredentialAuthorityError::InvalidUserPreparation); - } - match self.control.put_cas(0, 0, key.as_bytes(), &value, 0).await { - Ok(_) => return Ok(candidate.take().expect("candidate is present")), - Err(KvError::CasFailed { .. } | KvError::CasBusy | KvError::OutcomeUnknown) => {} - Err(error) => return Err(error.into()), - } - } - Err(KvError::OutcomeUnknown.into()) - } - - fn new_prepared(&self, user: &[u8]) -> Result { - let (issued, encrypted) = self.cipher.issue_user_token(user, 1)?; - let record = DurableCredentialRecord { + let mut digest = Sha256::new(); + digest.update(b"crowdb/s3/user-credential/v1\0"); + digest.update(user); + let access_key_id = format!("CROWU{}", hex::encode(digest.finalize())); + let (issued, encrypted) = self.cipher.issue_user_token_for_key(user, access_key_id, 1)?; + let key = credential_key(&issued.access_key_id); + let bytes = DurableCredentialRecord { user: user.to_vec(), encrypted, } .encode()?; - Ok(PreparedUser { - version: 1, - access_key_id: issued.access_key_id.clone(), - record, - }) + match self.control.put_cas(0, 0, &key, &bytes, 0).await { + Ok(_) => Ok(issued), + // Group-0 currently returns only the revision, not the observed value. + // This one authoritative GET is required; successful creation needs none. + Err(error @ (KvError::CasFailed { .. } | KvError::CasBusy | KvError::OutcomeUnknown)) => { + match self.control.get(0, 0, &key, ReadMode::Linearizable, None).await? { + GetOutcome::Found { value, .. } => self.token(user, &issued.access_key_id, &value), + GetOutcome::NotFound => Err(error.into()), + } + } + Err(error) => Err(error.into()), + } } - fn validate_prepared( + fn token( &self, user: &[u8], - prepared: &PreparedUser, - ) -> Result<(), CredentialAuthorityError> { - if prepared.version != 1 || prepared.access_key_id.is_empty() || prepared.access_key_id.len() > 80 { - return Err(CredentialAuthorityError::InvalidUserPreparation); - } - let record = DurableCredentialRecord::decode(&prepared.record)?; + access_key: &str, + value: &[u8], + ) -> Result { + let record = DurableCredentialRecord::decode(value)?; if record.user != user { - return Err(CredentialAuthorityError::InvalidUserPreparation); + return Err(CredentialAuthorityError::Collision); } - let credential = self - .cipher - .decrypt(user, &prepared.access_key_id, &record.encrypted)?; + let mut credential = self.cipher.decrypt(user, access_key, &record.encrypted)?; if !credential.enabled { return Err(CredentialAuthorityError::InvalidUserCredential); } - Ok(()) + Ok(IssuedUserToken { + access_key_id: access_key.to_owned(), + secret_key: String::from_utf8(std::mem::take(&mut credential.secret_key)) + .map_err(|_| CredentialAuthorityError::InvalidUserCredential)?, + }) } } diff --git a/app/crowdb-access-server/tests/credential_authority_test.rs b/app/crowdb-access-server/tests/credential_authority_test.rs index 45a71a49b..192a15aaa 100644 --- a/app/crowdb-access-server/tests/credential_authority_test.rs +++ b/app/crowdb-access-server/tests/credential_authority_test.rs @@ -11,6 +11,15 @@ const USER: &[u8] = b"console-root"; const PREPARED: &[u8] = b"crowdb/s3/user-bootstrap/"; const CREDENTIALS: &[u8] = b"crowdb/s3/credential/"; +fn legacy_key() -> Vec { + use sha2::Digest; + format!( + "crowdb/s3/user-bootstrap/{}", + hex::encode(sha2::Sha256::digest(USER)) + ) + .into_bytes() +} + #[tokio::test] async fn concurrent_bootstrap_callers_publish_one_encrypted_credential_and_reuse_it() { let test = TestCredentials::start().await; @@ -56,9 +65,8 @@ async fn concurrent_bootstrap_callers_publish_one_encrypted_credential_and_reuse ) .await .unwrap(); - assert_eq!(prepared.items.len(), 1); - let state: serde_json::Value = serde_json::from_slice(&prepared.items[0].1).unwrap(); - let encrypted: Vec = serde_json::from_value(state["record"].clone()).unwrap(); + assert!(prepared.items.is_empty()); + let encrypted = &durable.items[0].1; assert!(!encrypted .windows(tokens[0].secret_key.len()) .any(|bytes| bytes == tokens[0].secret_key.as_bytes())); @@ -71,31 +79,41 @@ async fn concurrent_bootstrap_callers_publish_one_encrypted_credential_and_reuse } #[tokio::test] -async fn interrupted_projection_resumes_the_original_encrypted_candidate_after_authority_restart() { +async fn deleted_final_credential_is_recreated_without_resurrecting_a_legacy_preparation() { let test = TestCredentials::start().await; let original = test.authority().ensure_user(USER).await.unwrap(); let key = format!("crowdb/s3/credential/{}", original.access_key_id); - let before = test + let GetOutcome::Found { value: before, .. } = test .control .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) .await - .unwrap(); - // Reproduce interruption after preparation but before its credential projection. + .unwrap() + else { + panic!() + }; + let legacy_key = legacy_key(); + let legacy = serde_json::to_vec(&serde_json::json!({"version": 1, + "access_key_id": original.access_key_id, "record": before.to_vec()})) + .unwrap(); + test.control.put(0, 0, &legacy_key, &legacy, None).await.unwrap(); test.control.delete(0, 0, key.as_bytes(), None).await.unwrap(); assert!(test.authority().lookup_user(USER).await.unwrap().is_none()); - let resumed = test.authority().ensure_user(USER).await.unwrap(); - assert_eq!(resumed.access_key_id, original.access_key_id); - assert_eq!(resumed.secret_key, original.secret_key); - match ( - before, - test.control - .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) - .await - .unwrap(), - ) { - (GetOutcome::Found { value: a, .. }, GetOutcome::Found { value: b, .. }) => assert_eq!(a, b), - _ => panic!("credential projection is absent"), - } + let replaced = test.authority().ensure_user(USER).await.unwrap(); + assert_eq!(replaced.access_key_id, original.access_key_id); + assert_ne!(replaced.secret_key, original.secret_key); + let GetOutcome::Found { value: after, .. } = test + .control + .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) + .await + .unwrap() + else { + panic!() + }; + assert_ne!(before, after); + assert_eq!( + test.authority().load_credentials().await.unwrap()[0].0, + replaced.access_key_id + ); } #[tokio::test] @@ -151,53 +169,67 @@ async fn legacy_credentials_are_preserved_and_existing_duplicates_are_never_repa } #[tokio::test] -async fn malformed_preparation_never_creates_or_replaces_a_user_credential() { +async fn malformed_legacy_process_records_do_not_authorize_or_block_credentials() { let test = TestCredentials::start().await; - let original = test.authority().ensure_user(USER).await.unwrap(); - test.control - .delete( - 0, - 0, - format!("crowdb/s3/credential/{}", original.access_key_id).as_bytes(), - None, - ) + let key = legacy_key(); + let bytes = b"not an encrypted credential"; + test.control.put(0, 0, &key, bytes, None).await.unwrap(); + assert!(test.authority().lookup_user(USER).await.unwrap().is_none()); + let issued = test.authority().ensure_user(USER).await.unwrap(); + assert_eq!( + test.authority() + .lookup_user(USER) + .await + .unwrap() + .unwrap() + .secret_key, + issued.secret_key + ); + assert_eq!(test.authority().load_credentials().await.unwrap().len(), 1); + let GetOutcome::Found { value, .. } = test + .control + .get(0, 0, &key, ReadMode::Linearizable, None) .await - .unwrap(); - let rows = test + .unwrap() + else { + panic!() + }; + assert_eq!(value.as_ref(), bytes); +} + +#[tokio::test] +async fn disabled_canonical_credentials_are_never_overwritten_by_bootstrap() { + let test = TestCredentials::start().await; + let issued = test.authority().ensure_user(USER).await.unwrap(); + let key = format!("crowdb/s3/credential/{}", issued.access_key_id); + let GetOutcome::Found { value, .. } = test .control - .scan( - 0, - 0, - PREPARED, - b"", - b"", - 100, - ReadMode::Linearizable, - None, - false, - None, - ) + .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) + .await + .unwrap() + else { + panic!() + }; + let mut record = crowdb_access_s3::auth::DurableCredentialRecord::decode(&value).unwrap(); + record.encrypted.enabled = false; + let disabled = record.encode().unwrap(); + test.control + .put(0, 0, key.as_bytes(), &disabled, None) .await .unwrap(); - let key = &rows.items[0].0; - let mut invalid: serde_json::Value = serde_json::from_slice(&rows.items[0].1).unwrap(); - invalid["version"] = serde_json::json!(2); - let bytes = serde_json::to_vec(&invalid).unwrap(); - test.control.put(0, 0, key, &bytes, None).await.unwrap(); assert!(matches!( test.authority().ensure_user(USER).await, - Err(CredentialAuthorityError::InvalidUserPreparation) + Err(CredentialAuthorityError::InvalidUserCredential) )); - assert!(test.authority().lookup_user(USER).await.unwrap().is_none()); let GetOutcome::Found { value, .. } = test .control - .get(0, 0, key, ReadMode::Linearizable, None) + .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) .await .unwrap() else { - panic!("preparation disappeared") + panic!() }; - assert_eq!(value.as_ref(), bytes.as_slice()); + assert_eq!(value.as_ref(), disabled); } #[tokio::test] diff --git a/doc/design/access-server/s3/design-crowdb-access-s3.md b/doc/design/access-server/s3/design-crowdb-access-s3.md index 2d057c6be..7e3636fb2 100644 --- a/doc/design/access-server/s3/design-crowdb-access-s3.md +++ b/doc/design/access-server/s3/design-crowdb-access-s3.md @@ -61,6 +61,26 @@ boundaries, not isolation between multiple accepted users on one listener. Presigned request expiry is enforced independently of the allowed future-clock skew; clock tolerance cannot extend an issued URL's lifetime. +User bootstrap reads the bounded final credential inventory to preserve existing +keys and reject already ambiguous or disabled users. If absent, it derives a +stable public access-key identifier from the user and conditionally creates one +final encrypted credential. The secret and encryption nonce remain random; +the public identifier is not an authentication secret. A collision cannot replace +another user: the losing caller reads the exact target and validates its user, +ciphertext and enabled state. Successful creation returns its known token without +another inventory scan. There is no persistent bootstrap preparation or response. +Group-0 CAS currently supplies only a conflicting revision, so that exceptional +path needs one authoritative GET. An absent value after an uncertain write remains +an error for the caller to retry. + +Explicit token issuance can still create multiple credentials for a user; +bootstrap does not provide an atomic uniqueness guarantee against that independent +administrative operation. Existing duplicates fail the inventory check. Unused +legacy preparation records are neither authentication data nor a source for +restoring a deleted or disabled final credential. Deleting the final credential +and invoking bootstrap creates a fresh random secret; ordinary lookup never +creates data. Credential snapshots continue to load only final encrypted records. + ## 3. HTTP data path PUT and multipart upload stream HTTP bodies into bounded CROWDB writers. GET diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index c9d6f13f9..540a81d89 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -391,14 +391,10 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 - [ ] **P14.1 审查 root/context/初始化回读**：记录正常请求内重复 GET 和初始化写，复用已验证 immutable 事实，保留撤销和 catalog reset 的准入边界。凭据持久化不是 reader session。Files: Iceberg `catalog/{repository,storage}.rs`、Access `credentials.rs`、`credentials/ensure.rs`。 -- [ ] **P14.2 删除凭据 bootstrap 准备状态**：现状 `ensure_user` 先扫全部 credential，GET/创建 - `user-bootstrap/` 的加密候选，再 CAS credential，最后再全量 scan 取回 token。 - 准备记录不是认证来源，唯一用途是多个 bootstrap 共用随机 access key。改为 user 派生的 - 稳定 access-key 目标，secret 仍随机；在现有 credential scope 一次 create-CAS 保存最终 - 加密资源。冲突读取唯一该目标并核对 user/密文/启用状态，成功直接返回本次已知 token。 - 保留 legacy inventory 查找与显式 issue-user 的多 token 语义；旧准备记录不恢复/覆盖正式 - 凭据。Group-0 `put_cas` 当前只返回 revision，不能假装有 observed value；需单独补返回值 - 或保留一次确有必要的直接 GET，未知结果不能当作明确冲突。Files: Access credentials、S3 auth。 +- [ ] **P14.3 补 Group-0 CAS observation**：`CrowdbKvClient::put_cas` 的 `CasFailed` 当前仅有 + current_revision；凭据 bootstrap loser 因此需要直接 GET，无法复用值。由比较时读取的 + 同一 revision/value 贯穿 group admission、RPC response、client error，禁止另读最新值 + 伪装成原 observation；兼容旧服务端与 unknown/busy。Files: KV group CAS、kv client wire/API、credentials。 ## 其他性能任务 @@ -471,6 +467,19 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，P14.2**：删除 PreparedUser、prepare_user、准备记录格式/验证和相应状态错误； + ensure-user 的稳定公开 key 由 user 域分离摘要确定，secret/nonce 仍随机。在原有 credential + scope 一次 create-CAS 保存正式加密资源，没有新索引或协调 key，成功直接返回已知 token。 + 初次正常路径由静态 2 scan + 1 prepare GET + 2 CAS 变为 1 scan + 1 CAS；保留一次 legacy + inventory 查找。Group-0 conflict/busy/unknown 只直接读唯一目标，校验 user/AEAD/启用状态， + absent 仍报原错误，不能假定成功；P14.3 追踪底层 observation。显式 issue-user 可签发 + 多 token，ensure-user 没有对该独立管理操作提供跨 key 原子唯一性，既有重复仍拒绝。 + 旧正式凭据保留，旧准备记录不再参与认证或恢复；实际旧 hash key 的损坏准备记录不阻塞 + 创建，删除正式资源后不会复活旧 secret，disabled 也不覆盖。token 解密结果移交 Vec， + 不额外 clone secret。真实 Group-0 16 并发调用选出同一 token、restart、legacy duplicate、 + 4097-row 不完整 inventory、disabled 共 6 项及 secret 5 项、两 crate all-target/all-feature + clippy、fmt/diff 通过。没有 routed 性能数字或 TPS 提升声明。 + - **2026-10-24，P14.1a**：Catalog management install/audit 复用 ledger locator 的 absent/value， authority publication 与退休 CAS 复用 observed，删除重复 GET。initialize 正常路径实测 19G + 7C（原 22G + 7C），重复已完成请求 2G + 0C（原 3G + 0C）；只是逻辑 store 成本， diff --git a/lib/crowdb-access-s3/src/auth/secrets.rs b/lib/crowdb-access-s3/src/auth/secrets.rs index 9ee8807f2..41b3bb448 100644 --- a/lib/crowdb-access-s3/src/auth/secrets.rs +++ b/lib/crowdb-access-s3/src/auth/secrets.rs @@ -143,10 +143,29 @@ impl CredentialCipher { generation: u64, ) -> Result<(IssuedUserToken, EncryptedCredentialRecord), SecretError> { let mut access_random = [0_u8; ACCESS_RANDOM_LENGTH]; - let mut secret = [0_u8; SECRET_LENGTH]; OsRng.fill_bytes(&mut access_random); - OsRng.fill_bytes(&mut secret); let access_key_id = format!("CROW{}", encode_hex(&access_random).to_ascii_uppercase()); + self.issue_user_token_for_key(user, access_key_id, generation) + } + + /// Creates a random secret bound to a caller-selected public key identifier. + /// The caller publishes the encrypted credential with create-if-absent. + /// # Errors + /// Rejects invalid public identifiers or failed encryption. + pub fn issue_user_token_for_key( + &self, + user: &[u8], + access_key_id: String, + generation: u64, + ) -> Result<(IssuedUserToken, EncryptedCredentialRecord), SecretError> { + if access_key_id.is_empty() + || access_key_id.len() > 80 + || !access_key_id.bytes().all(|byte| byte.is_ascii_alphanumeric()) + { + return Err(SecretError::InvalidRecord); + } + let mut secret = [0_u8; SECRET_LENGTH]; + OsRng.fill_bytes(&mut secret); let secret_key = URL_SAFE_NO_PAD.encode(secret); secret.zeroize(); let issued = IssuedUserToken { diff --git a/lib/crowdb-access-s3/tests/credential_secret_test.rs b/lib/crowdb-access-s3/tests/credential_secret_test.rs index c958dbaea..7af1749fb 100644 --- a/lib/crowdb-access-s3/tests/credential_secret_test.rs +++ b/lib/crowdb-access-s3/tests/credential_secret_test.rs @@ -63,3 +63,22 @@ fn durable_record_preserves_user_binding() { Err(SecretError::InvalidRecord) )); } + +#[test] +fn a_selected_public_key_still_gets_random_secrets_and_bound_ciphertext() { + let cipher = CredentialCipher::new(&MasterKey::from_hex(&"66".repeat(32)).unwrap()); + let key = format!("CROWU{}", "ab".repeat(32)); + let (first, encrypted) = cipher.issue_user_token_for_key(b"user", key.clone(), 1).unwrap(); + let (second, _) = cipher.issue_user_token_for_key(b"user", key.clone(), 1).unwrap(); + assert_eq!(first.access_key_id, second.access_key_id); + assert_ne!(first.secret_key, second.secret_key); + assert_eq!( + cipher.decrypt(b"user", &key, &encrypted).unwrap().secret_key, + first.secret_key.as_bytes() + ); + assert!(cipher.decrypt(b"other", &key, &encrypted).is_err()); + assert!(cipher.decrypt(b"user", "CROWOTHER", &encrypted).is_err()); + for invalid in [String::new(), "a".repeat(81), "a/b".into(), "a\0b".into()] { + assert!(cipher.issue_user_token_for_key(b"user", invalid, 1).is_err()); + } +} From f58a4bb936f78d75cf223dcbcfee21f41208abe2 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 12:55:16 +0800 Subject: [PATCH 107/258] Bound and rotate upload expiry cleanup without persisted cursors --- app/crowdb-access-server/src/dataset.rs | 39 +-- .../dataset/design-crowdb-access-dataset.md | 24 +- doc/working/plan-access-simplification.md | 20 +- lib/crowdb-access-dataset/Cargo.toml | 2 +- .../src/authority/access/sweep.rs | 14 + lib/crowdb-access-dataset/src/lib.rs | 4 +- lib/crowdb-access-dataset/src/upload.rs | 1 + .../src/upload/expiry.rs | 47 +--- .../src/upload/expiry/cursor.rs | 118 +++++++++ .../src/upload/expiry/discovery.rs | 61 +++++ .../src/upload/expiry/window.rs | 114 ++++++++ .../src/upload/ownership.rs | 115 +++++--- .../tests/common/expiry_payload.rs | 49 ++++ .../tests/common/snapshot_cleanup_stall.rs | 47 ++++ .../tests/deletion_grace_test.rs | 47 ++++ .../tests/upload_expiry_step_test.rs | 246 ++++++++++++++++++ .../tests/upload_registry_test.rs | 20 +- .../tests/upload_test.rs | 10 +- 18 files changed, 868 insertions(+), 110 deletions(-) create mode 100644 lib/crowdb-access-dataset/src/upload/expiry/cursor.rs create mode 100644 lib/crowdb-access-dataset/src/upload/expiry/discovery.rs create mode 100644 lib/crowdb-access-dataset/src/upload/expiry/window.rs create mode 100644 lib/crowdb-access-dataset/tests/common/expiry_payload.rs create mode 100644 lib/crowdb-access-dataset/tests/common/snapshot_cleanup_stall.rs create mode 100644 lib/crowdb-access-dataset/tests/upload_expiry_step_test.rs diff --git a/app/crowdb-access-server/src/dataset.rs b/app/crowdb-access-server/src/dataset.rs index 78d5f419b..81b9d3c42 100644 --- a/app/crowdb-access-server/src/dataset.rs +++ b/app/crowdb-access-server/src/dataset.rs @@ -11,7 +11,7 @@ use std::time::Duration; use crowdb_access_dataset::{ AuthorityError, ChunkKvDatasetStore, ChunkPayloadStorage, DatasetAuthority, DatasetReadService, - DatasetSnapshotRequest, DatasetUploads, UploadedChunkReader, + DatasetSnapshotRequest, DatasetUploads, UploadExpiryCursor, UploadedChunkReader, }; use crowdb_chunk_kv_client::{ ChunkKvClient, ChunkKvRpcTransport, ClientConfig as ChunkKvConfig, Group0ChunkKvRangeCatalogSource, @@ -229,21 +229,30 @@ fn start_cleanup( tokio::spawn(async move { let mut tick = tokio::time::interval(Duration::from_secs(15)); let mut after = None; - let mut upload_after = None; + let mut upload_cursor = UploadExpiryCursor::default(); + let mut next_upload = tokio::time::Instant::now(); loop { - tick.tick().await; - if let Err(error) = authority - .reclaim_due_page( - &uploads, - &mut after, - &crowdb_access_dataset::ReadCancellation::new(), - ) - .await - { - tracing::error!(%error, "Dataset snapshot cleanup failed; durable deletion marks retained for retry"); - } - if let Err(error) = uploads.expire(unix_seconds(), &mut upload_after).await { - tracing::error!(%error, "Dataset upload cleanup failed; retaining cleanup records for retry"); + tokio::select! { + _ = tick.tick() => { + if let Err(error) = authority.reclaim_due_page(&uploads, &mut after, + &crowdb_access_dataset::ReadCancellation::new()).await { + if matches!(error, crowdb_access_dataset::SnapshotReclaimError::Limit) { + tracing::debug!("Dataset snapshot cleanup yielded at its work deadline"); + } else { + tracing::error!(%error, "Dataset snapshot cleanup failed; durable deletion marks retained for retry"); + } + } + } + () = tokio::time::sleep_until(next_upload) => { + let result = uploads.expire_step(unix_seconds(), &mut upload_cursor).await; + let delay = if upload_cursor.has_more() { + if result.is_ok() { Duration::from_millis(50) } else { Duration::from_millis(500) } + } else { Duration::from_secs(15) }; + next_upload = tokio::time::Instant::now() + delay; + if let Err(error) = result { + tracing::error!(%error, "Dataset upload cleanup failed; retaining cleanup records for retry"); + } + } } } }) diff --git a/doc/design/access-server/dataset/design-crowdb-access-dataset.md b/doc/design/access-server/dataset/design-crowdb-access-dataset.md index 743fa423a..e19ea8590 100644 --- a/doc/design/access-server/dataset/design-crowdb-access-dataset.md +++ b/doc/design/access-server/dataset/design-crowdb-access-dataset.md @@ -233,6 +233,16 @@ digests cannot be concatenated into the whole-upload MD5. Expiry scans bounded pages of upload authorities directly. Each caller retains its scan cursor locally; restart repeats the scan and loses no cleanup obligation. +Discovery reads at most twenty keys / 1 MiB and seeks past each upload's child +subtree. A bounded local queue schedules uploads in turn; each cleanup window +scans at most 32 actual part records / 1 MiB and attempts at most 64 range +notifications. Discovery and cleanup each have a two-second deadline. Sparse +uploads do not perform a point GET for every declared but absent part. +Accepted range retirement permits deletion of that aborted upload's part record; +the upload cannot accept a new part or owner. Deferred or uncertain notification +retains the exact range. A full queue can discard a hint only after serving it, +leaving a discovery slot; removed part records ensure useful progress survives +hint eviction and restart. There is no persistent scan or queue record. Independent upload creation writes only its own authority. No shared registry tail, registration generation, registry fence or durable scan cursor participates in creation, part upload or owner release. Corrupt/failed entries are reported and @@ -387,12 +397,14 @@ session CAS that removes its ownership. A delayed claim for that operation then conflicts; other owners remain valid. Sessions retain at most 4096 retired owner IDs and report capacity explicitly. Ordinary `release` remains reversible. -Upload expiry scans a bounded page of upload authorities with a local continuation -cursor. Shared registry registration and removal are absent. Restart restores -sessions, accepted parts, progress and pending cleanup from the durable store; -restarting the scan cannot lose a cleanup obligation. Scan throughput and per-upload -cleanup work determine the delay beyond expiry; the grace deadline is not a -maximum reclamation-latency guarantee. +Upload expiry uses local continuation/queue hints and existing upload/part resource +facts. Shared registry registration and removal are absent. Snapshot deletion +sweeps also yield after a two-second page deadline, keeping durable deletion marks +and advancing past a stalled item. The listener schedules snapshot pages every +fifteen seconds and upload catch-up windows fifty milliseconds apart; failed +upload windows back off five hundred milliseconds while other work remains. +Idle discovery runs every fifteen seconds. Limits bound work per window rather +than promise a maximum reclamation latency or measured foreground p99. ### 10.4 Transport and compatibility diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 540a81d89..a150a5baa 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -380,12 +380,14 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ### P13 — GC / expiry / cleanup -- [ ] **P13.1 拆开有界发现与回收**：复核 15 秒/20 key 调度，避免一个最多 10,000-part 的 - upload 串行清理阻塞 snapshot sweep；分时间/字节/IO 预算公平推进。Files: Access Dataset `start_cleanup`、Dataset `upload/{expiry,ownership,reclaim}.rs`。 - [ ] **P13.2 审查 cleanup 持久化**：将 claim/phase/cursor/tombstone 分为可重做进度与不可丢的 义务/代际；即使重扫昂贵也先优化发现/预算；新增持久化信息必须 review，用途和额外前台写均写明。Files: Dataset `authority/access/`、Split/Composition cleanup、Iceberg `gc/`、S3 terminal cleanup。 - [ ] **P13.3 量化回收滞后**：稀疏过期、大 upload、多 cleaner、频繁重启、损坏记录下测 backlog、 前台 p99 和孤儿收敛；如需确定完成时限再讨论 D5。Files: 相关 cleanup 测试、负载工具。 +- [ ] **P13.4 Snapshot 回收复用 CAS observation**：`fence_reclaim` 与 `retire_snapshot_record` + 当前在每次 definite conflict 后重新 GET；移到循环外初读，再使用返回的 head/Snapshot， + 保留帮助已发布 head 的 commit evidence。前者只推进当前 head fence，不改变 latest/stable； + 后者只退休固定 Snapshot，不能把显式 expected-base 发布改成 rebase。Files: authority/access/reclaim.rs。 ### P14 — Catalog 与凭据 @@ -467,6 +469,20 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，P13.1**：删除旧整 upload expiry 入口；新的 `expire_step` 每次发现至多 + 20 key/1 MiB，跳过 upload 子树；每个清理窗口只扫描实际存在的至多 32 part/1 MiB， + 不逐个 GET 最多 10,000 个声明位置，通知预算至多 64 个范围。发现与清理分别限两秒； + timeout/unknown 保留尚未接受的精确范围。通知被接受后删除 Aborted upload 的 part 资源， + 该 upload 不能再接收 part/owner；没有增加 phase/progress/queue key。队列最多 32 项， + 满时仅丢弃已服务的本地 hint，始终给新发现留位；已删除 part 不会因 hint 丢失而反复扫描， + 永久 deferred 的旧任务不会挡住后来的小任务。扫描 hint 丢失从资源事实恢复。 + Snapshot due page 也有两秒总预算，超时行保留标记并让后续行推进。服务按独立到期时间 + 调度 Snapshot 15 秒、upload backlog 50 ms；失败且仍有工作时 500 ms，空闲发现 15 秒。 + 10,000 声明/33 实际 part、64 个 deferred 后新增小任务、队列容量、32-range part、超时、 + cursor loss、claimed/损坏行和 Snapshot slow IO 共 8 项通过；Dataset `--all-targets`、 + HTTP lifecycle observation、两 crate all-target/all-feature clippy、fmt/diff 通过。 + 这是预算和故障验证；未测真实 foreground p99/全量 backlog，P13.3 与 V4 仍未完成。 + - **2026-10-24，P14.2**：删除 PreparedUser、prepare_user、准备记录格式/验证和相应状态错误； ensure-user 的稳定公开 key 由 user 域分离摘要确定，secret/nonce 仍随机。在原有 credential scope 一次 create-CAS 保存正式加密资源，没有新索引或协调 key，成功直接返回已知 token。 diff --git a/lib/crowdb-access-dataset/Cargo.toml b/lib/crowdb-access-dataset/Cargo.toml index 9a1352587..10044e6ad 100644 --- a/lib/crowdb-access-dataset/Cargo.toml +++ b/lib/crowdb-access-dataset/Cargo.toml @@ -34,5 +34,5 @@ uuid = { version = "1", features = ["v4"] } [dev-dependencies] crowdb-access-dataset = { path = ".", features = ["test-util"] } -tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] } arc-swap = "1.9" diff --git a/lib/crowdb-access-dataset/src/authority/access/sweep.rs b/lib/crowdb-access-dataset/src/authority/access/sweep.rs index f9a85742f..8028668cb 100644 --- a/lib/crowdb-access-dataset/src/authority/access/sweep.rs +++ b/lib/crowdb-access-dataset/src/authority/access/sweep.rs @@ -20,6 +20,20 @@ impl DatasetAuthority { uploads: &DatasetUploads, after: &mut Option>, cancel: &ReadCancellation, + ) -> Result<(), SnapshotReclaimError> { + tokio::time::timeout( + Duration::from_secs(2), + self.reclaim_due_window(uploads, after, cancel), + ) + .await + .map_err(|_| SnapshotReclaimError::Limit)? + } + + async fn reclaim_due_window( + &self, + uploads: &DatasetUploads, + after: &mut Option>, + cancel: &ReadCancellation, ) -> Result<(), SnapshotReclaimError> { let page = self .store diff --git a/lib/crowdb-access-dataset/src/lib.rs b/lib/crowdb-access-dataset/src/lib.rs index 40c8d5569..02552cd6c 100644 --- a/lib/crowdb-access-dataset/src/lib.rs +++ b/lib/crowdb-access-dataset/src/lib.rs @@ -35,8 +35,8 @@ pub use payload::{ ChunkPayloadStorage, PayloadDurability, PayloadStorage, PayloadStream, UploadedChunkReader, }; pub use upload::{ - DatasetUploads, SnapshotUploadCleanup, UploadError, UploadPart, UploadReclaimError, UploadRecord, - UploadState, MAX_UPLOAD_PART_BYTES, + DatasetUploads, SnapshotUploadCleanup, UploadError, UploadExpiryCursor, UploadExpiryStep, UploadPart, + UploadReclaimError, UploadRecord, UploadState, MAX_UPLOAD_PART_BYTES, }; pub use upload::{UploadReadDescriptor, VerifiedUploadPart}; diff --git a/lib/crowdb-access-dataset/src/upload.rs b/lib/crowdb-access-dataset/src/upload.rs index 9f7f6df34..cd4302b70 100644 --- a/lib/crowdb-access-dataset/src/upload.rs +++ b/lib/crowdb-access-dataset/src/upload.rs @@ -4,6 +4,7 @@ //! Durable Dataset uploads with independent immutable part records. mod expiry; +pub use expiry::{UploadExpiryCursor, UploadExpiryStep}; mod index; mod ownership; mod parts; diff --git a/lib/crowdb-access-dataset/src/upload/expiry.rs b/lib/crowdb-access-dataset/src/upload/expiry.rs index 8b8a80c64..3e459bf17 100644 --- a/lib/crowdb-access-dataset/src/upload/expiry.rs +++ b/lib/crowdb-access-dataset/src/upload/expiry.rs @@ -3,52 +3,13 @@ use super::{decode_session, DatasetUploads, UploadError, UploadState}; use crate::{MetadataScan, MetadataScanBudget, ReadCancellation}; -use std::time::Duration; const PREFIX: &[u8] = b"DS\0U\0"; -impl DatasetUploads { - /// Scans one bounded page of upload records. The caller-owned cursor is an - /// optimization only; restarting the scan cannot lose a cleanup obligation. - /// # Errors - /// Reports corrupt records or cleanup/storage failures. An item failure - /// advances the cursor so unrelated uploads can progress before it is retried. - pub async fn expire(&self, now: u64, after: &mut Option>) -> Result<(), UploadError> { - let page = self - .store - .scan_page( - &MetadataScan { - prefix: PREFIX.to_vec(), - after: after.clone(), - budget: MetadataScanBudget { - items: 20, - bytes: 1024 * 1024, - timeout: Duration::from_secs(2), - }, - }, - &ReadCancellation::new(), - ) - .await?; - for (key, stored) in &page.entries { - *after = Some(key.clone()); - if !is_session_key(key)? { - continue; - } - let session = decode_session(&stored.bytes)?; - if session.cleanup_complete || !session.owners.is_empty() { - continue; - } - if session.state == UploadState::Aborted || now >= session.expires_at { - match self.abort(key).await { - Ok(_) | Err(UploadError::Conflict) => {} - Err(error) => return Err(error), - } - } - } - *after = page.after; - Ok(()) - } -} +mod cursor; +mod discovery; +mod window; +pub use cursor::{UploadExpiryCursor, UploadExpiryStep}; // Qualified identity = length-prefixed namespace segments, 0xff, name; // the upload ID follows it. Part and index records append six bytes. diff --git a/lib/crowdb-access-dataset/src/upload/expiry/cursor.rs b/lib/crowdb-access-dataset/src/upload/expiry/cursor.rs new file mode 100644 index 000000000..89fdac6ba --- /dev/null +++ b/lib/crowdb-access-dataset/src/upload/expiry/cursor.rs @@ -0,0 +1,118 @@ +use std::collections::VecDeque; + +use super::super::DatasetUploads; +use super::super::UploadError; +use std::time::Duration; + +const MAX_PENDING: usize = 32; +pub(super) const PART_READS: usize = 32; +pub(super) const RANGE_NOTIFICATIONS: usize = 64; + +/// Volatile scheduling hints only. Dropping this cursor restarts discovery; +/// upload authorities and exact part ranges remain the cleanup obligations. +#[derive(Debug, Default)] +pub struct UploadExpiryCursor { + pub(super) after: Option>, + pub(super) pending: VecDeque, +} + +#[derive(Debug)] +pub(super) struct ExpiredUpload { + pub location: Vec, + pub parts: Option, + pub after: Option>, + pub clear: bool, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct UploadExpiryStep { + pub scanned: usize, + pub part_reads: usize, + pub range_notifications: usize, + pub time_limited: bool, +} + +impl UploadExpiryCursor { + /// Locally scheduled uploads, not the complete storage-wide cleanup backlog. + #[must_use] + pub fn pending_uploads(&self) -> usize { + self.pending.len() + } + + #[must_use] + pub fn has_more(&self) -> bool { + self.after.is_some() || !self.pending.is_empty() + } + + pub(super) fn enqueue(&mut self, location: Vec) { + if !self.pending.iter().any(|item| item.location == location) { + self.pending.push_back(ExpiredUpload { + location, + parts: None, + after: None, + clear: true, + }); + } + } + + pub(super) fn full(&self) -> bool { + self.pending.len() >= MAX_PENDING + } + + fn requeue(&mut self, work: ExpiredUpload) { + // Always leave a discovery slot. Only a work item already served in this + // window may be evicted; accepted part retirements have removed their + // metadata, so even an evicted large upload makes progress after rescan. + if self.pending.len() < MAX_PENDING - 1 { + self.pending.push_back(work); + } + } +} + +impl DatasetUploads { + /// Discovers at most twenty keys and advances one round-robin upload by at + /// most 32 part reads / 64 range notification attempts. Discovery and cleanup + /// each have an independent two-second deadline. + /// No scan/progress state is persisted. Deferred ranges remain in their + /// resource records; a full cache may discard a served scheduling hint. + /// # Errors + /// Returns corrupt metadata and storage errors. Resource records retain every + /// range whose retirement has not been accepted. + pub async fn expire_step( + &self, + now: u64, + cursor: &mut UploadExpiryCursor, + ) -> Result { + // A bad discovery row must not starve work already queued behind it. + let discovered = tokio::time::timeout(Duration::from_secs(2), self.discover_expired(now, cursor)) + .await + .unwrap_or(Err(UploadError::Pending)); + let mut report = UploadExpiryStep { + scanned: *discovered.as_ref().unwrap_or(&0), + ..UploadExpiryStep::default() + }; + if let Some(mut work) = cursor.pending.pop_front() { + let mut ranges = RANGE_NOTIFICATIONS; + let result = tokio::time::timeout( + Duration::from_secs(2), + self.expire_window(&mut work, &mut report, &mut ranges), + ) + .await; + report.range_notifications = RANGE_NOTIFICATIONS - ranges; + match result { + Ok(Ok(true)) => (), + Ok(Ok(false)) => cursor.requeue(work), + Ok(Err(error)) => { + cursor.requeue(work); + return Err(error); + } + Err(_) => { + cursor.requeue(work); + report.time_limited = true; + } + } + } + discovered?; + Ok(report) + } +} diff --git a/lib/crowdb-access-dataset/src/upload/expiry/discovery.rs b/lib/crowdb-access-dataset/src/upload/expiry/discovery.rs new file mode 100644 index 000000000..a01d61708 --- /dev/null +++ b/lib/crowdb-access-dataset/src/upload/expiry/discovery.rs @@ -0,0 +1,61 @@ +use super::{ + is_session_key, DatasetUploads, MetadataScan, MetadataScanBudget, ReadCancellation, UploadError, + UploadExpiryCursor, UploadState, PREFIX, +}; +use std::time::Duration; + +impl DatasetUploads { + pub(super) async fn discover_expired( + &self, + now: u64, + cursor: &mut UploadExpiryCursor, + ) -> Result { + if cursor.full() { + return Ok(0); + } + let page = self + .store + .scan_page( + &MetadataScan { + prefix: PREFIX.to_vec(), + after: cursor.after.clone(), + budget: MetadataScanBudget { + items: 20, + bytes: 1024 * 1024, + timeout: Duration::from_secs(2), + }, + }, + &ReadCancellation::new(), + ) + .await?; + let scanned = page.entries.len(); + for (key, stored) in page.entries { + if cursor.full() { + return Ok(scanned); + } + if cursor.after.as_ref().is_some_and(|after| key <= *after) { + continue; + } + cursor.after = Some(key.clone()); + if !is_session_key(&key)? { + continue; + } + // Upload children append a zero separator; seek beyond that subtree + // instead of spending discovery pages on all of a large upload's parts. + let mut end = key.clone(); + end.push(0xff); + cursor.after = Some(end); + let session = super::decode_session(&stored.bytes)?; + if !session.cleanup_complete + && session.owners.is_empty() + && (session.state == UploadState::Aborted || now >= session.expires_at) + { + cursor.enqueue(key); + } + } + if page.after.is_none() { + cursor.after = None; + } + Ok(scanned) + } +} diff --git a/lib/crowdb-access-dataset/src/upload/expiry/window.rs b/lib/crowdb-access-dataset/src/upload/expiry/window.rs new file mode 100644 index 000000000..8a972e284 --- /dev/null +++ b/lib/crowdb-access-dataset/src/upload/expiry/window.rs @@ -0,0 +1,114 @@ +use super::super::ownership::PartCleanupMode; +use super::super::{decode_part, decode_session, part_count, part_key}; +use super::{ + cursor::{ExpiredUpload, PART_READS}, + DatasetUploads, MetadataScanBudget, ReadCancellation, UploadError, UploadExpiryStep, +}; +use crate::MetadataScan; +use std::time::Duration; + +impl DatasetUploads { + pub(super) async fn expire_window( + &self, + work: &mut ExpiredUpload, + report: &mut UploadExpiryStep, + ranges: &mut usize, + ) -> Result { + if work.parts.is_none() { + let Some(stored) = self + .store + .get_bounded(&work.location, budget(256 * 1024), &ReadCancellation::new()) + .await? + else { + return Ok(true); + }; + let record = match self.abort_record(&work.location, stored).await { + Ok((record, _)) => record, + Err(UploadError::Conflict | UploadError::NotFound) => return Ok(true), + Err(error) => return Err(error), + }; + if record.cleanup_complete { + return Ok(true); + } + work.parts = Some(part_count(&record)); + } + self.expire_parts(work, report, ranges).await + } + + async fn expire_parts( + &self, + work: &mut ExpiredUpload, + report: &mut UploadExpiryStep, + ranges: &mut usize, + ) -> Result { + let parts = work.parts.ok_or(UploadError::Integrity)?; + let mut prefix = work.location.clone(); + prefix.extend(b"\0P"); + let page = self + .store + .scan_page( + &MetadataScan { + prefix: prefix.clone(), + after: work.after.clone(), + budget: MetadataScanBudget { + items: PART_READS, + bytes: 1024 * 1024, + timeout: Duration::from_secs(2), + }, + }, + &ReadCancellation::new(), + ) + .await?; + report.part_reads += page.entries.len(); + if page.entries.is_empty() && page.after.is_some() { + return Err(UploadError::Integrity); + } + for (key, stored) in page.entries { + let part = decode_part(&stored.bytes)?; + if !key.starts_with(&prefix) + || part.part.index >= parts + || key != part_key(&work.location, part.part.index) + || work.after.as_ref().is_some_and(|after| key <= *after) + { + return Err(UploadError::Integrity); + } + if part.part.locations.len() > *ranges { + return Ok(false); + } + work.clear &= self + .cleanup_part(&key, part, ranges, PartCleanupMode::RemoveMetadata) + .await?; + work.after = Some(key); + } + if let Some(after) = page.after { + if !after.starts_with(&prefix) || work.after.as_ref().map_or(true, |last| after < *last) { + return Err(UploadError::Integrity); + } + work.after = Some(after); + return Ok(false); + } + if !work.clear { + work.after = None; + work.clear = true; + return Ok(false); + } + let Some(stored) = self + .store + .get_bounded(&work.location, budget(256 * 1024), &ReadCancellation::new()) + .await? + else { + return Ok(true); + }; + let record = decode_session(&stored.bytes)?; + self.finish_cleanup(&work.location, record, stored.bytes).await?; + Ok(true) + } +} + +fn budget(bytes: usize) -> MetadataScanBudget { + MetadataScanBudget { + items: 1, + bytes, + timeout: Duration::from_secs(2), + } +} diff --git a/lib/crowdb-access-dataset/src/upload/ownership.rs b/lib/crowdb-access-dataset/src/upload/ownership.rs index bce1a069d..283083c28 100644 --- a/lib/crowdb-access-dataset/src/upload/ownership.rs +++ b/lib/crowdb-access-dataset/src/upload/ownership.rs @@ -7,6 +7,11 @@ use super::{ }; use crate::{CasOutcome, OperationId}; +pub(super) enum PartCleanupMode { + RetainMetadata, + RemoveMetadata, +} + impl DatasetUploads { /// Pins a complete upload for one publication operation. /// # Errors @@ -80,7 +85,16 @@ impl DatasetUploads { /// # Errors /// Claimed data conflicts. Deferred cleanup notifications remain observable. pub async fn abort(&self, location: &[u8]) -> Result { - let mut stored = self.store.get(location).await?.ok_or(UploadError::NotFound)?; + let stored = self.store.get(location).await?.ok_or(UploadError::NotFound)?; + let (record, expected) = self.abort_record(location, stored).await?; + self.cleanup(location, record, expected).await + } + + pub(super) async fn abort_record( + &self, + location: &[u8], + mut stored: crate::StoredValue, + ) -> Result<(UploadRecord, Vec), UploadError> { for _ in 0..16 { let mut record = decode_session(&stored.bytes)?; if !record.owners.is_empty() { @@ -88,12 +102,13 @@ impl DatasetUploads { } if record.state != UploadState::Aborted { record.state = UploadState::Aborted; + let bytes = encode(&record)?; match self .store - .compare_exchange(location, Some(&stored.bytes), &encode(&record)?) + .compare_exchange(location, Some(&stored.bytes), &bytes) .await? { - CasOutcome::Applied(_) => {} + CasOutcome::Applied(_) => return Ok((record, bytes)), CasOutcome::Conflict(Some(observed)) => { stored = observed; continue; @@ -101,55 +116,78 @@ impl DatasetUploads { CasOutcome::Conflict(None) => return Err(UploadError::NotFound), } } - return self.cleanup(location, record).await; + return Ok((record, stored.bytes)); } Err(UploadError::Pending) } - async fn cleanup(&self, location: &[u8], mut record: UploadRecord) -> Result { + async fn cleanup( + &self, + location: &[u8], + mut record: UploadRecord, + expected: Vec, + ) -> Result { if record.cleanup_complete { record.parts = self.accepted_parts(location, &record).await?; return Ok(record); } let entries = self.entries(location, &record).await?; let mut complete = true; + let mut ranges = usize::MAX; for entry in entries { complete &= self - .cleanup_part(&part_key(location, entry.part.index), entry) + .cleanup_part( + &part_key(location, entry.part.index), + entry, + &mut ranges, + PartCleanupMode::RetainMetadata, + ) .await?; } if complete { - let mut expected = encode(&record)?; - for _ in 0..16 { - if record.state != UploadState::Aborted { - return Err(UploadError::Conflict); - } - if record.cleanup_complete { - break; - } - record.cleanup_complete = true; - match self - .store - .compare_exchange(location, Some(&expected), &encode(&record)?) - .await? - { - CasOutcome::Applied(_) => break, - CasOutcome::Conflict(Some(observed)) => { - record = decode_session(&observed.bytes)?; - expected = observed.bytes; - } - CasOutcome::Conflict(None) => return Err(UploadError::NotFound), - } - } - if !record.cleanup_complete { - return Err(UploadError::Pending); - } + record = self.finish_cleanup(location, record, expected).await?; } record.parts = self.accepted_parts(location, &record).await?; Ok(record) } - async fn cleanup_part(&self, key: &[u8], mut record: super::PartEntry) -> Result { + pub(super) async fn finish_cleanup( + &self, + location: &[u8], + mut record: UploadRecord, + mut expected: Vec, + ) -> Result { + for _ in 0..16 { + if record.state != UploadState::Aborted || !record.owners.is_empty() { + return Err(UploadError::Conflict); + } + if record.cleanup_complete { + return Ok(record); + } + record.cleanup_complete = true; + match self + .store + .compare_exchange(location, Some(&expected), &encode(&record)?) + .await? + { + CasOutcome::Applied(_) => return Ok(record), + CasOutcome::Conflict(Some(observed)) => { + record = decode_session(&observed.bytes)?; + expected = observed.bytes; + } + CasOutcome::Conflict(None) => return Err(UploadError::NotFound), + } + } + Err(UploadError::Pending) + } + + pub(super) async fn cleanup_part( + &self, + key: &[u8], + mut record: super::PartEntry, + ranges: &mut usize, + mode: PartCleanupMode, + ) -> Result { let mut expected = encode(&record)?; for _ in 0..16 { if record.state != PartState::Reclaimed { @@ -166,11 +204,24 @@ impl DatasetUploads { } } if record.part.locations.is_empty() { + if matches!(mode, PartCleanupMode::RemoveMetadata) { + self.store.delete(key).await?; + } return Ok(true); } + if *ranges < record.part.locations.len() { + return Ok(false); + } + *ranges -= record.part.locations.len(); if !self.payloads.discard(&record.part.locations).await? { return Ok(false); } + if matches!(mode, PartCleanupMode::RemoveMetadata) { + // The aborted upload cannot acquire new owners or accept another + // part. ChunkDB now owns the accepted range retirement obligation. + self.store.delete(key).await?; + return Ok(true); + } record.part.locations.clear(); match self .store diff --git a/lib/crowdb-access-dataset/tests/common/expiry_payload.rs b/lib/crowdb-access-dataset/tests/common/expiry_payload.rs new file mode 100644 index 000000000..17764021a --- /dev/null +++ b/lib/crowdb-access-dataset/tests/common/expiry_payload.rs @@ -0,0 +1,49 @@ +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_access_dataset::{PayloadDurability, PayloadStorage, UploadError}; +use crowdb_chunk_client::SmallWriteIntent; +use crowdb_protocol::chunkdb::rpc::Location; +use std::sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, +}; + +#[derive(Default)] +pub struct TestExpiryPayload { + inner: crate::upload::TestPayload, + pub deferred_through: AtomicU64, + pub attempted_ranges: AtomicU64, + pub stall: AtomicBool, +} + +#[async_trait] +impl PayloadStorage for TestExpiryPayload { + async fn write( + &self, + key: &[u8], + data: Bytes, + intent: Arc, + ) -> Result, UploadError> { + self.inner.write(key, data, intent).await + } + async fn read(&self, locations: &[Location]) -> Result { + self.inner.read(locations).await + } + async fn durability(&self, locations: &[Location]) -> Result { + self.inner.durability(locations).await + } + async fn discard(&self, locations: &[Location]) -> Result { + self.attempted_ranges + .fetch_add(locations.len() as u64, Ordering::SeqCst); + if self.stall.load(Ordering::SeqCst) { + futures::future::pending::<()>().await; + } + if locations + .iter() + .any(|location| location.chunk_id.unwrap().low <= self.deferred_through.load(Ordering::SeqCst)) + { + return Ok(false); + } + self.inner.discard(locations).await + } +} diff --git a/lib/crowdb-access-dataset/tests/common/snapshot_cleanup_stall.rs b/lib/crowdb-access-dataset/tests/common/snapshot_cleanup_stall.rs new file mode 100644 index 000000000..95437b12e --- /dev/null +++ b/lib/crowdb-access-dataset/tests/common/snapshot_cleanup_stall.rs @@ -0,0 +1,47 @@ +use async_trait::async_trait; +use crowdb_access_dataset::*; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; + +pub struct TestSnapshotCleanupStall { + pub inner: Arc, + pub stall: AtomicBool, +} + +#[async_trait] +impl DatasetStore for TestSnapshotCleanupStall { + async fn get(&self, key: &[u8]) -> Result, StoreError> { + if key.starts_with(b"DS\0S\0") && self.stall.swap(false, Ordering::SeqCst) { + futures::future::pending::<()>().await; + } + self.inner.get(key).await + } + async fn get_bounded( + &self, + key: &[u8], + budget: MetadataScanBudget, + cancel: &ReadCancellation, + ) -> Result, StoreError> { + self.inner.get_bounded(key, budget, cancel).await + } + async fn scan_page( + &self, + scan: &MetadataScan, + cancel: &ReadCancellation, + ) -> Result { + self.inner.scan_page(scan, cancel).await + } + async fn compare_exchange( + &self, + key: &[u8], + expected: Option<&[u8]>, + value: &[u8], + ) -> Result { + self.inner.compare_exchange(key, expected, value).await + } + async fn delete(&self, key: &[u8]) -> Result<(), StoreError> { + self.inner.delete(key).await + } +} diff --git a/lib/crowdb-access-dataset/tests/deletion_grace_test.rs b/lib/crowdb-access-dataset/tests/deletion_grace_test.rs index 2a6cebceb..5174a2d0d 100644 --- a/lib/crowdb-access-dataset/tests/deletion_grace_test.rs +++ b/lib/crowdb-access-dataset/tests/deletion_grace_test.rs @@ -7,12 +7,59 @@ mod access_store; mod deletion; #[path = "common/publication.rs"] mod publication; +#[path = "common/snapshot_cleanup_stall.rs"] +mod stall; #[path = "common/upload.rs"] #[allow(dead_code)] mod upload; use crowdb_access_dataset::*; use std::sync::{atomic::Ordering, Arc}; +#[tokio::test(start_paused = true)] +async fn a_slow_snapshot_yields_the_sweep_and_keeps_its_durable_cleanup_obligation() { + let fixture = deletion::TestDeletionFixture::new().await; + fixture + .authority + .begin_reclaim(&fixture.identity, fixture.snapshot) + .await + .unwrap(); + fixture.clock.store(1300, Ordering::Release); + let store = Arc::new(stall::TestSnapshotCleanupStall { + inner: fixture.store.clone(), + stall: std::sync::atomic::AtomicBool::new(true), + }); + let authority = DatasetAuthority::new(store.clone()).with_reclaim_clock_for_tests(fixture.clock.clone()); + let uploads = DatasetUploads::new(store, Arc::new(upload::TestPayload::default())); + let mut after = None; + assert!(matches!( + authority + .reclaim_due_page(&uploads, &mut after, &ReadCancellation::new()) + .await, + Err(SnapshotReclaimError::Limit) + )); + assert!( + after.is_some(), + "a timed-out row must yield to the next deletion mark" + ); + // Losing the local cursor does not lose the persistent retirement fact. + after = None; + for _ in 0..32 { + authority + .reclaim_due_page(&uploads, &mut after, &ReadCancellation::new()) + .await + .unwrap(); + if authority + .inspect_snapshot_lifecycle(&fixture.identity, fixture.snapshot, &ReadCancellation::new()) + .await + .unwrap() + .completed + { + return; + } + } + panic!("snapshot cleanup did not recover after the stalled request was canceled"); +} + #[tokio::test] async fn deletion_rejects_new_reads_and_keeps_the_grace_deadline_across_restarts() { let deletion::TestDeletionFixture { diff --git a/lib/crowdb-access-dataset/tests/upload_expiry_step_test.rs b/lib/crowdb-access-dataset/tests/upload_expiry_step_test.rs new file mode 100644 index 000000000..578532c7e --- /dev/null +++ b/lib/crowdb-access-dataset/tests/upload_expiry_step_test.rs @@ -0,0 +1,246 @@ +#[path = "common/expiry_payload.rs"] +mod payload; +#[path = "common/upload.rs"] +#[allow(dead_code)] +mod upload; + +use bytes::Bytes; +use crowdb_access_dataset::*; +use md5::{Digest, Md5}; +use std::{ + collections::BTreeSet, + sync::{atomic::Ordering, Arc}, +}; + +fn location(seed: u8) -> Vec { + DatasetUploads::location( + &DatasetIdentity::new(NamespacePath::root(), "expiry-step").unwrap(), + OperationId::from_bytes(&[seed; 16]).unwrap(), + ) + .unwrap() +} + +async fn keys(store: &upload::TestStore) -> BTreeSet> { + let page = store + .scan_page( + &MetadataScan { + prefix: b"DS\0".to_vec(), + after: None, + budget: MetadataScanBudget { + items: 1024, + bytes: 16 * 1024 * 1024, + timeout: std::time::Duration::from_secs(2), + }, + }, + &ReadCancellation::new(), + ) + .await + .unwrap(); + assert!(page.after.is_none()); + page.entries.into_iter().map(|(key, _)| key).collect() +} + +#[tokio::test] +async fn a_sparse_ten_thousand_part_upload_and_deferred_notifications_do_not_starve_a_small_one() { + let store = Arc::new(upload::TestStore::default()); + let payload = Arc::new(payload::TestExpiryPayload::default()); + let uploads = DatasetUploads::new(store.clone(), payload.clone()); + let large = location(1); + let small = location(2); + uploads.create(&large, 10_000, 1, [0; 16], 10).await.unwrap(); + for index in 0..33 { + uploads + .put_part(&large, index, Bytes::from_static(b"x"), 1) + .await + .unwrap(); + } + uploads + .create(&small, 1, 1, Md5::digest(b"y").into(), 10) + .await + .unwrap(); + uploads + .put_part(&small, 0, Bytes::from_static(b"y"), 1) + .await + .unwrap(); + payload.deferred_through.store(33, Ordering::SeqCst); + let before = keys(&store).await; + let mut cursor = UploadExpiryCursor::default(); + for _ in 0..3 { + let attempts = payload.attempted_ranges.load(Ordering::SeqCst); + let step = uploads.expire_step(20, &mut cursor).await.unwrap(); + assert!(step.scanned <= 20 && step.part_reads <= 32 && step.range_notifications <= 64); + assert_eq!( + payload.attempted_ranges.load(Ordering::SeqCst) - attempts, + step.range_notifications as u64 + ); + assert!(cursor.pending_uploads() <= 32); + } + assert!(uploads.metadata(&small).await.unwrap().cleanup_complete); + assert!(!uploads.metadata(&large).await.unwrap().cleanup_complete); + assert!(cursor.pending_uploads() != 0); + payload.deferred_through.store(0, Ordering::SeqCst); + // A new Access instance discards every in-memory scheduling hint. + let uploads = DatasetUploads::new(store.clone(), payload); + let mut cursor = UploadExpiryCursor::default(); + for _ in 0..320 { + uploads.expire_step(20, &mut cursor).await.unwrap(); + if uploads.metadata(&large).await.unwrap().cleanup_complete { + break; + } + } + assert!(uploads.metadata(&large).await.unwrap().cleanup_complete); + assert!( + keys(&store).await.is_subset(&before), + "GC scheduling must not create progress records" + ); +} + +#[tokio::test(start_paused = true)] +async fn timed_out_notifications_retain_exact_ranges_and_recover_after_cursor_loss() { + let store = Arc::new(upload::TestStore::default()); + let payload = Arc::new(payload::TestExpiryPayload::default()); + let uploads = DatasetUploads::new(store.clone(), payload.clone()); + let key = location(3); + uploads.create(&key, 1, 1, [0; 16], 10).await.unwrap(); + uploads + .put_part(&key, 0, Bytes::from_static(b"x"), 1) + .await + .unwrap(); + payload.stall.store(true, Ordering::SeqCst); + let mut cursor = UploadExpiryCursor::default(); + let step = uploads.expire_step(20, &mut cursor).await.unwrap(); + assert!(step.time_limited); + assert_eq!(step.range_notifications, 1); + assert_eq!(step.part_reads, 1); + assert_eq!(cursor.pending_uploads(), 1); + assert!(!uploads.metadata(&key).await.unwrap().cleanup_complete); + payload.stall.store(false, Ordering::SeqCst); + let mut restarted = UploadExpiryCursor::default(); + uploads.expire_step(20, &mut restarted).await.unwrap(); + assert!(uploads.metadata(&key).await.unwrap().cleanup_complete); + assert_eq!(payload.attempted_ranges.load(Ordering::SeqCst), 2); +} + +#[tokio::test] +async fn claimed_resources_and_corrupt_discovery_rows_do_not_hide_later_expired_uploads() { + let store = Arc::new(upload::TestStore::default()); + let uploads = DatasetUploads::new(store.clone(), Arc::new(payload::TestExpiryPayload::default())); + let held = location(4); + uploads + .create(&held, 1, 1, Md5::digest(b"x").into(), 10) + .await + .unwrap(); + uploads + .put_part(&held, 0, Bytes::from_static(b"x"), 1) + .await + .unwrap(); + uploads.complete(&held, 2).await.unwrap(); + uploads.claim(&held, OperationId::random(), 3).await.unwrap(); + let corrupt = location(5); + store + .compare_exchange(&corrupt, None, b"invalid metadata") + .await + .unwrap(); + let expired = location(6); + uploads.create(&expired, 1, 1, [0; 16], 10).await.unwrap(); + let mut cursor = UploadExpiryCursor::default(); + assert!(matches!( + uploads.expire_step(20, &mut cursor).await, + Err(UploadError::Integrity) + )); + uploads.expire_step(20, &mut cursor).await.unwrap(); + assert!(uploads.metadata(&expired).await.unwrap().cleanup_complete); + assert_eq!(uploads.metadata(&held).await.unwrap().state, UploadState::Claimed); +} + +#[tokio::test] +async fn multi_extent_parts_respect_the_notification_budget_before_completion() { + let store = Arc::new(upload::TestStore::default()); + let payload = Arc::new(payload::TestExpiryPayload::default()); + let uploads = DatasetUploads::new(store.clone(), payload.clone()); + let location = location(7); + uploads.create(&location, 96, 32, [0; 16], 10).await.unwrap(); + for index in 0_u32..3 { + let mut key = location.clone(); + key.extend(b"\0P"); + key.extend(index.to_be_bytes()); + let locations: Vec<_> = (0..32) + .map(|offset| crowdb_protocol::chunkdb::rpc::Location { + chunk_id: Some(crowdb_protocol::common::ChunkId { + high: 1, + low: 1000 + u64::from(index) * 32 + offset, + }), + offset: 0, + length: 4096, + logical_offset: offset, + logical_length: 1, + }) + .collect(); + let record = serde_json::json!({"version":1,"owner":OperationId::random(),"deadline":10, + "state":"Accepted","part":{"index":index,"length":32,"md5":([0_u8;16]),"locations":locations}}); + store + .compare_exchange(&key, None, &serde_json::to_vec(&record).unwrap()) + .await + .unwrap(); + } + let mut cursor = UploadExpiryCursor::default(); + let step = uploads.expire_step(20, &mut cursor).await.unwrap(); + assert_eq!(step.range_notifications, 64); + assert_eq!(step.part_reads, 3); + assert_eq!(payload.attempted_ranges.load(Ordering::SeqCst), 64); + assert!(!uploads.metadata(&location).await.unwrap().cleanup_complete); + let step = uploads.expire_step(20, &mut cursor).await.unwrap(); + assert_eq!(step.range_notifications, 32); + assert!(uploads.metadata(&location).await.unwrap().cleanup_complete); +} + +#[tokio::test] +async fn queue_capacity_does_not_drop_obligations_when_more_than_thirty_two_uploads_are_expired() { + let store = Arc::new(upload::TestStore::default()); + let payload = Arc::new(payload::TestExpiryPayload::default()); + let uploads = DatasetUploads::new(store, payload.clone()); + for seed in 1..=64 { + let key = location(seed); + uploads.create(&key, 1, 1, [0; 16], 10).await.unwrap(); + uploads + .put_part(&key, 0, Bytes::from_static(b"x"), 1) + .await + .unwrap(); + } + payload.deferred_through.store(64, Ordering::SeqCst); + let mut cursor = UploadExpiryCursor::default(); + let mut high_watermark = 0; + for _ in 0..20 { + uploads.expire_step(20, &mut cursor).await.unwrap(); + high_watermark = high_watermark.max(cursor.pending_uploads()); + assert!(cursor.pending_uploads() <= 32); + } + assert_eq!(high_watermark, 31); + let later = location(65); + uploads.create(&later, 1, 1, [0; 16], 10).await.unwrap(); + uploads + .put_part(&later, 0, Bytes::from_static(b"y"), 1) + .await + .unwrap(); + for _ in 0..100 { + uploads.expire_step(20, &mut cursor).await.unwrap(); + if uploads.metadata(&later).await.unwrap().cleanup_complete { + break; + } + } + assert!( + uploads.metadata(&later).await.unwrap().cleanup_complete, + "a full cache of deferred ranges must leave room for new discovery" + ); + assert!(!uploads.metadata(&location(1)).await.unwrap().cleanup_complete); + payload.deferred_through.store(0, Ordering::SeqCst); + for _ in 0..200 { + uploads.expire_step(20, &mut cursor).await.unwrap(); + if !cursor.has_more() { + break; + } + } + for seed in 1..=64 { + assert!(uploads.metadata(&location(seed)).await.unwrap().cleanup_complete); + } +} diff --git a/lib/crowdb-access-dataset/tests/upload_registry_test.rs b/lib/crowdb-access-dataset/tests/upload_registry_test.rs index a9553bb93..1e95a09af 100644 --- a/lib/crowdb-access-dataset/tests/upload_registry_test.rs +++ b/lib/crowdb-access-dataset/tests/upload_registry_test.rs @@ -40,10 +40,10 @@ async fn independent_uploads_need_one_create_write_and_no_shared_registry() { let owner = OperationId::random(); uploads.claim(&keys[0], owner, 3).await.unwrap(); let restarted = DatasetUploads::new(store, payloads); - let mut after = None; + let mut cursor = UploadExpiryCursor::default(); loop { - restarted.expire(20, &mut after).await.unwrap(); - if after.is_none() { + restarted.expire_step(20, &mut cursor).await.unwrap(); + if !cursor.has_more() { break; } } @@ -53,8 +53,8 @@ async fn independent_uploads_need_one_create_write_and_no_shared_registry() { } restarted.release(&keys[0], owner).await.unwrap(); loop { - restarted.expire(20, &mut after).await.unwrap(); - if after.is_none() { + restarted.expire_step(20, &mut cursor).await.unwrap(); + if !cursor.has_more() { break; } } @@ -80,11 +80,17 @@ async fn legacy_registry_fields_are_unnecessary_for_expiry_after_restart() { .await .unwrap(); let restarted = DatasetUploads::new(store.clone(), payloads); - restarted.expire(20, &mut None).await.unwrap(); + restarted + .expire_step(20, &mut crowdb_access_dataset::UploadExpiryCursor::default()) + .await + .unwrap(); assert!(restarted.get(&location).await.unwrap().cleanup_complete); let writes = store.writes.load(Ordering::Relaxed); restarted.abort(&location).await.unwrap(); - restarted.expire(20, &mut None).await.unwrap(); + restarted + .expire_step(20, &mut crowdb_access_dataset::UploadExpiryCursor::default()) + .await + .unwrap(); assert_eq!(store.writes.load(Ordering::Relaxed), writes); } diff --git a/lib/crowdb-access-dataset/tests/upload_test.rs b/lib/crowdb-access-dataset/tests/upload_test.rs index a136f0d5a..0874e0f4e 100644 --- a/lib/crowdb-access-dataset/tests/upload_test.rs +++ b/lib/crowdb-access-dataset/tests/upload_test.rs @@ -109,7 +109,10 @@ async fn independent_owners_and_deferred_cleanup_survive_restart() { assert!(!aborted.cleanup_complete); let restarted = DatasetUploads::new(store, payloads.clone()); payloads.defer_reclaim.store(false, Ordering::Release); - restarted.expire(20, &mut None).await.unwrap(); + restarted + .expire_step(20, &mut crowdb_access_dataset::UploadExpiryCursor::default()) + .await + .unwrap(); assert!(restarted.get(&key).await.unwrap().cleanup_complete); } #[tokio::test] @@ -129,7 +132,10 @@ async fn invalid_length_digest_and_expiry_do_not_publish() { uploads.complete(&key, 2).await, Err(UploadError::Integrity) )); - uploads.expire(11, &mut None).await.unwrap(); + uploads + .expire_step(11, &mut crowdb_access_dataset::UploadExpiryCursor::default()) + .await + .unwrap(); assert_eq!(uploads.get(&key).await.unwrap().state, UploadState::Aborted); } From 8b2aa5ed2a3f19cd99dcd55956edf4dddb9cae0f Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 13:02:20 +0800 Subject: [PATCH 108/258] Retry snapshot reclamation with returned comparison observations --- doc/working/plan-access-simplification.md | 12 +- .../src/authority/access/reclaim.rs | 53 ++++---- .../tests/common/reclaim_observed.rs | 127 ++++++++++++++++++ .../tests/reclaim_observed_test.rs | 97 +++++++++++++ 4 files changed, 260 insertions(+), 29 deletions(-) create mode 100644 lib/crowdb-access-dataset/tests/common/reclaim_observed.rs create mode 100644 lib/crowdb-access-dataset/tests/reclaim_observed_test.rs diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index a150a5baa..07790ef36 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -384,10 +384,6 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 义务/代际；即使重扫昂贵也先优化发现/预算；新增持久化信息必须 review，用途和额外前台写均写明。Files: Dataset `authority/access/`、Split/Composition cleanup、Iceberg `gc/`、S3 terminal cleanup。 - [ ] **P13.3 量化回收滞后**：稀疏过期、大 upload、多 cleaner、频繁重启、损坏记录下测 backlog、 前台 p99 和孤儿收敛；如需确定完成时限再讨论 D5。Files: 相关 cleanup 测试、负载工具。 -- [ ] **P13.4 Snapshot 回收复用 CAS observation**：`fence_reclaim` 与 `retire_snapshot_record` - 当前在每次 definite conflict 后重新 GET；移到循环外初读，再使用返回的 head/Snapshot， - 保留帮助已发布 head 的 commit evidence。前者只推进当前 head fence，不改变 latest/stable； - 后者只退休固定 Snapshot，不能把显式 expected-base 发布改成 rebase。Files: authority/access/reclaim.rs。 ### P14 — Catalog 与凭据 @@ -469,6 +465,14 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，P13.4**：Snapshot 回收 head fence 与固定 Snapshot 退休都只初读一次， + CAS conflict 直接接 observed，不再循环 GET。四次实际 head nonce 竞争及一次竞争退休 + 中禁止读取对应 key，仍成功；合计省去五次逻辑 GET。fence 不改变 latest/stable，固定 + Snapshot 不换目标；head 的未收尾 commit evidence 先保留，竞争后的新 nonce 仍有该证明。 + missing 与 unknown 不额外查询或谎称成功；未知结果可由下一次独立请求恢复。 + 新增 observed 两项、candidate publication 六项、reclaim race 一项、deletion grace 三项、 + branch reclaim 三项及 Dataset all-target/all-feature clippy、fmt/diff 通过。 + - **2026-10-24，P13.1**：删除旧整 upload expiry 入口；新的 `expire_step` 每次发现至多 20 key/1 MiB，跳过 upload 子树；每个清理窗口只扫描实际存在的至多 32 part/1 MiB， 不逐个 GET 最多 10,000 个声明位置，通知预算至多 64 个范围。发现与清理分别限两秒； diff --git a/lib/crowdb-access-dataset/src/authority/access/reclaim.rs b/lib/crowdb-access-dataset/src/authority/access/reclaim.rs index 2b643647a..10993c231 100644 --- a/lib/crowdb-access-dataset/src/authority/access/reclaim.rs +++ b/lib/crowdb-access-dataset/src/authority/access/reclaim.rs @@ -29,8 +29,8 @@ impl DatasetAuthority { self.fence_split_publication(identity, snapshot).await?; // Invalidate publishers that read the old head before checking the admission fence. let key = key::head(identity)?; + let mut old = self.store.get(&key).await?; for _ in 0..64 { - let old = self.store.get(&key).await?; if let Some(old) = &old { self.help_head_commit(identity, old).await?; } @@ -42,17 +42,17 @@ impl DatasetAuthority { latest: None, stable: None, }); - if matches!( - self.store - .compare_exchange( - &key, - old.as_ref().map(|v| v.bytes.as_slice()), - &encode_head(head)? - ) - .await?, - CasOutcome::Applied(_) - ) { - return Ok(Some(owner)); + match self + .store + .compare_exchange( + &key, + old.as_ref().map(|v| v.bytes.as_slice()), + &encode_head(head)?, + ) + .await? + { + CasOutcome::Applied(_) => return Ok(Some(owner)), + CasOutcome::Conflict(observed) => old = observed, } } Err(AuthorityError::HeadConflict) @@ -151,26 +151,29 @@ impl DatasetAuthority { snapshot: SnapshotId, ) -> Result { let key = key::snapshot(identity, snapshot)?; + let mut old = self + .store + .get(&key) + .await? + .ok_or(AuthorityError::SnapshotNotFound)?; for _ in 0..64 { - let old = self - .store - .get(&key) - .await? - .ok_or(AuthorityError::SnapshotNotFound)?; let mut record: crate::SnapshotRecord = decode(&old.bytes)?; if record.publication.state == crate::PublicationState::Aborted { self.mark_snapshot_retired(identity, snapshot).await?; return Ok(record); } record.publication.state = crate::PublicationState::Aborted; - if matches!( - self.store - .compare_exchange(&key, Some(&old.bytes), &crate::authority::encode(&record)?) - .await?, - CasOutcome::Applied(_) - ) { - self.mark_snapshot_retired(identity, snapshot).await?; - return Ok(record); + match self + .store + .compare_exchange(&key, Some(&old.bytes), &crate::authority::encode(&record)?) + .await? + { + CasOutcome::Applied(_) => { + self.mark_snapshot_retired(identity, snapshot).await?; + return Ok(record); + } + CasOutcome::Conflict(Some(observed)) => old = observed, + CasOutcome::Conflict(None) => return Err(AuthorityError::SnapshotNotFound), } } Err(AuthorityError::HeadConflict) diff --git a/lib/crowdb-access-dataset/tests/common/reclaim_observed.rs b/lib/crowdb-access-dataset/tests/common/reclaim_observed.rs new file mode 100644 index 000000000..a834881a6 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/common/reclaim_observed.rs @@ -0,0 +1,127 @@ +use async_trait::async_trait; +use crowdb_access_dataset::*; +use std::sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, +}; + +#[derive(Clone, Copy)] +pub enum TestSnapshotReply { + Conflict, + Missing, + Unknown, +} + +pub struct TestReclaimObserved { + pub inner: Arc, + pub snapshot: Vec, + pub head: HeadRecord, + pub head_conflicts: AtomicUsize, + pub snapshot_conflicts: AtomicUsize, + pub reply: TestSnapshotReply, + head_retry: AtomicBool, + snapshot_retry: AtomicBool, +} + +impl TestReclaimObserved { + pub fn new( + inner: Arc, + snapshot: Vec, + head: HeadRecord, + reply: TestSnapshotReply, + ) -> Self { + Self { + inner, + snapshot, + head, + head_conflicts: AtomicUsize::new(4), + snapshot_conflicts: AtomicUsize::new(0), + reply, + head_retry: AtomicBool::new(false), + snapshot_retry: AtomicBool::new(false), + } + } + fn check_read(&self, key: &[u8]) { + assert!( + !(key.starts_with(b"DS\0H\0") && self.head_retry.load(Ordering::SeqCst)), + "head conflict must reuse the observed value" + ); + assert!( + !(key == self.snapshot && self.snapshot_retry.load(Ordering::SeqCst)), + "snapshot mutation must not issue a confirmation GET" + ); + } +} + +#[async_trait] +impl DatasetStore for TestReclaimObserved { + async fn get(&self, key: &[u8]) -> Result, StoreError> { + self.check_read(key); + self.inner.get(key).await + } + async fn get_bounded( + &self, + key: &[u8], + budget: MetadataScanBudget, + cancel: &ReadCancellation, + ) -> Result, StoreError> { + self.check_read(key); + self.inner.get_bounded(key, budget, cancel).await + } + async fn scan_page( + &self, + scan: &MetadataScan, + cancel: &ReadCancellation, + ) -> Result { + self.inner.scan_page(scan, cancel).await + } + async fn compare_exchange( + &self, + key: &[u8], + expected: Option<&[u8]>, + value: &[u8], + ) -> Result { + if key.starts_with(b"DS\0H\0") + && self + .head_conflicts + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) + .is_ok() + { + // Another reclaimer advances only the physical head nonce. + let bytes = bincode::serialize(&(2_u16, &self.head, OperationId::random())).unwrap(); + assert!(matches!( + self.inner.compare_exchange(key, expected, &bytes).await?, + CasOutcome::Applied(_) + )); + let observed = self.inner.get(key).await?; + self.head_retry.store(true, Ordering::SeqCst); + return Ok(CasOutcome::Conflict(observed)); + } + if key == self.snapshot { + self.snapshot_conflicts.fetch_add(1, Ordering::SeqCst); + self.snapshot_retry.store(true, Ordering::SeqCst); + return match self.reply { + TestSnapshotReply::Conflict => { + assert!(matches!( + self.inner.compare_exchange(key, expected, value).await?, + CasOutcome::Applied(_) + )); + Ok(CasOutcome::Conflict(self.inner.get(key).await?)) + } + TestSnapshotReply::Missing => { + self.inner.delete(key).await?; + Ok(CasOutcome::Conflict(None)) + } + TestSnapshotReply::Unknown => Err(StoreError::Rejected), + }; + } + let result = self.inner.compare_exchange(key, expected, value).await; + if key.starts_with(b"DS\0H\0") && matches!(result, Ok(CasOutcome::Applied(_))) { + self.head_retry.store(false, Ordering::SeqCst); + } + result + } + async fn delete(&self, key: &[u8]) -> Result<(), StoreError> { + self.inner.delete(key).await + } +} diff --git a/lib/crowdb-access-dataset/tests/reclaim_observed_test.rs b/lib/crowdb-access-dataset/tests/reclaim_observed_test.rs new file mode 100644 index 000000000..dacf86969 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/reclaim_observed_test.rs @@ -0,0 +1,97 @@ +#[path = "common/access_store.rs"] +mod access_store; +#[path = "common/deletion.rs"] +mod deletion; +#[path = "common/reclaim_observed.rs"] +mod observed; +#[path = "common/publication.rs"] +mod publication; + +use crowdb_access_dataset::*; +use std::sync::{atomic::Ordering, Arc}; + +fn scoped(prefix: &[u8], fixture: &deletion::TestDeletionFixture, id: &[u8]) -> Vec { + let mut key = prefix.to_vec(); + key.extend(fixture.identity.key().unwrap()); + key.extend(id); + key +} + +#[tokio::test] +async fn reclaim_reuses_four_head_observations_and_a_terminal_snapshot_without_confirmation_gets() { + let fixture = deletion::TestDeletionFixture::new().await; + assert!(!fixture.held.samples.is_empty()); + let head_key = scoped(b"DS\0H\0", &fixture, &[]); + let head_value = fixture.store.get(&head_key).await.unwrap().unwrap(); + let (version, head, _, attempt): (u16, HeadRecord, OperationId, CommitAttempt) = + bincode::deserialize(&head_value.bytes).unwrap(); + assert_eq!(version, 3); + let receipt = scoped(b"DS\0CR\0", &fixture, attempt.id.as_bytes()); + let snapshot_receipt = scoped(b"DS\0SC\0", &fixture, fixture.next.as_bytes()); + fixture.store.delete(&receipt).await.unwrap(); + fixture.store.delete(&snapshot_receipt).await.unwrap(); + let snapshot = scoped(b"DS\0S\0", &fixture, fixture.snapshot.as_bytes()); + let store = Arc::new(observed::TestReclaimObserved::new( + fixture.store.clone(), + snapshot, + head.clone(), + observed::TestSnapshotReply::Conflict, + )); + let authority = DatasetAuthority::new(store.clone()).with_reclaim_clock_for_tests(fixture.clock.clone()); + let retired = authority + .begin_reclaim(&fixture.identity, fixture.snapshot) + .await + .unwrap(); + assert_eq!(retired.publication.state, PublicationState::Aborted); + assert_eq!(store.head_conflicts.load(Ordering::SeqCst), 0); + assert_eq!(store.snapshot_conflicts.load(Ordering::SeqCst), 1); + let current = fixture.authority.inspect_head(&fixture.identity).await.unwrap(); + assert_eq!(current.latest, head.latest); + assert_eq!(current.stable, head.stable); + assert_ne!( + current.generation.unwrap(), + CommitGeneration::from_head(&head_value.bytes) + ); + assert!(fixture.store.get(&receipt).await.unwrap().is_some()); + assert!(fixture.store.get(&snapshot_receipt).await.unwrap().is_some()); +} + +#[tokio::test] +async fn missing_and_unknown_snapshot_cas_results_are_not_reclassified_by_another_get() { + for reply in [ + observed::TestSnapshotReply::Missing, + observed::TestSnapshotReply::Unknown, + ] { + let fixture = deletion::TestDeletionFixture::new().await; + let known = fixture.authority.inspect_head(&fixture.identity).await.unwrap(); + let snapshot = scoped(b"DS\0S\0", &fixture, fixture.snapshot.as_bytes()); + let store = Arc::new(observed::TestReclaimObserved::new( + fixture.store.clone(), + snapshot, + HeadRecord { + latest: known.latest, + stable: known.stable, + }, + reply, + )); + let authority = + DatasetAuthority::new(store.clone()).with_reclaim_clock_for_tests(fixture.clock.clone()); + let result = authority.begin_reclaim(&fixture.identity, fixture.snapshot).await; + assert!(result.is_err()); + if matches!(reply, observed::TestSnapshotReply::Missing) { + assert!(matches!(result, Err(AuthorityError::SnapshotNotFound))); + } else { + assert_eq!( + fixture + .authority + .begin_reclaim(&fixture.identity, fixture.snapshot) + .await + .unwrap() + .publication + .state, + PublicationState::Aborted + ); + } + assert_eq!(store.snapshot_conflicts.load(Ordering::SeqCst), 1); + } +} From 208ebc00556c333fe5ca0e5037042653bbe0db31 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 13:14:31 +0800 Subject: [PATCH 109/258] Avoid repeated legacy shuffle hashing and full window copies --- .../dataset/design-crowdb-access-dataset.md | 19 +- doc/working/plan-access-simplification.md | 10 + lib/crowdb-access-dataset/src/authority.rs | 17 +- lib/crowdb-access-dataset/src/planner.rs | 103 +++++--- lib/crowdb-access-dataset/src/shuffle.rs | 65 ++--- lib/crowdb-access-dataset/src/transport.rs | 32 ++- .../tests/legacy_planning_test.rs | 226 ++++++++++++++++++ 7 files changed, 373 insertions(+), 99 deletions(-) create mode 100644 lib/crowdb-access-dataset/tests/legacy_planning_test.rs diff --git a/doc/design/access-server/dataset/design-crowdb-access-dataset.md b/doc/design/access-server/dataset/design-crowdb-access-dataset.md index e19ea8590..d6d71c781 100644 --- a/doc/design/access-server/dataset/design-crowdb-access-dataset.md +++ b/doc/design/access-server/dataset/design-crowdb-access-dataset.md @@ -313,12 +313,19 @@ Partial data and failed streams provide no proof. Multiple parts still require the independent whole-field digest, and inline fields remain checked directly. The internal legacy scan adapter also returns a caller-owned cursor without loading or persisting progress. Requests have no shared inactivity TTL. - -Binding reuses its committed-Snapshot admission. Inspection-backed selection, -preview and nonempty batch delivery perform admission inside their bounded -resolver; wrappers do not repeat the same deletion lookup. Empty and legacy -batch entry points retain independent admission. Selection and delivery remain -separate bounded operations, each with its own admission. +Legacy scan planning retains one flat ordered ID vector, extracts only the +current window by moving its IDs, and releases the remaining vector before +payload delivery. Batch adapters move IDs into windows. Selection and budget +accounting share one pass; sample shuffle computes its plan identity once and +orders by the existing MD5 seed/epoch/plan/ID key with binary-ID tie breaking. +The legacy path still scans its full selection; flat planning does not provide +the incremental metadata behavior of inspection-backed reads. + +Binding reuses its committed-Snapshot admission. Independent selection, preview +and batch calls retain their own admission. Combined continuation selection and +delivery share one request-local Snapshot/schema/membership proof under separate +work budgets; wrappers do not repeat the same deletion lookup. Empty and legacy +batch entry points retain independent admission. No proof is cached across requests. Legacy Snapshots without a stored visibility proof are checked against head and anchor ancestry using bounded, cancellable reads. The traversal validates the diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 07790ef36..3003090bc 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -465,6 +465,16 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **F4.1 / F9.2（2026-10-24）**：legacy scan 只移交当前窗口的 ID，public batch adapter + 移交 ID allocation；ReadPlan limits/shuffle 一次选择累计预算，shuffle 每次执行只计算一次 + plan identity，删除逐 sample 的选中集合反查和逐 ID 的 plan 序列化/哈希。保持既有 MD5 + 顺序、二进制 ID、各入口独立的预算口径和 group/offset/end cursor 合同。legacy 全量 + Manifest/选择集规划仍未消除，由 F4 跟踪；未增加跨请求缓存或持久化。 + `legacy_planning_test` 新增两例独立参考排序、边界预算和实际 service 分页验证； + Dataset `--all-targets`、Access `dataset_contract_test`/`dataset_read_query_contract_test` + 共六个 HTTP 用例通过；Dataset all-target/all-feature clippy、workspace fmt、diff check 通过。 + 本批为 CPU/复制路径改进，没有减少正常请求 RPC 或测得端到端 TPS 提升的声明。 + - **2026-10-24，P13.4**：Snapshot 回收 head fence 与固定 Snapshot 退休都只初读一次， CAS conflict 直接接 observed，不再循环 GET。四次实际 head nonce 竞争及一次竞争退休 中禁止读取对应 key，仍成功；合计省去五次逻辑 GET。fence 不改变 latest/stable，固定 diff --git a/lib/crowdb-access-dataset/src/authority.rs b/lib/crowdb-access-dataset/src/authority.rs index 85cde0a3b..f4a925bb3 100644 --- a/lib/crowdb-access-dataset/src/authority.rs +++ b/lib/crowdb-access-dataset/src/authority.rs @@ -487,6 +487,18 @@ impl DatasetAuthority { plan: &crate::ReadPlan, limits: crate::ReadLimits, ) -> Result>>, AuthorityError> { + Ok(crate::planner::into_batches( + self.plan_ids(identity, plan, limits).await?, + plan.batch_size, + )) + } + + pub(crate) async fn plan_ids( + &self, + identity: &DatasetIdentity, + plan: &crate::ReadPlan, + limits: crate::ReadLimits, + ) -> Result>, AuthorityError> { plan.validate()?; limits.validate()?; let binding = self @@ -523,11 +535,10 @@ impl DatasetAuthority { } } ids.sort(); - let batches: Vec<_> = ids.chunks(plan.batch_size).map(<[Vec]>::to_vec).collect(); - if batches.len() > limits.max_batches { + if ids.len().div_ceil(plan.batch_size) > limits.max_batches { return Err(DatasetError::ReadLimitExceeded.into()); } - Ok(batches) + Ok(ids) } /// Deletes one unreachable snapshot's Dataset metadata idempotently. diff --git a/lib/crowdb-access-dataset/src/planner.rs b/lib/crowdb-access-dataset/src/planner.rs index aa5b812e3..a5fe90420 100644 --- a/lib/crowdb-access-dataset/src/planner.rs +++ b/lib/crowdb-access-dataset/src/planner.rs @@ -171,19 +171,27 @@ impl ReadPlan { /// # Errors /// Propagates plan or manifest validation failures. pub fn scan_ids(&self, manifest: &ManifestRecord) -> Result>, DatasetError> { + let mut ids: Vec<_> = self + .selected_samples(manifest)? + .map(|sample| sample.sample_id.clone()) + .collect(); + ids.sort(); + Ok(ids) + } + + pub(crate) fn selected_samples<'a>( + &'a self, + manifest: &'a ManifestRecord, + ) -> Result + 'a, DatasetError> { self.validate()?; manifest.validate()?; if manifest.snapshot != self.snapshot { return Err(DatasetError::InvalidManifest); } - let mut ids: Vec<_> = manifest + Ok(manifest .samples .iter() - .filter(|sample| self.selection.matches(sample)) - .map(|sample| sample.sample_id.clone()) - .collect(); - ids.sort(); - Ok(ids) + .filter(move |sample| self.selection.matches(sample))) } /// Splits a deterministic scan into bounded batches without reading field @@ -193,7 +201,7 @@ impl ReadPlan { /// Propagates plan and manifest validation failures. pub fn scan_batches(&self, manifest: &ManifestRecord) -> Result>>, DatasetError> { let ids = self.scan_ids(manifest)?; - Ok(ids.chunks(self.batch_size).map(<[Vec]>::to_vec).collect()) + Ok(into_batches(ids, self.batch_size)) } /// Returns a stable identity for retries, cursors, and surface adapters. @@ -223,36 +231,69 @@ impl ReadPlan { limits: ReadLimits, ) -> Result>>, DatasetError> { limits.validate()?; - let batches = self.scan_batches(manifest)?; - if batches.len() > limits.max_batches { - return Err(DatasetError::ReadLimitExceeded); + let mut ids = Vec::new(); + let mut metadata_bytes = 0_usize; + for sample in self.selected_samples(manifest)? { + if ids.len() >= limits.max_samples { + return Err(DatasetError::ReadLimitExceeded); + } + for field in &sample.fields { + let size = match &field.value { + crate::FieldLocator::Inline { value, .. } => value.len(), + crate::FieldLocator::Chunk { location, .. } => location.len(), + crate::FieldLocator::Tombstone => 0, + }; + metadata_bytes = metadata_bytes.saturating_add(size); + if metadata_bytes > limits.max_metadata_bytes { + return Err(DatasetError::ReadLimitExceeded); + } + } + ids.push(sample.sample_id.clone()); } - let sample_count = batches.iter().map(Vec::len).sum::(); - if sample_count > limits.max_samples { + if ids.len().div_ceil(self.batch_size) > limits.max_batches { return Err(DatasetError::ReadLimitExceeded); } if self.batch_size > limits.in_flight.saturating_mul(MAX_BATCH_SIZE) { return Err(DatasetError::ReadLimitExceeded); } - let metadata_bytes = manifest - .samples - .iter() - .filter(|sample| batches.iter().flatten().any(|id| id == &sample.sample_id)) - .map(|sample| { - sample - .fields - .iter() - .map(|field| match &field.value { - crate::FieldLocator::Inline { value, .. } => value.len(), - crate::FieldLocator::Chunk { location, .. } => location.len(), - crate::FieldLocator::Tombstone => 0, - }) - .sum::() - }) - .sum::(); - if metadata_bytes > limits.max_metadata_bytes { - return Err(DatasetError::ReadLimitExceeded); + ids.sort(); + Ok(into_batches(ids, self.batch_size)) + } +} + +pub(crate) fn into_batches(ids: Vec>, batch_size: usize) -> Vec>> { + let mut batches = Vec::with_capacity(ids.len().div_ceil(batch_size)); + let mut remaining = ids.into_iter(); + loop { + let batch: Vec<_> = remaining.by_ref().take(batch_size).collect(); + if batch.is_empty() { + return batches; } - Ok(batches) + batches.push(batch); + } +} + +pub(crate) fn batch_range( + total: usize, + batch_size: usize, + group: u32, + offset: u32, +) -> Result, DatasetError> { + let group = usize::try_from(group).map_err(|_| DatasetError::CursorMismatch)?; + let offset = usize::try_from(offset).map_err(|_| DatasetError::CursorMismatch)?; + let windows = total.div_ceil(batch_size); + if group > windows || (group == windows && offset != 0) { + return Err(DatasetError::CursorMismatch); + } + if group == windows { + return Ok(total..total); + } + let start = group + .checked_mul(batch_size) + .ok_or(DatasetError::CursorMismatch)?; + let end = start.saturating_add(batch_size).min(total); + if offset > end - start { + return Err(DatasetError::CursorMismatch); } + Ok(start + offset..end) } diff --git a/lib/crowdb-access-dataset/src/shuffle.rs b/lib/crowdb-access-dataset/src/shuffle.rs index 7dc7d2b77..3505ea683 100644 --- a/lib/crowdb-access-dataset/src/shuffle.rs +++ b/lib/crowdb-access-dataset/src/shuffle.rs @@ -68,29 +68,27 @@ impl SampleShuffle { spec: &ShuffleSpec, ) -> Result>, DatasetError> { spec.validate()?; - let ids = plan.scan_ids(manifest)?; - if ids.len() > spec.max_shuffle_objects { - return Err(DatasetError::ReadLimitExceeded); - } - let metadata_bytes = manifest - .samples - .iter() - .filter(|sample| ids.iter().any(|id| id == &sample.sample_id)) - .map(|sample| { - sample.sample_id.len() - + sample - .fields - .iter() - .map(|field| field.name.len() + locator_size(&field.value)) - .sum::() - }) - .sum::(); - if metadata_bytes > spec.max_shuffle_bytes { - return Err(DatasetError::ReadLimitExceeded); + let mut ids = Vec::new(); + let mut metadata_bytes = 0_usize; + for sample in plan.selected_samples(manifest)? { + if ids.len() >= spec.max_shuffle_objects { + return Err(DatasetError::ReadLimitExceeded); + } + metadata_bytes = metadata_bytes.saturating_add(sample.sample_id.len()); + for field in &sample.fields { + metadata_bytes = metadata_bytes + .saturating_add(field.name.len()) + .saturating_add(locator_size(&field.value)); + } + if metadata_bytes > spec.max_shuffle_bytes { + return Err(DatasetError::ReadLimitExceeded); + } + ids.push(sample.sample_id.clone()); } + let identity = plan.identity(); let mut keyed = ids .into_iter() - .map(|id| (shuffle_key(spec, plan, &id), id)) + .map(|id| (shuffle_key(spec, &identity, &id), id)) .collect::>(); keyed.sort_unstable_by(|left, right| left.0.cmp(&right.0).then_with(|| left.1.cmp(&right.1))); Ok(keyed.into_iter().map(|(_, id)| id).collect()) @@ -108,7 +106,7 @@ impl SampleShuffle { spec: &ShuffleSpec, ) -> Result>>, DatasetError> { let ids = Self::execute(plan, manifest, spec)?; - Ok(ids.chunks(plan.batch_size).map(<[Vec]>::to_vec).collect()) + Ok(crate::planner::into_batches(ids, plan.batch_size)) } /// Groups selected IDs using the indexed prefix grouping rule. @@ -153,24 +151,9 @@ impl SampleShuffle { cursor: &ReadCursor, ) -> Result>, DatasetError> { cursor.validate_for(plan)?; - let windows = Self::execute_windows(plan, manifest, spec)?; - let group = usize::try_from(cursor.group).map_err(|_| DatasetError::CursorMismatch)?; - if group > windows.len() { - return Err(DatasetError::CursorMismatch); - } - if group == windows.len() { - if cursor.offset != 0 { - return Err(DatasetError::CursorMismatch); - } - return Ok(Vec::new()); - } - let offset = usize::try_from(cursor.offset).map_err(|_| DatasetError::CursorMismatch)?; - if offset > windows[group].len() { - return Err(DatasetError::CursorMismatch); - } - let mut result = windows[group][offset..].to_vec(); - result.extend(windows.iter().skip(group + 1).flatten().cloned()); - Ok(result) + let ids = Self::execute(plan, manifest, spec)?; + let range = crate::planner::batch_range(ids.len(), plan.batch_size, cursor.group, cursor.offset)?; + Ok(ids.into_iter().skip(range.start).collect()) } } @@ -182,11 +165,11 @@ fn locator_size(locator: &crate::FieldLocator) -> usize { } } -fn shuffle_key(spec: &ShuffleSpec, plan: &ReadPlan, id: &[u8]) -> [u8; 16] { +fn shuffle_key(spec: &ShuffleSpec, identity: &[u8; 16], id: &[u8]) -> [u8; 16] { let mut digest = Md5::new(); digest.update(spec.seed.to_le_bytes()); digest.update(spec.epoch.to_le_bytes()); - digest.update(plan.identity()); + digest.update(identity); digest.update(id); digest.finalize().into() } diff --git a/lib/crowdb-access-dataset/src/transport.rs b/lib/crowdb-access-dataset/src/transport.rs index 4c8cdbb4f..707c9d69b 100644 --- a/lib/crowdb-access-dataset/src/transport.rs +++ b/lib/crowdb-access-dataset/src/transport.rs @@ -109,17 +109,19 @@ impl DatasetReadService { self.authority .check_snapshot_readable(&self.identity, request.plan.snapshot) .await?; - let windows = match request.shuffle.as_ref() { + let cursor = request.cursor.unwrap_or_else(|| ReadCursor::start(&request.plan)); + cursor.validate_for(&request.plan)?; + let planned = match request.shuffle.as_ref() { Some(spec) => { let manifest = self .authority .get_manifest(&self.identity, request.plan.snapshot) .await?; - SampleShuffle::execute_windows(&request.plan, &manifest, spec)? + SampleShuffle::execute(&request.plan, &manifest, spec)? } None => { self.authority - .plan_batches( + .plan_ids( &self.identity, &request.plan, crate::ReadLimits { @@ -133,21 +135,15 @@ impl DatasetReadService { .await? } }; - let cursor = request.cursor.unwrap_or_else(|| ReadCursor::start(&request.plan)); - cursor.validate_for(&request.plan)?; + let windows = planned.len().div_ceil(request.plan.batch_size); let group = usize::try_from(cursor.group).map_err(|_| DatasetError::CursorMismatch)?; - if group > windows.len() || (group == windows.len() && cursor.offset != 0) { - return Err(DatasetError::CursorMismatch.into()); - } - let ids = if group == windows.len() { - Vec::new() - } else { - let offset = usize::try_from(cursor.offset).map_err(|_| DatasetError::CursorMismatch)?; - if offset > windows[group].len() { - return Err(DatasetError::CursorMismatch.into()); - } - windows[group][offset..].to_vec() - }; + let range = crate::planner::batch_range( + planned.len(), + request.plan.batch_size, + cursor.group, + cursor.offset, + )?; + let ids: Vec<_> = planned.into_iter().skip(range.start).take(range.len()).collect(); let fields: Vec<&str> = request.plan.projection.iter().map(String::as_str).collect(); let samples = if ids.is_empty() { Vec::new() @@ -166,7 +162,7 @@ impl DatasetReadService { snapshot: request.plan.snapshot, samples, cursor: next, - end: group >= windows.len().saturating_sub(1), + end: group >= windows.saturating_sub(1), }) } diff --git a/lib/crowdb-access-dataset/tests/legacy_planning_test.rs b/lib/crowdb-access-dataset/tests/legacy_planning_test.rs new file mode 100644 index 000000000..8ea12ced0 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/legacy_planning_test.rs @@ -0,0 +1,226 @@ +#[path = "common/access_store.rs"] +mod access_store; +#[path = "common/publication.rs"] +mod publication; + +use crowdb_access_dataset::*; +use md5::{Digest, Md5}; +use std::sync::Arc; + +fn manifest() -> ManifestRecord { + let mut manifest: ManifestRecord = bincode::deserialize(&publication::publication_manifest(7)).unwrap(); + manifest.samples = (0_u8..16) + .rev() + .map(|number| { + let value = vec![number; usize::from(number) + 1]; + SampleRecord { + sample_id: vec![b's', number, 0xff], + fields: vec![FieldRecord { + name: "marker".into(), + value: FieldLocator::Inline { + md5: Md5::digest(&value).into(), + value, + }, + }], + } + }) + .collect(); + manifest +} + +fn plan(snapshot: SnapshotId) -> ReadPlan { + ReadPlan { + snapshot, + selection: Selection::Prefix(b"s".to_vec()), + ordering: crowdb_access_dataset::Ordering::SampleId, + projection: vec!["marker".into()], + batch_size: 3, + } +} + +#[test] +fn planner_and_shuffle_keep_binary_order_and_each_independent_budget_boundary() { + let manifest = manifest(); + let plan = plan(manifest.snapshot); + let mut ordered: Vec<_> = manifest.samples.iter().map(|row| row.sample_id.clone()).collect(); + ordered.sort(); + let limits = ReadLimits { + max_samples: 16, + max_metadata_bytes: 136, + max_batches: 6, + prefetch: 1, + in_flight: 1, + }; + assert_eq!( + plan.scan_batches_with_limits(&manifest, limits).unwrap().concat(), + ordered + ); + for limits in [ + ReadLimits { + max_samples: 15, + ..limits + }, + ReadLimits { + max_metadata_bytes: 135, + ..limits + }, + ReadLimits { + max_batches: 5, + ..limits + }, + ] { + assert_eq!( + plan.scan_batches_with_limits(&manifest, limits), + Err(DatasetError::ReadLimitExceeded) + ); + } + let spec = ShuffleSpec { + seed: 17, + epoch: 3, + max_shuffle_objects: 16, + max_shuffle_bytes: 280, + }; + let identity = plan.identity(); + let mut reference: Vec<_> = ordered + .iter() + .map(|id| { + let mut digest = Md5::new(); + digest.update(spec.seed.to_le_bytes()); + digest.update(spec.epoch.to_le_bytes()); + digest.update(identity); + digest.update(id); + (<[u8; 16]>::from(digest.finalize()), id.clone()) + }) + .collect(); + reference.sort(); + let expected: Vec<_> = reference.into_iter().map(|(_, id)| id).collect(); + assert_eq!(SampleShuffle::execute(&plan, &manifest, &spec).unwrap(), expected); + assert_eq!( + SampleShuffle::execute_windows(&plan, &manifest, &spec) + .unwrap() + .concat(), + expected + ); + for spec in [ + ShuffleSpec { + max_shuffle_objects: 15, + ..spec.clone() + }, + ShuffleSpec { + max_shuffle_bytes: 279, + ..spec.clone() + }, + ] { + assert_eq!( + SampleShuffle::execute(&plan, &manifest, &spec), + Err(DatasetError::ReadLimitExceeded) + ); + } + assert_shuffle_resume(&plan, &manifest, &spec, &expected); +} + +fn assert_shuffle_resume( + plan: &ReadPlan, + manifest: &ManifestRecord, + spec: &ShuffleSpec, + expected: &[Vec], +) { + let windows = expected.len().div_ceil(plan.batch_size); + for group in 0..=windows { + let length = if group == windows { + 0 + } else { + (expected.len() - group * plan.batch_size).min(plan.batch_size) + }; + for offset in 0..=length { + let cursor = ReadCursor { + group: u32::try_from(group).unwrap(), + offset: u32::try_from(offset).unwrap(), + ..ReadCursor::start(plan) + }; + let start = (group * plan.batch_size).min(expected.len()) + offset; + assert_eq!( + SampleShuffle::resume(plan, manifest, spec, &cursor).unwrap(), + expected[start..] + ); + } + } + for (group, offset) in [(0, 4), (6, 1), (7, 0), (u32::MAX, 0)] { + let cursor = ReadCursor { + group, + offset, + ..ReadCursor::start(plan) + }; + assert_eq!( + SampleShuffle::resume(plan, manifest, spec, &cursor), + Err(DatasetError::CursorMismatch) + ); + } +} + +#[tokio::test] +async fn legacy_scan_delivers_only_the_current_window_and_preserves_cursor_edges() { + let store = access_store::TestAccessStore::shared(); + let authority = Arc::new(DatasetAuthority::new(store)); + let identity = DatasetIdentity::new(NamespacePath::root(), "legacy-planning").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + let snapshot = authority + .publish_snapshot(&identity, None, bincode::serialize(&manifest()).unwrap()) + .await + .unwrap(); + let manifest = authority.get_manifest(&identity, snapshot).await.unwrap(); + let plan = plan(snapshot); + let service = DatasetReadService::new(authority, identity, 1).unwrap(); + for shuffle in [None, Some(ShuffleSpec::default())] { + let windows = if let Some(spec) = &shuffle { + SampleShuffle::execute_windows(&plan, &manifest, spec).unwrap() + } else { + plan.scan_batches(&manifest).unwrap() + }; + for group in 0..=windows.len() { + let length = windows.get(group).map_or(0, Vec::len); + for offset in 0..=length { + let cursor = ReadCursor { + group: u32::try_from(group).unwrap(), + offset: u32::try_from(offset).unwrap(), + ..ReadCursor::start(&plan) + }; + let result = service + .scan(DatasetScanRequest { + plan: plan.clone(), + shuffle: shuffle.clone(), + cursor: Some(cursor), + }) + .await + .unwrap(); + let actual: Vec<_> = result + .samples + .into_iter() + .map(|sample| sample.sample_id) + .collect(); + let expected = windows.get(group).map_or(&[][..], |ids| &ids[offset..]); + assert_eq!(actual, expected); + assert_eq!(result.end, group >= windows.len().saturating_sub(1)); + assert_eq!( + result.cursor.group as usize, + group + usize::from(!expected.is_empty()) + ); + assert_eq!(result.cursor.offset, 0); + assert_eq!(service.in_flight(), 0); + } + } + let cursor = ReadCursor { + group: u32::MAX, + ..ReadCursor::start(&plan) + }; + assert!(service + .scan(DatasetScanRequest { + plan: plan.clone(), + shuffle, + cursor: Some(cursor) + }) + .await + .is_err()); + assert_eq!(service.in_flight(), 0); + } +} From 7c61e5c0e240db24525ce0a31ed0b53de298c6fd Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 13:25:38 +0800 Subject: [PATCH 110/258] Return versioned comparison observations from group conditional writes --- .../src/credentials/ensure.rs | 10 +- .../src/control_store.rs | 2 +- .../tests/group0_control_plane_test.rs | 2 +- .../design-crowdb-access-server.md | 16 +- doc/design/kv/design-crowdb-kv.md | 14 ++ doc/working/plan-access-simplification.md | 22 ++- e2e-test/tests/kv/client_data.rs | 2 +- .../src/binding/chunk_slots.rs | 1 + .../src/binding/chunk_slots/handoff.rs | 17 +- .../src/binding/chunk_slots/publication.rs | 1 + .../binding/chunk_slots/publication/epochs.rs | 2 + .../src/client/core/conditional.rs | 22 ++- lib/crowdb-kv-client/src/client/core/owned.rs | 1 + lib/crowdb-kv-client/src/error.rs | 6 +- lib/crowdb-kv-client/src/lib.rs | 1 + lib/crowdb-kv-client/src/transport.rs | 1 + .../src/transport/comparison.rs | 36 +++++ .../src/transport/rpc_transport.rs | 16 +- .../tests/comparison_wire_test.rs | 84 ++++++++++ .../tests/conditional_observation_test.rs | 151 ++++++++++++++++++ lib/crowdb-kv/src/cluster/group.rs | 2 +- lib/crowdb-kv/src/cluster/group_cas.rs | 7 +- lib/crowdb-kv/src/cluster/group_operations.rs | 8 +- .../src/cluster/group_owner_fence.rs | 2 +- lib/crowdb-kv/src/cluster/group_propose.rs | 16 +- .../src/cluster/px_kv_store_handler.rs | 10 +- .../src/cluster/px_kv_store_owner.rs | 10 +- lib/crowdb-kv/src/rpc/kv_rpc_service.rs | 20 +++ .../src/rpc/kv_rpc_service/comparison.rs | 25 +++ lib/crowdb-kv/tests/group_test/cas_test.rs | 9 +- .../tests/group_test/kv_slot_retry_test.rs | 33 ++++ .../tests/store_test/group_operations_test.rs | 2 +- .../src/fb_wrappers/kv_client.rs | 3 + lib/crowdb-protocol/src/fbs/kv_client.fbs | 11 +- lib/crowdb-protocol/src/types/kv_client.rs | 43 +++++ .../tests/kv_client_wrappers_test.rs | 2 + 36 files changed, 558 insertions(+), 52 deletions(-) create mode 100644 lib/crowdb-kv-client/src/transport/comparison.rs create mode 100644 lib/crowdb-kv-client/tests/comparison_wire_test.rs create mode 100644 lib/crowdb-kv-client/tests/conditional_observation_test.rs create mode 100644 lib/crowdb-kv/src/rpc/kv_rpc_service/comparison.rs diff --git a/app/crowdb-access-server/src/credentials/ensure.rs b/app/crowdb-access-server/src/credentials/ensure.rs index 359f7e66c..6a0693e7d 100644 --- a/app/crowdb-access-server/src/credentials/ensure.rs +++ b/app/crowdb-access-server/src/credentials/ensure.rs @@ -32,8 +32,14 @@ impl CredentialAuthority { .encode()?; match self.control.put_cas(0, 0, &key, &bytes, 0).await { Ok(_) => Ok(issued), - // Group-0 currently returns only the revision, not the observed value. - // This one authoritative GET is required; successful creation needs none. + Err(KvError::CasFailed { + observation: Some(observation), + .. + }) if observation.key.as_ref() == key.as_slice() && observation.revision != 0 => { + self.token(user, &issued.access_key_id, &observation.value) + } + // Older peers and uncertain/busy outcomes have no comparison value. + // Reconcile only the final resource, never a preparation record. Err(error @ (KvError::CasFailed { .. } | KvError::CasBusy | KvError::OutcomeUnknown)) => { match self.control.get(0, 0, &key, ReadMode::Linearizable, None).await? { GetOutcome::Found { value, .. } => self.token(user, &issued.access_key_id, &value), diff --git a/app/crowdb-chunk-kv-server/src/control_store.rs b/app/crowdb-chunk-kv-server/src/control_store.rs index 6b25ad1f8..0bb88108d 100644 --- a/app/crowdb-chunk-kv-server/src/control_store.rs +++ b/app/crowdb-chunk-kv-server/src/control_store.rs @@ -141,7 +141,7 @@ impl Group0Kv for CrowdbKvClient { fn map_client_error(error: KvClientError) -> Group0KvError { match error { - KvClientError::CasFailed { current_revision } => Group0KvError::CasFailed { current_revision }, + KvClientError::CasFailed { current_revision, .. } => Group0KvError::CasFailed { current_revision }, KvClientError::OutcomeUnknown => Group0KvError::OutcomeUnknown, other => Group0KvError::Unavailable(other.to_string()), } diff --git a/app/crowdb-kv-server/tests/group0_control_plane_test.rs b/app/crowdb-kv-server/tests/group0_control_plane_test.rs index 301bc90ea..423c6f0c4 100644 --- a/app/crowdb-kv-server/tests/group0_control_plane_test.rs +++ b/app/crowdb-kv-server/tests/group0_control_plane_test.rs @@ -45,7 +45,7 @@ async fn facade_reads_scans_and_revision_checks_without_rpc_loopback() { .unwrap_err(); assert!(matches!( error, - KvGroupOperationError::CompareFailed { current_revision } if current_revision == first_revision + KvGroupOperationError::CompareFailed { observation } if observation.revision == first_revision )); } diff --git a/doc/design/access-server/design-crowdb-access-server.md b/doc/design/access-server/design-crowdb-access-server.md index c6e1f57a8..100dbd90d 100644 --- a/doc/design/access-server/design-crowdb-access-server.md +++ b/doc/design/access-server/design-crowdb-access-server.md @@ -271,9 +271,10 @@ Frontend query failures do not mutate these backend health observations. Coordination follows the semantic owner, rather than the Access Server process: -- Dataset read admission is read-only. Explicit session acknowledgements persist - application progress; normal fetching and snapshot activity do not. Draft - journals recover one draft's partial work and fence late writes after cleanup. +- Dataset read admission is read-only. Clients retain continuation progress; + fetching, acknowledgements and snapshot activity create no reader records. + Existing write-side draft journals recover partial work and fence late writes + after cleanup. - Dataset upload expiry scans upload authorities; creation does not enter a global registry. Shared payload owner records prevent one snapshot's retirement from reclaiming another snapshot's bytes. @@ -284,9 +285,12 @@ Coordination follows the semantic owner, rather than the Access Server process: recovery state preserves that strict contract and is not a reader lease. - S3 multipart coordinates accounting and exact completion selection within one upload. Ordinary object reads do not register persistent activity. -- A failed CAS observation can supply the next base for an allowed retry. Unknown - transport outcomes require reconciliation with the same mutation identity; - application preconditions must not silently move to a newer base. +- A failed CAS observation supplies the compared key, revision and value for an + allowed retry. Unknown transport outcomes follow the operation's retry contract; + application preconditions must not silently move to a newer base. Credential + bootstrap publishes one encrypted final resource through create-if-absent; + comparison losers reuse that value, while older peers and busy/unknown replies + require a direct resource read. It has no persisted preparation record. ## 9. Direction and risks diff --git a/doc/design/kv/design-crowdb-kv.md b/doc/design/kv/design-crowdb-kv.md index 123690cc3..e65f50eaf 100644 --- a/doc/design/kv/design-crowdb-kv.md +++ b/doc/design/kv/design-crowdb-kv.md @@ -223,6 +223,20 @@ higher ballot, adopting an accepted value or filling NoOp; failure to resolve closes conditional admission for that leader tenure and returns `OutcomeUnknown`. +`CasFailed` carries the comparison's key, revision and complete value, taken +from the same versioned engine lookup. Revision zero means observed absent; +an existing empty value has a nonzero revision. No confirmation GET is issued +to build the reply. After adopting a foreign slot or resolving an abandoned +slot, a renewed comparison returns that lookup's observation. It may be stale +again by receipt; clients may retry legal updates with its revision, while +explicit expected-base operations retain their original precondition. + +The response's optional `compare_observation` is absent on old peers, local +validation rejection, `CasBusy` and `OutcomeUnknown`. Clients preserve these +distinct outcomes. An owner-guarded write also identifies whether the owner +key or business-record key failed; one key's revision cannot be used for the +other condition. Success adds no observation read or persisted record. + The group-owned task survives RPC cancellation. New leaders admit reads and CAS only after bulk Phase 1 has resolved and locally applied every slot through its election ceiling. Blind writes do not participate in the guard, so the CAS diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 3003090bc..94d76eb33 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -389,10 +389,6 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 - [ ] **P14.1 审查 root/context/初始化回读**：记录正常请求内重复 GET 和初始化写，复用已验证 immutable 事实，保留撤销和 catalog reset 的准入边界。凭据持久化不是 reader session。Files: Iceberg `catalog/{repository,storage}.rs`、Access `credentials.rs`、`credentials/ensure.rs`。 -- [ ] **P14.3 补 Group-0 CAS observation**：`CrowdbKvClient::put_cas` 的 `CasFailed` 当前仅有 - current_revision；凭据 bootstrap loser 因此需要直接 GET，无法复用值。由比较时读取的 - 同一 revision/value 贯穿 group admission、RPC response、client error，禁止另读最新值 - 伪装成原 observation；兼容旧服务端与 unknown/busy。Files: KV group CAS、kv client wire/API、credentials。 ## 其他性能任务 @@ -438,6 +434,11 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 无条件覆盖与 expected-base 重试测试。断言总 GET+写，不仅是零写。Files: Dataset `tests/*cost*`、`read_query*`、upload/Composition/Split/inspection；S3/Iceberg 对应 repository 测试。 - [ ] **V2 HTTP/native/SDK 集成**：各受影响 route 的鉴权、错误、协议/SDK 一致性；保留删除准入、 迟到写者、非空 namespace、共享 owner、held buffer、cancel/drop 和损坏内容测试。Files: Access/native tests、`sdk/tests/`、native Python consumer。 +- [ ] **V2.1 迁移旧容器 fixture 并验证当前源码**：`cargo check -p e2e-test --tests` 已发现 + catalog fixture 仍调用已删除 RetryLedger、旧 property journal/响应 API，ChunkDB fixture + 仍断言 DeleteChunkRange Unimplemented。改为验证资源结果、无新过程记录、legacy 解码与 + GC 兼容，以及已接受的范围退休语义；不恢复生产 ledger 来通过旧测试。 + Files: `e2e-test/tests/access_iceberg/catalog/{journal,property}.rs`、`catalog.rs`、`chunkdb/lifecycle.rs`。 - [ ] **V3 工程门禁与文档**：每批执行适用 fmt/clippy/测试，全部经 `pixi run`；同步 owning design/spec/OpenAPI/SDK。只修改文档时检查 diff、引用和任务状态，不为此跑整套服务测试。 - [ ] **V4 同源真实负载验收**：重新构建当前源码的镜像，运行现有 HTTP/native Rust/native Python @@ -465,6 +466,19 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **P14.3（2026-10-24）**：Group CAS 的首次比较、foreign slot 后重比较、失败 slot + 结算后重比较及 owner value 比较，均直接移交同一 engine lookup 的 key/revision/value。 + 可选 `FBKvCompareObservation` 贯穿 typed operations、RPC 和 client error；旧 peer 与 + 非比较拒绝为 None，revision=0 的显式 observation 才表示观察到缺失，空值可保留非零 + revision。owner/record 条件带实际失败 key。成功不加 GET/写/状态；错误附加数据仅在 + 比较失败时分配，避免放大每个成功 proposal 的 enum/response。凭据 definite loser 直接 + 校验 observation，不再确认 GET；busy/unknown/旧 peer 仍按已有合同读最终资源。 + 真实 RPC 测试覆盖连续四次冲突无 GET、缺失/空值/删除重建、owner/record 两种失败； + synthetic wire 覆盖旧 peer、非法值/代际、busy/unknown 和缺字段。三副本测试覆盖实际 + prior accepted foreign value，异步 apply fence、内部 Group-0、凭据六例均通过。 + Protocol wrapper 十三例及五个受影响 crate 的 all-target/all-feature clippy 通过。 + E2E 编译另发现此前清理留下的旧 fixture API，列 V2.1 继续处理；未声称容器验收已通过。 + - **F4.1 / F9.2（2026-10-24）**：legacy scan 只移交当前窗口的 ID，public batch adapter 移交 ID allocation；ReadPlan limits/shuffle 一次选择累计预算，shuffle 每次执行只计算一次 plan identity，删除逐 sample 的选中集合反查和逐 ID 的 plan 序列化/哈希。保持既有 MD5 diff --git a/e2e-test/tests/kv/client_data.rs b/e2e-test/tests/kv/client_data.rs index ef50e38ad..08a6ae3f1 100644 --- a/e2e-test/tests/kv/client_data.rs +++ b/e2e-test/tests/kv/client_data.rs @@ -70,7 +70,7 @@ async fn revision_cas_and_conditional_batch_round_trip() { assert!(matches!( client.put_cas(STORE_ID, GROUP_ID, b"guard", b"stale", 0).await, Err(crowdb_kv_client::Error::CasFailed { - current_revision + current_revision, .. }) if current_revision == created.revision )); diff --git a/lib/crowdb-kv-client/src/binding/chunk_slots.rs b/lib/crowdb-kv-client/src/binding/chunk_slots.rs index 9f0f71cfd..20cc9a803 100644 --- a/lib/crowdb-kv-client/src/binding/chunk_slots.rs +++ b/lib/crowdb-kv-client/src/binding/chunk_slots.rs @@ -125,6 +125,7 @@ impl ChunkSlotMapClient { if head != after || revision != after_revision { return Err(Error::CasFailed { current_revision: after_revision, + observation: None, }); } ChunkSlotMap::new(head, bindings).map_err(|error| invalid(&key, &error.to_string())) diff --git a/lib/crowdb-kv-client/src/binding/chunk_slots/handoff.rs b/lib/crowdb-kv-client/src/binding/chunk_slots/handoff.rs index 80cb1ff79..cc24aa969 100644 --- a/lib/crowdb-kv-client/src/binding/chunk_slots/handoff.rs +++ b/lib/crowdb-kv-client/src/binding/chunk_slots/handoff.rs @@ -74,10 +74,16 @@ impl ChunkSlotMapClient { GetOutcome::Found { revision, .. } => { return Err(Error::CasFailed { current_revision: revision, + observation: None, }) } GetOutcome::NotFound if transfer.previous.is_none() => 0, - GetOutcome::NotFound => return Err(Error::CasFailed { current_revision: 0 }), + GetOutcome::NotFound => { + return Err(Error::CasFailed { + current_revision: 0, + observation: None, + }) + } }; match self .kv @@ -263,9 +269,15 @@ impl ChunkSlotMapClient { GetOutcome::Found { revision, .. } => { return Err(Error::CasFailed { current_revision: revision, + observation: None, + }) + } + GetOutcome::NotFound => { + return Err(Error::CasFailed { + current_revision: 0, + observation: None, }) } - GetOutcome::NotFound => return Err(Error::CasFailed { current_revision: 0 }), } } Ok(()) @@ -278,6 +290,7 @@ impl ChunkSlotMapClient { } else { Err(Error::CasFailed { current_revision: current.revision, + observation: None, }) } } diff --git a/lib/crowdb-kv-client/src/binding/chunk_slots/publication.rs b/lib/crowdb-kv-client/src/binding/chunk_slots/publication.rs index e51ee8d59..5311eba96 100644 --- a/lib/crowdb-kv-client/src/binding/chunk_slots/publication.rs +++ b/lib/crowdb-kv-client/src/binding/chunk_slots/publication.rs @@ -96,6 +96,7 @@ impl ChunkSlotMapClient { if before != after || before.as_ref().map(|(head, _)| head) != Some(service.head()) { return Err(crate::Error::CasFailed { current_revision: after.map_or(0, |(_, revision)| revision), + observation: None, }); } ChunkServiceSnapshot::validate(service, authority) diff --git a/lib/crowdb-kv-client/src/binding/chunk_slots/publication/epochs.rs b/lib/crowdb-kv-client/src/binding/chunk_slots/publication/epochs.rs index 7beaeb1ab..5b2371095 100644 --- a/lib/crowdb-kv-client/src/binding/chunk_slots/publication/epochs.rs +++ b/lib/crowdb-kv-client/src/binding/chunk_slots/publication/epochs.rs @@ -114,6 +114,7 @@ impl ChunkSlotMapClient { if self.read_head::().await? != Some(before.clone()) || service.head() != &before.0 { return Err(Error::CasFailed { current_revision: before.1, + observation: None, }); } if next.head().generation @@ -137,6 +138,7 @@ impl ChunkSlotMapClient { if next.head().generation <= before.0.generation { return Err(Error::CasFailed { current_revision: before.1, + observation: None, }); } return Err(invalid( diff --git a/lib/crowdb-kv-client/src/client/core/conditional.rs b/lib/crowdb-kv-client/src/client/core/conditional.rs index 35e2d9d0e..536bd8726 100644 --- a/lib/crowdb-kv-client/src/client/core/conditional.rs +++ b/lib/crowdb-kv-client/src/client/core/conditional.rs @@ -67,9 +67,7 @@ impl CrowdbKvClient { continue; } return Err(match KvErrorCode::try_from(response.error_code) { - Ok(KvErrorCode::KvErrorCasFailed) => Error::CasFailed { - current_revision: response.revision, - }, + Ok(KvErrorCode::KvErrorCasFailed) => comparison_error(response, key, expected_revision), Ok(KvErrorCode::KvErrorCasBusy) => Error::CasBusy, Ok(KvErrorCode::KvErrorOutcomeUnknown) => Error::OutcomeUnknown, _ => Error::Server(response.error), @@ -156,9 +154,9 @@ impl CrowdbKvClient { continue; } return Err(match KvErrorCode::try_from(response.error_code) { - Ok(KvErrorCode::KvErrorCasFailed) => Error::CasFailed { - current_revision: response.revision, - }, + Ok(KvErrorCode::KvErrorCasFailed) => { + comparison_error(response, precondition_key, expected_revision) + } Ok(KvErrorCode::KvErrorCasBusy) => Error::CasBusy, Ok(KvErrorCode::KvErrorOutcomeUnknown) => Error::OutcomeUnknown, _ => Error::Server(response.error), @@ -166,3 +164,15 @@ impl CrowdbKvClient { } } } + +fn comparison_error(response: crowdb_kv::rpc::KvResponse, key: &[u8], expected_revision: u64) -> Error { + if response.compare_observation.as_ref().is_some_and(|observation| { + observation.key.as_ref() != key || observation.revision == expected_revision + }) { + return Error::OutcomeUnknown; + } + Error::CasFailed { + current_revision: response.revision, + observation: response.compare_observation, + } +} diff --git a/lib/crowdb-kv-client/src/client/core/owned.rs b/lib/crowdb-kv-client/src/client/core/owned.rs index a6fecb66c..a7db7423e 100644 --- a/lib/crowdb-kv-client/src/client/core/owned.rs +++ b/lib/crowdb-kv-client/src/client/core/owned.rs @@ -93,6 +93,7 @@ impl CrowdbKvClient { return Err(match KvErrorCode::try_from(response.error_code) { Ok(KvErrorCode::KvErrorCasFailed) => Error::CasFailed { current_revision: response.revision, + observation: response.compare_observation, }, Ok(KvErrorCode::KvErrorCasBusy) => Error::CasBusy, Ok(KvErrorCode::KvErrorOutcomeUnknown) => Error::OutcomeUnknown, diff --git a/lib/crowdb-kv-client/src/error.rs b/lib/crowdb-kv-client/src/error.rs index 491e174c7..49d7b6ca8 100644 --- a/lib/crowdb-kv-client/src/error.rs +++ b/lib/crowdb-kv-client/src/error.rs @@ -29,7 +29,11 @@ pub enum Error { JournalScanGcGap, #[error("compare-and-set failed; current revision is {current_revision}")] - CasFailed { current_revision: u64 }, + CasFailed { + current_revision: u64, + /// Exact failed comparison, absent for old servers or local validation. + observation: Option>, + }, #[error("compare-and-set key is busy")] CasBusy, diff --git a/lib/crowdb-kv-client/src/lib.rs b/lib/crowdb-kv-client/src/lib.rs index c3e14f990..6a8cfa8e7 100644 --- a/lib/crowdb-kv-client/src/lib.rs +++ b/lib/crowdb-kv-client/src/lib.rs @@ -57,6 +57,7 @@ pub use service::{ }; pub use transport::{KVClusterAdmin, KVClusterMetaClient, KvRpcTransport}; +pub use crowdb_kv::rpc::KvCompareObservation; /// Re-exported so callers don't need a direct `crowdb_kv` dependency just to /// pick a read mode or use snapshot DTOs. pub use crowdb_kv::rpc::ReadMode; diff --git a/lib/crowdb-kv-client/src/transport.rs b/lib/crowdb-kv-client/src/transport.rs index 6cde8444a..593222e1b 100644 --- a/lib/crowdb-kv-client/src/transport.rs +++ b/lib/crowdb-kv-client/src/transport.rs @@ -4,6 +4,7 @@ //! KV RPC transport and cluster metadata client. pub mod cluster; +mod comparison; pub mod rpc_transport; pub use cluster::{KVClusterAdmin, KVClusterMetaClient}; diff --git a/lib/crowdb-kv-client/src/transport/comparison.rs b/lib/crowdb-kv-client/src/transport/comparison.rs new file mode 100644 index 000000000..0017d0145 --- /dev/null +++ b/lib/crowdb-kv-client/src/transport/comparison.rs @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Validate the optional versioned value of a failed remote comparison. + +use bytes::Bytes; +use crowdb_protocol::fb_wrappers::kv_client::FBKvResponseRef; +use crowdb_protocol::kv_client_fb::FBKvClientRetCode; + +use crate::{Error, KvCompareObservation, Result}; + +pub(super) fn decode(response: &FBKvResponseRef<'_>) -> Result>> { + let Some(observation) = response.compare_observation() else { + return Ok(None); + }; + let invalid = || Error::Transport { + endpoint: String::new(), + status: "invalid comparison observation".into(), + }; + let key = observation.key().ok_or_else(invalid)?; + let value = observation.value().ok_or_else(invalid)?; + let revision = observation.revision(); + if response.ret_code() != FBKvClientRetCode::CasFailed + || response.ok() + || response.not_found() + || response.revision() != revision + || (revision == 0 && !value.is_empty()) + { + return Err(invalid()); + } + Ok(Some(Box::new(KvCompareObservation { + key: Bytes::copy_from_slice(key.bytes()), + revision, + value: Bytes::copy_from_slice(value.bytes()), + }))) +} diff --git a/lib/crowdb-kv-client/src/transport/rpc_transport.rs b/lib/crowdb-kv-client/src/transport/rpc_transport.rs index bb41167e1..9ddf16df8 100644 --- a/lib/crowdb-kv-client/src/transport/rpc_transport.rs +++ b/lib/crowdb-kv-client/src/transport/rpc_transport.rs @@ -1002,7 +1002,10 @@ fn check_client_ret_code(code: FBKvClientRetCode, msg: Option<&str>, endpoint: & status: msg.unwrap_or("unavailable").into(), }), FBKvClientRetCode::JournalScanGcGap => Err(Error::JournalScanGcGap), - FBKvClientRetCode::CasFailed => Err(Error::CasFailed { current_revision: 0 }), + FBKvClientRetCode::CasFailed => Err(Error::CasFailed { + current_revision: 0, + observation: None, + }), FBKvClientRetCode::CasBusy => Err(Error::CasBusy), FBKvClientRetCode::OutcomeUnknown => Err(Error::OutcomeUnknown), _ => Err(Error::Transport { @@ -1015,6 +1018,16 @@ fn check_client_ret_code(code: FBKvClientRetCode, msg: Option<&str>, endpoint: & // ── Response parsing ───────────────────────────────────────────── /// Parse a `FBKvResponse` flatbuffer into a `KvResponse`. +#[cfg(feature = "test-util")] +impl KvRpcTransport { + /// Decode a synthetic peer reply through the production parser. + /// # Errors + /// Rejects malformed control messages and inconsistent comparisons. + pub fn parse_response_for_tests(buf: &[u8]) -> Result { + parse_kv_response(buf) + } +} + fn parse_kv_response(buf: &[u8]) -> Result { let r = FBKvResponseRef::new(buf); if !r.valid() { @@ -1042,6 +1055,7 @@ fn parse_kv_response(buf: &[u8]) -> Result { read_slot: r.read_slot(), safe_slot: r.safe_slot(), error_code: fb_ret_code_to_kv_error_code(ret_code), + compare_observation: super::comparison::decode(&r)?, }) } diff --git a/lib/crowdb-kv-client/tests/comparison_wire_test.rs b/lib/crowdb-kv-client/tests/comparison_wire_test.rs new file mode 100644 index 000000000..506413c14 --- /dev/null +++ b/lib/crowdb-kv-client/tests/comparison_wire_test.rs @@ -0,0 +1,84 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_kv_client::KvRpcTransport; +use crowdb_protocol::kv_client_fb::{ + FBKvClientRetCode, FBKvCompareObservation, FBKvCompareObservationArgs, FBKvResponse, FBKvResponseArgs, +}; +use flatbuffers::FlatBufferBuilder; + +fn reply(code: FBKvClientRetCode, revision: u64, observed: Option<(u64, &[u8])>) -> Vec { + let mut builder = FlatBufferBuilder::new(); + let comparison = observed.map(|(revision, value)| { + let key = builder.create_vector(b"key\0\xff"); + let value = builder.create_vector(value); + FBKvCompareObservation::create( + &mut builder, + &FBKvCompareObservationArgs { + key: Some(key), + revision, + value: Some(value), + }, + ) + }); + let response = FBKvResponse::create( + &mut builder, + &FBKvResponseArgs { + id: 42, + ret_code: code, + revision, + compare_observation: comparison, + ..Default::default() + }, + ); + builder.finish(response, None); + builder.finished_data().to_vec() +} + +#[test] +fn legacy_failure_is_not_misread_as_an_observed_absent_key() { + let legacy = + KvRpcTransport::parse_response_for_tests(&reply(FBKvClientRetCode::CasFailed, 8, None)).unwrap(); + assert_eq!(legacy.revision, 8); + assert!(legacy.compare_observation.is_none()); + let absent = + KvRpcTransport::parse_response_for_tests(&reply(FBKvClientRetCode::CasFailed, 0, Some((0, b"")))) + .unwrap() + .compare_observation + .unwrap(); + assert_eq!(absent.revision, 0); + assert_eq!(absent.key.as_ref(), b"key\0\xff"); + assert!(absent.value.is_empty()); + let empty = + KvRpcTransport::parse_response_for_tests(&reply(FBKvClientRetCode::CasFailed, 9, Some((9, b"")))) + .unwrap() + .compare_observation + .unwrap(); + assert_eq!(empty.revision, 9); + assert!(empty.value.is_empty()); +} + +#[test] +fn invalid_observations_never_become_definite_comparison_results() { + for (code, revision, observed) in [ + (FBKvClientRetCode::CasFailed, 7, (8, b"value".as_slice())), + (FBKvClientRetCode::CasFailed, 0, (0, b"value".as_slice())), + (FBKvClientRetCode::CasBusy, 8, (8, b"value".as_slice())), + (FBKvClientRetCode::OutcomeUnknown, 8, (8, b"value".as_slice())), + (FBKvClientRetCode::Success, 8, (8, b"value".as_slice())), + ] { + assert!(KvRpcTransport::parse_response_for_tests(&reply(code, revision, Some(observed))).is_err()); + } + let mut builder = FlatBufferBuilder::new(); + let comparison = FBKvCompareObservation::create(&mut builder, &FBKvCompareObservationArgs::default()); + let response = FBKvResponse::create( + &mut builder, + &FBKvResponseArgs { + ret_code: FBKvClientRetCode::CasFailed, + compare_observation: Some(comparison), + ..Default::default() + }, + ); + builder.finish(response, None); + assert!(KvRpcTransport::parse_response_for_tests(builder.finished_data()).is_err()); +} diff --git a/lib/crowdb-kv-client/tests/conditional_observation_test.rs b/lib/crowdb-kv-client/tests/conditional_observation_test.rs new file mode 100644 index 000000000..bdfd05226 --- /dev/null +++ b/lib/crowdb-kv-client/tests/conditional_observation_test.rs @@ -0,0 +1,151 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/owner_fence_server.rs"] +mod owner_fence_server; + +use bytes::Bytes; +use crowdb_kv_client::{BatchOp, Error, KvCompareObservation, KvRpcTransport}; +use crowdb_protocol::chunk_slot::{ChunkServiceIncarnation, ChunkSlotAuthority}; +use owner_fence_server::TestOwnerFenceServer; +use std::time::Duration; + +fn observed(error: Error, key: &[u8]) -> KvCompareObservation { + let Error::CasFailed { + current_revision, + observation: Some(observation), + } = error + else { + panic!("expected an observed comparison: {error:?}"); + }; + assert_eq!(observation.key.as_ref(), key); + assert_eq!(observation.revision, current_revision); + *observation +} + +#[tokio::test] +async fn four_real_conflicts_reuse_versioned_values_without_any_get() { + let test = TestOwnerFenceServer::start().await; + test.store.set_get_delay_for_tests(Duration::from_secs(60)); + tokio::time::timeout(Duration::from_secs(5), async { + let key = b"counter\0\xff"; + let mut revision = 0; + for count in 1_u64..=4 { + let value = count.to_le_bytes(); + let competitor = test.client.put(1, 1, key, &value, None).await.unwrap(); + let error = test + .client + .put_cas(1, 1, key, b"stale", revision) + .await + .unwrap_err(); + let observation = observed(error, key); + assert_eq!(observation.revision, competitor.revision); + assert_eq!(observation.value.as_ref(), value); + revision = observation.revision; + } + test.client + .put_cas(1, 1, key, &5_u64.to_le_bytes(), revision) + .await + .unwrap(); + }) + .await + .expect("CAS observations must not need a delayed GET"); +} + +#[tokio::test] +async fn absent_and_existing_empty_values_remain_distinct_through_batch_delete() { + let test = TestOwnerFenceServer::start().await; + let key = b"empty"; + let raw = KvRpcTransport::new() + .send_put_cas(&test.endpoint, key, b"bad", 99, 901, 1, 1, 0, 1) + .await + .unwrap(); + assert!(!raw.ok && !raw.not_found); + assert_eq!(raw.revision, 0); + assert_eq!(raw.compare_observation.unwrap().key.as_ref(), key); + let missing = observed(test.client.put_cas(1, 1, key, b"bad", 99).await.unwrap_err(), key); + assert_eq!(missing.revision, 0); + assert!(missing.value.is_empty()); + let written = test + .client + .put_cas(1, 1, key, b"", missing.revision) + .await + .unwrap(); + let deletes = [BatchOp::Delete { + key: Bytes::from_static(key), + }]; + let empty = observed( + test.client + .batch_write_cas(1, 1, &deletes, key, 0) + .await + .unwrap_err(), + key, + ); + assert_eq!(empty.revision, written.revision); + assert_ne!(empty.revision, 0); + assert!(empty.value.is_empty()); + test.client + .batch_write_cas(1, 1, &deletes, key, empty.revision) + .await + .unwrap(); + let retired = observed( + test.client + .put_cas(1, 1, key, b"bad", empty.revision) + .await + .unwrap_err(), + key, + ); + assert_eq!(retired.revision, 0); + test.client + .put_cas(1, 1, key, b"recreated", retired.revision) + .await + .unwrap(); +} + +#[tokio::test] +async fn owner_and_record_conditions_identify_the_key_that_actually_failed() { + let test = TestOwnerFenceServer::start().await; + let fence = b"/chunkdb/ownership-fence/0"; + let current = ChunkSlotAuthority::new(4, ChunkServiceIncarnation::try_from([1; 16]).unwrap(), 1) + .unwrap() + .to_fence_value(); + let stale = ChunkSlotAuthority::new(3, ChunkServiceIncarnation::try_from([2; 16]).unwrap(), 1) + .unwrap() + .to_fence_value(); + let owner = test.client.put_cas(1, 1, fence, ¤t, 0).await.unwrap(); + let record = b"record"; + let version = test.client.put(1, 1, record, b"v1", None).await.unwrap(); + let ops = [BatchOp::Put { + key: Bytes::from_static(record), + value: Bytes::from_static(b"v2"), + }]; + let failed_owner = observed( + test.client + .batch_write_owned(1, 1, &ops, fence, &stale, Some((record, 0))) + .await + .unwrap_err(), + fence, + ); + assert_eq!(failed_owner.revision, owner.revision); + assert_eq!(failed_owner.value.as_ref(), current); + let failed_record = observed( + test.client + .batch_write_owned(1, 1, &ops, fence, ¤t, Some((record, 0))) + .await + .unwrap_err(), + record, + ); + assert_eq!(failed_record.revision, version.revision); + assert_eq!(failed_record.value.as_ref(), b"v1"); + test.client + .batch_write_owned( + 1, + 1, + &ops, + fence, + ¤t, + Some((record, failed_record.revision)), + ) + .await + .unwrap(); +} diff --git a/lib/crowdb-kv/src/cluster/group.rs b/lib/crowdb-kv/src/cluster/group.rs index 5f887262d..1bb131239 100644 --- a/lib/crowdb-kv/src/cluster/group.rs +++ b/lib/crowdb-kv/src/cluster/group.rs @@ -1036,7 +1036,7 @@ pub enum ProposeResult { /// Distinct from `Err` so the KV layer can surface a retryable signal. Busy, CasFailed { - current_revision: u64, + observation: Box, }, CasBusy, OutcomeUnknown, diff --git a/lib/crowdb-kv/src/cluster/group_cas.rs b/lib/crowdb-kv/src/cluster/group_cas.rs index 69b1169c0..e4f199c9d 100644 --- a/lib/crowdb-kv/src/cluster/group_cas.rs +++ b/lib/crowdb-kv/src/cluster/group_cas.rs @@ -135,7 +135,12 @@ impl PxGroup { Ok(value) => { let current_revision = value.as_ref().map_or(0, |(revision, _)| *revision); if current_revision != expected_revision { - ProposeResult::CasFailed { current_revision } + ProposeResult::CasFailed { + observation: Box::new(crate::rpc::KvCompareObservation::from_versioned( + precondition_key, + value, + )), + } } else if !self.cas_admission_ready(tenure, required_term) { self.cas_not_ready_result() } else { diff --git a/lib/crowdb-kv/src/cluster/group_operations.rs b/lib/crowdb-kv/src/cluster/group_operations.rs index 77ad47ab8..b0a3004cc 100644 --- a/lib/crowdb-kv/src/cluster/group_operations.rs +++ b/lib/crowdb-kv/src/cluster/group_operations.rs @@ -92,7 +92,9 @@ pub enum KvGroupOperationError { #[error("proposal admission is busy")] Busy, #[error("compare-and-set precondition failed")] - CompareFailed { current_revision: u64 }, + CompareFailed { + observation: Box, + }, #[error("compare-and-set admission is busy")] CompareBusy, #[error("proposal outcome is unknown")] @@ -358,8 +360,8 @@ impl KvGroupOperations { } ProposeResult::NotLeader { leader_hint } => Err(KvGroupOperationError::NotLeader { leader_hint }), ProposeResult::Busy => Err(KvGroupOperationError::Busy), - ProposeResult::CasFailed { current_revision } => { - Err(KvGroupOperationError::CompareFailed { current_revision }) + ProposeResult::CasFailed { observation } => { + Err(KvGroupOperationError::CompareFailed { observation }) } ProposeResult::CasBusy => Err(KvGroupOperationError::CompareBusy), ProposeResult::OutcomeUnknown => Err(KvGroupOperationError::OutcomeUnknown), diff --git a/lib/crowdb-kv/src/cluster/group_owner_fence.rs b/lib/crowdb-kv/src/cluster/group_owner_fence.rs index a3f3c2e69..3e100548c 100644 --- a/lib/crowdb-kv/src/cluster/group_owner_fence.rs +++ b/lib/crowdb-kv/src/cluster/group_owner_fence.rs @@ -193,7 +193,7 @@ impl PxGroup { Ok(Some((revision, value))) if value == expected_value => revision, Ok(value) => { return ProposeResult::CasFailed { - current_revision: value.map_or(0, |(revision, _)| revision), + observation: Box::new(crate::rpc::KvCompareObservation::from_versioned(key, value)), } } Err(error) => return ProposeResult::Err(error), diff --git a/lib/crowdb-kv/src/cluster/group_propose.rs b/lib/crowdb-kv/src/cluster/group_propose.rs index 855afd494..8f9595f5c 100644 --- a/lib/crowdb-kv/src/cluster/group_propose.rs +++ b/lib/crowdb-kv/src/cluster/group_propose.rs @@ -342,7 +342,14 @@ impl PxGroup { let current_revision = value.as_ref().map_or(0, |(revision, _)| *revision); if current_revision != expected_revision { - return ProposeResult::CasFailed { current_revision }; + return ProposeResult::CasFailed { + observation: Box::new( + crate::rpc::KvCompareObservation::from_versioned( + bytes::Bytes::copy_from_slice(key), + value, + ), + ), + }; } } Err(error) => return ProposeResult::Err(error), @@ -421,7 +428,12 @@ impl PxGroup { Ok(value) => { let current_revision = value.as_ref().map_or(0, |(revision, _)| *revision); if current_revision != expected_revision { - return ProposeResult::CasFailed { current_revision }; + return ProposeResult::CasFailed { + observation: Box::new(crate::rpc::KvCompareObservation::from_versioned( + bytes::Bytes::copy_from_slice(key), + value, + )), + }; } } Err(error) => return ProposeResult::Err(error), diff --git a/lib/crowdb-kv/src/cluster/px_kv_store_handler.rs b/lib/crowdb-kv/src/cluster/px_kv_store_handler.rs index 41ee8b17b..208ee5072 100644 --- a/lib/crowdb-kv/src/cluster/px_kv_store_handler.rs +++ b/lib/crowdb-kv/src/cluster/px_kv_store_handler.rs @@ -73,13 +73,9 @@ fn operation_error_response( request_id, request_create_ms, ), - KvGroupOperationError::CompareFailed { current_revision } => crate::rpc::KvResponse::cas_error( - crate::rpc::KvErrorCode::KvErrorCasFailed, - current_revision, - "compare-and-set precondition failed", - request_id, - request_create_ms, - ), + KvGroupOperationError::CompareFailed { observation } => { + crate::rpc::KvResponse::compare_failed(observation, request_id, request_create_ms) + } KvGroupOperationError::CompareBusy => crate::rpc::KvResponse::cas_error( crate::rpc::KvErrorCode::KvErrorCasBusy, 0, diff --git a/lib/crowdb-kv/src/cluster/px_kv_store_owner.rs b/lib/crowdb-kv/src/cluster/px_kv_store_owner.rs index b4cb99f18..c5cd82381 100644 --- a/lib/crowdb-kv/src/cluster/px_kv_store_owner.rs +++ b/lib/crowdb-kv/src/cluster/px_kv_store_owner.rs @@ -48,13 +48,9 @@ pub(super) fn write_response( Err(KvGroupOperationError::NotLeader { leader_hint }) => { crate::rpc::KvResponse::not_leader(leader_hint, request_id, request_create_ms) } - Err(KvGroupOperationError::CompareFailed { current_revision }) => crate::rpc::KvResponse::cas_error( - crate::rpc::KvErrorCode::KvErrorCasFailed, - current_revision, - "compare-and-set precondition failed", - request_id, - request_create_ms, - ), + Err(KvGroupOperationError::CompareFailed { observation }) => { + crate::rpc::KvResponse::compare_failed(observation, request_id, request_create_ms) + } Err(KvGroupOperationError::Busy | KvGroupOperationError::CompareBusy) => { crate::rpc::KvResponse::cas_error( crate::rpc::KvErrorCode::KvErrorCasBusy, diff --git a/lib/crowdb-kv/src/rpc/kv_rpc_service.rs b/lib/crowdb-kv/src/rpc/kv_rpc_service.rs index 61e2ccebd..467341e08 100644 --- a/lib/crowdb-kv/src/rpc/kv_rpc_service.rs +++ b/lib/crowdb-kv/src/rpc/kv_rpc_service.rs @@ -31,6 +31,7 @@ //! (`KvClientRpcForwarder`) lives in `crowdb-kv` itself (not //! `crowdb-kv-client`) to avoid a crate cycle. +mod comparison; mod owner_fence; mod response_correlation; mod watch; @@ -1285,6 +1286,14 @@ fn build_kv_response(req_id: u64, create_nano: u64, resp: &crate::rpc::KvRespons } else { Some(builder.create_vector(&resp.value)) }; + let compare_observation = resp.compare_observation.as_ref().map(|observation| { + comparison::encode( + &mut builder, + &observation.key, + observation.revision, + &observation.value, + ) + }); let args = FBKvResponseArgs { id: req_id, rpc_create_nano: create_nano, @@ -1300,6 +1309,7 @@ fn build_kv_response(req_id: u64, create_nano: u64, resp: &crate::rpc::KvRespons value, read_slot: resp.read_slot, safe_slot: resp.safe_slot, + compare_observation, }; let fb = FBKvResponse::create(&mut builder, &args); builder.finish(fb, None); @@ -1587,6 +1597,14 @@ fn patch_not_leader_hint(ctrl: &mut (Vec, usize), endpoint: &str, req_id: u6 let error_msg = view.error_msg().map(|s| builder.create_string(s)); let hint = builder.create_string(endpoint); let value = view.value().map(|v| builder.create_vector(v)); + let compare_observation = view.compare_observation().map(|observation| { + comparison::encode( + &mut builder, + observation.key().map_or(&[], |key| key.bytes()), + observation.revision(), + observation.value().map_or(&[], |value| value.bytes()), + ) + }); let args = FBKvResponseArgs { id: req_id, rpc_create_nano: create_nano, @@ -1602,6 +1620,7 @@ fn patch_not_leader_hint(ctrl: &mut (Vec, usize), endpoint: &str, req_id: u6 value, read_slot: view.read_slot(), safe_slot: view.safe_slot(), + compare_observation, }; let fb = FBKvResponse::create(&mut builder, &args); builder.finish(fb, None); @@ -1802,6 +1821,7 @@ fn submit_kv_error( value: None, read_slot: 0, safe_slot: 0, + compare_observation: None, }; let resp = FBKvResponse::create(&mut builder, &args); builder.finish(resp, None); diff --git a/lib/crowdb-kv/src/rpc/kv_rpc_service/comparison.rs b/lib/crowdb-kv/src/rpc/kv_rpc_service/comparison.rs new file mode 100644 index 000000000..eb8fab0e3 --- /dev/null +++ b/lib/crowdb-kv/src/rpc/kv_rpc_service/comparison.rs @@ -0,0 +1,25 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Serialize a comparison observation without reading the engine again. + +use crowdb_protocol::kv_client_fb::{FBKvCompareObservation, FBKvCompareObservationArgs}; +use flatbuffers::{FlatBufferBuilder, WIPOffset}; + +pub(super) fn encode<'a>( + builder: &mut FlatBufferBuilder<'a>, + key: &[u8], + revision: u64, + value: &[u8], +) -> WIPOffset> { + let key = builder.create_vector(key); + let value = builder.create_vector(value); + FBKvCompareObservation::create( + builder, + &FBKvCompareObservationArgs { + key: Some(key), + revision, + value: Some(value), + }, + ) +} diff --git a/lib/crowdb-kv/tests/group_test/cas_test.rs b/lib/crowdb-kv/tests/group_test/cas_test.rs index ec0bc9519..df558214b 100644 --- a/lib/crowdb-kv/tests/group_test/cas_test.rs +++ b/lib/crowdb-kv/tests/group_test/cas_test.rs @@ -44,7 +44,8 @@ async fn cas_matches_current_revision_and_rejects_stale_revision() { )); assert!(matches!( group.propose_cas(encode_put(&key, b"stale"), key, 1, 9, 3).await, - ProposeResult::CasFailed { current_revision: 2 } + ProposeResult::CasFailed { observation } if observation.revision == 2 + && observation.key.as_ref() == b"cas-key" && observation.value.as_ref() == b"v2" )); } @@ -62,7 +63,8 @@ async fn create_if_absent_cas_succeeds_once() { group .propose_cas(encode_put(&key, b"second"), key, 0, 11, 1) .await, - ProposeResult::CasFailed { current_revision: 1 } + ProposeResult::CasFailed { observation } if observation.revision == 1 + && observation.key.as_ref() == b"new-key" && observation.value.as_ref() == b"first" )); } @@ -138,7 +140,8 @@ async fn create_if_absent_waits_for_acknowledged_async_apply() { .unwrap() .unwrap(); assert!( - matches!(result, ProposeResult::CasFailed { current_revision: 1 }), + matches!(&result, ProposeResult::CasFailed { observation } if observation.revision == 1 + && observation.key.as_ref() == b"async-cas-key" && observation.value.as_ref() == b"winner"), "{result:?}" ); let value = group diff --git a/lib/crowdb-kv/tests/group_test/kv_slot_retry_test.rs b/lib/crowdb-kv/tests/group_test/kv_slot_retry_test.rs index 98a74c6fe..5df48d2a5 100644 --- a/lib/crowdb-kv/tests/group_test/kv_slot_retry_test.rs +++ b/lib/crowdb-kv/tests/group_test/kv_slot_retry_test.rs @@ -3,6 +3,7 @@ use crate::common::cluster::start_cluster_classic; use bytes::Bytes; +use crowdb_kv::cluster::group::ProposeResult; use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; use crowdb_kv::rpc::KvSetRequest; @@ -17,6 +18,38 @@ fn encode_put_payload(key: &[u8], value: &[u8]) -> Vec { payload } +#[tokio::test] +async fn conditional_foreign_slot_returns_the_value_from_the_renewed_comparison() { + let cluster = start_cluster_classic(&[0, 1, 2], 0).await; + let key = Bytes::from_static(b"guard"); + let accepted = encode_put_payload(&key, b"foreign"); + for follower in cluster.followers() { + let group = follower.get_group(1).unwrap(); + let entry = PxLogEntry { + slot: 1, + ballot: PxBallot::new(10, 99), + term: 0, + payload: Bytes::from(accepted.clone()), + }; + assert!(matches!( + group.local_replica().on_accept(&entry).await, + crowdb_kv::paxos::roles::PxAcceptReply::Accepted { .. } + )); + } + let group = cluster.leader().get_group(1).unwrap(); + let result = group + .propose_cas(encode_put_payload(&key, b"original"), key.clone(), 0, 499, 1) + .await; + let ProposeResult::CasFailed { observation } = result else { + panic!("{result:?}") + }; + assert_eq!(observation.key, key); + assert_eq!(observation.revision, 1); + assert_eq!(observation.value.as_ref(), b"foreign"); + assert_eq!(group.local_replica().contiguous_chosen(), 1); + cluster.shutdown().await; +} + #[tokio::test] async fn kv_put_retries_next_slot_when_slot_has_prior_accepted_value() { let cluster = start_cluster_classic(&[0, 1, 2], 0).await; diff --git a/lib/crowdb-kv/tests/store_test/group_operations_test.rs b/lib/crowdb-kv/tests/store_test/group_operations_test.rs index 355310296..a9318be34 100644 --- a/lib/crowdb-kv/tests/store_test/group_operations_test.rs +++ b/lib/crowdb-kv/tests/store_test/group_operations_test.rs @@ -143,6 +143,6 @@ async fn conditional_write_uses_the_same_apply_fence_and_revision() { .unwrap_err(); assert!(matches!( error, - KvGroupOperationError::CompareFailed { current_revision } if current_revision == write.chosen_slot + KvGroupOperationError::CompareFailed { observation } if observation.revision == write.chosen_slot )); } diff --git a/lib/crowdb-protocol/src/fb_wrappers/kv_client.rs b/lib/crowdb-protocol/src/fb_wrappers/kv_client.rs index 48ba77d35..5f72b027f 100644 --- a/lib/crowdb-protocol/src/fb_wrappers/kv_client.rs +++ b/lib/crowdb-protocol/src/fb_wrappers/kv_client.rs @@ -66,6 +66,9 @@ impl<'a> FBKvResponseRef<'a> { pub fn safe_slot(&self) -> u64 { self.root.map_or(0, |r| r.safe_slot()) } + pub fn compare_observation(&self) -> Option> { + self.root.and_then(|r| r.compare_observation()) + } } // ── FBKvScanResponseRef ───────────────────────────────────────── diff --git a/lib/crowdb-protocol/src/fbs/kv_client.fbs b/lib/crowdb-protocol/src/fbs/kv_client.fbs index 40d0fd426..d1cef6c41 100644 --- a/lib/crowdb-protocol/src/fbs/kv_client.fbs +++ b/lib/crowdb-protocol/src/fbs/kv_client.fbs @@ -11,7 +11,8 @@ namespace crowdb.kv_client.proto; // as its first two fields, same as kv_consensus.fbs / diskdb.fbs. // - Every response carries `ret_code` (FBKvClientRetCode) + optional // `error_msg`. On Success the data fields are populated; on error -// the data fields are empty and `error_msg` describes the failure. +// `error_msg` describes the failure. CasFailed may carry the exact key and +// versioned value from the comparison in compare_observation. // - `NotLeaderHint` is NOT a separate message — it is a string field // (`not_leader_hint`) on the response tables. The client reads it // to detect a non-leader replica and redirects via its topology @@ -139,6 +140,13 @@ table FBKvBatchWriteRequest { owner_fence:FBKvOwnerFence; } +// revision zero means observed absent, distinct from an absent optional table. +table FBKvCompareObservation { + key:[ubyte]; + revision:uint64; + value:[ubyte]; +} + // Unified mutation + get response (mirrors kv.proto KvResponse). table FBKvResponse { id:uint64; @@ -155,6 +163,7 @@ table FBKvResponse { value:[ubyte]; read_slot:uint64; safe_slot:uint64; + compare_observation:FBKvCompareObservation; } // ── Scan ──────────────────────────────────────────────────────── diff --git a/lib/crowdb-protocol/src/types/kv_client.rs b/lib/crowdb-protocol/src/types/kv_client.rs index 10034fbf4..dc59ff6a1 100644 --- a/lib/crowdb-protocol/src/types/kv_client.rs +++ b/lib/crowdb-protocol/src/types/kv_client.rs @@ -158,6 +158,24 @@ pub struct KvRevisionPrecondition { pub expected_revision: u64, } +/// The key and versioned value read by a failed comparison. Revision zero +/// means absent; an existing empty value has a nonzero revision. This is an +/// observation at comparison time, not a promise that the key remains unchanged. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct KvCompareObservation { + pub key: Bytes, + pub revision: u64, + pub value: Bytes, +} + +impl KvCompareObservation { + #[must_use] + pub fn from_versioned(key: Bytes, observed: Option<(u64, Bytes)>) -> Self { + let (revision, value) = observed.unwrap_or_default(); + Self { key, revision, value } + } +} + /// Unified mutation response. #[derive(Clone, Debug, PartialEq, Default)] pub struct KvResponse { @@ -182,6 +200,8 @@ pub struct KvResponse { pub safe_slot: u64, /// Structured error code. Default 0 = NONE (old server). pub error_code: i32, + /// None on old servers or errors without a completed comparison. + pub compare_observation: Option>, } impl KvResponse { @@ -206,6 +226,7 @@ impl KvResponse { read_slot: 0, safe_slot: 0, error_code: KvErrorCode::KvErrorNone as i32, + compare_observation: None, } } @@ -234,6 +255,7 @@ impl KvResponse { read_slot: 0, safe_slot: 0, error_code: KvErrorCode::KvErrorNone as i32, + compare_observation: None, } } @@ -260,6 +282,7 @@ impl KvResponse { read_slot: 0, safe_slot: 0, error_code: KvErrorCode::KvErrorNone as i32, + compare_observation: None, } } @@ -279,6 +302,7 @@ impl KvResponse { read_slot: 0, safe_slot: 0, error_code: KvErrorCode::KvErrorNone as i32, + compare_observation: None, } } @@ -299,6 +323,7 @@ impl KvResponse { read_slot: 0, safe_slot: 0, error_code: KvErrorCode::KvErrorNotLeader as i32, + compare_observation: None, } } @@ -318,6 +343,7 @@ impl KvResponse { read_slot: 0, safe_slot: 0, error_code: KvErrorCode::KvErrorInternal as i32, + compare_observation: None, } } @@ -334,6 +360,23 @@ impl KvResponse { response.error_code = code as i32; response } + + #[must_use] + pub fn compare_failed( + observation: Box, + request_id: u64, + request_create_ms: u64, + ) -> Self { + let mut response = Self::cas_error( + KvErrorCode::KvErrorCasFailed, + observation.revision, + "compare-and-set precondition failed", + request_id, + request_create_ms, + ); + response.compare_observation = Some(observation); + response + } } /// Prefix scan over the learner store. V1: local-replica read like diff --git a/lib/crowdb-protocol/tests/kv_client_wrappers_test.rs b/lib/crowdb-protocol/tests/kv_client_wrappers_test.rs index 22bb33728..18faaee91 100644 --- a/lib/crowdb-protocol/tests/kv_client_wrappers_test.rs +++ b/lib/crowdb-protocol/tests/kv_client_wrappers_test.rs @@ -43,6 +43,7 @@ fn kv_response_round_trip() { value: Some(value), read_slot: 7, safe_slot: 5, + compare_observation: None, }, ); fbb.finish(resp, None); @@ -81,6 +82,7 @@ fn kv_response_not_leader_ret_code() { value: None, read_slot: 0, safe_slot: 0, + compare_observation: None, }, ); fbb.finish(resp, None); From fa4ea275ecdc64bf4674a832150e2d6824160f24 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 13:55:15 +0800 Subject: [PATCH 111/258] Skip empty Dataset worker windows in Python iteration --- sdk/README.md | 3 ++ sdk/crowdb_dataset/iterable.py | 14 +++++-- sdk/test_iterable.py | 75 +++++++++++++++++++++++++++++++++- 3 files changed, 87 insertions(+), 5 deletions(-) diff --git a/sdk/README.md b/sdk/README.md index 638d63961..3572fd063 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -93,6 +93,9 @@ with DatasetClient("http://localhost:9093", "training", "token") as client: The iterable acknowledges locally and makes no progress RPC. Continuing without acknowledgement fails. Save the cursor returned by `batch.acknowledge()` after processing succeeds; pass it as `checkpoint` to a new iterable to resume. +The iterable skips empty selection or worker windows and advances their local +continuation. Nonempty samples with zero-length fields are still delivered and +require acknowledgement. The lower-level `next_batch` retains empty windows. `bind_read(plan)` pins immutable versions and returns the initial cursor. `next_batch(cursor)` replays that cursor on any server. Drop or cancel stops only the current stream; there is no read ID, session TTL or remote close. Use streams diff --git a/sdk/crowdb_dataset/iterable.py b/sdk/crowdb_dataset/iterable.py index 1c99e38ed..ec41931e6 100644 --- a/sdk/crowdb_dataset/iterable.py +++ b/sdk/crowdb_dataset/iterable.py @@ -2,6 +2,8 @@ # Licensed under the Apache License, Version 2.0. """Server-partitioned batches with explicit application acknowledgement.""" +from .client import IntegrityError + class WorkerPartition: def __init__(self, worker=0, workers=1, rank=0, world_size=1): @@ -50,9 +52,15 @@ def __iter__(self): while True: with self.client.next_batch(self.checkpoint) as stream: batch = DatasetBatch(stream.batch()) - yield batch - if not batch.acknowledged: - raise RuntimeError("acknowledge the consumed batch before requesting the next batch") + following = batch.header["next"] + if (batch.header["end"] != (following is None) + or following == self.checkpoint + or (not batch.header["sample_ids"] and batch.fields)): + raise IntegrityError({"code": "integrity", "message": "invalid read continuation"}) + if batch.header["sample_ids"]: + yield batch + if not batch.acknowledged: + raise RuntimeError("acknowledge the consumed batch before requesting the next batch") self.checkpoint = batch.header["next"] self.complete = batch.header["end"] if self.complete: diff --git a/sdk/test_iterable.py b/sdk/test_iterable.py index 787ecc91a..c5b2934bc 100644 --- a/sdk/test_iterable.py +++ b/sdk/test_iterable.py @@ -1,13 +1,13 @@ # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. import unittest -from crowdb_dataset import Batch, DatasetIterable, WorkerPartition +from crowdb_dataset import Batch, DatasetIterable, IntegrityError, WorkerPartition class TestStream: def __enter__(self): return self def __exit__(self, *_): pass - def batch(self): return Batch({"next": None, "end": True}, [b"owned"]) + def batch(self): return Batch({"sample_ids": ["sample"], "next": None, "end": True}, [b"owned"]) class TestClient: @@ -39,5 +39,76 @@ def test_continuing_without_acknowledgement_is_rejected(self): next(iterator) with self.assertRaises(RuntimeError): next(iterator) + def test_empty_windows_advance_locally_and_nonempty_data_still_requires_ack(self): + client = TestWindowClient([ + Batch({"sample_ids": [], "next": "empty-2", "end": False}, []), + Batch({"sample_ids": [], "next": "data", "end": False}, []), + Batch({"sample_ids": ["sample"], "next": None, "end": True}, [b""]), + ]) + reader = DatasetIterable(client, {}) + iterator = iter(reader) + batch = next(iterator) + self.assertEqual(client.cursors, ["old", "empty-2", "data"]) + self.assertEqual(batch.fields, [b""]) + self.assertEqual(reader.checkpoint, "data") + with self.assertRaises(RuntimeError): next(iterator) + self.assertEqual(reader.checkpoint, "data") + self.assertFalse(reader.complete) + + def test_empty_terminal_partition_completes_without_yield_or_rebinding(self): + client = TestWindowClient([Batch({"sample_ids": [], "next": None, "end": True}, [])]) + reader = DatasetIterable(client, {}) + self.assertEqual(list(reader), []) + self.assertTrue(reader.complete) + self.assertIsNone(reader.checkpoint) + self.assertEqual(list(reader), []) + self.assertEqual(client.cursors, ["old"]) + self.assertEqual(client.binds, 1) + + def test_acknowledgement_then_empty_tail_reaches_eof(self): + client = TestWindowClient([ + Batch({"sample_ids": [], "next": "data", "end": False}, []), + Batch({"sample_ids": ["sample"], "next": "tail", "end": False}, [b"value"]), + Batch({"sample_ids": [], "next": None, "end": True}, []), + ]) + reader = DatasetIterable(client, {}) + iterator = iter(reader) + batch = next(iterator) + self.assertEqual(batch.acknowledge(), "tail") + self.assertEqual(reader.checkpoint, "data") + self.assertEqual(list(iterator), []) + self.assertTrue(reader.complete) + self.assertEqual(client.cursors, ["old", "data", "tail"]) + + def test_empty_window_cannot_replay_forever_or_hide_payload(self): + for header, fields in [ + ({"sample_ids": [], "next": "old", "end": False}, []), + ({"sample_ids": [], "next": None, "end": False}, []), + ({"sample_ids": [], "next": "next", "end": True}, []), + ({"sample_ids": [], "next": None, "end": True}, [b"hidden"]), + ]: + with self.subTest(header=header, fields=fields): + reader = DatasetIterable(TestWindowClient([Batch(header, fields)]), {}) + with self.assertRaises(IntegrityError): next(iter(reader)) + self.assertEqual(reader.checkpoint, "old") + + +class TestWindowStream: + def __init__(self, batch): self._batch = batch + def __enter__(self): return self + def __exit__(self, *_): pass + def batch(self): return self._batch + + +class TestWindowClient(TestClient): + def __init__(self, windows): + self.windows, self.cursors, self.binds = iter(windows), [], 0 + def bind_read(self, plan): + self.binds += 1 + return super().bind_read(plan) + def next_batch(self, cursor): + self.cursors.append(cursor) + return TestWindowStream(next(self.windows)) + if __name__ == "__main__": unittest.main() From aa87a872f6d7df0f317752682b8b1c4759d52446 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 13:55:15 +0800 Subject: [PATCH 112/258] Migrate container fixtures to stateless access contracts --- doc/working/plan-access-simplification.md | 46 +++++++- e2e-test/clients/iceberg-python/namespace.py | 5 +- e2e-test/tests/access_iceberg/catalog.rs | 79 +------------ .../tests/access_iceberg/catalog/fault.rs | 2 +- .../tests/access_iceberg/catalog/journal.rs | 78 ++++++++++--- .../tests/access_iceberg/catalog/property.rs | 51 ++++----- e2e-test/tests/access_iceberg/catalog/scan.rs | 107 ++++++++++++++++++ e2e-test/tests/chunkdb/lifecycle.rs | 63 ++++++++++- .../src/authority/access.rs | 2 + 9 files changed, 302 insertions(+), 131 deletions(-) create mode 100644 e2e-test/tests/access_iceberg/catalog/scan.rs diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 94d76eb33..0112e4af2 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -130,6 +130,24 @@ CAS 不再满足只证明“本次条件不成立”，不能谎称“第一次 不为每次请求添加历史查询或回执；只有某个具体调用者必须识别被后续提交遮住的历史成功， 才提出该场景需要的最小方案及代价，交用户 review。 +**资源可见性约束 review（2026-10-24，等待决定）**：现代码把“内容已完整生成”与“曾经成功 +切换 head”区分。Snapshot 的 `Published` 标记在 head CAS 之前写，不能单独证明成功； +`require_committed_snapshot`、历史读取、parent/stable 准入和 Split revision 历史依赖 +`DS\0SC\0` / `DS\0SRC\0`。每次发布还写 `DS\0CA\0` attempt、`DS\0CR\0` response receipt； +同一成功证据写两份，head 被替换前还帮前一个写者补齐。CA/CR 是请求账本；SC/SRC 在当前 +合同下承担资源可见性，不能把三者一起删后仍声称保留原合同。 + +- 场景：完整结果 S1 已经生成；客户端以 H0 发布 S1 的 CAS 输给 S2。现在 S1 既不可按 ID + 读取，也不可成为新 Snapshot 的 parent/stable。另一个场景是 S1 曾赢 head、回复丢失、 + 又被 S2 替换；S1 必须可作历史资源。为区分两者，现实现保存上述成功证明。 +- 建议：完整不可变结果按资源 ID 即可读取/引用；head CAS 只决定 latest/stable 指针是否 + 按指定 base 更新。未完成内容仍不可读；未引用结果按明确期限过期，过期请求失败并重做。 + 这可删除“曾切换 head”证明、attempt/response 账本与相应 helper；GC/期限隔离仍须按 R4 + 单独证明，不能仅凭本地超时回收。它改变现有可见性合同，不能未经决定直接放开。 +- 保持严格语义的备选：保留每个资源一份最小成功事实，删除 CA/CR 及按 attempt 查询。 + 沿用当前独立键存储时，每个发布仍需额外一次条件写成功事实及覆盖 head 前的帮助检查； + 这份记录仅用于历史资源可见性，读取不登记，不保存编辑/计算过程或原响应。 + - [ ] **R2 审查条件发布的回复丢失窗口**：用 H0→H1→H2 测试区分条件失败、相同结果与历史未知； 列出确实依赖历史精确结果的调用者，没有这种需求就移除相应 ledger/response replay。 Files: Iceberg `commit/direct.rs`、Dataset `authority/publication/`、Split head、SDK。 @@ -434,11 +452,6 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 无条件覆盖与 expected-base 重试测试。断言总 GET+写，不仅是零写。Files: Dataset `tests/*cost*`、`read_query*`、upload/Composition/Split/inspection；S3/Iceberg 对应 repository 测试。 - [ ] **V2 HTTP/native/SDK 集成**：各受影响 route 的鉴权、错误、协议/SDK 一致性；保留删除准入、 迟到写者、非空 namespace、共享 owner、held buffer、cancel/drop 和损坏内容测试。Files: Access/native tests、`sdk/tests/`、native Python consumer。 -- [ ] **V2.1 迁移旧容器 fixture 并验证当前源码**：`cargo check -p e2e-test --tests` 已发现 - catalog fixture 仍调用已删除 RetryLedger、旧 property journal/响应 API，ChunkDB fixture - 仍断言 DeleteChunkRange Unimplemented。改为验证资源结果、无新过程记录、legacy 解码与 - GC 兼容，以及已接受的范围退休语义；不恢复生产 ledger 来通过旧测试。 - Files: `e2e-test/tests/access_iceberg/catalog/{journal,property}.rs`、`catalog.rs`、`chunkdb/lifecycle.rs`。 - [ ] **V3 工程门禁与文档**：每批执行适用 fmt/clippy/测试，全部经 `pixi run`；同步 owning design/spec/OpenAPI/SDK。只修改文档时检查 diff、引用和任务状态，不为此跑整套服务测试。 - [ ] **V4 同源真实负载验收**：重新构建当前源码的镜像，运行现有 HTTP/native Rust/native Python @@ -446,6 +459,11 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 记录 p50/p99、TPS、CPU/RSS、RPC、复制/摘要、清理积压，不缩小负载掩盖失败。 Files: `tools/sdk/dataset_workload.py`、相关 examples、`e2e-test/`、本机 HCA runner。 物理多主机验证继续留在独立计划，不并回此处。 +- [~] **V4.1 同源 TCP 混合负载与 native 绑定验收**：使用当前镜像运行 + `tools/sdk/dataset_workload.py` 完整模式；client CPU 2、server CPU 4–15、metadata 节点 + 2 GiB，三条路径同 Snapshot、相同 0–4 MiB size 集合、并发 1/3、完整摘要校验及 restart。 + 保留原 trial 数与全部失败日志，记录延迟/CPU/RSS/复制及精确 image/source 身份；本机 HCA + 和改动前后吞吐对比仍由 V4 跟踪，不把本项当作全部性能验收。 ## 成本口径与集中代码索引 @@ -466,6 +484,24 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **V2.1（2026-10-24）**：迁移旧 RetryLedger/property journal fixture，不恢复生产请求账本。 + property 在实际 authority CAS 丢回复后检查资源已落地且 key 集合未增加，重启后重试返回 + 当前 missing 状态；手工构造 legacy inline/paged response 验证存储兼容。范围通知保留精确 + revision/byte range，重复接受不声称物理容量释放，dedicated chunk 仍拒绝错误范围调用。 + 新源码镜像 `crowdb-node:access-cleanup-5195dfff`，OCI digest + `sha256:7bffacfc3824f55e06d6b109070207965af5ad2c4bee0de25594a9f7971c271b`，source tree + `95d97a857d52552128d439baf08f00e0b1988120`；Docker import identity + `sha256:31f9acf7749663e987d0de8ee143d87f1af8d727aaed35725525f8582b3e509f`。 + catalog 重启 `run-67ab0f42fd984da290d8d222fea37b38`、ChunkDB lifecycle/range 通知 + `run-bd3471d493434467b60c6679e60f51f1`、全部五个 Dataset HTTP/Rust/安装后 Python SDK 用例 + `run-0731e2669f5042db81212c94a59e6ab2`、namespace CRUD/restart + `run-97eb01b45c2c4ebe91af5f73e66eef84` 全部通过。记录位于 + `.crowdb-runtime/artifacts/container-e2e/`。E2E check/clippy、fmt/diff 通过。 + 首轮 catalog 失败是 Python fixture 仍要求已删除的 idempotency 广告，已改为检查不广告 + 和重复删除的当前资源效果。首轮 Dataset 发现 Python iterable 把合法 worker 空窗口交给 + 应用；已修复本地推进，保留非空和零长度字段的显式 ack，补空终页/无进展/隐藏 payload + 回归，重新打包安装 wheel 后整套通过。两个失败 run 均保留，未增 timeout 或降低校验。 + - **P14.3（2026-10-24）**：Group CAS 的首次比较、foreign slot 后重比较、失败 slot 结算后重比较及 owner value 比较，均直接移交同一 engine lookup 的 key/revision/value。 可选 `FBKvCompareObservation` 贯穿 typed operations、RPC 和 client error；旧 peer 与 diff --git a/e2e-test/clients/iceberg-python/namespace.py b/e2e-test/clients/iceberg-python/namespace.py index f468f3d15..0e7ada86a 100644 --- a/e2e-test/clients/iceberg-python/namespace.py +++ b/e2e-test/clients/iceberg-python/namespace.py @@ -53,7 +53,7 @@ def main(): "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials", } assert set(response.json()["endpoints"]) == expected_endpoints, response.json()["endpoints"] - assert response.json()["idempotency-key-lifetime"] == "PT24H" + assert "idempotency-key-lifetime" not in response.json() catalog = load_catalog("crowdb", **properties) namespaces = catalog.list_namespaces() assert isinstance(namespaces, list) @@ -90,6 +90,9 @@ def verify_namespaces(uri, properties): update = writer.update_namespace_properties(namespace, removals={"owner", "absent"}, updates={"owner2": "retained"}) assert update.removed == ["owner"] and update.missing == ["absent"] assert writer.load_namespace_properties(namespace) == {"owner2": "retained"} + repeated = writer.update_namespace_properties(namespace, removals={"owner", "absent"}, updates={"owner2": "retained"}) + assert repeated.removed == [] and repeated.missing == ["absent", "owner"] + assert writer.load_namespace_properties(namespace) == {"owner2": "retained"} try: writer.drop_namespace(namespace) except NamespaceNotEmptyError: diff --git a/e2e-test/tests/access_iceberg/catalog.rs b/e2e-test/tests/access_iceberg/catalog.rs index 13b3acc28..94576fe71 100644 --- a/e2e-test/tests/access_iceberg/catalog.rs +++ b/e2e-test/tests/access_iceberg/catalog.rs @@ -12,6 +12,7 @@ use super::frontend; mod journal; mod namespace; mod property; +mod scan; use super::{ native::{self, TestNativeStorage}, now_ms, @@ -126,7 +127,7 @@ async fn catalog_recovery_survives_real_chunk_kv_restart() { assert_eq!(execute(&repository, initialize).await, original); assert_eq!(repository.status().await.unwrap().0.context.activation_epoch, 3); native::activate(&repository).await; - verify_retry_scan(&stack, &repository).await; + scan::verify_legacy_records(&stack, &repository).await; frontend.stop().await; second_frontend.stop().await; namespace::verify_name_index(&stack, latest.catalog).await; @@ -178,79 +179,3 @@ async fn verify_interrupted_clear(stack: &TestNativeStorage, repository: &Catalo assert_eq!(execute(repository, clear).await, recovered); } } - -async fn verify_retry_scan(stack: &TestNativeStorage, repository: &CatalogRepository) { - use crowdb_access_iceberg::key::IcebergKey; - use crowdb_access_iceberg::operation::{RetryAdmission, RetryLedger, RetryRecord}; - use crowdb_chunk_kv_client::MultiScanRequest; - use crowdb_protocol::chunk_kv::ScanDirection; - let store = stack.store().await; - let ledger = RetryLedger::new(store.clone()); - let context = repository.status().await.unwrap().0.context; - for _ in 0..3 { - let request = RetryRecord { - identity: RequestIdentity { - operation: OperationId::random(), - issued_ms: now_ms(), - }, - principal: "reader".into(), - route: "test mutation".into(), - digest: [1; 32], - context, - retained_until_ms: 0, - status: 0, - body: Vec::new(), - }; - assert!(matches!( - ledger.begin(request.clone(), now_ms()).await.unwrap(), - RetryAdmission::New(_) - )); - assert!(!ledger - .finish(request.clone(), 503, Vec::new(), now_ms()) - .await - .unwrap()); - assert!(matches!( - ledger.begin(request.clone(), now_ms()).await.unwrap(), - RetryAdmission::Resume(_) - )); - ledger - .finish(request.clone(), 409, b"conflict".to_vec(), now_ms()) - .await - .unwrap(); - let RetryAdmission::Replay(result) = RetryLedger::new(stack.store().await) - .begin(request, now_ms()) - .await - .unwrap() - else { - panic!("durable result replay") - }; - assert_eq!(result.status, 409); - assert_eq!(result.body, b"conflict"); - } - let range = IcebergKey::catalog_range(context.catalog); - let mut scan = MultiScanRequest { - start: Some(range.start.clone()), - end: Some(range.end.clone()), - direction: ScanDirection::Forward, - max_items: 1, - max_bytes: 64 * 1024, - continuation: None, - }; - let mut count = 0; - loop { - let page = store.scan(scan.clone()).await.unwrap(); - assert!(page.items.len() <= 1); - for item in &page.items { - assert!(range.contains(&item.key)); - } - count += page.items.len(); - scan.continuation = page.continuation; - if scan.continuation.is_none() { - break; - } - assert!(count <= 4); - } - assert_eq!(count, 4); - scan.start = None; - assert!(store.scan(scan).await.is_err()); -} diff --git a/e2e-test/tests/access_iceberg/catalog/fault.rs b/e2e-test/tests/access_iceberg/catalog/fault.rs index 134e7514e..1ded41ad4 100644 --- a/e2e-test/tests/access_iceberg/catalog/fault.rs +++ b/e2e-test/tests/access_iceberg/catalog/fault.rs @@ -52,7 +52,7 @@ impl CatalogStore for TestFaultStore { let intercept = matches!(&record, StorageRecord::Active(root) if root.state == RootState::Fencing) || (self.mode.load(Ordering::SeqCst) == 3 && expected.is_some() - && matches!(&record, StorageRecord::NamespaceAuthority(authority) if authority.pending_operation.is_some())); + && matches!(&record, StorageRecord::NamespaceAuthority(authority) if authority.property_revision == 2 && authority.pending_operation.is_none())); let mode = if intercept { self.mode.swap(0, Ordering::SeqCst) } else { diff --git a/e2e-test/tests/access_iceberg/catalog/journal.rs b/e2e-test/tests/access_iceberg/catalog/journal.rs index f5e63b95b..8ab4701bf 100644 --- a/e2e-test/tests/access_iceberg/catalog/journal.rs +++ b/e2e-test/tests/access_iceberg/catalog/journal.rs @@ -1,16 +1,19 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use crowdb_access_iceberg::catalog::{CatalogContext, CatalogStore}; +use crowdb_access_iceberg::catalog::{CatalogContext, CatalogStore, RoutedCatalogStore}; use crowdb_access_iceberg::key::{NamespaceId, OperationId}; use crowdb_access_iceberg::namespace::{ NamespaceAction, NamespaceIdentifier, NamespaceJournal, NamespaceOperation, NamespacePhase, }; use crowdb_access_iceberg::operation::{ - PayloadStore, RequestIdentity, RetryAdmission, RetryLedger, RetryRecord, + mutation_identity, PayloadStore, RequestIdentity, RetryRecord, RetryResult, }; use super::{now_ms, TestNativeStorage}; +use crowdb_access_iceberg::key::SystemScope; +use crowdb_access_iceberg::record::StorageRecord; +use std::sync::Arc; pub async fn verify_recovery(stack: &mut TestNativeStorage, context: CatalogContext) { let store = stack.store().await; @@ -20,7 +23,7 @@ pub async fn verify_recovery(stack: &mut TestNativeStorage, context: CatalogCont let identity = fresh_identity(store.as_ref()).await; let body = vec![23; 70 * 1024]; let input = PayloadStore::new(store.clone()) - .put(context.catalog, identity.operation, &body) + .put(context.catalog, identity.operation, b"{}") .await .unwrap(); let mut operation = NamespaceOperation { @@ -51,16 +54,11 @@ pub async fn verify_recovery(stack: &mut TestNativeStorage, context: CatalogCont route: "POST /namespaces".into(), digest: operation.input.digest, context, - retained_until_ms: 0, - status: 0, + retained_until_ms: now_ms() + 300_000, + status: 200, body: Vec::new(), }; - let ledger = RetryLedger::new(store); - ledger.begin(request.clone(), now_ms()).await.unwrap(); - ledger - .finish(request.clone(), 200, body.clone(), now_ms()) - .await - .unwrap(); + let legacy = seed_legacy_result(store, &request, &body).await; stack.restart_catalog().await; let recovered_store = stack.store().await; super::property::verify(recovered_store.clone(), &property_request).await; @@ -75,15 +73,24 @@ pub async fn verify_recovery(stack: &mut TestNativeStorage, context: CatalogCont .unwrap(), operation ); - let RetryAdmission::Replay(result) = RetryLedger::new(recovered_store) - .begin(request, now_ms()) + let key = request.result_key(); + let value = recovered_store + .get(&key.encode().unwrap()) .await .unwrap() - else { - panic!("large response replay") + .unwrap(); + assert_eq!(StorageRecord::decode(&key, &value.bytes).unwrap(), legacy); + let StorageRecord::RetryResult(result) = legacy else { + panic!("legacy paged response") }; - assert_eq!(result.body, body); - assert_eq!(result.principal, "writer"); + assert_eq!(result.binding, request); + assert_eq!( + PayloadStore::new(recovered_store) + .get(&result.body) + .await + .unwrap(), + body + ); } async fn fresh_identity(store: &dyn CatalogStore) -> RequestIdentity { @@ -105,3 +112,40 @@ async fn fresh_identity(store: &dyn CatalogStore) -> RequestIdentity { } panic!("no free retry slot for fixture"); } + +async fn seed_legacy_result( + store: Arc, + request: &RetryRecord, + body: &[u8], +) -> StorageRecord { + let response = PayloadStore::new(store.clone()) + .put(request.context.catalog, request.identity.operation, body) + .await + .unwrap(); + let legacy = StorageRecord::RetryResult(Box::new(RetryResult { + binding: request.clone(), + body: response, + })); + for (key, record) in [ + ( + crowdb_access_iceberg::operation::ledger_key( + SystemScope::RetryBinding, + request.identity.operation, + ) + .unwrap(), + StorageRecord::Retry(Box::new(request.clone())), + ), + (request.result_key(), legacy.clone()), + ] { + let key = key.encode().unwrap(); + let value = record.encode().unwrap(); + assert!(matches!( + store + .compare_exchange(&key, None, &value, mutation_identity(&key, None, &value)) + .await + .unwrap(), + crowdb_access_iceberg::catalog::CasOutcome::Applied(_) + )); + } + legacy +} diff --git a/e2e-test/tests/access_iceberg/catalog/property.rs b/e2e-test/tests/access_iceberg/catalog/property.rs index 7ba860a68..6b82c3cc8 100644 --- a/e2e-test/tests/access_iceberg/catalog/property.rs +++ b/e2e-test/tests/access_iceberg/catalog/property.rs @@ -7,26 +7,21 @@ use std::sync::{atomic::AtomicU8, Arc}; use crowdb_access_iceberg::catalog::{CatalogContext, CatalogStore, RoutedCatalogStore}; use crowdb_access_iceberg::key::{NamespaceId, OperationId}; use crowdb_access_iceberg::namespace::{ - authority_key, name_key, NamespaceAuthority, NamespaceIdentifier, NamespaceJournal, NamespaceLifecycle, - NamespaceMapping, NamespaceMappingState, NamespacePhase, NamespaceProperties, NamespacePropertyRequest, - NamespaceRepository, PropertyChanges, + authority_key, name_key, NamespaceAuthority, NamespaceIdentifier, NamespaceLifecycle, NamespaceMapping, + NamespaceMappingState, NamespaceProperties, NamespacePropertyRequest, NamespaceRepository, + PropertyChanges, }; -use crowdb_access_iceberg::operation::{mutation_identity, PayloadStore, RequestIdentity}; +use crowdb_access_iceberg::operation::mutation_identity; use crowdb_access_iceberg::record::StorageRecord; -use super::{fault::TestFaultStore, now_ms}; +use super::fault::TestFaultStore; pub async fn prepare(store: Arc, context: CatalogContext) -> NamespacePropertyRequest { let request = NamespacePropertyRequest { context, - identity: RequestIdentity { - operation: OperationId::random(), - issued_ms: now_ms(), - }, - principal: "writer".into(), identifier: NamespaceIdentifier::new(vec!["property-recovery".into()]).unwrap(), changes: PropertyChanges { - removals: Vec::new(), + removals: vec!["gone".into()], updates: BTreeMap::from([("owner".into(), "survives-restart".into())]), }, }; @@ -41,7 +36,8 @@ pub async fn prepare(store: Arc, context: CatalogContext) -> mutation_revision: 1, lifecycle: NamespaceLifecycle::Ready, pending_operation: None, - properties: NamespaceProperties::default(), + rename: None, + properties: NamespaceProperties::new(BTreeMap::from([("gone".into(), "old".into())])).unwrap(), }; let mapping = NamespaceMapping { catalog: context.catalog, @@ -69,6 +65,7 @@ pub async fn prepare(store: Arc, context: CatalogContext) -> .await .unwrap(); } + let before = super::scan::catalog_keys(&store, context).await; let fault = Arc::new(TestFaultStore { inner: store.clone(), mode: AtomicU8::new(3), @@ -77,29 +74,25 @@ pub async fn prepare(store: Arc, context: CatalogContext) -> .update_properties(&request) .await .is_err()); - assert_eq!( - NamespaceJournal::new(store) - .load(context, request.identity.operation) - .await - .unwrap() - .unwrap() - .phase, - NamespacePhase::Publishing - ); + let applied = NamespaceRepository::new(store.clone()) + .load(context, &request.identifier) + .await + .unwrap() + .unwrap(); + assert_eq!(applied.property_revision, 2); + assert!(applied.pending_operation.is_none()); + assert!(applied.rename.is_none()); + assert!(!applied.properties.entries().contains_key("gone")); + assert_eq!(before, super::scan::catalog_keys(&store, context).await); request } pub async fn verify(store: Arc, request: &NamespacePropertyRequest) { let repository = NamespaceRepository::new(store.clone()); let outcome = repository.update_properties(request).await.unwrap().unwrap(); - assert_eq!(outcome.status, 200); - let body = PayloadStore::new(store).get(&outcome.body).await.unwrap(); - assert_eq!( - serde_json::from_slice::(&body).unwrap(), - serde_json::json!({ - "removed": [], "updated": ["owner"], "missing": [], - }) - ); + assert!(outcome.removed.is_empty()); + assert_eq!(outcome.updated, ["owner"]); + assert_eq!(outcome.missing, ["gone"]); assert_eq!( repository.update_properties(request).await.unwrap(), Some(outcome) diff --git a/e2e-test/tests/access_iceberg/catalog/scan.rs b/e2e-test/tests/access_iceberg/catalog/scan.rs new file mode 100644 index 000000000..7ce366e28 --- /dev/null +++ b/e2e-test/tests/access_iceberg/catalog/scan.rs @@ -0,0 +1,107 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Historical response bytes remain decodable; new API calls use no ledger. + +use crowdb_access_iceberg::catalog::{ + CasOutcome, CatalogContext, CatalogRepository, CatalogStore, RoutedCatalogStore, +}; +use crowdb_access_iceberg::key::{IcebergKey, OperationId}; +use crowdb_access_iceberg::operation::{mutation_identity, RequestIdentity, RetryRecord}; +use crowdb_access_iceberg::record::StorageRecord; +use crowdb_chunk_kv_client::MultiScanRequest; +use crowdb_protocol::chunk_kv::ScanDirection; + +use super::{now_ms, TestNativeStorage}; + +pub(super) async fn catalog_keys( + store: &RoutedCatalogStore, + context: CatalogContext, +) -> std::collections::BTreeSet> { + let range = IcebergKey::catalog_range(context.catalog); + let mut request = MultiScanRequest { + start: Some(range.start.clone()), + end: Some(range.end.clone()), + direction: ScanDirection::Forward, + max_items: 20, + max_bytes: 1024 * 1024, + continuation: None, + }; + let mut keys = std::collections::BTreeSet::new(); + loop { + let page = store.scan(request.clone()).await.unwrap(); + for item in page.items { + assert!(range.contains(&item.key)); + assert!(keys.insert(item.key)); + } + assert!( + keys.len() <= 10_000, + "fixture catalog exceeded its inventory bound" + ); + request.continuation = page.continuation; + if request.continuation.is_none() { + return keys; + } + } +} + +pub async fn verify_legacy_records(stack: &TestNativeStorage, repository: &CatalogRepository) { + let store = stack.store().await; + let context = repository.status().await.unwrap().0.context; + for _ in 0..3 { + let record = RetryRecord { + identity: RequestIdentity { + operation: OperationId::random(), + issued_ms: now_ms(), + }, + principal: "reader".into(), + route: "legacy fixture mutation".into(), + digest: [1; 32], + context, + retained_until_ms: now_ms() + 300_000, + status: 409, + body: b"conflict".to_vec(), + }; + let key = record.result_key(); + let bytes = StorageRecord::Retry(Box::new(record.clone())).encode().unwrap(); + let encoded = key.encode().unwrap(); + assert!(matches!( + store + .compare_exchange(&encoded, None, &bytes, mutation_identity(&encoded, None, &bytes)) + .await + .unwrap(), + CasOutcome::Applied(_) + )); + let value = stack.store().await.get(&encoded).await.unwrap().unwrap(); + assert_eq!( + StorageRecord::decode(&key, &value.bytes).unwrap(), + StorageRecord::Retry(Box::new(record)) + ); + } + let range = IcebergKey::catalog_range(context.catalog); + let mut scan = MultiScanRequest { + start: Some(range.start.clone()), + end: Some(range.end.clone()), + direction: ScanDirection::Forward, + max_items: 1, + max_bytes: 64 * 1024, + continuation: None, + }; + let mut count = 0; + loop { + let page = store.scan(scan.clone()).await.unwrap(); + assert!(page.items.len() <= 1); + for item in &page.items { + assert!(range.contains(&item.key)); + } + count += page.items.len(); + scan.continuation = page.continuation; + if scan.continuation.is_none() { + break; + } + assert!(count <= 4); + } + assert_eq!(count, 4); + scan.start = None; + assert!(store.scan(scan).await.is_err()); +} diff --git a/e2e-test/tests/chunkdb/lifecycle.rs b/e2e-test/tests/chunkdb/lifecycle.rs index 66a214ca6..7d2bcb991 100644 --- a/e2e-test/tests/chunkdb/lifecycle.rs +++ b/e2e-test/tests/chunkdb/lifecycle.rs @@ -75,10 +75,71 @@ async fn allocate_seal_delete_preserves_physical_placement_and_free_facts() { chunk_id: Some(id), chunk_offset: 0, chunk_size: 1, + expected_modify_ts: queried.modify_ts, }) .await, - Err(ChunkdbClientError::Unimplemented(_)) + Err(ChunkdbClientError::InvalidArgument(_)) )); + verify_range_notification(&client).await; +} + +async fn verify_range_notification(client: &ChunkdbClient) { + let allocated = client + .allocate_chunk(AllocateChunkRequest { + write_granularity: 1, + strip_count: 1, + strip_type: StripType::Mirror as i32, + copy_count: 3, + chunk_type: ChunkType::S3 as i32, + writer_epoch: 17, + writer_lease_ms: 60_000, + ..Default::default() + }) + .await + .unwrap() + .chunk + .unwrap(); + let id = allocated.id.unwrap(); + let sealed = client + .seal_chunk(SealChunkRequest { + chunk_id: Some(id), + seal_bytes: 8192, + ..Default::default() + }) + .await + .unwrap() + .chunk + .unwrap(); + let request = DeleteChunkRangeRequest { + chunk_id: Some(id), + chunk_offset: 134, + chunk_size: 234, + expected_modify_ts: sealed.modify_ts, + }; + assert!( + !client + .delete_chunk_range(request.clone()) + .await + .unwrap() + .reclaimed + ); + assert!(!client.delete_chunk_range(request).await.unwrap().reclaimed); + assert_eq!( + client + .query_chunk(QueryChunkRequest { chunk_id: Some(id) }) + .await + .unwrap() + .chunk + .unwrap(), + sealed + ); + let retired = client + .delete_chunk(DeleteChunkRequest { chunk_id: Some(id) }) + .await + .unwrap() + .chunk + .unwrap(); + assert_eq!(retired.state, ChunkState::Deleted as i32); } async fn assert_free_facts(segments: &[crowdb_protocol::diskdb::rpc::Segment]) { diff --git a/lib/crowdb-access-dataset/src/authority/access.rs b/lib/crowdb-access-dataset/src/authority/access.rs index 27941e60d..e6f99bc1f 100644 --- a/lib/crowdb-access-dataset/src/authority/access.rs +++ b/lib/crowdb-access-dataset/src/authority/access.rs @@ -187,6 +187,8 @@ impl DatasetAuthority { Ok(()) } + // The per-authority clock override exists only in test builds. + #[cfg_attr(not(feature = "test-util"), allow(clippy::unused_self))] fn reclaim_time(&self) -> u64 { #[cfg(feature = "test-util")] if let Some(clock) = &self.reclaim_clock { From 2aebe54b4fc2c9f6229ad6c32579d1e19ac99878 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 14:13:02 +0800 Subject: [PATCH 113/258] Overlap bounded Dataset delivery metadata reads --- .../dataset/design-dataset-control.md | 11 + doc/working/plan-access-simplification.md | 40 ++- lib/crowdb-access-dataset/src/inspection.rs | 1 + .../src/inspection/evaluate.rs | 13 +- .../src/inspection/materialize.rs | 43 ++- .../src/inspection/read.rs | 91 ++--- .../src/inspection/read_fields.rs | 111 ++++++ .../src/inspection/work.rs | 62 ++++ .../tests/common/inspection_reads.rs | 140 ++++++++ .../tests/inspection_parallel_read_test.rs | 321 ++++++++++++++++++ 10 files changed, 757 insertions(+), 76 deletions(-) create mode 100644 lib/crowdb-access-dataset/src/inspection/read_fields.rs create mode 100644 lib/crowdb-access-dataset/tests/common/inspection_reads.rs create mode 100644 lib/crowdb-access-dataset/tests/inspection_parallel_read_test.rs diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index 802f9dfe0..9599cb161 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -119,6 +119,17 @@ results. Explicit batch reads remain independent admitted requests. HTTP deliver moves inline payload owners from resolved locators into output buffers without cloning the payload. +Delivery resolves membership and field bindings in windows of at most sixteen +cells. Independent point reads overlap within one ancestry level; unresolved +fields proceed to the parent only after its read-only admission. Ancestor summaries +remain local to the request. Each read reserves its response cap before dispatch, +with at most sixteen requests and 2 MiB of response capacity in flight. Completed +rows consume the aggregate metadata budget, including responses waiting behind +an earlier read. Unused reservations are released before admitting a smaller final +read. Request order, missing fields, tombstones and locator integrity checks remain +independent of completion order. All windows share the request deadline and +cancellation; no reader state is persisted. + Old snapshots without projections require explicit resumable inspection preparation. Unsupported or oversized legacy state is reported, never represented as an empty Snapshot. Field preview is explicit and capped at 64 KiB after read-only admission; it diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 0112e4af2..5ec440929 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -422,9 +422,11 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 与 token 大小；不恢复持久化 session。Files: Dataset `read_query/page.rs`、`inspection/evaluate.rs`、SDK iterable。 - [ ] **F4 替换 legacy scan 全量规划**：`scan_surface` 每页重建所有窗口，造成 CPU/内存放大； 追踪调用者后迁移到有界 inspection 或增量 planner。Files: Dataset `transport.rs`、`planner/`、legacy API 和 scan 测试。 -- [ ] **F6 批量或并发 locator 读取**：membership + 每 sample/field/ancestor 元数据 + value - 读取目前串行；bounded point read 在 chunk store 内还转成单 key scan。保留 materialize 前 - 字节/期限限制、结果顺序和 missing/tombstone。Files: Dataset `inspection/`、`chunk_store/metadata.rs`、相关 RPC 支持。 +- [ ] **F6 剩余 inspection 并发与 routed batch**：delivery 的 membership/field/locator 已有界 + 并发；matrix/selection/composition source 仍有串行步骤，bounded point read 在 chunk store + 内还转成单 key scan。比较有字节界限的 routed batch，保留 materialize 前的聚合预算、 + 共同截止时间、结果顺序和 missing/tombstone；不可把并发当作 RPC 数下降。 + Files: Dataset `inspection/`、`chunk_store/metadata.rs`、相关 RPC 支持。 - [ ] **F7 比较预读与 create-CAS 总成本**：新 immutable 页的 GET+CAS 可能比直接 CAS 更贵， replay 则可能相反；比较新写/重放比例，复用本请求已知值，不能无证明改普通 PUT。Files: Dataset Composition/Split receipts、Iceberg `operation/payload.rs`、journal validation。 - [ ] **F8 减少 payload 复制**：审查 HTTP 收整 part、`BytesMut` 拼接、inline copy、兼容 @@ -459,11 +461,6 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 记录 p50/p99、TPS、CPU/RSS、RPC、复制/摘要、清理积压，不缩小负载掩盖失败。 Files: `tools/sdk/dataset_workload.py`、相关 examples、`e2e-test/`、本机 HCA runner。 物理多主机验证继续留在独立计划，不并回此处。 -- [~] **V4.1 同源 TCP 混合负载与 native 绑定验收**：使用当前镜像运行 - `tools/sdk/dataset_workload.py` 完整模式；client CPU 2、server CPU 4–15、metadata 节点 - 2 GiB，三条路径同 Snapshot、相同 0–4 MiB size 集合、并发 1/3、完整摘要校验及 restart。 - 保留原 trial 数与全部失败日志，记录延迟/CPU/RSS/复制及精确 image/source 身份；本机 HCA - 和改动前后吞吐对比仍由 V4 跟踪，不把本项当作全部性能验收。 ## 成本口径与集中代码索引 @@ -484,6 +481,33 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **F6.1（2026-10-24）**：delivery 按最多 16 个 sample-field cell 分窗，同层 field/祖先和 + value 分阶段并发，membership 独立分窗；相同祖先只准入一次。每条 bounded read 发起前 + 预留 response cap，在途最多 16 条/2 MiB，已完成但等前序的响应仍占预留；释放未用额度后 + 才缩小最后一条读取的 cap，防止临时预留导致错误 Limit。全请求共用字节预算和绝对期限， + 返回保持 ID/字段绑定；零长度 inline、Chunk 长度/摘要、墓碑及继承原有校验均保留。 + 没有增加 GET/写、跨请求缓存、session 或锁；32 sample/三字段/两代 fixture 仍为 + 32 membership + 160 field + 32 value GET，祖先 summary 各一次，并发峰值不超过 16。 + 新增五例验证乱序完成、预算前置、接近预算的合法前缀、缺失成员、错误摘要/Chunk 长度、 + storage failure、取消/超时释放和 HTTP 所需 Send future。Dataset `--all-targets`、HTTP + batch/read-query/contract 七例、native digest/payload/read 八例通过；三个 crate 的 + all-target/all-feature clippy、fmt、diff check 通过。新增测试先修正了父 Snapshot fixture， + 集成编译发现跨 await 借用 iterator 的 Send 约束，已用有界具体窗口修复并加编译期断言。 + 本项未取得新镜像吞吐结果，V4.1 数字是本项之前的基线。 + +- **V4.1（2026-10-24）**：`tools/sdk/dataset_workload.py` 完整模式退出 0，三条路径同一 + Snapshot，size 为 0/128/513/4096/65535/262144/1048576/4194321 字节，每路径并发 1/3、 + 每档三次 128 请求，另有 warmup；全部校验且 errors=0，native Python parity/held lease/ + cancellation 和存储重启后实际 payload 校验通过。client CPU 2、server CPU 4–15、metadata + 每节点 2 GiB；镜像身份同 V2.1，客户端源码 `3992a23c`。完整 trial、安装 wheel、cgroup、 + 日志和 manifest 位于 `.crowdb-runtime/artifacts/dataset-workload/36ba3a8f4e84427db86f7761e4ddd67d/`。 + 每档三次中位吞吐（MiB/s），并发 1：HTTP 166.59 / native Rust 208.09 / native Python 208.83； + 并发 3：260.96 / 274.56 / 265.55。client CPU 每完成请求约 2.19–2.52 ms，最大 RSS 分别 + 17/31/56 MiB；Python `bytes()` 复制包含在读耗时/CPU，runtime compatibility copy 为 0。 + 4 KiB 中位 p50（并发 1）为 586/790/727 µs；大消息占用和并发使小消息尾延迟明显增长， + 继续按 F10 分析。各 size 每次仅 16 个延迟样本，p99 不作稳定生产结论。该运行是同主机 + TCP 短负载，未得到历史版本对照、逐请求真实 RPC/复制数或 HCA 数据，V4/F1 仍待完成。 + - **V2.1（2026-10-24）**：迁移旧 RetryLedger/property journal fixture，不恢复生产请求账本。 property 在实际 authority CAS 丢回复后检查资源已落地且 key 集合未增加，重启后重试返回 当前 missing 状态；手工构造 legacy inline/paged response 验证存储兼容。范围通知保留精确 diff --git a/lib/crowdb-access-dataset/src/inspection.rs b/lib/crowdb-access-dataset/src/inspection.rs index 5116b35df..9b0b397cf 100644 --- a/lib/crowdb-access-dataset/src/inspection.rs +++ b/lib/crowdb-access-dataset/src/inspection.rs @@ -27,6 +27,7 @@ mod exact; mod materialize; mod matrix; mod read; +mod read_fields; pub(crate) use read::PreparedInspectionRead; mod uploads; pub use matrix::InspectionSample; diff --git a/lib/crowdb-access-dataset/src/inspection/evaluate.rs b/lib/crowdb-access-dataset/src/inspection/evaluate.rs index 106ec3aef..8a8004b06 100644 --- a/lib/crowdb-access-dataset/src/inspection/evaluate.rs +++ b/lib/crowdb-access-dataset/src/inspection/evaluate.rs @@ -87,19 +87,10 @@ impl DatasetAuthority { .get(&key, 512 * 1024) .await? .ok_or(InspectionError::Corrupt)?; - let locator: FieldLocator = - serde_json::from_slice(&stored.bytes).map_err(|_| InspectionError::Corrupt)?; - let FieldLocator::Inline { value, md5 } = &locator else { - return Err(InspectionError::Corrupt); - }; - if value.len() > crate::manifest::MAX_INLINE_BYTES - || value.len() as u64 != metadata.length - || Some(*md5) != metadata.md5 - { + if metadata.storage != FieldStorageKind::Inline { return Err(InspectionError::Corrupt); } - locator.verify_payload(value).map_err(AuthorityError::from)?; - Ok(locator) + super::materialize::decode_locator(metadata, &stored.bytes) } } diff --git a/lib/crowdb-access-dataset/src/inspection/materialize.rs b/lib/crowdb-access-dataset/src/inspection/materialize.rs index 19ed76b7a..92fd8c38c 100644 --- a/lib/crowdb-access-dataset/src/inspection/materialize.rs +++ b/lib/crowdb-access-dataset/src/inspection/materialize.rs @@ -91,24 +91,35 @@ impl DatasetAuthority { .get(&key, 512 * 1024) .await? .ok_or(InspectionError::Corrupt)?; - let locator: FieldLocator = - serde_json::from_slice(&stored.bytes).map_err(|_| InspectionError::Corrupt)?; - let FieldLocator::Chunk { + decode_locator(metadata, &stored.bytes) + } +} + +pub(super) fn decode_locator( + metadata: &FieldMetadata, + bytes: &[u8], +) -> Result { + let locator: FieldLocator = serde_json::from_slice(bytes).map_err(|_| InspectionError::Corrupt)?; + match &locator { + FieldLocator::Inline { value, md5 } + if metadata.storage == FieldStorageKind::Inline + && value.len() <= crate::manifest::MAX_INLINE_BYTES + && value.len() as u64 == metadata.length + && Some(*md5) == metadata.md5 => + { + locator.verify_payload(value).map_err(AuthorityError::from)?; + } + FieldLocator::Chunk { location, length, md5, - } = &locator - else { - return Err(InspectionError::Corrupt); - }; - if location.is_empty() - || location.len() > 64 * 1024 - || *length == 0 - || *length != metadata.length - || Some(*md5) != metadata.md5 - { - return Err(InspectionError::Corrupt); - } - Ok(locator) + } if metadata.storage == FieldStorageKind::Chunk + && !location.is_empty() + && location.len() <= 64 * 1024 + && *length != 0 + && *length == metadata.length + && Some(*md5) == metadata.md5 => {} + _ => return Err(InspectionError::Corrupt), } + Ok(locator) } diff --git a/lib/crowdb-access-dataset/src/inspection/read.rs b/lib/crowdb-access-dataset/src/inspection/read.rs index 5317953c5..ddeaf46cd 100644 --- a/lib/crowdb-access-dataset/src/inspection/read.rs +++ b/lib/crowdb-access-dataset/src/inspection/read.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{key, work::InspectionWork, InspectionError}; +use super::{key, read_fields::FieldRequest, work::InspectionWork, InspectionError}; use crate::{ AuthorityError, DatasetAuthority, DatasetError, DatasetIdentity, MetadataScanBudget, ReadCancellation, SampleView, SnapshotId, @@ -89,52 +89,61 @@ impl PreparedInspectionRead<'_> { { return Err(AuthorityError::Invalid(DatasetError::InvalidManifest).into()); } - let mut output = Vec::new(); - for id in ids { - if id.is_empty() || id.len() > 16 * 1024 { - return Err(InspectionError::Limit); - } - if !known_members { - let mut member = - key::scope(self.identity, self.snapshot, b'M').map_err(AuthorityError::from)?; - key::component(&mut member, id); - let value = work - .get(&member, 128 * 1024) - .await? - .ok_or(AuthorityError::Invalid(DatasetError::SampleNotFound))?; - let stored: Vec = - serde_json::from_slice(&value.bytes).map_err(|_| InspectionError::Corrupt)?; - if stored != *id { - return Err(InspectionError::Corrupt); - } - } - let mut sample = SampleView { + if ids.iter().any(|id| id.is_empty() || id.len() > 16 * 1024) { + return Err(InspectionError::Limit); + } + if !known_members { + self.members(ids, work).await?; + } + let mut output: Vec<_> = ids + .iter() + .map(|id| SampleView { sample_id: id.clone(), fields: BTreeMap::new(), values: BTreeMap::new(), - }; - for field in fields { - if let Some(metadata) = self - .authority - .effective_field_metadata( - self.identity, - self.snapshot, - id, - field, - &mut self.summaries, - work, - ) - .await? - { - let locator = self - .authority - .composition_field_locator(self.identity, id, &metadata, work) - .await?; - sample.fields.insert(field.clone(), locator); + }) + .collect(); + for start in (0..ids.len() * fields.len()).step_by(16) { + let end = (start + 16).min(ids.len() * fields.len()); + let requests: Vec<_> = (start..end) + .map(|index| FieldRequest { + sample: &ids[index / fields.len()], + field: &fields[index % fields.len()], + }) + .collect(); + let locators = self.fields(&requests, work).await?; + for (index, (request, locator)) in (start..end).zip(requests.into_iter().zip(locators)) { + if let Some(locator) = locator { + output[index / fields.len()] + .fields + .insert(request.field.to_owned(), locator); } } - output.push(sample); } Ok(output) } + + async fn members(&self, ids: &[Vec], work: &mut InspectionWork<'_>) -> Result<(), InspectionError> { + let prefix = key::scope(self.identity, self.snapshot, b'M').map_err(AuthorityError::from)?; + for window in ids.chunks(16) { + let keys = window + .iter() + .map(|id| { + let mut key = prefix.clone(); + key::component(&mut key, id); + key + }) + .collect(); + let rows = work.get_many(keys, 128 * 1024).await?; + for (id, row) in window.iter().zip(rows) { + let row = row.ok_or(AuthorityError::Invalid(DatasetError::SampleNotFound))?; + let stored: Vec = + serde_json::from_slice(&row.bytes).map_err(|_| InspectionError::Corrupt)?; + if stored != *id { + return Err(InspectionError::Corrupt); + } + } + } + Ok(()) + } } diff --git a/lib/crowdb-access-dataset/src/inspection/read_fields.rs b/lib/crowdb-access-dataset/src/inspection/read_fields.rs new file mode 100644 index 000000000..cf22303b2 --- /dev/null +++ b/lib/crowdb-access-dataset/src/inspection/read_fields.rs @@ -0,0 +1,111 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + key, materialize::decode_locator, read::PreparedInspectionRead, work::InspectionWork, FieldMetadata, + FieldStorageKind, InspectionError, +}; +use crate::{AuthorityError, FieldLocator, SnapshotId}; +use std::collections::{btree_map::Entry, HashSet}; + +pub(super) struct FieldRequest<'a> { + pub(super) sample: &'a [u8], + pub(super) field: &'a str, +} + +impl PreparedInspectionRead<'_> { + /// Resolve a small field window one ancestry level at a time. Shared ancestors + /// are admitted once, and explicit tombstones stop inheritance without value IO. + pub(super) async fn fields( + &mut self, + requests: &[FieldRequest<'_>], + work: &mut InspectionWork<'_>, + ) -> Result>, InspectionError> { + let metadata = self.metadata(requests, work).await?; + let keys = requests.iter().zip(&metadata).filter_map(|(request, metadata)| { + metadata.as_ref().and_then(|metadata| { + (metadata.storage != FieldStorageKind::Tombstone) + .then(|| field_key(self.identity, metadata.source_snapshot, b'V', request)) + }) + }); + // Key construction can fail before any read; this window contains at most 16 cells. + let keys: Vec<_> = keys.collect::>()?; + let mut rows = work.get_many(keys, 512 * 1024).await?.into_iter(); + metadata + .into_iter() + .map(|metadata| { + metadata + .map(|metadata| { + if metadata.storage == FieldStorageKind::Tombstone { + return Ok(FieldLocator::Tombstone); + } + let row = rows.next().flatten().ok_or(InspectionError::Corrupt)?; + decode_locator(&metadata, &row.bytes) + }) + .transpose() + }) + .collect() + } + + async fn metadata( + &mut self, + requests: &[FieldRequest<'_>], + work: &mut InspectionWork<'_>, + ) -> Result>, InspectionError> { + let mut output = vec![None; requests.len()]; + let mut unresolved: Vec<_> = (0..requests.len()).collect(); + let mut current = Some(self.snapshot); + let mut seen = HashSet::new(); + while let Some(snapshot) = current { + if unresolved.is_empty() { + break; + } + if !seen.insert(snapshot) { + return Err(AuthorityError::AncestryCycle.into()); + } + if seen.len() > 64 { + return Err(InspectionError::Limit); + } + if let Entry::Vacant(entry) = self.summaries.entry(snapshot) { + entry.insert( + self.authority + .ready_inspection(self.identity, snapshot, work) + .await?, + ); + } + let keys = unresolved + .iter() + .map(|index| field_key(self.identity, snapshot, b'F', &requests[*index])) + .collect::, _>>()?; + let values = work.get_many(keys, 128 * 1024).await?; + let mut missing = Vec::new(); + for (index, value) in unresolved.into_iter().zip(values) { + if let Some(value) = value { + let metadata: FieldMetadata = + serde_json::from_slice(&value.bytes).map_err(|_| InspectionError::Corrupt)?; + if metadata.name != requests[index].field || metadata.source_snapshot != snapshot { + return Err(InspectionError::Corrupt); + } + output[index] = Some(metadata); + } else { + missing.push(index); + } + } + unresolved = missing; + current = self.summaries[&snapshot].parent; + } + Ok(output) + } +} + +fn field_key( + identity: &crate::DatasetIdentity, + snapshot: SnapshotId, + kind: u8, + request: &FieldRequest<'_>, +) -> Result, InspectionError> { + let mut key = key::scope(identity, snapshot, kind).map_err(AuthorityError::from)?; + key::component(&mut key, request.sample); + key::component(&mut key, request.field.as_bytes()); + Ok(key) +} diff --git a/lib/crowdb-access-dataset/src/inspection/work.rs b/lib/crowdb-access-dataset/src/inspection/work.rs index 1514e10ea..4f2159c52 100644 --- a/lib/crowdb-access-dataset/src/inspection/work.rs +++ b/lib/crowdb-access-dataset/src/inspection/work.rs @@ -3,6 +3,7 @@ use super::InspectionError; use crate::{DatasetStore, MetadataPage, MetadataScan, MetadataScanBudget, ReadCancellation, StoredValue}; +use futures::{stream::FuturesOrdered, StreamExt}; use std::time::Instant; /// One aggregate storage byte/deadline budget, including visibility lookups. @@ -57,6 +58,67 @@ impl<'a> InspectionWork<'a> { self.remaining = self.remaining.checked_sub(used).ok_or(InspectionError::Limit)?; Ok(value) } + /// Reserve response capacity before polling each read, including completed responses + /// waiting behind an earlier request. Returned rows consume the shared request budget. + pub(crate) async fn get_many( + &mut self, + keys: Vec>, + cap: usize, + ) -> Result>, InspectionError> { + let mut keys = keys.into_iter().peekable(); + let mut pending = FuturesOrdered::new(); + let mut reserved = 0; + let mut output = Vec::new(); + loop { + while pending.len() < 16 && keys.peek().is_some() { + let available = self + .remaining + .saturating_sub(reserved) + .min(2 * 1024 * 1024 - reserved); + if available == 0 { + break; + } + // Reclaim unused reservations before issuing a smaller final read; + // otherwise a legal row could fail only because peers are in flight. + if available < cap && !pending.is_empty() { + break; + } + let budget = self.budget(cap.min(available), 1)?; + let key = keys.next().ok_or(InspectionError::Corrupt)?; + reserved += budget.bytes; + let store = self.store; + let cancel = self.cancel; + let deadline = tokio::time::Instant::from_std(self.deadline); + pending.push_back(async move { + let value = tokio::time::timeout_at(deadline, store.get_bounded(&key, budget, cancel)) + .await + .map_err(|_| InspectionError::Limit)?; + Ok::<_, InspectionError>((key.len(), budget.bytes, value?)) + }); + } + let Some(result) = pending.next().await else { + return if keys.peek().is_some() { + Err(InspectionError::Limit) + } else { + Ok(output) + }; + }; + let (key_len, capacity, value) = result?; + if self.cancel.is_cancelled() { + return Err(InspectionError::Cancelled); + } + if Instant::now() >= self.deadline { + return Err(InspectionError::Limit); + } + let used = key_len + .checked_add(value.as_ref().map_or(0, |v| v.bytes.len())) + .filter(|used| *used <= capacity) + .ok_or(InspectionError::Limit)?; + reserved -= capacity; + self.remaining = self.remaining.checked_sub(used).ok_or(InspectionError::Limit)?; + output.push(value); + } + } pub(crate) async fn scan( &mut self, prefix: Vec, diff --git a/lib/crowdb-access-dataset/tests/common/inspection_reads.rs b/lib/crowdb-access-dataset/tests/common/inspection_reads.rs new file mode 100644 index 000000000..afad29f37 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/common/inspection_reads.rs @@ -0,0 +1,140 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::dataset_store::TestDatasetStore; +use crowdb_access_dataset::*; +use std::{ + collections::BTreeMap, + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Mutex, + }, + time::Duration, +}; + +#[derive(Default)] +pub struct TestIndexReads { + pub inner: TestDatasetStore, + pub reading: AtomicBool, + pub stalled: AtomicBool, + pub fail_value: AtomicBool, + pub pending: AtomicUsize, + reserved: AtomicUsize, + pub peak_pending: AtomicUsize, + pub peak_reserved: AtomicUsize, + pub total_bytes: AtomicUsize, + pub reads: Mutex, usize>>, +} + +struct TestReadGuard<'a> { + store: &'a TestIndexReads, + bytes: usize, +} +impl Drop for TestReadGuard<'_> { + fn drop(&mut self) { + self.store.pending.fetch_sub(1, Ordering::Relaxed); + self.store.reserved.fetch_sub(self.bytes, Ordering::Relaxed); + } +} + +impl TestIndexReads { + pub fn begin(&self) { + self.reads.lock().unwrap().clear(); + self.total_bytes.store(0, Ordering::Relaxed); + self.reading.store(true, Ordering::Relaxed); + } + pub fn key(identity: &DatasetIdentity, snapshot: SnapshotId, kind: u8, suffix: &[&[u8]]) -> Vec { + let mut key = b"DS\0I\0".to_vec(); + key.extend(identity.key().unwrap()); + key.extend(snapshot.as_bytes()); + key.push(kind); + for part in suffix { + for byte in *part { + if *byte == 0 { + key.extend([0, 255]); + } else { + key.push(*byte); + } + } + key.extend([0, 0]); + } + key + } +} + +#[async_trait::async_trait] +impl DatasetStore for TestIndexReads { + async fn get(&self, key: &[u8]) -> Result, StoreError> { + assert!(!self.reading.load(Ordering::Relaxed), "unbounded inspection GET"); + self.inner.get(key).await + } + async fn get_bounded( + &self, + key: &[u8], + budget: MetadataScanBudget, + cancel: &ReadCancellation, + ) -> Result, StoreError> { + assert!(!key.starts_with(b"DS\0S\0") && !key.starts_with(b"DS\0M\0")); + *self.reads.lock().unwrap().entry(key.to_vec()).or_default() += 1; + let _guard = TestReadGuard { + store: self, + bytes: budget.bytes, + }; + self.peak_pending.fetch_max( + self.pending.fetch_add(1, Ordering::Relaxed) + 1, + Ordering::Relaxed, + ); + self.peak_reserved.fetch_max( + self.reserved.fetch_add(budget.bytes, Ordering::Relaxed) + budget.bytes, + Ordering::Relaxed, + ); + if self.reading.load(Ordering::Relaxed) { + // Different completion times exercise caller order and held response capacity. + let delay = if key.last().is_some_and(|b| b % 3 == 0) { + 6 + } else { + 1 + }; + tokio::time::sleep(Duration::from_millis(delay)).await; + while self.stalled.load(Ordering::Relaxed) + && matches!(budget.bytes, 131_072 | 524_288) + && !cancel.is_cancelled() + { + tokio::time::sleep(Duration::from_millis(1)).await; + } + } + let value = self.inner.get_bounded(key, budget, cancel).await?; + if self.fail_value.load(Ordering::Relaxed) && budget.bytes == 512 * 1024 { + return Err(StoreError::Rejected); + } + self.total_bytes.fetch_add( + key.len() + value.as_ref().map_or(0, |v| v.bytes.len()), + Ordering::Relaxed, + ); + Ok(value) + } + async fn scan_page( + &self, + request: &MetadataScan, + cancel: &ReadCancellation, + ) -> Result { + let page = self.inner.scan_page(request, cancel).await?; + self.total_bytes.fetch_add( + page.entries + .iter() + .map(|(key, value)| key.len() + value.bytes.len()) + .sum::(), + Ordering::Relaxed, + ); + Ok(page) + } + async fn compare_exchange( + &self, + key: &[u8], + expected: Option<&[u8]>, + value: &[u8], + ) -> Result { + assert!(!self.reading.load(Ordering::Relaxed), "inspection mutation"); + self.inner.compare_exchange(key, expected, value).await + } +} diff --git a/lib/crowdb-access-dataset/tests/inspection_parallel_read_test.rs b/lib/crowdb-access-dataset/tests/inspection_parallel_read_test.rs new file mode 100644 index 000000000..dd4079bf7 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/inspection_parallel_read_test.rs @@ -0,0 +1,321 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/dataset_store.rs"] +#[allow(dead_code)] +mod dataset_store; +#[path = "common/inspection_reads.rs"] +mod inspection_reads; +use crowdb_access_dataset::{inspection::InspectionError, *}; +use inspection_reads::TestIndexReads; +use md5::{Digest, Md5}; +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; + +fn require_send(value: T) -> T { + value +} + +fn manifest(count: u8, bytes: usize) -> ManifestRecord { + let value = vec![255; bytes]; + ManifestRecord { + version: 1, + snapshot: SnapshotId::random(), + schema: SchemaRecord { + version: 1, + fields: ["body", "hidden", "absent"] + .into_iter() + .map(|name| FieldDefinition { + name: name.into(), + required: false, + }) + .collect(), + }, + samples: (0..count) + .map(|i| SampleRecord { + sample_id: vec![i, 0, 255], + fields: vec![ + FieldRecord { + name: "body".into(), + value: FieldLocator::Inline { + md5: Md5::digest(&value).into(), + value: value.clone(), + }, + }, + FieldRecord { + name: "hidden".into(), + value: FieldLocator::Tombstone, + }, + ], + }) + .collect(), + } +} + +async fn fixture( + count: u8, + bytes: usize, +) -> ( + Arc, + DatasetAuthority, + DatasetIdentity, + SnapshotId, + ManifestRecord, +) { + let store = Arc::new(TestIndexReads::default()); + let authority = DatasetAuthority::new(store.clone()); + let identity = DatasetIdentity::new(NamespacePath::root(), "parallel-read").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + let manifest = manifest(count, bytes); + let snapshot = authority + .publish_snapshot(&identity, None, bincode::serialize(&manifest).unwrap()) + .await + .unwrap(); + store.begin(); + (store, authority, identity, snapshot, manifest) +} + +#[tokio::test] +async fn delivery_overlaps_bounded_reads_preserving_order_inheritance_and_tombstones() { + let (store, authority, identity, parent, mut manifest) = fixture(32, 13).await; + store.reading.store(false, Ordering::Relaxed); + for sample in &mut manifest.samples { + sample.fields.retain(|f| f.name != "body"); + } + let child = authority + .publish_snapshot(&identity, Some(parent), bincode::serialize(&manifest).unwrap()) + .await + .unwrap(); + let ids: Vec<_> = manifest + .samples + .iter() + .rev() + .map(|s| s.sample_id.clone()) + .collect(); + store.begin(); + let rows = require_send(authority.resolve_inspection_batch_locators( + &identity, + child, + &ids, + &["body".into(), "hidden".into(), "absent".into()], + &ReadCancellation::new(), + )) + .await + .unwrap(); + for (row, id) in rows.iter().zip(&ids) { + assert_eq!(&row.sample_id, id); + assert!(matches!(&row.fields["body"], FieldLocator::Inline { value, .. } if value == &vec![255; 13])); + assert_eq!(row.fields["hidden"], FieldLocator::Tombstone); + assert!(!row.fields.contains_key("absent")); + } + let peak = store.peak_pending.load(Ordering::Relaxed); + assert!((2..=16).contains(&peak), "peak reads: {peak}"); + assert!(store.peak_reserved.load(Ordering::Relaxed) <= 2 * 1024 * 1024); + let reads = store.reads.lock().unwrap(); + for snapshot in [parent, child] { + assert_eq!(reads[&TestIndexReads::key(&identity, snapshot, b'H', &[])], 1); + } + let kind_offset = b"DS\0I\0".len() + identity.key().unwrap().len() + 16; + for (kind, expected) in [(b'M', 32), (b'F', 160), (b'V', 32)] { + assert_eq!( + reads + .iter() + .filter(|(key, _)| key.starts_with(b"DS\0I\0") && key[kind_offset] == kind) + .map(|(_, count)| count) + .sum::(), + expected + ); + } + assert!(!reads + .keys() + .any(|key| key == &TestIndexReads::key(&identity, parent, b'V', &[&ids[0], b"hidden"]))); +} + +#[tokio::test] +async fn mixed_locators_keep_integrity_checks_and_storage_failure_drops_peers() { + let (store, authority, identity, parent, mut manifest) = fixture(20, 13).await; + store.reading.store(false, Ordering::Relaxed); + for (index, sample) in manifest.samples.iter_mut().enumerate() { + sample.fields[0].value = if index % 2 == 0 { + FieldLocator::Chunk { + location: vec![0, 255, u8::try_from(index).unwrap()], + length: 4097, + md5: [7; 16], + } + } else { + FieldLocator::Inline { + value: Vec::new(), + md5: Md5::digest([]).into(), + } + }; + } + let snapshot = authority + .publish_snapshot(&identity, Some(parent), bincode::serialize(&manifest).unwrap()) + .await + .unwrap(); + let ids: Vec<_> = manifest.samples.iter().map(|s| s.sample_id.clone()).collect(); + store.begin(); + let fields = ["body".into()]; + let rows = authority + .resolve_inspection_batch_locators(&identity, snapshot, &ids, &fields, &ReadCancellation::new()) + .await + .unwrap(); + for (row, expected) in rows.iter().zip(&manifest.samples) { + assert_eq!(row.fields["body"], expected.fields[0].value); + } + store.fail_value.store(true, Ordering::Relaxed); + assert!(matches!( + authority + .resolve_inspection_batch_locators(&identity, snapshot, &ids, &fields, &ReadCancellation::new()) + .await, + Err(InspectionError::Store(StoreError::Rejected)) + )); + assert_eq!(store.pending.load(Ordering::Relaxed), 0); + store.fail_value.store(false, Ordering::Relaxed); + let key = TestIndexReads::key(&identity, snapshot, b'V', &[&ids[0], b"body"]); + let old = store.inner.get(&key).await.unwrap().unwrap(); + let bad = FieldLocator::Chunk { + location: vec![1], + length: 4096, + md5: [7; 16], + }; + store + .inner + .compare_exchange(&key, Some(&old.bytes), &serde_json::to_vec(&bad).unwrap()) + .await + .unwrap(); + assert!(matches!( + authority + .resolve_inspection_batch_locators(&identity, snapshot, &ids, &fields, &ReadCancellation::new()) + .await, + Err(InspectionError::Corrupt) + )); + assert_eq!(store.pending.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn delivery_enforces_aggregate_bytes_before_materialization_and_releases_failed_reads() { + let (store, authority, identity, snapshot, manifest) = fixture(72, 64 * 1024).await; + let ids: Vec<_> = manifest.samples.iter().map(|s| s.sample_id.clone()).collect(); + assert!(matches!( + authority + .resolve_inspection_batch_locators( + &identity, + snapshot, + &ids, + &["body".into()], + &ReadCancellation::new() + ) + .await, + Err(InspectionError::Limit | InspectionError::Store(StoreError::Limit)) + )); + assert!(store.total_bytes.load(Ordering::Relaxed) <= 16 * 1024 * 1024); + assert!(store.peak_reserved.load(Ordering::Relaxed) <= 2 * 1024 * 1024); + assert_eq!(store.pending.load(Ordering::Relaxed), 0); + // The exact fitting prefix must not fail just because unused reservations are outstanding. + store.begin(); + assert_eq!( + authority + .resolve_inspection_batch_locators( + &identity, + snapshot, + &ids[..63], + &["body".into()], + &ReadCancellation::new() + ) + .await + .unwrap() + .len(), + 63 + ); + assert_eq!(store.pending.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn cancellation_and_deadline_drop_all_pending_metadata_reads() { + let (store, authority, identity, snapshot, manifest) = fixture(20, 1).await; + let ids: Vec<_> = manifest.samples.iter().map(|s| s.sample_id.clone()).collect(); + let cancel = ReadCancellation::new(); + store.stalled.store(true, Ordering::Relaxed); + let stop = async { + while store.pending.load(Ordering::Relaxed) < 2 { + tokio::time::sleep(Duration::from_millis(1)).await; + } + cancel.cancel(); + }; + let fields = ["body".into()]; + let read = authority.resolve_inspection_batch_locators(&identity, snapshot, &ids, &fields, &cancel); + let (result, ()) = tokio::time::timeout(Duration::from_secs(1), async { tokio::join!(read, stop) }) + .await + .unwrap(); + assert!(result.is_err()); + assert_eq!(store.pending.load(Ordering::Relaxed), 0); + let started = std::time::Instant::now(); + assert!(matches!( + authority + .resolve_inspection_batch_locators( + &identity, + snapshot, + &ids, + &["body".into()], + &ReadCancellation::new() + ) + .await, + Err(InspectionError::Limit) + )); + assert!(started.elapsed() < Duration::from_secs(3)); + assert_eq!(store.pending.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn missing_members_and_corrupt_inline_payloads_fail_without_partial_delivery() { + let (store, authority, identity, snapshot, manifest) = fixture(20, 13).await; + let mut ids: Vec<_> = manifest.samples.iter().map(|s| s.sample_id.clone()).collect(); + ids[10] = b"missing".to_vec(); + assert!(matches!( + authority + .resolve_inspection_batch_locators( + &identity, + snapshot, + &ids, + &["body".into()], + &ReadCancellation::new() + ) + .await, + Err(InspectionError::Authority(AuthorityError::Invalid( + DatasetError::SampleNotFound + ))) + )); + assert!(!store + .reads + .lock() + .unwrap() + .keys() + .any(|key| key == &TestIndexReads::key(&identity, snapshot, b'V', &[&ids[0], b"body"]))); + let id = &manifest.samples[0].sample_id; + let key = TestIndexReads::key(&identity, snapshot, b'V', &[id, b"body"]); + let old = store.inner.get(&key).await.unwrap().unwrap(); + let mut bad: FieldLocator = serde_json::from_slice(&old.bytes).unwrap(); + let FieldLocator::Inline { value, .. } = &mut bad else { + panic!("inline fixture") + }; + value[0] = 0; + store + .inner + .compare_exchange(&key, Some(&old.bytes), &serde_json::to_vec(&bad).unwrap()) + .await + .unwrap(); + assert!(authority + .resolve_inspection_batch_locators( + &identity, + snapshot, + std::slice::from_ref(id), + &["body".into()], + &ReadCancellation::new() + ) + .await + .is_err()); + assert_eq!(store.pending.load(Ordering::Relaxed), 0); +} From f3b7eaa1102d5269d7b7769a1bad4f60f0e1aeef Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 14:17:50 +0800 Subject: [PATCH 114/258] Confine legacy multipart credit creation to recovery fixtures --- .../iceberge/design-crowdb-iceberg.md | 2 ++ doc/working/plan-access-simplification.md | 12 +++++-- e2e-test/Cargo.toml | 2 +- .../access_iceberg/native_files/worker.rs | 4 +-- .../src/file/multipart_credits.rs | 20 +++++++---- .../tests/multipart_admission_test.rs | 34 +++++++++++-------- .../tests/multipart_repository_test.rs | 15 +++++++- 7 files changed, 62 insertions(+), 27 deletions(-) diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg.md b/doc/design/access-server/iceberge/design-crowdb-iceberg.md index eca8e4dfd..a9309d9c6 100644 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg.md +++ b/doc/design/access-server/iceberge/design-crowdb-iceberg.md @@ -562,6 +562,8 @@ New multipart sessions publish their own authority without a catalog-wide credit reserve or release. Persisted per-session limits bound staged bytes and part count. Legacy admission journals and credit receipts remain recoverable in the background; ordinary new sessions never contend on those shared records. +Creating or reserving a legacy global credit policy is available only through +explicit compatibility test hooks. Production admission exposes no such path. The legacy tree multipart repository reserves one part mutation in the session before changing its part authority. A bounded before/after snapshot and monotonically increasing revisions make the write and fence release recoverable across servers. diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 5ec440929..076dec026 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -349,8 +349,6 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 ### P6 — Iceberg 旧 multipart / 全局 credits -- [ ] **P6.1 追踪活跃与恢复入口**：新 part 前台已是准入后 `1G + 1C`，旧 `part` helper 可有 - `5G`；分别计数。列出旧记录兼容/结算责任，不能把“无前台调用”当成“可以删”。Files: Iceberg `file/multipart{,_credits,_recovery,_repository}.rs` 及子模块、Access file 路由。 - [ ] **P6.2 删除确证无用的协调**：没有调用/恢复义务的代码可删；有旧记录则给出迁移/结算方案。 只有仍有全局硬配额需求才触发 D4。Files: `file/multipart_admission.rs`、`file/multipart_credits/settle.rs` 及恢复测试。 @@ -481,6 +479,16 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **P6.1（2026-10-24）**：核实新前台上传不调用全局 credits；旧 `initialize/reserve` 创建 + 接口限于 test-util 并命名 `*_for_tests`，生产仅保留历史资源的 load/settle/release。 + 不删除未结算的旧 pending reserve：该记录可能是尚未建立 MultipartSession 的唯一恢复入口； + 旧 terminal release 仍需防止计数重复返还。它们不再由新上传创建，后续迁移继续由 P6.2 审查。 + 旧 `part` 前后各 load（root+session）总计 5G，仍服务 legacy tree completion/publication； + 新 admitted streamed part 为 1G+1C。正常 begin 新增断言 1G+1C/无 credit，两条 streamed + part 为 2G+2C，key 集合没有 policy/OperationPayload。六项旧 admission、三项 record、 + 八项 repository、六项 recovery/budget 测试通过；prod-only clippy 与 Iceberg/E2E + all-target/all-feature clippy、fmt、diff check 通过。没有引入替代配额或放宽业务资源边界。 + - **F6.1（2026-10-24）**：delivery 按最多 16 个 sample-field cell 分窗，同层 field/祖先和 value 分阶段并发，membership 独立分窗；相同祖先只准入一次。每条 bounded read 发起前 预留 response cap，在途最多 16 条/2 MiB，已完成但等前序的响应仍占预留；释放未用额度后 diff --git a/e2e-test/Cargo.toml b/e2e-test/Cargo.toml index 9ac7a4eeb..ce25102b1 100644 --- a/e2e-test/Cargo.toml +++ b/e2e-test/Cargo.toml @@ -21,7 +21,7 @@ sha2 = "0.10" percent-encoding = "2" crowdb-console-shared = { path = "../lib/crowdb-console-shared" } crowdb-access-server = { path = "../app/crowdb-access-server" } -crowdb-access-iceberg = { path = "../lib/crowdb-access-iceberg" } +crowdb-access-iceberg = { path = "../lib/crowdb-access-iceberg", features = ["test-util"] } crowdb-access-dataset = { path = "../lib/crowdb-access-dataset" } crowdb-access-s3 = { path = "../lib/crowdb-access-s3" } crowdb-sdk-native = { path = "../lib/crowdb-sdk-native" } diff --git a/e2e-test/tests/access_iceberg/native_files/worker.rs b/e2e-test/tests/access_iceberg/native_files/worker.rs index 5353e0233..ae8a56177 100644 --- a/e2e-test/tests/access_iceberg/native_files/worker.rs +++ b/e2e-test/tests/access_iceberg/native_files/worker.rs @@ -45,7 +45,7 @@ pub async fn verify(stack: &TestNativeStorage, context: CatalogContext, table: T }; let admission = MultipartAdmission::new(store.clone()); let policy = admission - .initialize( + .initialize_for_tests( context, MultipartAdmissionLimits { max_sessions: 2, @@ -54,7 +54,7 @@ pub async fn verify(stack: &TestNativeStorage, context: CatalogContext, table: T ) .await .unwrap(); - assert!(admission.reserve(&policy, &initial, 1).await.unwrap()); + assert!(admission.reserve_for_tests(&policy, &initial, 1).await.unwrap()); let repository = MultipartRepository::new(store); let admitted = repository.load(context, initial.upload).await.unwrap().unwrap(); let part = MultipartPart { diff --git a/lib/crowdb-access-iceberg/src/file/multipart_credits.rs b/lib/crowdb-access-iceberg/src/file/multipart_credits.rs index 92269e3cb..f931f4ddd 100644 --- a/lib/crowdb-access-iceberg/src/file/multipart_credits.rs +++ b/lib/crowdb-access-iceberg/src/file/multipart_credits.rs @@ -2,13 +2,17 @@ use std::sync::Arc; use crate::catalog::{check_context, CasOutcome, CatalogContext, CatalogError, CatalogStore}; use crate::error::ValidationError; -use crate::key::{CatalogScope, IcebergKey, OperationId}; +#[cfg(feature = "test-util")] +use crate::key::OperationId; +use crate::key::{CatalogScope, IcebergKey}; use crate::operation::{mutation_identity, PayloadStore}; use crate::record::StorageRecord; +#[cfg(feature = "test-util")] +use super::MultipartAdmissionLimits; use super::{ - MultipartAdmissionLimits, MultipartAdmissionRecord, MultipartCredit, MultipartCreditAction, - MultipartCreditMutation, MultipartPhase, MultipartRepository, MultipartSession, + MultipartAdmissionRecord, MultipartCredit, MultipartCreditAction, MultipartCreditMutation, + MultipartPhase, MultipartRepository, MultipartSession, }; mod settle; @@ -27,10 +31,11 @@ impl MultipartAdmission { } } - /// Installs immutable catalog-wide admission limits or checks the existing policy. + /// Reproduces a legacy policy for compatibility and recovery tests. /// # Errors /// Rejects invalid bounds, conflicting policy, retired contexts and storage failures. - pub async fn initialize( + #[cfg(feature = "test-util")] + pub async fn initialize_for_tests( &self, context: CatalogContext, limits: MultipartAdmissionLimits, @@ -95,10 +100,11 @@ impl MultipartAdmission { Ok(record) } - /// Reserves the session's complete staged-byte ceiling before creating its authority. + /// Reproduces legacy reservation writes, including interrupted journals, for recovery tests. /// # Errors /// Rejects stale or occupied policies, expired requests, exhausted credits and identity reuse. - pub async fn reserve( + #[cfg(feature = "test-util")] + pub async fn reserve_for_tests( &self, record: &MultipartAdmissionRecord, session: &MultipartSession, diff --git a/lib/crowdb-access-iceberg/tests/multipart_admission_test.rs b/lib/crowdb-access-iceberg/tests/multipart_admission_test.rs index 2829ea3b9..2d3b88f31 100644 --- a/lib/crowdb-access-iceberg/tests/multipart_admission_test.rs +++ b/lib/crowdb-access-iceberg/tests/multipart_admission_test.rs @@ -25,7 +25,7 @@ async fn setup() -> (file::TestFile, MultipartSession, MultipartAdmissionRecord) initial.owner.file = fixture.record("file", b"{}").file; initial.location = fixture.table.file("file").unwrap(); let record = MultipartAdmission::new(fixture.store.clone()) - .initialize( + .initialize_for_tests( fixture.context, MultipartAdmissionLimits { max_sessions: 2, @@ -58,7 +58,7 @@ async fn reservation_reply_loss_recovers_once_across_instances_at_every_write() fixture.store.writes.load(Ordering::SeqCst) + lost, Ordering::SeqCst, ); - assert!(admission.reserve(&record, &initial, 100).await.is_err()); + assert!(admission.reserve_for_tests(&record, &initial, 100).await.is_err()); fixture.store.fail_after.store(0, Ordering::SeqCst); let admission = MultipartAdmission::new(fixture.store.clone()); let record = policy(&admission, &initial).await; @@ -66,7 +66,7 @@ async fn reservation_reply_loss_recovers_once_across_instances_at_every_write() assert!(admission.settle(&record).await.unwrap()); } let record = policy(&admission, &initial).await; - assert!(admission.reserve(&record, &initial, 100).await.unwrap()); + assert!(admission.reserve_for_tests(&record, &initial, 100).await.unwrap()); let record = policy(&admission, &initial).await; assert_eq!((record.sessions, record.reserved_bytes), (1, 1500)); assert!(record.pending.is_none()); @@ -75,7 +75,10 @@ async fn reservation_reply_loss_recovers_once_across_instances_at_every_write() assert_eq!(admitted.credit.unwrap().policy, record.policy); assert!(!admitted.credit.unwrap().released); let writes = fixture.store.writes.load(Ordering::SeqCst); - assert!(admission.reserve(&record, &initial, 5000).await.unwrap()); + assert!(admission + .reserve_for_tests(&record, &initial, 5000) + .await + .unwrap()); assert_eq!(fixture.store.writes.load(Ordering::SeqCst), writes); } } @@ -86,7 +89,7 @@ async fn terminal_release_reply_loss_never_returns_credits_twice() { let (fixture, initial, record) = setup().await; let admission = MultipartAdmission::new(fixture.store.clone()); let repository = MultipartRepository::new(fixture.store.clone()); - assert!(admission.reserve(&record, &initial, 100).await.unwrap()); + assert!(admission.reserve_for_tests(&record, &initial, 100).await.unwrap()); let active = session(&repository, &initial).await; let record = policy(&admission, &initial).await; assert!(matches!( @@ -127,19 +130,22 @@ async fn admission_fences_competing_reservations_and_persisted_policy_changes() let mut other = initial.clone(); other.upload = OperationId::random(); let (left, right) = tokio::join!( - first.reserve(&record, &initial, 100), - second.reserve(&record, &other, 100) + first.reserve_for_tests(&record, &initial, 100), + second.reserve_for_tests(&record, &other, 100) ); assert_ne!(left.unwrap(), right.unwrap()); let current = policy(&first, &initial).await; assert_eq!(current.sessions, 1); assert_eq!( - first.initialize(initial.context, record.limits).await.unwrap(), + first + .initialize_for_tests(initial.context, record.limits) + .await + .unwrap(), current ); assert!(matches!( first - .initialize( + .initialize_for_tests( initial.context, MultipartAdmissionLimits { max_sessions: 3, @@ -160,11 +166,11 @@ async fn admission_fences_competing_reservations_and_persisted_policy_changes() } else { &other }; - assert!(first.reserve(¤t, missing, 100).await.unwrap()); + assert!(first.reserve_for_tests(¤t, missing, 100).await.unwrap()); let mut overflow = initial.clone(); overflow.upload = OperationId::random(); assert!(first - .reserve(&policy(&first, &initial).await, &overflow, 100) + .reserve_for_tests(&policy(&first, &initial).await, &overflow, 100) .await .is_err()); assert!(repository @@ -182,7 +188,7 @@ async fn sweep_recovers_precreation_credit_then_expires_and_releases_without_del .store .fail_after .store(fixture.store.writes.load(Ordering::SeqCst) + 2, Ordering::SeqCst); - assert!(admission.reserve(&record, &initial, 100).await.is_err()); + assert!(admission.reserve_for_tests(&record, &initial, 100).await.is_err()); fixture.store.fail_after.store(0, Ordering::SeqCst); let repository = MultipartRepository::new(fixture.store.clone()); assert!(repository @@ -234,7 +240,7 @@ async fn superseded_helpers_cannot_recreate_released_sessions_or_change_new_cred .store .fail_after .store(fixture.store.writes.load(Ordering::SeqCst) + 2, Ordering::SeqCst); - assert!(admission.reserve(&record, &initial, 100).await.is_err()); + assert!(admission.reserve_for_tests(&record, &initial, 100).await.is_err()); fixture.store.fail_after.store(0, Ordering::SeqCst); let reserved = policy(&admission, &initial).await; assert!(admission.settle(&reserved).await.unwrap()); @@ -256,7 +262,7 @@ async fn superseded_helpers_cannot_recreate_released_sessions_or_change_new_cred let mut other = initial.clone(); other.upload = OperationId::random(); let current = policy(&admission, &initial).await; - assert!(admission.reserve(¤t, &other, 100).await.unwrap()); + assert!(admission.reserve_for_tests(¤t, &other, 100).await.unwrap()); let current = policy(&admission, &initial).await; let writes = fixture.store.writes.load(Ordering::SeqCst); assert!(!admission.settle(&reserved).await.unwrap()); diff --git a/lib/crowdb-access-iceberg/tests/multipart_repository_test.rs b/lib/crowdb-access-iceberg/tests/multipart_repository_test.rs index cbe34e0b5..3d523e225 100644 --- a/lib/crowdb-access-iceberg/tests/multipart_repository_test.rs +++ b/lib/crowdb-access-iceberg/tests/multipart_repository_test.rs @@ -86,7 +86,12 @@ fn stream_part(session: &MultipartSession, number: u16, chunk_low: u64) -> Multi async fn streamed_parts_commit_independently_without_session_mutations() { let (fixture, initial) = setup().await; let repository = MultipartRepository::new(fixture.store.clone()); - repository.begin(&initial, 100).await.unwrap(); + let writes = fixture.store.writes.load(Ordering::SeqCst); + let reads = fixture.store.reads.load(Ordering::SeqCst); + let created = repository.begin(&initial, 100).await.unwrap(); + assert!(created.credit.is_none()); + assert_eq!(fixture.store.writes.load(Ordering::SeqCst) - writes, 1); + assert_eq!(fixture.store.reads.load(Ordering::SeqCst) - reads, 1); let writes = fixture.store.writes.load(Ordering::SeqCst); let reads = fixture.store.reads.load(Ordering::SeqCst); let first_input = stream_part(&initial, 1, 1); @@ -101,6 +106,14 @@ async fn streamed_parts_commit_independently_without_session_mutations() { assert_eq!(second.revision, 1); assert_eq!(fixture.store.writes.load(Ordering::SeqCst) - writes, 2); assert_eq!(fixture.store.reads.load(Ordering::SeqCst) - reads, 2); + assert!(!fixture.store.values.load().keys().any(|key| matches!( + crowdb_access_iceberg::key::IcebergKey::decode(key), + Ok(crowdb_access_iceberg::key::IcebergKey::Catalog { + scope: crowdb_access_iceberg::key::CatalogScope::MultipartAdmission + | crowdb_access_iceberg::key::CatalogScope::OperationPayload, + .. + }) + ))); assert_eq!(load(&repository, &initial).await, initial); let replacement = repository .put_stream_part(&initial, &stream_part(&initial, 1, 3), 102) From 6c9bda14338216808cb6953794d0d3ae5d77d83d Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 14:24:44 +0800 Subject: [PATCH 115/258] Retain complete Dataset part buffers without copying --- .../dataset/design-dataset-control.md | 8 ++ doc/working/plan-access-simplification.md | 11 +++ lib/crowdb-access-dataset/src/lib.rs | 2 + lib/crowdb-access-dataset/src/payload.rs | 31 ++++--- .../src/payload/stream.rs | 35 ++++++++ .../tests/common/payload_collect.rs | 73 +++++++++++++++ .../tests/payload_collect_test.rs | 88 +++++++++++++++++++ 7 files changed, 238 insertions(+), 10 deletions(-) create mode 100644 lib/crowdb-access-dataset/tests/common/payload_collect.rs create mode 100644 lib/crowdb-access-dataset/tests/payload_collect_test.rs diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index 9599cb161..a3f5f97f2 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -130,6 +130,14 @@ read. Request order, missing fields, tombstones and locator integrity checks rem independent of completion order. All windows share the request deadline and cancellation; no reader state is persisted. +The bounded compatibility part reader checks the declared logical length before +allocation. A single buffer covering that complete object is transferred with its +allocation owner after successful EOF. Fragmented objects allocate their final +buffer once; each source buffer is copied and released before the next storage +pull, so a one-slot reader can make progress. Short, oversized and failed terminal +reads remain errors. Whole-part digest verification still runs on the returned +bytes before acceptance. + Old snapshots without projections require explicit resumable inspection preparation. Unsupported or oversized legacy state is reported, never represented as an empty Snapshot. Field preview is explicit and capped at 64 KiB after read-only admission; it diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 076dec026..a4d529c40 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -479,6 +479,17 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **F8.3（2026-10-24）**：`ChunkPayloadStorage::read` 在读前按逻辑长度限制最大 4 MiB， + 单块完整结果确认 EOF 后直接移交 Bytes owner；多块只按最终长度分配一次，复制当前片段后 + 立即释放其读 credit 再 pull，避免单 slot 死等。没有新增 metadata/RPC/持久化，仍检查 + 空/短/超长片段和迟到 EOF 错误，后续 whole-part MD5 校验保留。三项 collector 回归用 + 实际 allocation pointer 验证单块未复制、owner 随返回 Bytes 存活；多块模拟单 credit， + 覆盖非整齐长度、4 MiB、失败后释放。非法“完整对象后额外数据”的故障流单独允许越界输出， + 不套用合法对象的 credit 假设；collector 仍明确拒绝，未降低该错误断言。 + upload read/stream/完整性共十三例、HTTP batch 与独立 SDK payload 两例、native + digest/payload/read 八例通过；Dataset all-target/all-feature clippy、fmt/diff 通过。 + 尚无本项的同源性能测量，F8 的 HTTP 收整 body 和其他产品复制继续审查。 + - **P6.1（2026-10-24）**：核实新前台上传不调用全局 credits；旧 `initialize/reserve` 创建 接口限于 test-util 并命名 `*_for_tests`，生产仅保留历史资源的 load/settle/release。 不删除未结算的旧 pending reserve：该记录可能是尚未建立 MultipartSession 的唯一恢复入口； diff --git a/lib/crowdb-access-dataset/src/lib.rs b/lib/crowdb-access-dataset/src/lib.rs index 02552cd6c..03106e9df 100644 --- a/lib/crowdb-access-dataset/src/lib.rs +++ b/lib/crowdb-access-dataset/src/lib.rs @@ -18,6 +18,8 @@ mod key; mod manifest; mod namespace; mod payload; +#[cfg(feature = "test-util")] +pub use payload::collect_payload_for_tests; mod planner; mod publication; mod record; diff --git a/lib/crowdb-access-dataset/src/payload.rs b/lib/crowdb-access-dataset/src/payload.rs index 474110276..c7eb255aa 100644 --- a/lib/crowdb-access-dataset/src/payload.rs +++ b/lib/crowdb-access-dataset/src/payload.rs @@ -124,19 +124,19 @@ impl PayloadStorage for ChunkPayloadStorage { } async fn read(&self, locations: &[Location]) -> Result { - let mut stream = self + let length = locations + .iter() + .filter(|location| location.length != 0) + .try_fold(0usize, |total, location| { + total.checked_add(usize::try_from(location.logical_length).ok()?) + }) + .filter(|length| *length > 0 && *length <= MAX_UPLOAD_PART_BYTES) + .ok_or(UploadError::Integrity)?; + let stream = self .chunks .read_stream(locations) .map_err(|error| UploadError::Storage(error.to_string()))?; - let mut output = bytes::BytesMut::new(); - while let Some(data) = stream.next_chunk().await { - let data = data.map_err(|error| UploadError::Storage(error.to_string()))?; - if output.len().saturating_add(data.len()) > MAX_UPLOAD_PART_BYTES { - return Err(UploadError::Integrity); - } - output.extend_from_slice(&data); - } - Ok(output.freeze()) + stream::collect(&mut stream::ChunkStream(stream), length).await } async fn durability(&self, locations: &[Location]) -> Result { @@ -199,3 +199,14 @@ impl PayloadStorage for ChunkPayloadStorage { Ok(complete) } } + +/// Exercises the compatibility collector with bounded storage streams. +/// # Errors +/// Returns invalid lengths, incomplete/oversized streams and storage failures. +#[cfg(feature = "test-util")] +pub async fn collect_payload_for_tests( + stream: &mut dyn PayloadStream, + length: usize, +) -> Result { + stream::collect(stream, length).await +} diff --git a/lib/crowdb-access-dataset/src/payload/stream.rs b/lib/crowdb-access-dataset/src/payload/stream.rs index 36a9d97ec..36e7a46ec 100644 --- a/lib/crowdb-access-dataset/src/payload/stream.rs +++ b/lib/crowdb-access-dataset/src/payload/stream.rs @@ -31,3 +31,38 @@ impl PayloadStream for ChunkStream { .map_err(|error| UploadError::Storage(error.to_string())) } } + +pub(super) async fn collect( + stream: &mut dyn PayloadStream, + length: usize, +) -> Result { + if length == 0 || length > crate::MAX_UPLOAD_PART_BYTES { + return Err(UploadError::Integrity); + } + let first = stream.next().await?.ok_or(UploadError::Integrity)?; + if first.is_empty() || first.len() > length { + return Err(UploadError::Integrity); + } + if first.len() == length { + // A complete logical object needs no further IO credit to reach EOF. + // Still pull once: an error after the last data frame must not become success. + if stream.next().await?.is_some() { + return Err(UploadError::Integrity); + } + return Ok(first.into_bytes()); + } + let mut output = bytes::BytesMut::with_capacity(length); + output.extend_from_slice(&first); + // A subsequent fetch may require this allocation's read credit. + drop(first); + while let Some(data) = stream.next().await? { + if data.is_empty() || data.len() > length - output.len() { + return Err(UploadError::Integrity); + } + output.extend_from_slice(&data); + } + if output.len() != length { + return Err(UploadError::Integrity); + } + Ok(output.freeze()) +} diff --git a/lib/crowdb-access-dataset/tests/common/payload_collect.rs b/lib/crowdb-access-dataset/tests/common/payload_collect.rs new file mode 100644 index 000000000..54488ebe5 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/common/payload_collect.rs @@ -0,0 +1,73 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_access_dataset::{PayloadStream, UploadError}; +use crowdb_rpc_ffi::OwnedPayload; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +pub struct TestPayloadCollector { + chunks: std::vec::IntoIter>, + pub active: Arc, + pub pulls: usize, + pub first_pointer: usize, + pub fail_eof: bool, + pub enforce_credit: bool, +} +impl TestPayloadCollector { + pub fn new(chunks: Vec>) -> Self { + Self { + chunks: chunks.into_iter(), + active: Arc::new(AtomicUsize::new(0)), + pulls: 0, + first_pointer: 0, + fail_eof: false, + enforce_credit: true, + } + } +} +struct TestOwner { + data: Vec, + active: Arc, +} +impl AsRef<[u8]> for TestOwner { + fn as_ref(&self) -> &[u8] { + &self.data + } +} +impl Drop for TestOwner { + fn drop(&mut self) { + self.active.fetch_sub(1, Ordering::Relaxed); + } +} +#[async_trait::async_trait] +impl PayloadStream for TestPayloadCollector { + async fn next(&mut self) -> Result, UploadError> { + self.pulls += 1; + let Some(data) = self.chunks.next() else { + return if self.fail_eof { + Err(UploadError::Storage("failed final read".into())) + } else { + Ok(None) + }; + }; + if self.enforce_credit { + assert_eq!( + self.active.load(Ordering::Relaxed), + 0, + "next IO needs the previous buffer's credit" + ); + } + if self.first_pointer == 0 { + self.first_pointer = data.as_ptr() as usize; + } + self.active.fetch_add(1, Ordering::Relaxed); + Ok(Some(OwnedPayload::from(Bytes::from_owner(TestOwner { + data, + active: Arc::clone(&self.active), + })))) + } +} diff --git a/lib/crowdb-access-dataset/tests/payload_collect_test.rs b/lib/crowdb-access-dataset/tests/payload_collect_test.rs new file mode 100644 index 000000000..01197e0fa --- /dev/null +++ b/lib/crowdb-access-dataset/tests/payload_collect_test.rs @@ -0,0 +1,88 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/payload_collect.rs"] +mod payload_collect; +use crowdb_access_dataset::{collect_payload_for_tests, UploadError, MAX_UPLOAD_PART_BYTES}; +use payload_collect::TestPayloadCollector; +use std::sync::atomic::Ordering; + +#[tokio::test] +async fn complete_single_buffer_moves_the_same_allocation_and_keeps_its_owner() { + for length in [1, 513, 4096, 65_513] { + let mut stream = TestPayloadCollector::new(vec![vec![7; length]]); + let bytes = collect_payload_for_tests(&mut stream, length).await.unwrap(); + assert_eq!(bytes.as_ptr() as usize, stream.first_pointer); + assert_eq!(bytes.as_ref(), vec![7; length]); + assert_eq!(stream.pulls, 2, "successful EOF is required"); + assert_eq!(stream.active.load(Ordering::Relaxed), 1); + let active = stream.active.clone(); + drop(stream); + assert_eq!(active.load(Ordering::Relaxed), 1); + drop(bytes); + assert_eq!(active.load(Ordering::Relaxed), 0); + } +} + +#[tokio::test] +async fn fragmented_payload_releases_each_credit_before_fetching_the_next_buffer() { + for lengths in [ + vec![1, 4096, 513], + vec![65_502; 16], + vec![1, MAX_UPLOAD_PART_BYTES - 1], + ] { + let chunks: Vec<_> = lengths + .iter() + .enumerate() + .map(|(i, len)| vec![u8::try_from(i).unwrap(); *len]) + .collect(); + let expected = chunks.concat(); + let mut stream = TestPayloadCollector::new(chunks); + let output = collect_payload_for_tests(&mut stream, expected.len()) + .await + .unwrap(); + assert_eq!(output.as_ref(), expected); + assert_ne!(output.as_ptr() as usize, stream.first_pointer); + assert_eq!(stream.active.load(Ordering::Relaxed), 0); + assert_eq!(stream.pulls, lengths.len() + 1); + } +} + +#[tokio::test] +async fn invalid_boundaries_and_late_eof_errors_never_return_success() { + for (chunks, length) in [ + (vec![], 1), + (vec![vec![]], 1), + (vec![vec![1; 4]], 3), + (vec![vec![1; 2]], 3), + (vec![vec![1; 2], vec![]], 3), + (vec![vec![1; 2], vec![2; 2]], 3), + (vec![vec![1; 3], vec![2]], 3), + ] { + let mut stream = TestPayloadCollector::new(chunks); + // These streams deliberately violate the declared logical object bounds. + stream.enforce_credit = false; + assert!(matches!( + collect_payload_for_tests(&mut stream, length).await, + Err(UploadError::Integrity) + )); + assert_eq!(stream.active.load(Ordering::Relaxed), 0); + } + for length in [0, MAX_UPLOAD_PART_BYTES + 1] { + let mut stream = TestPayloadCollector::new(vec![vec![1]]); + assert!(matches!( + collect_payload_for_tests(&mut stream, length).await, + Err(UploadError::Integrity) + )); + assert_eq!(stream.pulls, 0); + } + for chunks in [vec![vec![1; 3]], vec![vec![1], vec![2; 2]]] { + let mut stream = TestPayloadCollector::new(chunks); + stream.fail_eof = true; + assert!(matches!( + collect_payload_for_tests(&mut stream, 3).await, + Err(UploadError::Storage(_)) + )); + assert_eq!(stream.active.load(Ordering::Relaxed), 0); + } +} From 4981260aafba27ebd3ab7a03f5c741b360305f01 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 14:33:38 +0800 Subject: [PATCH 116/258] Record access cleanup decisions and validation status --- doc/working/plan-access-simplification.md | 33 +++++++++++++++---- .../tests/read_query_cost_test.rs | 6 ++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index a4d529c40..965724675 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -19,6 +19,8 @@ Upstream: [Dataset SDK](../backlog/R220-dataset-fat-client-plan.md)、 静态发现、提出方案、代码完成、性能验收不能互相代替。 - 当前实施基线为 `489ae507`；旧性能审查来自 `878ee901`， 涉及已删除 session 的结论已移除。下面的成本数字只描述注明的函数路径，需动态复核。 +- 用户要求暂时离开后继续。交接时 46 个未完成 checkbox， + 包含条件决策、交叉审查和总体验收，不代表 46 个独立功能；没有正在运行的实施步骤。 ## 已确定的约束 @@ -31,6 +33,10 @@ Upstream: [Dataset SDK](../backlog/R220-dataset-fat-client-plan.md)、 - **非空 Iceberg namespace 删除必须失败**，已由用户决定，不再提出放宽选项。 - **用户决定（2026-10-24）**：允许 rename 在参与父目录中保存最小有界占位，用于严格非空 删除和迟到 CAS 隔离；删除新 rename 的完整 journal、输入和响应。普通读取不登记。 +- **用户决定（2026-10-24）**：完整 Snapshot/Split 结果即使没有赢 head CAS，也允许按 ID + 读取和引用。head CAS 仅切换 latest/stable/currentSplit，失败仍返回冲突，不覆盖后来者。 + 未完成资源不开放；无正式引用的结果按期限清理，GC 隔离按 R4 实施。移除“曾发布”证明 + 与请求账本，不能只放宽入口后遗留 CA/CR/SC/SRC/helper。 - definite CAS conflict 立即使用返回的 observed value 计算合法重试，无“三次后才更新”。 返回值可能随后又变旧，因此仍需期限/重试上限；transport timeout 不等于 CAS 失败。 明确 expected revision 的业务操作不能擅自改 base 重放。 @@ -130,7 +136,7 @@ CAS 不再满足只证明“本次条件不成立”，不能谎称“第一次 不为每次请求添加历史查询或回执；只有某个具体调用者必须识别被后续提交遮住的历史成功， 才提出该场景需要的最小方案及代价，交用户 review。 -**资源可见性约束 review（2026-10-24，等待决定）**：现代码把“内容已完整生成”与“曾经成功 +**资源可见性约束 review（2026-10-24，用户已选择完整结果按 ID 可用）**：旧实现把“内容已完整生成”与“曾经成功 切换 head”区分。Snapshot 的 `Published` 标记在 head CAS 之前写，不能单独证明成功； `require_committed_snapshot`、历史读取、parent/stable 准入和 Split revision 历史依赖 `DS\0SC\0` / `DS\0SRC\0`。每次发布还写 `DS\0CA\0` attempt、`DS\0CR\0` response receipt； @@ -140,13 +146,21 @@ CAS 不再满足只证明“本次条件不成立”，不能谎称“第一次 - 场景：完整结果 S1 已经生成；客户端以 H0 发布 S1 的 CAS 输给 S2。现在 S1 既不可按 ID 读取，也不可成为新 Snapshot 的 parent/stable。另一个场景是 S1 曾赢 head、回复丢失、 又被 S2 替换；S1 必须可作历史资源。为区分两者，现实现保存上述成功证明。 -- 建议：完整不可变结果按资源 ID 即可读取/引用；head CAS 只决定 latest/stable 指针是否 +- 已确认：完整不可变结果按资源 ID 即可读取/引用；head CAS 只决定 latest/stable 指针是否 按指定 base 更新。未完成内容仍不可读；未引用结果按明确期限过期，过期请求失败并重做。 这可删除“曾切换 head”证明、attempt/response 账本与相应 helper；GC/期限隔离仍须按 R4 - 单独证明，不能仅凭本地超时回收。它改变现有可见性合同，不能未经决定直接放开。 -- 保持严格语义的备选：保留每个资源一份最小成功事实，删除 CA/CR 及按 attempt 查询。 - 沿用当前独立键存储时，每个发布仍需额外一次条件写成功事实及覆盖 head 前的帮助检查； - 这份记录仅用于历史资源可见性，读取不登记，不保存编辑/计算过程或原响应。 + 单独证明，不能仅凭本地超时回收。用户在 S0→S1/S2 竞争和回复丢失例子后明确选择此合同， + 不采用“必须曾发布才能读”的备选。 + +- [ ] **R2.1 统一完整资源准入**：只在 Manifest/inspection、初始 Split 与必要 payload owner + 已完整就绪后按 ID 开放；准备记录本身不等于结果完整。Snapshot/Split 按 ID、parent/stable、 + history/inventory、continuation 与 legacy 路径统一，删除状态仍拒绝新请求。 + 不能只将 SC/SRC lookup 改成 true：Composition 在 CB 之后还准备初始 Split、claim upload + owner，obsolete/abandon cleanup 和 child 保护也依赖旧“曾发布”区分，须同步调整。 +- [ ] **R2.2 删除发布过程账本**：移除新 CA/CR/SC/SRC 写入、覆盖 head 前的帮助与按 attempt + 历史回复查询；Composition/Split 的 Committing 恢复与 P2/P3 一起删除，客户端保留完整输入。 + 旧记录只作为兼容解码/GC 输入，不让旧账本控制新请求。head 保留 expected base；同一 + candidate 的当前结果可判断相同，已被后续 head 遮住时不保证历史请求结果重放。 - [ ] **R2 审查条件发布的回复丢失窗口**：用 H0→H1→H2 测试区分条件失败、相同结果与历史未知； 列出确实依赖历史精确结果的调用者，没有这种需求就移除相应 ledger/response replay。 @@ -459,6 +473,13 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 记录 p50/p99、TPS、CPU/RSS、RPC、复制/摘要、清理积压，不缩小负载掩盖失败。 Files: `tools/sdk/dataset_workload.py`、相关 examples、`e2e-test/`、本机 HCA runner。 物理多主机验证继续留在独立计划，不并回此处。 +- [ ] **V4.2 验证并发 metadata 与单块 owner 移交的真实链路**：由 `acd25325` 生产源码重新 + 构建独立本地镜像，运行 Dataset 全部容器用例与同参数 TCP workload。与 V4.1 使用相同 + client/server CPU、metadata 内存、size 集合、并发和校验，保留精确 artifact/source 身份； + 不把单 sample-field 的当前 workload 当作宽投影并发性能证明，后者继续由 F6/F1 验证。 + 首次 build 的 release/C++/frontend 均完成，但构建期间修改了本计划与成本测试，触发 + `Source tree changed while building; rebuild the shared OCI image`，退出 1；没有生成新 OCI。 + 恢复时先提交完整稳定树，重跑 build 期间不改任何非忽略文件，保留 source guard。 ## 成本口径与集中代码索引 diff --git a/lib/crowdb-access-dataset/tests/read_query_cost_test.rs b/lib/crowdb-access-dataset/tests/read_query_cost_test.rs index 8f8167de9..adbbd4155 100644 --- a/lib/crowdb-access-dataset/tests/read_query_cost_test.rs +++ b/lib/crowdb-access-dataset/tests/read_query_cost_test.rs @@ -74,13 +74,18 @@ async fn legacy_scan_replays_only_the_caller_checkpoint_without_writes() { shuffle: None, cursor: None, }; + let admissions = store.admission_reads.load(Ordering::Relaxed); let first = DatasetReadService::new(authority.clone(), identity.clone(), 1) .unwrap() .scan(request.clone()) .await .unwrap(); + assert_eq!(store.admission_reads.load(Ordering::Relaxed) - admissions, 1); let restarted = DatasetReadService::new(authority, identity, 1).unwrap(); + let admissions = store.admission_reads.load(Ordering::Relaxed); assert_eq!(restarted.scan(request.clone()).await.unwrap(), first); + assert_eq!(store.admission_reads.load(Ordering::Relaxed) - admissions, 1); + let admissions = store.admission_reads.load(Ordering::Relaxed); let completed = restarted .scan(DatasetScanRequest { cursor: Some(first.cursor), @@ -88,6 +93,7 @@ async fn legacy_scan_replays_only_the_caller_checkpoint_without_writes() { }) .await .unwrap(); + assert_eq!(store.admission_reads.load(Ordering::Relaxed) - admissions, 1); assert!(completed.samples.is_empty()); assert!(completed.end); assert_eq!(store.writes.load(Ordering::Relaxed), writes); From 933e5b2603820ae2a026bd749040857143985ec4 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 16:58:25 +0800 Subject: [PATCH 117/258] Resume access cleanup validation --- doc/working/plan-access-simplification.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 965724675..43ee71a53 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -19,8 +19,8 @@ Upstream: [Dataset SDK](../backlog/R220-dataset-fat-client-plan.md)、 静态发现、提出方案、代码完成、性能验收不能互相代替。 - 当前实施基线为 `489ae507`；旧性能审查来自 `878ee901`， 涉及已删除 session 的结论已移除。下面的成本数字只描述注明的函数路径，需动态复核。 -- 用户要求暂时离开后继续。交接时 46 个未完成 checkbox， - 包含条件决策、交叉审查和总体验收，不代表 46 个独立功能；没有正在运行的实施步骤。 +- 用户已恢复工作；本地交接已读入并按要求删除。交接时 46 个未完成 checkbox， + 包含条件决策、交叉审查和总体验收，不代表 46 个独立功能。 ## 已确定的约束 @@ -473,7 +473,7 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 记录 p50/p99、TPS、CPU/RSS、RPC、复制/摘要、清理积压，不缩小负载掩盖失败。 Files: `tools/sdk/dataset_workload.py`、相关 examples、`e2e-test/`、本机 HCA runner。 物理多主机验证继续留在独立计划，不并回此处。 -- [ ] **V4.2 验证并发 metadata 与单块 owner 移交的真实链路**：由 `acd25325` 生产源码重新 +- [~] **V4.2 验证并发 metadata 与单块 owner 移交的真实链路**：由 `acd25325` 生产源码重新 构建独立本地镜像，运行 Dataset 全部容器用例与同参数 TCP workload。与 V4.1 使用相同 client/server CPU、metadata 内存、size 集合、并发和校验，保留精确 artifact/source 身份； 不把单 sample-field 的当前 workload 当作宽投影并发性能证明，后者继续由 F6/F1 验证。 From a907680022156103c253f41e0f48299a4a4f60f0 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 17:22:21 +0800 Subject: [PATCH 118/258] Record current-source dataset workload results --- doc/working/plan-access-simplification.md | 24 ++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 43ee71a53..bb593b0e8 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -473,13 +473,6 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 记录 p50/p99、TPS、CPU/RSS、RPC、复制/摘要、清理积压，不缩小负载掩盖失败。 Files: `tools/sdk/dataset_workload.py`、相关 examples、`e2e-test/`、本机 HCA runner。 物理多主机验证继续留在独立计划，不并回此处。 -- [~] **V4.2 验证并发 metadata 与单块 owner 移交的真实链路**：由 `acd25325` 生产源码重新 - 构建独立本地镜像，运行 Dataset 全部容器用例与同参数 TCP workload。与 V4.1 使用相同 - client/server CPU、metadata 内存、size 集合、并发和校验，保留精确 artifact/source 身份； - 不把单 sample-field 的当前 workload 当作宽投影并发性能证明，后者继续由 F6/F1 验证。 - 首次 build 的 release/C++/frontend 均完成，但构建期间修改了本计划与成本测试，触发 - `Source tree changed while building; rebuild the shared OCI image`，退出 1；没有生成新 OCI。 - 恢复时先提交完整稳定树，重跑 build 期间不改任何非忽略文件，保留 source guard。 ## 成本口径与集中代码索引 @@ -500,6 +493,23 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **V4.2（2026-10-24）**：稳定源码树重新构建成功；本地镜像 + `crowdb-node:access-cleanup-acd25325`，revision `d82721f1`，source tree + `7e19feabf2c73f469fa93156b6239cb79b040244`，OCI digest + `sha256:7ddb31ac6ebd3c1cab0fb0c5b118ab0209d0b4c1139bd3cca32fb4572ddfce55`， + Docker import identity `sha256:21d9ec146e42f4c3c36453232d7c0566766372ebe283e55984b10a3641e39ba0`。 + 全部五个 Dataset 容器用例通过，记录在 `container-e2e/run-e2c9c096a5b34e7e9d1ea0aefe5d0757`； + 同 V4.1 参数的完整 TCP workload、native Python 安装/parity/held buffer/cancel 与存储重启 + 校验均通过，18 档试验 errors=0，记录在 `dataset-workload/e538038bee3f4153b78d01bca7829aac`。 + 两个路径均相对于 `.crowdb-runtime/artifacts/`。并发 1 的中位 MiB/s 为 HTTP 169.52 / + Rust 185.83 / Python 189.66，并发 3 为 258.62 / 254.11 / 264.70；最大 client RSS + 分别约 19/32/56 MiB。相对 V4.1，HTTP 接近，Rust 低约 7%–11%，Python 并发 1 低约 9%、 + 并发 3 接近；这次短负载没有证明吞吐提高。4 KiB 并发 1 中位 p50 为 578/712/829 µs， + 每档每尺寸仅 16 个延迟样本，不作尾延迟生产结论。当前 workload 单 sample-field，不能 + 证明宽投影并发收益，也没有逐请求 RPC、复制计数或 HCA 对比，V4/F1/F6 继续跟踪。 + 首次构建因中途修改非忽略文档/测试触发 source guard 失败，重跑保留 guard；成功记录不 + 覆盖失败记录。构建/import/容器/workload 命令全部通过 `pixi run`，镜像仅导入本地。 + - **F8.3（2026-10-24）**：`ChunkPayloadStorage::read` 在读前按逻辑长度限制最大 4 MiB， 单块完整结果确认 EOF 后直接移交 Bytes owner；多块只按最终长度分配一次，复制当前片段后 立即释放其读 credit 再 pull，避免单 slot 死等。没有新增 metadata/RPC/持久化，仍检查 From 3d17c614f09faa09b57a9f50c9f2b0a3792c2f2a Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 17:39:23 +0800 Subject: [PATCH 119/258] Allow complete split results to be used before publication --- .../tests/dataset_split_contract_test.rs | 15 +- .../dataset_split_recovery_contract_test.rs | 11 + .../dataset/design-dataset-control.md | 12 +- .../dataset/openapi-crowdb-dataset-v2.yaml | 2 + .../dataset/spec-crowdb-dataset.md | 5 + doc/working/plan-access-simplification.md | 22 ++ .../src/authority/publication/visibility.rs | 25 -- .../src/split/cleanup.rs | 22 +- .../src/split/cleanup/revision.rs | 76 +++++ lib/crowdb-access-dataset/src/split/read.rs | 90 ++---- .../tests/common/split_store.rs | 26 +- .../tests/split_cleanup_test.rs | 32 +- .../tests/split_complete_result_test.rs | 301 ++++++++++++++++++ .../tests/split_inheritance_test.rs | 28 +- .../tests/split_publication_test.rs | 21 ++ .../tests/split_read_query_test.rs | 9 +- .../tests/split_read_test.rs | 28 +- sdk/README.md | 3 + .../examples/split_control.rs | 11 + 19 files changed, 597 insertions(+), 142 deletions(-) create mode 100644 lib/crowdb-access-dataset/src/split/cleanup/revision.rs create mode 100644 lib/crowdb-access-dataset/tests/split_complete_result_test.rs diff --git a/app/crowdb-access-server/tests/dataset_split_contract_test.rs b/app/crowdb-access-server/tests/dataset_split_contract_test.rs index ad99a2b40..12255336f 100644 --- a/app/crowdb-access-server/tests/dataset_split_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_split_contract_test.rs @@ -23,11 +23,19 @@ async fn split_revision_flow_preserves_data_head_and_historical_members() { response(client.post(format!("{base}/split-drafts")).json(&begin), 200).await; let prepared = common::prepare(&client, &draft, "1").await; let revision = prepared["revision"].as_str().unwrap(); - response(client.get(format!("{base}/split-revisions/{revision}")), 404).await; assert_eq!( - response(client.get(format!("{base}/split-revisions")), 200).await["entries"], - json!([]) + response(client.get(format!("{base}/split-revisions/{revision}")), 200).await, + prepared + ); + assert_eq!( + response(client.get(format!("{base}/split-revisions")), 200).await["entries"][0]["revision"], + revision ); + let before_publish = members( + &client, + &format!("{base}/split-revisions/{revision}/splits/train%2Fa%20b/members"), + ) + .await; let receipt = common::publish( &client, &draft, @@ -68,6 +76,7 @@ async fn split_revision_flow_preserves_data_head_and_historical_members() { &format!("{base}/split-revisions/{revision}/splits/test/members"), ) .await; + assert_eq!(train, before_publish); assert!(train.is_disjoint(&test_ids)); assert_eq!(train.len() + test_ids.len(), 45); assert_eq!( diff --git a/app/crowdb-access-server/tests/dataset_split_recovery_contract_test.rs b/app/crowdb-access-server/tests/dataset_split_recovery_contract_test.rs index da6348d67..8728a3efc 100644 --- a/app/crowdb-access-server/tests/dataset_split_recovery_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_split_recovery_contract_test.rs @@ -68,6 +68,17 @@ async fn evaluation_resumes_after_server_restart_and_stale_publish_keeps_candida first_receipt ); assert_eq!(response(client.get(&head_url), 200).await, current); + assert_eq!( + response( + client.get(format!( + "{root}/snapshots/{snapshot}/split-revisions/{}", + b.1["revision"].as_str().unwrap() + )), + 200 + ) + .await, + b.1 + ); let historic = response( client.get(format!( "{root}/snapshots/{snapshot}/split-revisions/{}", diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index a3f5f97f2..96366e7b8 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -89,6 +89,10 @@ not a promise of exact cardinality. Overlap/coverage validation remains independ Membership pages are immutable and checksum-bound. Preparation persists the whole revision before currentSplit CAS. A conflict preserves the candidate and caller inputs; retry never silently rebases definitions or overwrites another revision. +The complete `SplitRevisionRecord` is the final result: an explicit revision ID +can be read, listed or used as an inherited source without winning currentSplit. +Read admission still checks the owning Snapshot. Reads load that immutable record +directly without Split publication proofs, currentSplit lookups or helper writes. ## 4. Bounded observation and reads @@ -167,6 +171,12 @@ Helping an already stored edit receipt only reads and validates it; retries do n submit a doomed CAS for the same bytes. Evaluation journals and cleanup cursors are scoped to one draft, not to unrelated readers. They retain exact partial-work recovery and prevent a delayed writer from reviving cleaned candidate data. +Split cleanup competes with preparation on the final revision key. If the complete +root already exists, including a root returned by a failed cleanup CAS, cleanup +preserves its member pages and summary while retiring evaluation intents. If +cleanup seals an absent root first, a delayed preparation cannot publish it. +Abandoning the draft does not delete an already complete revision. Retiring the +owning Snapshot still permits revision cleanup and closes new read admission. Sources use read-only availability checks. The compatibility `source_lease_deadline` field is zero for new drafts and edits. @@ -175,7 +185,7 @@ ownership under a terminal owner fence. A durable five-minute grace deadline separates deletion admission from cleanup. After that deadline it retires upload owners, Snapshot-scoped Split drafts, possible index writes and ordinary metadata in phases. Completion reports logical retirement; physical upload deletion remains deferred -to the bounded upload-authority scan. There is no independent committed Split revision pruning. +to the bounded upload-authority scan. There is no independent complete Split revision pruning. `GET snapshots/{snapshot}/reclaim` is bounded, read-only observation. It returns publication operation identity where available, latest/stable/retained flags, diff --git a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml index 4f662c4ae..ac4612ded 100644 --- a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml +++ b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml @@ -302,6 +302,7 @@ paths: /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/split-revisions: get: operationId: splitHistory + description: Lists complete revisions, including results that have not updated currentSplit. Snapshot read admission applies. parameters: - in: path name: dataset @@ -324,6 +325,7 @@ paths: /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/split-revisions/{revision}: get: operationId: splitRevision + description: Reads a complete immutable revision by ID without requiring currentSplit publication. Snapshot read admission applies. parameters: - in: path name: dataset diff --git a/doc/design/access-server/dataset/spec-crowdb-dataset.md b/doc/design/access-server/dataset/spec-crowdb-dataset.md index c1d75e5df..71cff9f0b 100644 --- a/doc/design/access-server/dataset/spec-crowdb-dataset.md +++ b/doc/design/access-server/dataset/spec-crowdb-dataset.md @@ -195,6 +195,11 @@ a reviewed retry uses a new attempt with a newly observed generation. Publishing an independent Split revision MUST update only its Snapshot's currentSplit pointer with an expected revision/head CAS. It MUST NOT create a data Snapshot or change latest/stable. Old revisions remain addressable until owner reclamation. +Every complete immutable Split revision MUST also be readable by ID, enumerable +and usable as a frozen source before publication or after a publication conflict. +Abandoning a draft MUST NOT remove its complete result. A missing or cleanup-sealed +final revision record is unavailable; incomplete membership pages alone do not +constitute a result. Snapshot deletion admission still applies to each new request. ## 7. Selection, plans, and cursors diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index bb593b0e8..fedccbd48 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -161,6 +161,10 @@ CAS 不再满足只证明“本次条件不成立”，不能谎称“第一次 历史回复查询；Composition/Split 的 Committing 恢复与 P2/P3 一起删除，客户端保留完整输入。 旧记录只作为兼容解码/GC 输入，不让旧账本控制新请求。head 保留 expected base；同一 candidate 的当前结果可判断相同，已被后续 head 遮住时不保证历史请求结果重放。 +- [~] **R2.2a 独立 Split 发布不写请求账本**：从完整 revision 和显式 expected base 更新 + currentSplit；删除新 CA/CR/SRC、Committing/Published 草稿写入及覆盖 head 前的 helper。 + 同一目标仍是当前结果时返回当前事实；后续代际已遮住时返回条件冲突，不回放旧响应或重置 + 后来者。旧账本仅兼容读取/清理。补 H0→H1→H2、未知回复、并发 CAS 与实际 mutation 成本。 - [ ] **R2 审查条件发布的回复丢失窗口**：用 H0→H1→H2 测试区分条件失败、相同结果与历史未知； 列出确实依赖历史精确结果的调用者，没有这种需求就移除相应 ledger/response replay。 @@ -270,6 +274,8 @@ S3 普通对象是直接 PUT，不能未经设计就套用 Dataset head 机制 - [ ] **R4.2 实现提交期限和回收隔离**：part 结果携带期限，提交权威边界验证；覆盖期限前准入 但期限后落地的 RPC、GC 与最终引用并发、成功发布后的长期存活。若需新增持久化资源事实， 先列出最小字段和每请求 RPC 成本供 review。Files: Dataset publication/upload、Iceberg file、GC。 + 完整但没有正式引用的 Snapshot/Split 也需独立过期与引用隔离；目前完整 Split 保留到 owner + Snapshot 回收，不能把“放弃草稿不删完整结果”当作独立期限 GC 已完成。 - [ ] **R4.5 简化通知入口的前置查询**：当前接口要求 expected_modify_ts，reclaim_location 因此先 QueryChunk，并兼容旧的 dedicated DeleteChunk 分支。与后续 ChunkDB 接口实现一起 收敛为 chunk_id/offset/length 的单次 DeleteChunkRange 通知；布局、封口与物理回收判断归 @@ -493,6 +499,22 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **R2.1a（2026-10-24）**：完整 Split 的 ID 读取、history、冻结继承和 continuation + 直接使用最终 `SplitRevisionRecord`；移除读取 SRC/currentSplit 的门槛和闲置 initial-Split + proof helper，Snapshot 准入与成员页校验保持。无新持久化记录、锁或后台 reader 状态。 + scratch cleanup 与最终 root 在同一 key 上 create-if-absent：已有完整 root 或冲突返回完整 + root 就保留成员/summary，仅清理 evaluation intent；清理先封口时迟到 prepare 失败。 + CAS 返回值直接使用，不再为确认结果读第二次 root；storage unknown 原样返回。 + 新增四例覆盖 CAS 输家、最终 root 丢回复、清理 CAS 竞争、迟到准备、未发布结果继承及 + owner 删除；增加 head CAS 丢回复后的直接读取。无发布 helper 的 ID 读取实际为 3 个 bounded + GET（Snapshot 删除准入、Snapshot 现有成功记录、Split root）+0 mutation；root 只取一次。 + Dataset `cargo test --all-targets`、七个 HTTP/独立 Rust SDK 用例、三个相关 crate 的 + all-target/all-feature clippy、SDK example clippy、两 workspace fmt、SDK boundary 与 + diff check 通过，全部经 `pixi run`。HTTP 首次因沙箱禁止本地 bind 失败，提升权限后相同 + 用例全部通过；继承旧测试仍要求未发布结果不可用，已按用户选定合同更新并保留缺失名称 + 与删除拒绝断言。文档同步 control design/spec/OpenAPI/SDK。Snapshot 完整准入、 + 发布账本删除、独立结果期限 GC 尚未完成；此批未构建新镜像或声称吞吐提升。 + - **V4.2（2026-10-24）**：稳定源码树重新构建成功；本地镜像 `crowdb-node:access-cleanup-acd25325`，revision `d82721f1`，source tree `7e19feabf2c73f469fa93156b6239cb79b040244`，OCI digest diff --git a/lib/crowdb-access-dataset/src/authority/publication/visibility.rs b/lib/crowdb-access-dataset/src/authority/publication/visibility.rs index f72434a60..3ab2425db 100644 --- a/lib/crowdb-access-dataset/src/authority/publication/visibility.rs +++ b/lib/crowdb-access-dataset/src/authority/publication/visibility.rs @@ -85,31 +85,6 @@ impl DatasetAuthority { } } -impl DatasetAuthority { - pub(crate) async fn inspection_initial_split( - &self, - identity: &DatasetIdentity, - snapshot: SnapshotId, - work: &mut InspectionWork<'_>, - ) -> Result, InspectionError> { - let Some(value) = work - .get( - &super::candidate::candidate_key(identity, snapshot).map_err(crate::AuthorityError::from)?, - 4096, - ) - .await? - else { - return Ok(None); - }; - let binding: super::candidate::CandidateBinding = - serde_json::from_slice(&value.bytes).map_err(|_| InspectionError::Corrupt)?; - if binding.version != 1 || binding.snapshot != snapshot { - return Err(InspectionError::Corrupt); - } - Ok(binding.initial_split) - } -} - impl DatasetAuthority { pub(crate) async fn inspection_snapshot_operation( &self, diff --git a/lib/crowdb-access-dataset/src/split/cleanup.rs b/lib/crowdb-access-dataset/src/split/cleanup.rs index 1332ff6e3..5c3818348 100644 --- a/lib/crowdb-access-dataset/src/split/cleanup.rs +++ b/lib/crowdb-access-dataset/src/split/cleanup.rs @@ -13,6 +13,8 @@ use crate::{ use serde::{Deserialize, Serialize}; use std::time::Duration; +mod revision; + pub(crate) const SEALED: &[u8] = b"split-retired-v1"; const END: &[u8] = b"split-retired-end-v1"; const RECORD_CAP: usize = 1024 * 1024 + 64 * 1024; @@ -32,7 +34,7 @@ struct Cursor { item: usize, } impl DatasetAuthority { - /// Retires obsolete uncommitted membership windows, preserving committed revision data. + /// Retires obsolete incomplete membership windows, preserving complete revision data. /// Each call processes at most twenty page references and retains small write fences. /// # Errors /// Rejects legacy drafts without work intents, live initial drafts, corruption and bounded-work failures. @@ -99,11 +101,6 @@ impl DatasetAuthority { } if cursor.revision < end { let revision = super::candidate_revision(operation, cursor.revision)?; - let record_key = super::key::revision(identity, state.record.snapshot, revision) - .map_err(AuthorityError::from)?; - // Preparation is immutable and can race abandonment after reading the root. - // Fence its record before retiring the pages it could otherwise reference. - self.seal_split_key(&record_key, &mut work).await?; let summary = super::SplitRevisionSummary { version: 1, snapshot: state.record.snapshot, @@ -111,15 +108,15 @@ impl DatasetAuthority { operation, split_count: 0, }; - let summary_key = super::key::revision_summary(identity, state.record.snapshot, revision) - .map_err(AuthorityError::from)?; - self.seal_split_value(&summary_key, &super::codec::encode(&summary)?, &mut work) + let preserve = self + .fence_split_cleanup_revision(identity, &summary, retired, &mut work) .await?; let intent_key = work_key(identity, operation, cursor.revision, cursor.page)?; self.retire_split_intent( &intent_key, &super::key::membership_prefix(identity, state.record.snapshot, revision) .map_err(AuthorityError::from)?, + !preserve, &mut cursor, &mut work, ) @@ -145,6 +142,7 @@ impl DatasetAuthority { &self, key: &[u8], prefix: &[u8], + retire_members: bool, cursor: &mut Cursor, work: &mut InspectionWork<'_>, ) -> Result<(), SplitLifecycleError> { @@ -174,8 +172,10 @@ impl DatasetAuthority { return Err(SplitLifecycleError::Corrupt); } let stop = (cursor.item + 20).min(intent.pages.len()); - for page in &intent.pages[cursor.item..stop] { - self.seal_split_key(&page.key, work).await?; + if retire_members { + for page in &intent.pages[cursor.item..stop] { + self.seal_split_key(&page.key, work).await?; + } } cursor.item = stop; if stop < intent.pages.len() { diff --git a/lib/crowdb-access-dataset/src/split/cleanup/revision.rs b/lib/crowdb-access-dataset/src/split/cleanup/revision.rs new file mode 100644 index 000000000..1400fb96c --- /dev/null +++ b/lib/crowdb-access-dataset/src/split/cleanup/revision.rs @@ -0,0 +1,76 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::{ + inspection::work::InspectionWork, split::SplitLifecycleError, split::SplitRevisionRecord, AuthorityError, + CasOutcome, DatasetAuthority, DatasetIdentity, OperationId, SnapshotId, SplitRevisionId, +}; + +impl DatasetAuthority { + pub(in crate::split::cleanup) async fn fence_split_cleanup_revision( + &self, + identity: &DatasetIdentity, + summary: &crate::split::SplitRevisionSummary, + retired: bool, + work: &mut InspectionWork<'_>, + ) -> Result { + let preserve = if retired { + let key = crate::split::key::revision(identity, summary.snapshot, summary.revision) + .map_err(AuthorityError::from)?; + self.seal_split_key(&key, work).await?; + false + } else { + self.fence_incomplete_split_revision( + identity, + summary.snapshot, + summary.revision, + summary.operation, + work, + ) + .await? + }; + if !preserve { + let key = crate::split::key::revision_summary(identity, summary.snapshot, summary.revision) + .map_err(AuthorityError::from)?; + self.seal_split_value(&key, &crate::split::codec::encode(summary)?, work) + .await?; + } + Ok(preserve) + } + /// A complete root and a cleanup fence compete on the same immutable key. + /// Once the root wins, scratch cleanup cannot retire its member pages. + async fn fence_incomplete_split_revision( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + revision: SplitRevisionId, + operation: OperationId, + work: &mut InspectionWork<'_>, + ) -> Result { + let key = crate::split::key::revision(identity, snapshot, revision).map_err(AuthorityError::from)?; + if let Some(value) = work.get(&key, super::RECORD_CAP).await? { + return complete(&value.bytes, snapshot, revision, operation); + } + match self.store.compare_exchange(&key, None, super::SEALED).await? { + CasOutcome::Applied(_) => Ok(false), + CasOutcome::Conflict(Some(observed)) => complete(&observed.bytes, snapshot, revision, operation), + CasOutcome::Conflict(None) => Err(SplitLifecycleError::Conflict), + } + } +} + +fn complete( + bytes: &[u8], + snapshot: SnapshotId, + revision: SplitRevisionId, + operation: OperationId, +) -> Result { + if bytes == super::SEALED { + return Ok(false); + } + let record = SplitRevisionRecord::decode(bytes)?; + if record.snapshot != snapshot || record.revision != revision || record.operation != operation { + return Err(SplitLifecycleError::Corrupt); + } + Ok(true) +} diff --git a/lib/crowdb-access-dataset/src/split/read.rs b/lib/crowdb-access-dataset/src/split/read.rs index ad340a5a8..85effb270 100644 --- a/lib/crowdb-access-dataset/src/split/read.rs +++ b/lib/crowdb-access-dataset/src/split/read.rs @@ -1,14 +1,11 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{ - head::CommittedSplitHead, key, MembershipPage, SplitHead, SplitLifecycleError, SplitName, - SplitRevisionRecord, -}; +use super::{key, MembershipPage, SplitLifecycleError, SplitName, SplitRevisionRecord}; use crate::{ inspection::{cursor::CursorScope, work::InspectionWork, InspectionPage}, - AuthorityError, CommitTarget, DatasetAuthority, DatasetIdentity, MetadataScanBudget, OperationId, - ReadCancellation, SnapshotId, SplitRevisionId, SuccessfulCommit, + AuthorityError, DatasetAuthority, DatasetIdentity, MetadataScanBudget, OperationId, ReadCancellation, + SnapshotId, SplitRevisionId, }; use serde::{Deserialize, Serialize}; @@ -22,7 +19,7 @@ pub struct SplitRevisionSummary { pub split_count: u32, } impl DatasetAuthority { - /// Lists only committed revision summaries; prepared revision indexes remain invisible. + /// Lists complete immutable revisions independently of the current Split pointer. /// # Errors /// Rejects unavailable owners, invalid cursors, corrupt records and bounded-work limits. pub async fn split_revision_history( @@ -52,10 +49,15 @@ impl DatasetAuthority { { return Err(SplitLifecycleError::Corrupt); } - if self - .split_revision_visible(identity, snapshot, summary.revision, &mut work) + if let Some(record) = self + .load_admitted_split_revision(identity, snapshot, summary.revision, &mut work) .await? { + if record.operation != summary.operation + || record.splits.len() != summary.split_count as usize + { + return Err(SplitLifecycleError::Corrupt); + } entries.push(summary); } } @@ -64,9 +66,9 @@ impl DatasetAuthority { after: cursor.encode(page.after)?, }) } - /// Loads immutable definitions/counts/provenance and page bindings for a committed revision. + /// Loads complete immutable definitions/counts/provenance and page bindings by ID. /// # Errors - /// Rejects uncommitted revisions, unavailable owners, corruption and work limits. + /// Rejects incomplete revisions, unavailable owners, corruption and work limits. pub async fn get_split_revision( &self, identity: &DatasetIdentity, @@ -100,24 +102,35 @@ impl DatasetAuthority { revision: SplitRevisionId, work: &mut InspectionWork<'_>, ) -> Result { - if !self - .split_revision_visible(identity, snapshot, revision, work) + self.load_admitted_split_revision(identity, snapshot, revision, work) .await? - { - return Err(SplitLifecycleError::NotFound); - } - let value = work + .ok_or(SplitLifecycleError::NotFound) + } + + async fn load_admitted_split_revision( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + revision: SplitRevisionId, + work: &mut InspectionWork<'_>, + ) -> Result, SplitLifecycleError> { + let Some(value) = work .get( &key::revision(identity, snapshot, revision).map_err(AuthorityError::from)?, 1024 * 1024, ) .await? - .ok_or(SplitLifecycleError::Corrupt)?; + else { + return Ok(None); + }; + if value.bytes == super::cleanup::SEALED { + return Ok(None); + } let record = SplitRevisionRecord::decode(&value.bytes)?; if record.snapshot != snapshot || record.revision != revision { return Err(SplitLifecycleError::Corrupt); } - Ok(record) + Ok(Some(record)) } async fn require_split_reader( &self, @@ -133,45 +146,6 @@ impl DatasetAuthority { } Ok(()) } - pub(super) async fn split_revision_visible( - &self, - identity: &DatasetIdentity, - snapshot: SnapshotId, - revision: SplitRevisionId, - work: &mut InspectionWork<'_>, - ) -> Result { - if self.inspection_initial_split(identity, snapshot, work).await? == Some(revision) { - return Ok(true); - } - let target = CommitTarget::SplitRevision { snapshot, revision }; - if let Some(value) = work - .get( - &key::committed_revision(identity, snapshot, revision).map_err(AuthorityError::from)?, - 8192, - ) - .await? - { - if SuccessfulCommit::decode(&value.bytes)?.attempt.target != target { - return Err(SplitLifecycleError::Corrupt); - } - return Ok(true); - } - let Some(value) = work - .get( - &key::head(identity, snapshot).map_err(AuthorityError::from)?, - 8192, - ) - .await? - else { - return Ok(false); - }; - if SplitHead::decode(&value.bytes).is_ok() { - return Ok(false); - } - let head = CommittedSplitHead::decode(&value.bytes)?; - Ok(head.attempt.target == target) - } - /// Returns exact ordered member IDs from checksum-bound immutable pages. /// # Errors /// Rejects wrong revision/name/cursors, missing pages, corrupt bindings and work limits. diff --git a/lib/crowdb-access-dataset/tests/common/split_store.rs b/lib/crowdb-access-dataset/tests/common/split_store.rs index c09902bd3..fe901981e 100644 --- a/lib/crowdb-access-dataset/tests/common/split_store.rs +++ b/lib/crowdb-access-dataset/tests/common/split_store.rs @@ -4,7 +4,7 @@ #[allow(dead_code)] mod dataset_store; use crowdb_access_dataset::*; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; #[derive(Default)] pub(super) struct TestDraftStore { @@ -17,6 +17,11 @@ pub(super) struct TestDraftStore { pub inspect_only: AtomicBool, pub metadata_only: AtomicBool, pub cancel_scan: AtomicBool, + pub revision_reads: AtomicUsize, + pub bounded_reads: AtomicUsize, + pub hide_revision_once: AtomicBool, + pub lose_revision_reply: AtomicBool, + pub reject_split_proof_reads: AtomicBool, } #[async_trait::async_trait] impl DatasetStore for TestDraftStore { @@ -39,6 +44,19 @@ impl DatasetStore for TestDraftStore { budget: MetadataScanBudget, cancel: &ReadCancellation, ) -> Result, StoreError> { + self.bounded_reads.fetch_add(1, Ordering::Relaxed); + if self.reject_split_proof_reads.load(Ordering::Relaxed) { + assert!( + !key.starts_with(b"DS\0SRC\0") && !key.starts_with(b"DS\0SH\0"), + "explicit Split reads queried publication evidence" + ); + } + if key.starts_with(b"DS\0SR\0") { + self.revision_reads.fetch_add(1, Ordering::Relaxed); + if self.hide_revision_once.swap(false, Ordering::Relaxed) { + return Ok(None); + } + } if self.inspect_only.load(Ordering::Relaxed) || self.metadata_only.load(Ordering::Relaxed) { assert!( !key.starts_with(b"DS\0S\0") && !key.starts_with(b"DS\0M\0"), @@ -80,6 +98,12 @@ impl DatasetStore for TestDraftStore { return Err(StoreError::Rejected); } let result = self.inner.compare_exchange(key, expected, value).await?; + if key.starts_with(b"DS\0SR\0") + && matches!(result, CasOutcome::Applied(_)) + && self.lose_revision_reply.swap(false, Ordering::Relaxed) + { + return Err(StoreError::Rejected); + } if key.starts_with(b"DS\0H\0") && matches!(result, CasOutcome::Applied(_)) && self.lose_data_reply.swap(false, Ordering::Relaxed) diff --git a/lib/crowdb-access-dataset/tests/split_cleanup_test.rs b/lib/crowdb-access-dataset/tests/split_cleanup_test.rs index ce38895a0..c5858a72c 100644 --- a/lib/crowdb-access-dataset/tests/split_cleanup_test.rs +++ b/lib/crowdb-access-dataset/tests/split_cleanup_test.rs @@ -21,7 +21,7 @@ async fn finish(authority: &DatasetAuthority, identity: &DatasetIdentity, operat } #[tokio::test] -async fn abandoning_prepared_split_retires_membership_and_fences_late_preparation() { +async fn abandoning_complete_split_preserves_result_and_membership() { let (store, authority, identity, snapshot) = split_store::fixture().await; let record = split_store::prepared(&authority, &identity, snapshot, None).await; let record_key = key::revision(&identity, snapshot, record.revision).unwrap(); @@ -33,7 +33,7 @@ async fn abandoning_prepared_split_retires_membership_and_fences_late_preparatio .await .unwrap(); finish(&authority, &identity, record.operation).await; - assert!(store.get(&page_key).await.unwrap().unwrap().bytes.len() < original.bytes.len()); + assert_eq!(store.get(&page_key).await.unwrap().unwrap().bytes, original.bytes); assert!(matches!( store .compare_exchange(&page_key, None, &original.bytes) @@ -48,12 +48,28 @@ async fn abandoning_prepared_split_retires_membership_and_fences_late_preparatio .unwrap(), CasOutcome::Conflict(_) )); - assert!(authority - .split_revision_history(&identity, snapshot, None, budget(), &ReadCancellation::new()) - .await - .unwrap() - .entries - .is_empty()); + assert_eq!( + authority + .split_revision_history(&identity, snapshot, None, budget(), &ReadCancellation::new()) + .await + .unwrap() + .entries + .len(), + 1 + ); + assert_eq!( + authority + .get_split_revision( + &identity, + snapshot, + record.revision, + budget(), + &ReadCancellation::new() + ) + .await + .unwrap(), + record + ); finish(&DatasetAuthority::new(store), &identity, record.operation).await; } diff --git a/lib/crowdb-access-dataset/tests/split_complete_result_test.rs b/lib/crowdb-access-dataset/tests/split_complete_result_test.rs new file mode 100644 index 000000000..869a26854 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/split_complete_result_test.rs @@ -0,0 +1,301 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/composition.rs"] +#[allow(dead_code)] +mod common; +#[path = "common/publication.rs"] +mod publication; +#[path = "common/split_store.rs"] +mod split_store; + +use crowdb_access_dataset::{composition::*, split::*, *}; +use std::sync::atomic::Ordering; + +async fn cleanup(authority: &DatasetAuthority, identity: &DatasetIdentity, operation: OperationId) { + for _ in 0..100 { + if authority + .cleanup_split_scratch_step(identity, operation, &ReadCancellation::new()) + .await + .unwrap() + .complete + { + return; + } + } + panic!("Split cleanup did not finish"); +} + +#[tokio::test] +async fn losing_pointer_cas_keeps_result_readable_without_split_proofs_or_writes() { + let (store, authority, identity, snapshot) = split_store::fixture().await; + let winner = split_store::prepared(&authority, &identity, snapshot, None).await; + let candidate = split_store::prepared(&authority, &identity, snapshot, None).await; + let attempt = split_store::attempt(&authority, &identity, &candidate).await; + split_store::publish_revision(&authority, &identity, &winner).await; + assert!(matches!( + authority + .publish_split_draft(&identity, candidate.operation, &attempt) + .await, + Err(SplitLifecycleError::Conflict) + )); + let head = authority.inspect_split_head(&identity, snapshot).await.unwrap(); + assert_eq!(head.revision, Some(winner.revision)); + authority + .abandon_split_draft(&identity, candidate.operation, 1) + .await + .unwrap(); + cleanup(&authority, &identity, candidate.operation).await; + store.reject_split_proof_reads.store(true, Ordering::Relaxed); + store.inspect_only.store(true, Ordering::Relaxed); + let before = store.revision_reads.load(Ordering::Relaxed); + let total = store.bounded_reads.load(Ordering::Relaxed); + let restarted = DatasetAuthority::new(store.clone()); + assert_eq!( + restarted + .get_split_revision( + &identity, + snapshot, + candidate.revision, + common::budget(), + &ReadCancellation::new() + ) + .await + .unwrap(), + candidate + ); + assert_eq!(store.revision_reads.load(Ordering::Relaxed) - before, 1); + assert_eq!(store.bounded_reads.load(Ordering::Relaxed) - total, 3); + let split = candidate.splits.iter().find(|s| s.count > 0).unwrap(); + let mut after = None; + let mut members = Vec::new(); + loop { + let page = restarted + .split_members_page( + &identity, + snapshot, + candidate.revision, + &split.name, + after, + MetadataScanBudget { + items: 1, + ..common::budget() + }, + &ReadCancellation::new(), + ) + .await + .unwrap(); + members.extend(page.entries); + after = page.after; + if after.is_none() { + break; + } + } + assert_eq!(members.len() as u64, split.count); + store.inspect_only.store(false, Ordering::Relaxed); + store.reject_split_proof_reads.store(false, Ordering::Relaxed); + assert_eq!( + restarted.inspect_split_head(&identity, snapshot).await.unwrap(), + head + ); +} + +#[tokio::test] +async fn complete_root_with_lost_reply_survives_abandonment_and_cleanup_cas_race() { + let (store, authority, identity, snapshot) = split_store::fixture().await; + let operation = OperationId::random(); + authority + .begin_split_draft( + &identity, + operation, + snapshot, + None, + SplitPolicy::default(), + split_store::partition_splits(), + ) + .await + .unwrap(); + let preview = authority + .evaluate_split_draft(&identity, operation, 1, 0, &ReadCancellation::new()) + .await + .unwrap(); + assert!(preview.complete); + store.lose_revision_reply.store(true, Ordering::Relaxed); + assert!(authority + .prepare_split_draft(&identity, operation, 1) + .await + .is_err()); + let roots = store + .scan_page( + &MetadataScan { + prefix: b"DS\0SR\0".to_vec(), + after: None, + budget: common::budget(), + }, + &ReadCancellation::new(), + ) + .await + .unwrap(); + assert_eq!(roots.entries.len(), 1); + let record = SplitRevisionRecord::decode(&roots.entries[0].1.bytes).unwrap(); + authority + .abandon_split_draft(&identity, operation, 1) + .await + .unwrap(); + // Cleanup reads absence, then loses its create-if-absent to the completed root. + store.hide_revision_once.store(true, Ordering::Relaxed); + let reads = store.revision_reads.load(Ordering::Relaxed); + authority + .cleanup_split_scratch_step(&identity, operation, &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(store.revision_reads.load(Ordering::Relaxed) - reads, 1); + cleanup(&DatasetAuthority::new(store.clone()), &identity, operation).await; + let restarted = DatasetAuthority::new(store.clone()); + store.inspect_only.store(true, Ordering::Relaxed); + store.reject_split_proof_reads.store(true, Ordering::Relaxed); + assert_eq!( + restarted + .get_split_revision( + &identity, + snapshot, + record.revision, + common::budget(), + &ReadCancellation::new() + ) + .await + .unwrap(), + record + ); + for split in &record.splits { + let page = restarted + .split_members_page( + &identity, + snapshot, + record.revision, + &split.name, + None, + common::budget(), + &ReadCancellation::new(), + ) + .await + .unwrap(); + assert_eq!(page.entries.len() as u64, split.count); + } +} + +#[tokio::test] +async fn cleanup_fences_missing_final_root_before_retiring_pages() { + let (store, authority, identity, snapshot) = split_store::fixture().await; + let record = split_store::prepared(&authority, &identity, snapshot, None).await; + let root = key::revision(&identity, snapshot, record.revision).unwrap(); + // Recreate a producer paused before its final immutable write. + store.delete(&root).await.unwrap(); + store + .delete(&key::revision_summary(&identity, snapshot, record.revision).unwrap()) + .await + .unwrap(); + authority + .abandon_split_draft(&identity, record.operation, 1) + .await + .unwrap(); + cleanup(&authority, &identity, record.operation).await; + assert!(matches!( + store + .compare_exchange(&root, None, &record.encode().unwrap()) + .await + .unwrap(), + CasOutcome::Conflict(Some(_)) + )); + assert!(matches!( + authority + .get_split_revision( + &identity, + snapshot, + record.revision, + common::budget(), + &ReadCancellation::new() + ) + .await, + Err(SplitLifecycleError::NotFound) + )); + for split in &record.splits { + for page in &split.pages { + let key = + key::membership_page(&identity, snapshot, record.revision, &split.name, page.page).unwrap(); + assert!(MembershipPage::decode(&store.get(&key).await.unwrap().unwrap().bytes).is_err()); + } + } +} + +#[tokio::test] +async fn unpublished_revision_can_supply_frozen_child_membership() { + let (_, authority, identity, parent) = split_store::fixture().await; + let source = split_store::prepared(&authority, &identity, parent, None).await; + let operation = OperationId::random(); + let input = CompositionInput { + parent_split_revision: Some(source.revision), + inherit_parent_splits: true, + ..common::input(parent) + }; + common::begin_evaluated(&authority, &identity, operation, &input).await; + let draft = common::prepare(&authority, &identity, operation, 1).await; + let child = draft.preparation.unwrap().candidate; + authority + .publish_composition( + &identity, + operation, + &common::attempt(&authority, &identity, child).await, + ) + .await + .unwrap(); + let revision = authority + .inspect_split_head(&identity, child) + .await + .unwrap() + .revision + .unwrap(); + let child_splits = authority + .get_split_revision( + &identity, + child, + revision, + common::budget(), + &ReadCancellation::new(), + ) + .await + .unwrap(); + assert_eq!( + child_splits.splits.iter().map(|s| s.count).sum::(), + source.splits.iter().map(|s| s.count).sum::() + ); + assert!(child_splits.splits.iter().all(|s| matches!(&s.origin, MembershipOrigin::Inherited(origin) if origin.snapshot == parent && origin.revision == source.revision))); + let root = authority + .prepare_snapshot_candidate( + &identity, + None, + publication::publication_manifest(7), + OperationId::random(), + ) + .await + .unwrap(); + authority + .publish_snapshot_candidate(&identity, &common::attempt(&authority, &identity, root).await) + .await + .unwrap(); + assert!(matches!( + authority.begin_reclaim(&identity, parent).await, + Err(AuthorityError::SnapshotProtected) + )); + authority.begin_reclaim(&identity, child).await.unwrap(); + assert!(authority + .get_split_revision( + &identity, + child, + revision, + common::budget(), + &ReadCancellation::new() + ) + .await + .is_err()); +} diff --git a/lib/crowdb-access-dataset/tests/split_inheritance_test.rs b/lib/crowdb-access-dataset/tests/split_inheritance_test.rs index 6f0cc6495..3bc423496 100644 --- a/lib/crowdb-access-dataset/tests/split_inheritance_test.rs +++ b/lib/crowdb-access-dataset/tests/split_inheritance_test.rs @@ -86,7 +86,7 @@ async fn frozen_child_membership_ignores_new_parent_revision_and_new_matching_sa } #[tokio::test] -async fn inherited_sources_require_a_committed_parent_revision_and_exact_name() { +async fn inherited_sources_accept_complete_parent_revisions_and_require_exact_names() { let (_, authority, identity, parent) = fixture().await; let source = split_store::prepared(&authority, &identity, parent, None).await; let child = authority @@ -102,27 +102,17 @@ async fn inherited_sources_require_a_committed_parent_revision_and_exact_name() name: SplitName::new("train").unwrap(), }), }; - assert!(matches!( - authority - .begin_split_draft( - &identity, - OperationId::random(), - child, - None, - SplitPolicy::default(), - vec![inherited.clone()] - ) - .await, - Err(SplitLifecycleError::NotFound) - )); - authority - .publish_split_draft( + assert!(authority + .begin_split_draft( &identity, - source.operation, - &attempt(&authority, &identity, &source).await, + OperationId::random(), + child, + None, + SplitPolicy::default(), + vec![inherited.clone()] ) .await - .unwrap(); + .is_ok()); let mut missing_name = inherited.clone(); if let MembershipOrigin::Inherited(source) = &mut missing_name.origin { source.name = SplitName::new("missing").unwrap(); diff --git a/lib/crowdb-access-dataset/tests/split_publication_test.rs b/lib/crowdb-access-dataset/tests/split_publication_test.rs index bb2b69c89..8cf61756e 100644 --- a/lib/crowdb-access-dataset/tests/split_publication_test.rs +++ b/lib/crowdb-access-dataset/tests/split_publication_test.rs @@ -71,6 +71,27 @@ async fn lost_split_cas_reply_is_recoverable_and_pending_commit_cannot_be_abando .publish_split_draft(&identity, record.operation, &attempt) .await .is_err()); + store.inspect_only.store(true, Ordering::Relaxed); + store.reject_split_proof_reads.store(true, Ordering::Relaxed); + assert_eq!( + authority + .get_split_revision( + &identity, + snapshot, + record.revision, + MetadataScanBudget { + items: 1, + bytes: 1024 * 1024, + timeout: std::time::Duration::from_secs(2) + }, + &ReadCancellation::new() + ) + .await + .unwrap(), + record + ); + store.inspect_only.store(false, Ordering::Relaxed); + store.reject_split_proof_reads.store(false, Ordering::Relaxed); assert!(matches!( authority .abandon_split_draft(&identity, record.operation, 1) diff --git a/lib/crowdb-access-dataset/tests/split_read_query_test.rs b/lib/crowdb-access-dataset/tests/split_read_query_test.rs index 5c9d87234..847c349a5 100644 --- a/lib/crowdb-access-dataset/tests/split_read_query_test.rs +++ b/lib/crowdb-access-dataset/tests/split_read_query_test.rs @@ -11,16 +11,17 @@ use crowdb_access_dataset::*; use std::sync::Arc; #[tokio::test] -async fn continuation_pins_snapshot_and_split_across_instances() { +async fn continuation_pins_unpublished_split_across_pointer_changes_and_instances() { let (_, authority, identity, parent) = split_store::fixture().await; let snapshot = split_store::publish_binary_samples(&authority, &identity, parent).await; let first = split_store::prepared(&authority, &identity, snapshot, None).await; - split_store::publish_revision(&authority, &identity, &first).await; let authority = Arc::new(authority); let reads = DatasetReads::new(authority.clone()); - let binding = reads.bind(&identity, common::spec("train")).await.unwrap(); + let mut spec = common::spec("train"); + spec.split.as_mut().unwrap().revision = Some(first.revision); + let binding = reads.bind(&identity, spec).await.unwrap(); assert_eq!(binding.split.as_ref().unwrap().revision, first.revision); - let next = split_store::prepared(&authority, &identity, snapshot, Some(first.revision)).await; + let next = split_store::prepared(&authority, &identity, snapshot, None).await; split_store::publish_revision(&authority, &identity, &next).await; authority .publish_snapshot(&identity, Some(snapshot), publication::publication_manifest(2)) diff --git a/lib/crowdb-access-dataset/tests/split_read_test.rs b/lib/crowdb-access-dataset/tests/split_read_test.rs index 2dd865085..d65a13547 100644 --- a/lib/crowdb-access-dataset/tests/split_read_test.rs +++ b/lib/crowdb-access-dataset/tests/split_read_test.rs @@ -115,22 +115,26 @@ async fn members_are_scoped_ordered_and_bounded() { } #[tokio::test] -async fn history_excludes_prepared_revisions_and_retains_committed_revisions() { +async fn history_includes_complete_revisions_before_and_after_publication() { let (store, authority, identity, snapshot) = fixture().await; let first = prepared(&authority, &identity, snapshot, None).await; let cancel = ReadCancellation::new(); - assert!(authority - .split_revision_history(&identity, snapshot, None, budget(20), &cancel) - .await - .unwrap() - .entries - .is_empty()); - assert!(matches!( + assert_eq!( + authority + .split_revision_history(&identity, snapshot, None, budget(20), &cancel) + .await + .unwrap() + .entries + .len(), + 1 + ); + assert_eq!( authority .get_split_revision(&identity, snapshot, first.revision, budget(20), &cancel) - .await, - Err(SplitLifecycleError::NotFound) - )); + .await + .unwrap(), + first + ); authority .publish_split_draft( &identity, @@ -154,7 +158,7 @@ async fn history_excludes_prepared_revisions_and_retains_committed_revisions() { .split_revision_history(&identity, snapshot, None, budget(20), &cancel) .await .unwrap(); - assert_eq!(history.entries.len(), 2); + assert_eq!(history.entries.len(), 3); assert_eq!( authority .get_split_revision(&identity, snapshot, first.revision, budget(20), &cancel) diff --git a/sdk/README.md b/sdk/README.md index 3572fd063..9e1e0e1d9 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -204,6 +204,9 @@ Existing `DatasetClient` methods retain their v1 contracts. `samples` return lazy `PageCursor`s. Each `next_page` fetches at most 100 items; it never eagerly collects the Dataset. Persist `continuation()` for scoped resume. Dropping the cursor stops further requests. +- Complete Split revisions can be read or used as frozen sources before updating + currentSplit, after a head conflict, or after abandoning their draft. Each new + read still checks Snapshot availability; incomplete revisions remain unavailable. - `bind_read` pins Snapshot and optional Split revision without writes; `next_batch(cursor)` uses verified streaming framing. Save the next cursor after successful processing. The required capability is `stateless-reads`; old session diff --git a/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs b/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs index 46fa6e98b..edccfc24d 100644 --- a/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs +++ b/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs @@ -242,6 +242,17 @@ async fn revise_splits( }, ) .await?; + assert_eq!( + client + .split_revision(snapshot, &revision.revision) + .await? + .revision, + revision.revision + ); + assert_eq!( + client.split_head(snapshot).await?.revision, + Some(previous.clone()) + ); client .publish_split( &begin.operation, From 462f5a907883e5d44f7f79a6bc8e54af8c65ba27 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 17:56:37 +0800 Subject: [PATCH 120/258] Publish split pointers without request ledgers --- .../tests/dataset_split_contract_test.rs | 10 +- .../dataset_split_recovery_contract_test.rs | 14 +- .../dataset/design-dataset-control.md | 12 + .../dataset/openapi-crowdb-dataset-v2.yaml | 2 + .../dataset/spec-crowdb-dataset.md | 9 +- doc/working/plan-access-simplification.md | 25 +- e2e-test/tests/access_dataset_split.rs | 25 ++ lib/crowdb-access-dataset/src/split/head.rs | 53 ---- .../src/split/publication.rs | 170 ++++++------ .../src/split/recovery.rs | 41 +-- .../tests/common/split_store.rs | 47 ++++ .../tests/split_publication_cost_test.rs | 243 ++++++++++++++++++ .../tests/split_publication_test.rs | 77 +++--- .../tests/split_reclaim_test.rs | 2 +- sdk/README.md | 8 +- 15 files changed, 514 insertions(+), 224 deletions(-) create mode 100644 lib/crowdb-access-dataset/tests/split_publication_cost_test.rs diff --git a/app/crowdb-access-server/tests/dataset_split_contract_test.rs b/app/crowdb-access-server/tests/dataset_split_contract_test.rs index 12255336f..019d1c0ce 100644 --- a/app/crowdb-access-server/tests/dataset_split_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_split_contract_test.rs @@ -62,7 +62,15 @@ async fn split_revision_flow_preserves_data_head_and_historical_members() { ); assert_eq!( response(client.get(&draft), 200).await["progress"]["state"], - "published" + "prepared" + ); + assert_eq!( + response( + client.get(format!("{root}/commits/03030303030303030303030303030303")), + 200 + ) + .await["receipt"], + Value::Null ); let current = response(client.get(format!("{base}/split-head")), 200).await; assert_eq!(current["revision"], revision); diff --git a/app/crowdb-access-server/tests/dataset_split_recovery_contract_test.rs b/app/crowdb-access-server/tests/dataset_split_recovery_contract_test.rs index 8728a3efc..bc8b1c0dd 100644 --- a/app/crowdb-access-server/tests/dataset_split_recovery_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_split_recovery_contract_test.rs @@ -14,7 +14,7 @@ use std::sync::Arc; async fn evaluation_resumes_after_server_restart_and_stale_publish_keeps_candidate() { let client = Client::new(); let (test, root, snapshot, draft, first_prepared) = restart_preparation(&client).await; - let first_receipt = common::publish( + common::publish( &client, &draft, &first_prepared, @@ -56,17 +56,7 @@ async fn evaluation_resumes_after_server_restart_and_stale_publish_keeps_candida response(client.get(&b.0), 200).await["progress"]["state"], "prepared" ); - assert_eq!( - common::publish( - &client, - &draft, - &first_prepared, - "22222222222222222222222222222222", - Value::Null - ) - .await, - first_receipt - ); + response(client.post(format!("{draft}/publish")).json(&json!({"attempt":"22222222222222222222222222222222", "revision":first_prepared["revision"], "expected_head":Value::Null})), 409).await; assert_eq!(response(client.get(&head_url), 200).await, current); assert_eq!( response( diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index 96366e7b8..31f79ea06 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -93,6 +93,16 @@ The complete `SplitRevisionRecord` is the final result: an explicit revision ID can be read, listed or used as an inherited source without winning currentSplit. Read admission still checks the owning Snapshot. Reads load that immutable record directly without Split publication proofs, currentSplit lookups or helper writes. +Standalone Split publication writes only currentSplit. It creates no attempt, +response receipt, per-revision success proof or draft committing/published state. +A retry returns the current generation when it still names the same revision; +after a later revision replaces it, the stale request returns a precondition +conflict without replaying historical success or rewinding the pointer. A definite +CAS conflict uses its returned head; a transport error remains unknown. Legacy +receipts and attempt-bearing heads remain readable without backfilling receipts. +The draft publication route is a compatibility adapter for its prepared target and +expected revision; `publish_split_revision` accepts these caller-owned inputs +directly. New draft status remains Prepared after publication. ## 4. Bounded observation and reads @@ -167,6 +177,8 @@ late write keys, including currently absent keys, rather than deleting a value a allowing a stale CAS(None) writer to recreate it. Edit IDs are registered before payload writes so failed admissions can also be retired. Small receipt/fence records remain after bulk cleanup to preserve replay and non-resurrection guarantees. +Split publication does not create those replay records; existing records and +editing/evaluation cleanup remain compatibility inputs. Helping an already stored edit receipt only reads and validates it; retries do not submit a doomed CAS for the same bytes. Evaluation journals and cleanup cursors are scoped to one draft, not to unrelated readers. They retain exact partial-work diff --git a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml index ac4612ded..5111a0268 100644 --- a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml +++ b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml @@ -523,6 +523,7 @@ paths: /namespaces/dataset-ns/datasets/{dataset}/split-drafts/{operation}/publish: post: operationId: publishSplitDraft + description: Conditionally updates currentSplit without a request ledger or draft publication progress. The same current revision confirms the current effect; a later revision makes a stale retry conflict. Unknown outcomes propagate without a confirmation query. parameters: - in: path name: dataset @@ -755,6 +756,7 @@ paths: get: operationId: commitStatusV2 summary: Recover a successful publication receipt + description: Returns Snapshot or legacy publication receipts. New independent Split publication does not create historical receipts; retry the same conditional request to confirm its current effect. parameters: - $ref: '#/components/parameters/dataset' - in: path diff --git a/doc/design/access-server/dataset/spec-crowdb-dataset.md b/doc/design/access-server/dataset/spec-crowdb-dataset.md index 71cff9f0b..8ace93621 100644 --- a/doc/design/access-server/dataset/spec-crowdb-dataset.md +++ b/doc/design/access-server/dataset/spec-crowdb-dataset.md @@ -188,7 +188,7 @@ addressable by ID until reclamation removes its metadata under Section 9. Control v2 separates candidate creation from publication attempts. An attempt MUST bind the exact candidate and expected opaque head generation. Initial Split membership and candidate upload ownership MUST be ready before the head transition. -A successful attempt MUST have a durable recoverable receipt. Retry of a historical +A successful Snapshot attempt MUST have a durable recoverable receipt. Retry of a historical success MUST NOT rewind latest. A head conflict preserves the candidate and parent; a reviewed retry uses a new attempt with a newly observed generation. @@ -200,6 +200,13 @@ and usable as a frozen source before publication or after a publication conflict Abandoning a draft MUST NOT remove its complete result. A missing or cleanup-sealed final revision record is unavailable; incomplete membership pages alone do not constitute a result. Snapshot deletion admission still applies to each new request. +Independent Split publication MUST NOT persist an attempt, response receipt or +draft publication progress. A retry that observes the same current revision can +confirm the current effect; if a later revision has replaced it, the stale expected +base MUST fail without rewinding currentSplit. It does not promise historical +response replay. Unknown storage outcomes MUST propagate without a confirmation +query. Existing legacy receipts remain readable; new Split attempts are absent +from commit-status history. ## 7. Selection, plans, and cursors diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index fedccbd48..79abbc03e 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -161,10 +161,10 @@ CAS 不再满足只证明“本次条件不成立”，不能谎称“第一次 历史回复查询；Composition/Split 的 Committing 恢复与 P2/P3 一起删除，客户端保留完整输入。 旧记录只作为兼容解码/GC 输入，不让旧账本控制新请求。head 保留 expected base；同一 candidate 的当前结果可判断相同，已被后续 head 遮住时不保证历史请求结果重放。 -- [~] **R2.2a 独立 Split 发布不写请求账本**：从完整 revision 和显式 expected base 更新 - currentSplit；删除新 CA/CR/SRC、Committing/Published 草稿写入及覆盖 head 前的 helper。 - 同一目标仍是当前结果时返回当前事实；后续代际已遮住时返回条件冲突，不回放旧响应或重置 - 后来者。旧账本仅兼容读取/清理。补 H0→H1→H2、未知回复、并发 CAS 与实际 mutation 成本。 +- [~] **R2.2b Split 发布直接接收资源与 precondition**：补无 draft 的 REST/SDK 入口， + 消除当前兼容路由的两次 draft 读取；冲突返回已观察的 revision/generation，供客户端审阅后 + 直接组成下一次条件请求，不为取得相同 head 再 GET。不得自动 rebase；旧服务能力缺失时 + 明确失败。验证缺省/null、鉴权、冲突与丢回复、Rust/Python SDK、无账本与实际读写成本。 - [ ] **R2 审查条件发布的回复丢失窗口**：用 H0→H1→H2 测试区分条件失败、相同结果与历史未知； 列出确实依赖历史精确结果的调用者，没有这种需求就移除相应 ledger/response replay。 @@ -499,6 +499,23 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **R2.2a（2026-10-24）**：新增直接接受完整 revision/显式 expected revision+generation + 的 `publish_split_revision`，旧 draft 发布仅作适配；新发布只 CAS currentSplit，不再写 + CA/CR/SRC、Committing/Published，也不帮助补写前一发布。相同当前结果返回当前事实， + H0→H1→H2 后重试 H0→H1 返回条件冲突；未知 mutation 原样返回，不确认 GET。 + legacy attempt/receipt/head 解码与旧 Committing 恢复仅用于旧输入，新操作不创建它们。 + 直接首次发布为 5G+1C（含空 head 时 initial-Split 查询），已有 head 时 4G+1C；确认相同 + 结果或拒绝旧 base 为 4G+0C。实际 key 差集只新增 Split head，CAS definite loser 的返回 + 值直接解码，unknown 没有额外读。兼容 REST 当前仍多两次 draft GET，由 R2.2b 消除。 + 回收 fence 同时修正为复用 CAS observation；有/无 initial/current revision 均连续四次 + 冲突后只读 head 一次，缺省 initial binding 在本请求只查一次，不做跨请求缓存。 + 新增五个成本/故障用例及现有 publication/reclaim 合同回归；Dataset 全部 target 通过， + 新增 H1 丢回复再被 H2 覆盖、缺省 head 竞争的聚焦用例随后通过。七个 HTTP/Rust SDK + 用例及显式 `--ignored` 安装后 Python consumer 通过；Python 使用 V4.2 同一 SDK 源码的 + 已安装 wheel，当前服务为新代码。四个受影响 crate（含 e2e-test）all-target/all-feature + clippy、两 workspace fmt/diff 通过，均经 `pixi run`。容器 fixture 已增加未发布 ID 读取与 + 无新 Split receipt 断言，尚未运行新镜像；新真实验收继续由 V2/V4 跟踪。 + - **R2.1a（2026-10-24）**：完整 Split 的 ID 读取、history、冻结继承和 continuation 直接使用最终 `SplitRevisionRecord`；移除读取 SRC/currentSplit 的门槛和闲置 initial-Split proof helper，Snapshot 准入与成员页校验保持。无新持久化记录、锁或后台 reader 状态。 diff --git a/e2e-test/tests/access_dataset_split.rs b/e2e-test/tests/access_dataset_split.rs index 76edf62f8..45b565263 100644 --- a/e2e-test/tests/access_dataset_split.rs +++ b/e2e-test/tests/access_dataset_split.rs @@ -185,6 +185,20 @@ async fn revise(client: &Client, snapshot: &str) { 200, ) .await; + assert_eq!( + call( + client, + Method::GET, + &format!( + "/snapshots/{snapshot}/split-revisions/{}", + prepared["revision"].as_str().unwrap() + ), + None, + 200 + ) + .await, + prepared + ); restart().await; let publish = json!({"attempt":"25252525252525252525252525252525","revision":prepared["revision"],"expected_head":head["generation"]}); let receipt = call( @@ -208,6 +222,17 @@ async fn revise(client: &Client, snapshot: &str) { receipt ); assert_eq!(call(client, Method::GET, "/head", None, 200).await, data_head); + assert_eq!( + call( + client, + Method::GET, + "/commits/25252525252525252525252525252525", + None, + 200 + ) + .await["receipt"], + Value::Null + ); assert_eq!( call( client, diff --git a/lib/crowdb-access-dataset/src/split/head.rs b/lib/crowdb-access-dataset/src/split/head.rs index ccdedfd4d..3a106eabc 100644 --- a/lib/crowdb-access-dataset/src/split/head.rs +++ b/lib/crowdb-access-dataset/src/split/head.rs @@ -21,17 +21,6 @@ pub(super) struct CommittedSplitHead { pub attempt: CommitAttempt, } impl CommittedSplitHead { - pub(super) fn encode(head: SplitHead, attempt: CommitAttempt) -> Result, SplitLifecycleError> { - head.encode()?; - attempt.validate()?; - let value = Self { - version: 2, - head, - attempt, - }; - value.validate()?; - Ok(codec::encode(&value)?) - } pub(super) fn decode(bytes: &[u8]) -> Result { let value: Self = codec::decode(bytes)?; value.validate()?; @@ -80,47 +69,6 @@ impl DatasetAuthority { } Ok(observed) } - pub(super) async fn help_split_head( - &self, - identity: &DatasetIdentity, - stored: &StoredValue, - ) -> Result<(), SplitLifecycleError> { - if let Ok(head) = SplitHead::decode(&stored.bytes) { - if let Some(revision) = head.revision { - if self.initial_split_revision(identity, head.snapshot).await? == Some(revision) { - return Ok(()); - } - let stored = self - .split_state_get( - &key::committed_revision(identity, head.snapshot, revision) - .map_err(crate::AuthorityError::from)?, - ) - .await? - .ok_or(SplitLifecycleError::Corrupt)?; - let receipt = SuccessfulCommit::decode(&stored.bytes)?; - if receipt.attempt.target - != (CommitTarget::SplitRevision { - snapshot: head.snapshot, - revision, - }) - { - return Err(SplitLifecycleError::Corrupt); - } - } - return Ok(()); - } - let value = CommittedSplitHead::decode(&stored.bytes)?; - self.persist_commit_receipt( - identity, - &SuccessfulCommit { - version: 1, - attempt: value.attempt, - committed_head: CommitGeneration::from_head(&stored.bytes), - }, - ) - .await?; - Ok(()) - } pub(super) async fn split_revision_receipt( &self, identity: &DatasetIdentity, @@ -154,7 +102,6 @@ impl DatasetAuthority { if head.attempt.target != target { return Ok(None); } - self.help_split_head(identity, &stored).await?; Ok(Some(SuccessfulCommit { version: 1, attempt: head.attempt, diff --git a/lib/crowdb-access-dataset/src/split/publication.rs b/lib/crowdb-access-dataset/src/split/publication.rs index 38c29806b..7a14118e6 100644 --- a/lib/crowdb-access-dataset/src/split/publication.rs +++ b/lib/crowdb-access-dataset/src/split/publication.rs @@ -1,18 +1,16 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{ - head::CommittedSplitHead, key, state::DraftState, SplitHead, SplitLifecycleError, SplitRevisionRecord, -}; +use super::{key, state::DraftState, SplitHead, SplitLifecycleError, SplitRevisionRecord}; use crate::{ composition::DraftProgress, CasOutcome, CommitAttempt, CommitGeneration, CommitTarget, DatasetAuthority, - DatasetIdentity, OperationId, StoredValue, SuccessfulCommit, + DatasetIdentity, OperationId, SplitRevisionId, StoredValue, SuccessfulCommit, }; impl DatasetAuthority { - /// Publishes one prepared complete Split set without changing the Dataset data head. + /// Compatibility adapter for callers that prepared a complete result through a draft. /// # Errors - /// Rejects divergent attempts, stale heads, abandoned drafts and unavailable owners. + /// Rejects mismatched targets, initial Split drafts and unavailable results. pub async fn publish_split_draft( &self, identity: &DatasetIdentity, @@ -20,7 +18,7 @@ impl DatasetAuthority { attempt: &CommitAttempt, ) -> Result { attempt.validate()?; - let (mut old, mut state) = self.load_split_draft(identity, operation).await?; + let (_, state) = self.load_split_draft(identity, operation).await?; if state.initial { return Err(SplitLifecycleError::Frozen); } @@ -33,8 +31,34 @@ impl DatasetAuthority { if target != attempt.target { return Err(SplitLifecycleError::Conflict); } - let CommitTarget::SplitRevision { snapshot, revision } = target else { - return Err(SplitLifecycleError::Corrupt); + self.commit_split_revision(identity, attempt, state.record.expected_revision, Some(operation)) + .await + } + + /// Updates currentSplit from a complete immutable result and caller-owned preconditions. + /// Success describes the current effect, without persisting request or response records. + /// # Errors + /// Rejects missing results, unavailable owners, stale preconditions and unknown storage outcomes. + pub async fn publish_split_revision( + &self, + identity: &DatasetIdentity, + attempt: &CommitAttempt, + expected_revision: Option, + ) -> Result { + self.commit_split_revision(identity, attempt, expected_revision, None) + .await + } + + async fn commit_split_revision( + &self, + identity: &DatasetIdentity, + attempt: &CommitAttempt, + expected_revision: Option, + operation: Option, + ) -> Result { + attempt.validate()?; + let CommitTarget::SplitRevision { snapshot, revision } = attempt.target else { + return Err(SplitLifecycleError::Conflict); }; let stored = self .split_state_get( @@ -42,108 +66,56 @@ impl DatasetAuthority { ) .await? .ok_or(SplitLifecycleError::NotFound)?; + if stored.bytes == super::cleanup::SEALED { + return Err(SplitLifecycleError::NotFound); + } let candidate = SplitRevisionRecord::decode(&stored.bytes)?; if candidate.snapshot != snapshot || candidate.revision != revision - || candidate.operation != operation + || operation.is_some_and(|operation| candidate.operation != operation) { return Err(SplitLifecycleError::Corrupt); } - if let Some(receipt) = self.commit_status(identity, attempt.id).await? { - if receipt.attempt != *attempt { - return Err(SplitLifecycleError::Conflict); - } - self.finish_split_publication(identity, &receipt, &old, &state) - .await?; - return Ok(receipt); - } - self.persist_attempt(identity, attempt).await?; - if let Some(receipt) = self.split_revision_receipt(identity, snapshot, revision).await? { - self.finish_split_publication(identity, &receipt, &old, &state) - .await?; - return Ok(receipt); - } - self.check_split_draft_sources(identity, &state).await?; - match state.record.progress { - DraftProgress::Prepared { .. } => { - let mut next = state.clone(); - next.record.progress = DraftProgress::Committing { - target, - attempt: attempt.id, - }; - next.edit = None; - self.replace_split_draft(identity, &old, &state, &next).await?; - (old, state) = self.load_split_draft(identity, operation).await?; - } - DraftProgress::Committing { attempt: pending, .. } if pending == attempt.id => {} - _ => return Err(SplitLifecycleError::Conflict), - } - self.commit_split_head(identity, attempt, &old, &state).await - } - - async fn commit_split_head( - &self, - identity: &DatasetIdentity, - attempt: &CommitAttempt, - old: &StoredValue, - state: &DraftState, - ) -> Result { - let CommitTarget::SplitRevision { snapshot, revision } = attempt.target else { - return Err(SplitLifecycleError::Corrupt); - }; let key = key::head(identity, snapshot).map_err(crate::AuthorityError::from)?; let current = self.split_state_get(&key).await?; - if let Some(value) = ¤t { - self.help_split_head(identity, value).await?; - } - if let Some(receipt) = self.split_revision_receipt(identity, snapshot, revision).await? { - self.finish_split_publication(identity, &receipt, old, state) - .await?; - return Ok(receipt); - } + // Closing Snapshot admission invalidates this exact head generation before reclaim. + self.require_committed_snapshot(identity, snapshot).await?; let observed = self .resolve_split_reference(identity, current.as_ref(), snapshot) .await?; - if observed.generation != attempt.expected_head || observed.revision != state.record.expected_revision - { - self.reset_split_publication(identity, old, state).await?; + if observed.revision == Some(revision) && current.is_some() { + return current_success(attempt, current.as_ref()); + } + if observed.generation != attempt.expected_head || observed.revision != expected_revision { return Err(SplitLifecycleError::Conflict); } - // Reclaim must invalidate this Split head after closing owner admission. - self.require_committed_snapshot(identity, snapshot).await?; - let bytes = CommittedSplitHead::encode( - SplitHead { - version: 1, - snapshot, - revision: Some(revision), - fence: OperationId::random(), - }, - attempt.clone(), - )?; - let result = self + let bytes = SplitHead { + version: 1, + snapshot, + revision: Some(revision), + fence: OperationId::random(), + } + .encode()?; + match self .store .compare_exchange(&key, current.as_ref().map(|v| v.bytes.as_slice()), &bytes) - .await?; - let receipt = match result { - CasOutcome::Applied(_) => SuccessfulCommit { + .await? + { + CasOutcome::Applied(_) => Ok(SuccessfulCommit { version: 1, attempt: attempt.clone(), committed_head: CommitGeneration::from_head(&bytes), - }, - CasOutcome::Conflict(_) => { - let Some(receipt) = self.split_revision_receipt(identity, snapshot, revision).await? else { - self.reset_split_publication(identity, old, state).await?; + }), + CasOutcome::Conflict(observed) => { + if super::head::reference(observed.as_ref(), snapshot)?.revision != Some(revision) { return Err(SplitLifecycleError::Conflict); - }; - receipt + } + current_success(attempt, observed.as_ref()) } - }; - self.persist_commit_receipt(identity, &receipt).await?; - self.finish_split_publication(identity, &receipt, old, state) - .await?; - Ok(receipt) + } } - pub(super) async fn reset_split_publication( + + pub(super) async fn reset_legacy_split_publication( &self, identity: &DatasetIdentity, old: &StoredValue, @@ -156,7 +128,7 @@ impl DatasetAuthority { next.record.progress = DraftProgress::Prepared { target }; self.replace_split_draft(identity, old, state, &next).await } - pub(super) async fn finish_split_publication( + pub(super) async fn finish_legacy_split_publication( &self, identity: &DatasetIdentity, receipt: &SuccessfulCommit, @@ -169,8 +141,18 @@ impl DatasetAuthority { target: receipt.attempt.target, attempt: receipt.attempt.id, }; - self.replace_split_draft(identity, old, state, &next).await?; - - Ok(()) + self.replace_split_draft(identity, old, state, &next).await } } + +fn current_success( + attempt: &CommitAttempt, + current: Option<&StoredValue>, +) -> Result { + let current = current.ok_or(SplitLifecycleError::Conflict)?; + Ok(SuccessfulCommit { + version: 1, + attempt: attempt.clone(), + committed_head: CommitGeneration::from_head(¤t.bytes), + }) +} diff --git a/lib/crowdb-access-dataset/src/split/recovery.rs b/lib/crowdb-access-dataset/src/split/recovery.rs index 57b1a9577..b18e9add2 100644 --- a/lib/crowdb-access-dataset/src/split/recovery.rs +++ b/lib/crowdb-access-dataset/src/split/recovery.rs @@ -23,7 +23,7 @@ impl DatasetAuthority { return Err(SplitLifecycleError::Corrupt); } if let Some(receipt) = self.commit_status(identity, id).await? { - self.finish_split_publication(identity, &receipt, &old, &state) + self.finish_legacy_split_publication(identity, &receipt, &old, &state) .await?; return Ok(self.load_split_draft(identity, operation).await?.1.record); } @@ -33,10 +33,11 @@ impl DatasetAuthority { let observed = self.inspect_split_head(identity, snapshot).await?; if observed.generation != attempt.expected_head { if let Some(receipt) = self.commit_status(identity, id).await? { - self.finish_split_publication(identity, &receipt, &old, &state) + self.finish_legacy_split_publication(identity, &receipt, &old, &state) .await?; } else { - self.reset_split_publication(identity, &old, &state).await?; + self.reset_legacy_split_publication(identity, &old, &state) + .await?; } return Ok(self.load_split_draft(identity, operation).await?.1.record); } @@ -72,28 +73,32 @@ impl DatasetAuthority { snapshot: SnapshotId, ) -> Result<(), SplitLifecycleError> { let key = key::head(identity, snapshot).map_err(AuthorityError::from)?; + let mut old = self.store.get(&key).await?; + let mut initial = None; for _ in 0..64 { - let old = self.store.get(&key).await?; - if let Some(value) = &old { - self.help_split_head(identity, value).await?; - } - let observed = self - .resolve_split_reference(identity, old.as_ref(), snapshot) - .await?; + let observed = super::head::reference(old.as_ref(), snapshot)?; + let revision = if observed.revision.is_some() { + observed.revision + } else { + if initial.is_none() { + initial = Some(self.initial_split_revision(identity, snapshot).await?); + } + initial.flatten() + }; let bytes = SplitHead { version: 1, snapshot, - revision: observed.revision, + revision, fence: OperationId::random(), } .encode()?; - if matches!( - self.store - .compare_exchange(&key, old.as_ref().map(|v| v.bytes.as_slice()), &bytes) - .await?, - CasOutcome::Applied(_) - ) { - return Ok(()); + match self + .store + .compare_exchange(&key, old.as_ref().map(|v| v.bytes.as_slice()), &bytes) + .await? + { + CasOutcome::Applied(_) => return Ok(()), + CasOutcome::Conflict(observed) => old = observed, } } Err(SplitLifecycleError::Conflict) diff --git a/lib/crowdb-access-dataset/tests/common/split_store.rs b/lib/crowdb-access-dataset/tests/common/split_store.rs index fe901981e..0d8956a4e 100644 --- a/lib/crowdb-access-dataset/tests/common/split_store.rs +++ b/lib/crowdb-access-dataset/tests/common/split_store.rs @@ -19,6 +19,14 @@ pub(super) struct TestDraftStore { pub cancel_scan: AtomicBool, pub revision_reads: AtomicUsize, pub bounded_reads: AtomicUsize, + pub unbounded_reads: AtomicUsize, + pub mutations: AtomicUsize, + pub split_head_reads: AtomicUsize, + pub candidate_reads: AtomicUsize, + pub split_head_mutations: AtomicUsize, + pub split_head_conflicts: AtomicUsize, + pub clear_split_conflict_target: AtomicBool, + pub reject_publishing_ledgers: AtomicBool, pub hide_revision_once: AtomicBool, pub lose_revision_reply: AtomicBool, pub reject_split_proof_reads: AtomicBool, @@ -26,6 +34,13 @@ pub(super) struct TestDraftStore { #[async_trait::async_trait] impl DatasetStore for TestDraftStore { async fn get(&self, key: &[u8]) -> Result, StoreError> { + self.unbounded_reads.fetch_add(1, Ordering::Relaxed); + if key.starts_with(b"DS\0CB\0") { + self.candidate_reads.fetch_add(1, Ordering::Relaxed); + } + if key.starts_with(b"DS\0SH\0") { + self.split_head_reads.fetch_add(1, Ordering::Relaxed); + } assert!( !self.inspect_only.load(Ordering::Relaxed), "Split inspection used unbounded get" @@ -45,6 +60,9 @@ impl DatasetStore for TestDraftStore { cancel: &ReadCancellation, ) -> Result, StoreError> { self.bounded_reads.fetch_add(1, Ordering::Relaxed); + if key.starts_with(b"DS\0SH\0") { + self.split_head_reads.fetch_add(1, Ordering::Relaxed); + } if self.reject_split_proof_reads.load(Ordering::Relaxed) { assert!( !key.starts_with(b"DS\0SRC\0") && !key.starts_with(b"DS\0SH\0"), @@ -84,6 +102,35 @@ impl DatasetStore for TestDraftStore { expected: Option<&[u8]>, value: &[u8], ) -> Result { + self.mutations.fetch_add(1, Ordering::Relaxed); + if self.reject_publishing_ledgers.load(Ordering::Relaxed) { + assert!( + ![b"DS\0CA\0".as_slice(), b"DS\0CR\0", b"DS\0SRC\0"] + .iter() + .any(|prefix| key.starts_with(prefix)), + "Split publication wrote a request ledger" + ); + } + if key.starts_with(b"DS\0SH\0") { + self.split_head_mutations.fetch_add(1, Ordering::Relaxed); + if self + .split_head_conflicts + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |n| n.checked_sub(1)) + .is_ok() + { + let mut head = split::SplitHead::decode(value).unwrap(); + head.fence = OperationId::random(); + if self.clear_split_conflict_target.load(Ordering::Relaxed) { + head.revision = None; + } + assert!(matches!( + self.inner + .compare_exchange(key, expected, &head.encode().unwrap()) + .await?, + CasOutcome::Applied(_) + )); + } + } assert!( !self.inspect_only.load(Ordering::Relaxed), "Split inspection wrote metadata" diff --git a/lib/crowdb-access-dataset/tests/split_publication_cost_test.rs b/lib/crowdb-access-dataset/tests/split_publication_cost_test.rs new file mode 100644 index 000000000..883c570be --- /dev/null +++ b/lib/crowdb-access-dataset/tests/split_publication_cost_test.rs @@ -0,0 +1,243 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/composition.rs"] +#[allow(dead_code)] +mod common; +#[path = "common/publication.rs"] +mod publication; +#[path = "common/split_store.rs"] +mod split_store; + +use crowdb_access_dataset::{split::*, *}; +use std::{collections::BTreeSet, sync::atomic::Ordering}; + +fn counts(store: &split_store::TestDraftStore) -> (usize, usize) { + ( + store.bounded_reads.load(Ordering::Relaxed) + store.unbounded_reads.load(Ordering::Relaxed), + store.mutations.load(Ordering::Relaxed), + ) +} +fn delta(store: &split_store::TestDraftStore, before: (usize, usize)) -> (usize, usize) { + let now = counts(store); + (now.0 - before.0, now.1 - before.1) +} +async fn keys(store: &split_store::TestDraftStore) -> BTreeSet> { + store + .scan_page( + &MetadataScan { + prefix: b"DS\0".to_vec(), + after: None, + budget: common::budget(), + }, + &ReadCancellation::new(), + ) + .await + .unwrap() + .entries + .into_iter() + .map(|(key, _)| key) + .collect() +} + +#[tokio::test] +async fn publication_uses_one_head_mutation_and_retries_current_facts_without_writes() { + let (store, authority, identity, snapshot) = split_store::fixture().await; + let first = split_store::prepared(&authority, &identity, snapshot, None).await; + let attempt = split_store::attempt(&authority, &identity, &first).await; + let before_keys = keys(&store).await; + store.reject_publishing_ledgers.store(true, Ordering::Relaxed); + let before = counts(&store); + let receipt = authority + .publish_split_revision(&identity, &attempt, None) + .await + .unwrap(); + assert_eq!(delta(&store, before), (5, 1)); + let after_keys = keys(&store).await; + assert_eq!( + after_keys.difference(&before_keys).cloned().collect::>(), + vec![key::head(&identity, snapshot).unwrap()] + ); + let before = counts(&store); + assert_eq!( + authority + .publish_split_revision(&identity, &attempt, None) + .await + .unwrap(), + receipt + ); + assert_eq!(delta(&store, before), (4, 0)); + let second = split_store::prepared(&authority, &identity, snapshot, Some(first.revision)).await; + let next = split_store::attempt(&authority, &identity, &second).await; + let before = counts(&store); + authority + .publish_split_revision(&identity, &next, Some(first.revision)) + .await + .unwrap(); + assert_eq!(delta(&store, before), (4, 1)); + let before = counts(&store); + assert!(matches!( + authority.publish_split_revision(&identity, &attempt, None).await, + Err(SplitLifecycleError::Conflict) + )); + assert_eq!(delta(&store, before), (4, 0)); +} + +#[tokio::test] +async fn definite_cas_conflicts_use_the_observation_and_unknown_outcomes_do_not_query() { + for same_target in [true, false] { + let (store, authority, identity, snapshot) = split_store::fixture().await; + let record = split_store::prepared(&authority, &identity, snapshot, None).await; + let attempt = split_store::attempt(&authority, &identity, &record).await; + store.split_head_conflicts.store(1, Ordering::Relaxed); + store + .clear_split_conflict_target + .store(!same_target, Ordering::Relaxed); + store.reject_publishing_ledgers.store(true, Ordering::Relaxed); + let before = counts(&store); + let result = authority.publish_split_revision(&identity, &attempt, None).await; + assert_eq!(delta(&store, before), (5, 1)); + if same_target { + let receipt = result.unwrap(); + assert_eq!( + receipt.committed_head, + authority + .inspect_split_head(&identity, snapshot) + .await + .unwrap() + .generation + .unwrap() + ); + } else { + assert!(matches!(result, Err(SplitLifecycleError::Conflict))); + } + } + let (store, authority, identity, snapshot) = split_store::fixture().await; + let record = split_store::prepared(&authority, &identity, snapshot, None).await; + let attempt = split_store::attempt(&authority, &identity, &record).await; + store.lose_split_reply.store(true, Ordering::Relaxed); + store.reject_publishing_ledgers.store(true, Ordering::Relaxed); + let before = counts(&store); + assert!(matches!( + authority.publish_split_revision(&identity, &attempt, None).await, + Err(SplitLifecycleError::Store(StoreError::Rejected)) + )); + assert_eq!(delta(&store, before), (5, 1)); + let restarted = DatasetAuthority::new(store.clone()); + let before = counts(&store); + restarted + .publish_split_revision(&identity, &attempt, None) + .await + .unwrap(); + assert_eq!(delta(&store, before), (4, 0)); +} + +#[tokio::test] +async fn lost_success_hidden_by_a_later_revision_is_a_conflict_without_replay() { + let (store, authority, identity, snapshot) = split_store::fixture().await; + let first = split_store::prepared(&authority, &identity, snapshot, None).await; + let attempt = split_store::attempt(&authority, &identity, &first).await; + store.lose_split_reply.store(true, Ordering::Relaxed); + assert!(authority + .publish_split_revision(&identity, &attempt, None) + .await + .is_err()); + let second = split_store::prepared(&authority, &identity, snapshot, Some(first.revision)).await; + let next = split_store::attempt(&authority, &identity, &second).await; + authority + .publish_split_revision(&identity, &next, Some(first.revision)) + .await + .unwrap(); + let head = authority.inspect_split_head(&identity, snapshot).await.unwrap(); + let before = counts(&store); + assert!(matches!( + authority.publish_split_revision(&identity, &attempt, None).await, + Err(SplitLifecycleError::Conflict) + )); + assert_eq!(delta(&store, before), (4, 0)); + assert_eq!( + authority.inspect_split_head(&identity, snapshot).await.unwrap(), + head + ); +} + +#[tokio::test] +async fn reclaim_reuses_four_split_head_observations_without_reloading_or_helping() { + for has_split in [false, true] { + let (store, authority, identity, snapshot) = split_store::fixture().await; + if has_split { + let record = split_store::prepared(&authority, &identity, snapshot, None).await; + split_store::publish_revision(&authority, &identity, &record).await; + } + let root = authority + .prepare_snapshot_candidate( + &identity, + None, + publication::publication_manifest(2), + OperationId::random(), + ) + .await + .unwrap(); + authority + .publish_snapshot_candidate(&identity, &common::attempt(&authority, &identity, root).await) + .await + .unwrap(); + store.split_head_conflicts.store(4, Ordering::Relaxed); + let gets = store.split_head_reads.load(Ordering::Relaxed); + let cas = store.split_head_mutations.load(Ordering::Relaxed); + let candidate = store.candidate_reads.load(Ordering::Relaxed); + authority.begin_reclaim(&identity, snapshot).await.unwrap(); + assert_eq!(store.split_head_reads.load(Ordering::Relaxed) - gets, 1); + assert_eq!(store.split_head_mutations.load(Ordering::Relaxed) - cas, 5); + assert_eq!( + store.candidate_reads.load(Ordering::Relaxed) - candidate, + usize::from(!has_split) + ); + } +} + +#[tokio::test] +async fn legacy_head_attempts_remain_readable_without_creating_receipts() { + let (store, authority, identity, snapshot) = split_store::fixture().await; + let record = split_store::prepared(&authority, &identity, snapshot, None).await; + let attempt = split_store::attempt(&authority, &identity, &record).await; + authority + .publish_split_revision(&identity, &attempt, None) + .await + .unwrap(); + let key = key::head(&identity, snapshot).unwrap(); + let current = store.get(&key).await.unwrap().unwrap(); + let head = SplitHead::decode(¤t.bytes).unwrap(); + let legacy = serde_json::to_vec(&serde_json::json!({"version":2,"head":head,"attempt":attempt})).unwrap(); + store + .compare_exchange(&key, Some(¤t.bytes), &legacy) + .await + .unwrap(); + let legacy_key = [ + b"DS\0CA\0".as_slice(), + &identity.key().unwrap(), + attempt.id.as_bytes(), + ] + .concat(); + store + .compare_exchange(&legacy_key, None, &attempt.encode().unwrap()) + .await + .unwrap(); + let before = counts(&store); + let receipt = authority + .commit_status(&identity, attempt.id) + .await + .unwrap() + .unwrap(); + assert_eq!(delta(&store, before).1, 0); + assert_eq!(receipt.attempt, attempt); + assert_eq!(receipt.committed_head, CommitGeneration::from_head(&legacy)); + assert_eq!( + authority + .inspect_split_head(&identity, snapshot) + .await + .unwrap() + .revision, + Some(record.revision) + ); +} diff --git a/lib/crowdb-access-dataset/tests/split_publication_test.rs b/lib/crowdb-access-dataset/tests/split_publication_test.rs index 8cf61756e..664bba37c 100644 --- a/lib/crowdb-access-dataset/tests/split_publication_test.rs +++ b/lib/crowdb-access-dataset/tests/split_publication_test.rs @@ -9,14 +9,14 @@ use split_store::{attempt, fixture, prepared}; use std::sync::atomic::Ordering; #[tokio::test] -async fn independent_revision_cas_and_historical_retries_never_move_data_head() { +async fn independent_revision_cas_and_historical_conflicts_never_move_data_head() { let (_, authority, identity, snapshot) = fixture().await; let data_head = authority.inspect_head(&identity).await.unwrap(); let first = prepared(&authority, &identity, snapshot, None).await; let competing = prepared(&authority, &identity, snapshot, None).await; let first_attempt = attempt(&authority, &identity, &first).await; let stale = attempt(&authority, &identity, &competing).await; - let receipt = authority + authority .publish_split_draft(&identity, first.operation, &first_attempt) .await .unwrap(); @@ -47,13 +47,12 @@ async fn independent_revision_cas_and_historical_retries_never_move_data_head() .unwrap(); let current = authority.inspect_split_head(&identity, snapshot).await.unwrap(); assert_eq!(current.revision, Some(next.revision)); - assert_eq!( + assert!(matches!( authority .publish_split_draft(&identity, first.operation, &first_attempt) - .await - .unwrap(), - receipt - ); + .await, + Err(SplitLifecycleError::Conflict) + )); assert_eq!( authority.inspect_split_head(&identity, snapshot).await.unwrap(), current @@ -62,7 +61,7 @@ async fn independent_revision_cas_and_historical_retries_never_move_data_head() } #[tokio::test] -async fn lost_split_cas_reply_is_recoverable_and_pending_commit_cannot_be_abandoned() { +async fn lost_split_cas_reply_retries_current_effect_without_a_request_ledger() { let (store, authority, identity, snapshot) = fixture().await; let record = prepared(&authority, &identity, snapshot, None).await; let attempt = attempt(&authority, &identity, &record).await; @@ -92,18 +91,16 @@ async fn lost_split_cas_reply_is_recoverable_and_pending_commit_cannot_be_abando ); store.inspect_only.store(false, Ordering::Relaxed); store.reject_split_proof_reads.store(false, Ordering::Relaxed); - assert!(matches!( - authority - .abandon_split_draft(&identity, record.operation, 1) - .await, - Err(SplitLifecycleError::Frozen) - )); let head = authority.inspect_split_head(&identity, snapshot).await.unwrap(); let restarted = DatasetAuthority::new(store); - let status = restarted + assert!(restarted .commit_status(&identity, attempt.id) .await .unwrap() + .is_none()); + let status = restarted + .publish_split_draft(&identity, record.operation, &attempt) + .await .unwrap(); assert_eq!(status.committed_head, head.generation.unwrap()); assert_eq!( @@ -123,49 +120,53 @@ async fn lost_split_cas_reply_is_recoverable_and_pending_commit_cannot_be_abando .await .unwrap() .progress, - DraftProgress::Published { .. } + DraftProgress::Prepared { .. } )); + restarted + .abandon_split_draft(&identity, record.operation, 1) + .await + .unwrap(); + assert_eq!( + restarted + .publish_split_revision(&identity, &attempt, None) + .await + .unwrap(), + status + ); } #[tokio::test] -async fn receipt_failure_blocks_replacement_until_pending_success_is_helped() { +async fn replacing_current_split_does_not_write_or_help_receipts() { let (store, authority, identity, snapshot) = fixture().await; let first = prepared(&authority, &identity, snapshot, None).await; let first_attempt = attempt(&authority, &identity, &first).await; store.reject_receipts.store(true, Ordering::Relaxed); - assert!(authority + authority .publish_split_draft(&identity, first.operation, &first_attempt) .await - .is_err()); + .unwrap(); let old = authority.inspect_split_head(&identity, snapshot).await.unwrap(); let next = prepared(&authority, &identity, snapshot, Some(first.revision)).await; let next_attempt = attempt(&authority, &identity, &next).await; - assert!(authority + authority .publish_split_draft(&identity, next.operation, &next_attempt) .await - .is_err()); - assert_eq!( + .unwrap(); + assert_ne!( authority.inspect_split_head(&identity, snapshot).await.unwrap(), old ); - store.reject_receipts.store(false, Ordering::Relaxed); - authority - .publish_split_draft(&identity, next.operation, &next_attempt) + assert!(authority + .commit_status(&identity, first_attempt.id) .await - .unwrap(); - assert_eq!( + .unwrap() + .is_none()); + assert!(matches!( authority - .commit_status(&identity, first_attempt.id) - .await - .unwrap() - .unwrap() - .committed_head, - old.generation.unwrap() - ); - authority - .publish_split_draft(&identity, first.operation, &first_attempt) - .await - .unwrap(); + .publish_split_draft(&identity, first.operation, &first_attempt) + .await, + Err(SplitLifecycleError::Conflict) + )); assert_eq!( authority .inspect_split_head(&identity, snapshot) diff --git a/lib/crowdb-access-dataset/tests/split_reclaim_test.rs b/lib/crowdb-access-dataset/tests/split_reclaim_test.rs index 33f3e8fbb..5df8a5481 100644 --- a/lib/crowdb-access-dataset/tests/split_reclaim_test.rs +++ b/lib/crowdb-access-dataset/tests/split_reclaim_test.rs @@ -107,7 +107,7 @@ async fn retired_snapshot_cleans_all_revision_pages_without_touching_other_owner .commit_status(&identity, first_attempt.id) .await .unwrap() - .is_some()); + .is_none()); assert!(restarted.inspect_split_head(&identity, snapshot).await.is_err()); assert!(restarted.get_manifest(&identity, snapshot).await.is_err()); } diff --git a/sdk/README.md b/sdk/README.md index 9e1e0e1d9..d299eee3e 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -198,8 +198,12 @@ Existing `DatasetClient` methods retain their v1 contracts. `uploads.publication_ready` is true. Initial Splits publish with the Snapshot. - `publish_composition` advances latest; `publish_split` updates only currentSplit on its owning Snapshot. Null expected heads/revisions are explicit. On an - unknown outcome, retry the exact request or query `commit_status`; a deliberate - retry after a head conflict uses a new attempt and freshly observed generation. + unknown outcome, retry the exact request. `commit_status` exposes Snapshot and + legacy receipts; new independent Split publication saves no request ledger. + A Split retry confirms the same current revision or returns a condition conflict + after a later revision replaces it. A deliberate retry after a head conflict + uses newly reviewed inputs and an observed generation. Publishing a Split does + not change its compatibility draft from Prepared to Published. - `split_members`, `split_history`, `inventory`, `roots`, `children`, `schema` and `samples` return lazy `PageCursor`s. Each `next_page` fetches at most 100 items; it never eagerly collects the Dataset. Persist `continuation()` for scoped From f92bd82a08695bb177347ba42062d91fa40dd83f Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 18:24:51 +0800 Subject: [PATCH 121/258] Expose stateless split head updates and conflict observations --- .../src/dataset/api/error.rs | 12 +- .../src/dataset/api/v2.rs | 1 + .../src/dataset/api/v2/split_requests.rs | 9 ++ .../src/dataset/api/v2/split_responses.rs | 6 + .../src/dataset/api/v2/splits.rs | 76 ++++++++---- .../tests/dataset_split_head_contract_test.rs | 115 ++++++++++++++++++ .../tests/dataset_v2_contract_test.rs | 1 + .../dataset/design-dataset-control.md | 9 ++ .../dataset/openapi-crowdb-dataset-v2.yaml | 31 ++++- .../dataset/spec-crowdb-dataset.md | 6 + doc/working/plan-access-simplification.md | 36 +++++- lib/crowdb-access-dataset/src/split.rs | 2 +- lib/crowdb-access-dataset/src/split/head.rs | 7 ++ .../src/split/lifecycle.rs | 5 + .../src/split/publication.rs | 112 +++++++++++++---- .../src/split/recovery.rs | 4 +- .../tests/common/upload.rs | 2 + .../tests/split_complete_result_test.rs | 2 +- .../tests/split_publication_cost_test.rs | 6 +- .../tests/split_publication_test.rs | 6 +- sdk/README.md | 6 + sdk/crates/crowdb-sdk-core/src/error.rs | 3 + sdk/crates/crowdb-sdk-core/src/http.rs | 1 + .../crowdb-sdk-core/src/http/attempt.rs | 1 + sdk/crates/crowdb-sdk-core/src/http/stream.rs | 1 + .../examples/split_control.rs | 25 ++-- sdk/crates/crowdb-sdk-dataset/src/v2.rs | 2 + .../crowdb-sdk-dataset/src/v2/split_head.rs | 48 ++++++++ .../tests/split_head_test.rs | 97 +++++++++++++++ sdk/crates/crowdb-sdk-python/src/control.rs | 1 + sdk/crates/crowdb-sdk-python/src/lib.rs | 6 +- sdk/crowdb_dataset/client.py | 1 + sdk/crowdb_dataset/control.py | 1 + sdk/tests/split_consumer.py | 10 +- 34 files changed, 576 insertions(+), 75 deletions(-) create mode 100644 app/crowdb-access-server/tests/dataset_split_head_contract_test.rs create mode 100644 sdk/crates/crowdb-sdk-dataset/src/v2/split_head.rs create mode 100644 sdk/crates/crowdb-sdk-dataset/tests/split_head_test.rs diff --git a/app/crowdb-access-server/src/dataset/api/error.rs b/app/crowdb-access-server/src/dataset/api/error.rs index 7a447d646..6e6871693 100644 --- a/app/crowdb-access-server/src/dataset/api/error.rs +++ b/app/crowdb-access-server/src/dataset/api/error.rs @@ -10,6 +10,7 @@ pub(super) struct ApiError { status: StatusCode, code: &'static str, retryable: bool, + observed_head: Option>, } impl ApiError { pub(super) fn conflict() -> Self { @@ -20,6 +21,7 @@ impl ApiError { status: StatusCode::TOO_MANY_REQUESTS, code: "capacity", retryable: true, + observed_head: None, } } pub(super) fn invalid() -> Self { @@ -48,16 +50,24 @@ impl ApiError { status, code, retryable: status == StatusCode::SERVICE_UNAVAILABLE, + observed_head: None, } } pub(super) fn code(&self) -> &'static str { self.code } + pub(super) fn with_observed_head(mut self, head: serde_json::Value) -> Self { + self.observed_head = Some(Box::new(head)); + self + } pub(super) fn retryable(&self) -> bool { self.retryable } pub(super) fn response(&self, request_id: &str) -> Response { - let body = serde_json::json!({"code":self.code,"message":self.code,"request_id":request_id,"retryable":self.retryable}); + let mut body = serde_json::json!({"code":self.code,"message":self.code,"request_id":request_id,"retryable":self.retryable}); + if let Some(head) = &self.observed_head { + body["observed_head"] = head.as_ref().clone(); + } Response::builder() .status(self.status) .header("content-type", "application/json") diff --git a/app/crowdb-access-server/src/dataset/api/v2.rs b/app/crowdb-access-server/src/dataset/api/v2.rs index 9cd31221e..5c58045af 100644 --- a/app/crowdb-access-server/src/dataset/api/v2.rs +++ b/app/crowdb-access-server/src/dataset/api/v2.rs @@ -53,6 +53,7 @@ impl DatasetHttpService { "inspection-preparation", "dataset-management", "split-revisions", + "split-head-update", "stateless-reads", ]; if self.uploads.is_some() { diff --git a/app/crowdb-access-server/src/dataset/api/v2/split_requests.rs b/app/crowdb-access-server/src/dataset/api/v2/split_requests.rs index f909f0c95..edf2119cf 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/split_requests.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/split_requests.rs @@ -44,6 +44,15 @@ pub(super) struct Publish { #[serde(deserialize_with = "required_option")] pub expected_head: Option, } +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct UpdateHead { + pub revision: String, + #[serde(deserialize_with = "required_option")] + pub expected_revision: Option, + #[serde(deserialize_with = "required_option")] + pub expected_head: Option, +} pub(super) fn generation(value: Option<&str>) -> Result, ApiError> { value .map(|value| { diff --git a/app/crowdb-access-server/src/dataset/api/v2/split_responses.rs b/app/crowdb-access-server/src/dataset/api/v2/split_responses.rs index 97f76d251..07e01b471 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/split_responses.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/split_responses.rs @@ -65,6 +65,9 @@ pub(super) fn preview(p: split::SplitPreview) -> Value { pub(super) fn receipt(r: &SuccessfulCommit) -> Value { json!({"attempt":r.attempt.id.to_string(), "target":target(r.attempt.target), "expected_head":r.attempt.expected_head.map(|h|hex::encode(h.as_bytes())), "committed_head":hex::encode(r.committed_head.as_bytes())}) } +pub(super) fn head(snapshot: crowdb_access_dataset::SnapshotId, head: &split::SplitReference) -> Value { + json!({"snapshot":snapshot.to_string(),"revision":head.revision.map(|id|id.to_string()),"generation":head.generation.map(|g|hex::encode(g.as_bytes()))}) +} impl From for ApiError { fn from(error: SplitLifecycleError) -> Self { use hyper::StatusCode; @@ -74,6 +77,9 @@ impl From for ApiError { SplitLifecycleError::Inspection(error) => error.into(), SplitLifecycleError::NotFound => Self::not_found(), SplitLifecycleError::Conflict => Self::conflict(), + SplitLifecycleError::HeadConflict { snapshot, observed } => { + Self::conflict().with_observed_head(head(snapshot, &observed)) + } SplitLifecycleError::Corrupt => Self::integrity(), SplitLifecycleError::Limit | SplitLifecycleError::Split(split::SplitError::Limit) => { Self::new(StatusCode::PAYLOAD_TOO_LARGE, "read_limit") diff --git a/app/crowdb-access-server/src/dataset/api/v2/splits.rs b/app/crowdb-access-server/src/dataset/api/v2/splits.rs index 2e7bacc57..fd4a38c64 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/splits.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/splits.rs @@ -40,31 +40,17 @@ impl DatasetHttpService { let authority = self.reads.authority(); match (request.method().clone(), tail) { (Method::POST, ["snapshots", owner, "split-drafts"]) => { - let input: input::Begin = read_json(request).await?; - let splits = input - .splits - .into_iter() - .map(DraftSplit::native) - .collect::>()?; - let record = authority - .begin_split_draft( - identity, - operation(&input.operation)?, - snapshot(owner)?, - input.expected_revision.as_deref().map(revision).transpose()?, - input.policy.into(), - splits, - ) - .await?; - json(&output::draft(record)) + self.v2_begin_split(identity, owner, request).await } (_, ["split-drafts", ..]) => self.v2_split_draft(identity, tail, request).await, (Method::GET, ["snapshots", owner, "split-head"]) => { read_body(request, false).await?; - let head = authority.inspect_split_head(identity, snapshot(owner)?).await?; - json( - &serde_json::json!({"snapshot":owner, "revision":head.revision.map(|r|r.to_string()), "generation":head.generation.map(|h|hex::encode(h.as_bytes()))}), - ) + let snapshot = snapshot(owner)?; + let head = authority.inspect_split_head(identity, snapshot).await?; + json(&output::head(snapshot, &head)) + } + (Method::PUT, ["snapshots", owner, "split-head"]) => { + self.v2_update_split_head(identity, owner, request).await } (Method::GET, ["snapshots", owner, "split-revisions"]) => { read_body(request, false).await?; @@ -115,6 +101,54 @@ impl DatasetHttpService { _ => Err(ApiError::not_found()), } } + async fn v2_begin_split( + &self, + identity: &DatasetIdentity, + owner: &str, + request: Request, + ) -> ApiResult { + let input: input::Begin = read_json(request).await?; + let splits = input + .splits + .into_iter() + .map(DraftSplit::native) + .collect::>()?; + let record = self + .reads + .authority() + .begin_split_draft( + identity, + operation(&input.operation)?, + snapshot(owner)?, + input.expected_revision.as_deref().map(revision).transpose()?, + input.policy.into(), + splits, + ) + .await?; + json(&output::draft(record)) + } + + async fn v2_update_split_head( + &self, + identity: &DatasetIdentity, + owner: &str, + request: Request, + ) -> ApiResult { + let input: input::UpdateHead = read_json(request).await?; + let snapshot = snapshot(owner)?; + let update = crowdb_access_dataset::split::SplitHeadUpdate { + revision: revision(&input.revision)?, + expected_revision: input.expected_revision.as_deref().map(revision).transpose()?, + expected_head: input::generation(input.expected_head.as_deref())?, + }; + let head = self + .reads + .authority() + .update_split_head(identity, snapshot, &update) + .await?; + json(&output::head(snapshot, &head)) + } + async fn v2_split_draft( &self, identity: &DatasetIdentity, diff --git a/app/crowdb-access-server/tests/dataset_split_head_contract_test.rs b/app/crowdb-access-server/tests/dataset_split_head_contract_test.rs new file mode 100644 index 000000000..23958ab76 --- /dev/null +++ b/app/crowdb-access-server/tests/dataset_split_head_contract_test.rs @@ -0,0 +1,115 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/dataset_v2.rs"] +#[allow(dead_code)] +mod common; +#[path = "common/dataset.rs"] +mod dataset; + +use crowdb_access_dataset::{DatasetIdentity, DatasetStore, NamespacePath}; +use dataset::{TestDatasetHttp, TestStore}; +use reqwest::Client; +use serde_json::{json, Value}; +use std::sync::{atomic::Ordering, Arc}; + +fn counts(store: &TestStore) -> (u64, u64) { + ( + store.reads.load(Ordering::Relaxed) + store.bounded_reads.load(Ordering::Relaxed), + store.mutations.load(Ordering::Relaxed), + ) +} +fn delta(store: &TestStore, before: (u64, u64)) -> (u64, u64) { + let now = counts(store); + (now.0 - before.0, now.1 - before.1) +} + +#[tokio::test] +async fn update_uses_only_complete_results_and_returns_the_failed_head_condition() { + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start(store.clone()).await; + let client = Client::new(); + let (root, snapshot) = common::fixture(&client, &test.origin).await; + let mut prepared = Vec::new(); + for operation in ["22".repeat(16), "33".repeat(16)] { + common::response( + client + .post(format!("{root}/snapshots/{snapshot}/split-drafts")) + .json(&common::begin(&operation, &Value::Null)), + 200, + ) + .await; + let draft = format!("{root}/split-drafts/{operation}"); + prepared.push(common::prepare(&client, &draft, "1").await); + } + // Remove the editing identity; the final result and its members are independent. + let identity = DatasetIdentity::new(NamespacePath::root(), "splits").unwrap(); + for prefix in [b"DS\0SD\0".as_slice(), b"DS\0SB\0"] { + store + .delete(&[prefix, &identity.key().unwrap(), &[0x22; 16]].concat()) + .await + .unwrap(); + } + let url = format!("{root}/snapshots/{snapshot}/split-head"); + let first = json!({"revision":prepared[0]["revision"], "expected_revision":null, "expected_head":null}); + let before = counts(&store); + let head = common::response(client.put(&url).json(&first), 200).await; + assert_eq!(delta(&store, before), (5, 1)); + assert_eq!(head["snapshot"], snapshot); + assert_eq!(head["revision"], first["revision"]); + let before = counts(&store); + assert_eq!(common::response(client.put(&url).json(&first), 200).await, head); + assert_eq!(delta(&store, before), (4, 0)); + let second = json!({"revision":prepared[1]["revision"], "expected_revision":null, "expected_head":null}); + let before = counts(&store); + let conflict = common::response(client.put(&url).json(&second), 409).await; + assert_eq!(delta(&store, before), (4, 0)); + assert_eq!(conflict["observed_head"], head); + let mut reviewed = second; + reviewed["expected_revision"] = conflict["observed_head"]["revision"].clone(); + reviewed["expected_head"] = conflict["observed_head"]["generation"].clone(); + let before = counts(&store); + let current = common::response(client.put(&url).json(&reviewed), 200).await; + assert_eq!(delta(&store, before), (4, 1)); + assert_eq!(current["revision"], prepared[1]["revision"]); + assert_ne!(current["generation"], head["generation"]); + assert_eq!( + common::response(client.get(format!("{root}/head")), 200).await["latest"], + snapshot + ); + drop(client); + test.stop().await; +} + +#[tokio::test] +async fn update_requires_explicit_nullable_conditions_and_auth_before_storage() { + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start(store.clone()).await; + let client = Client::new(); + let (root, snapshot) = common::fixture(&client, &test.origin).await; + let url = format!("{root}/snapshots/{snapshot}/split-head"); + let body = json!({"revision":"22".repeat(16),"expected_revision":null,"expected_head":null}); + let before = counts(&store); + let denied = client.put(&url).json(&body).send().await.unwrap(); + assert_eq!(denied.status(), 401); + denied.bytes().await.unwrap(); + assert_eq!(delta(&store, before), (0, 0)); + for field in ["expected_revision", "expected_head"] { + let mut invalid = body.clone(); + invalid.as_object_mut().unwrap().remove(field); + let before = counts(&store); + common::response(client.put(&url).json(&invalid), 400).await; + assert_eq!(delta(&store, before), (0, 0)); + } + let mut invalid = body.clone(); + invalid["attempt"] = json!("33".repeat(16)); + common::response(client.put(&url).json(&invalid), 400).await; + invalid = body.clone(); + invalid["expected_head"] = json!("G".repeat(64)); + common::response(client.put(&url).json(&invalid), 400).await; + let before = counts(&store); + common::response(client.put(&url).json(&body), 404).await; + assert_eq!(delta(&store, before), (1, 0)); + drop(client); + test.stop().await; +} diff --git a/app/crowdb-access-server/tests/dataset_v2_contract_test.rs b/app/crowdb-access-server/tests/dataset_v2_contract_test.rs index 1f5257d40..96d04a5fc 100644 --- a/app/crowdb-access-server/tests/dataset_v2_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_v2_contract_test.rs @@ -38,6 +38,7 @@ async fn capabilities_use_existing_auth_and_match_configured_operations() { "inspection-preparation", "dataset-management", "split-revisions", + "split-head-update", "stateless-reads" ]) ); diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index 31f79ea06..1281a70d5 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -103,6 +103,15 @@ receipts and attempt-bearing heads remain readable without backfilling receipts. The draft publication route is a compatibility adapter for its prepared target and expected revision; `publish_split_revision` accepts these caller-owned inputs directly. New draft status remains Prepared after publication. +`PUT snapshots/{snapshot}/split-head` accepts the target revision and both explicit +nullable expected revision/generation. It requires the `split-head-update` +capability and has no draft or attempt identity. The failed head condition is +returned as `observed_head`, including an observed absence. It is an observation, +not a promise that another writer has not changed the head since. The caller may +review the new base and use it in a new conditional request; no rebase is implicit. +Rust exposes the observation through `SplitHead::from_conflict`; Python exposes +`ConflictError.observed_head`. Missing legacy observations do not trigger an +automatic head query or a mutation downgrade. ## 4. Bounded observation and reads diff --git a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml index 5111a0268..08a798899 100644 --- a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml +++ b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml @@ -12,7 +12,7 @@ info: available under their original routes. Metadata GET routes require bounded-inspection; explicit POST inspection preparation requires inspection-preparation. Dataset creation, stable/retention changes and receipts - require dataset-management; Split lifecycle requires split-revisions; versioned + require dataset-management; Split lifecycle requires split-revisions; direct Split head updates require split-head-update; versioned stateless reads require stateless-reads. Upload, composition and Snapshot retirement require uploads, composition and snapshot-reclaim respectively, advertised only when the upload service is configured. Query parameters are strict and duplicate @@ -280,6 +280,26 @@ paths: default: $ref: '#/components/responses/InspectionError' /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/split-head: + put: + operationId: updateSplitHead + summary: Conditionally update currentSplit from a complete resource + description: Requires split-head-update. Caller supplies the target and both explicit nullable preconditions, without a draft or attempt ID. A conflict carries observed_head for caller review; the server does not rebase or query again to confirm a CAS outcome. + parameters: + - {$ref: '#/components/parameters/dataset'} + - {$ref: '#/components/parameters/snapshot'} + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/UpdateSplitHead'} + responses: + '200': + description: Current effect confirmed, without a historical response receipt. + content: + application/json: + schema: {$ref: '#/components/schemas/SplitHead'} + default: + $ref: '#/components/responses/InspectionError' get: operationId: splitHead parameters: @@ -1130,6 +1150,7 @@ components: message: {type: string} request_id: {type: string} retryable: {type: boolean} + observed_head: {$ref: '#/components/schemas/SplitHead'} securitySchemes: bearerAuth: type: http @@ -1420,6 +1441,14 @@ components: properties: expected_revision: {$ref: '#/components/schemas/Decimal'} expected_page: {$ref: '#/components/schemas/Decimal'} + UpdateSplitHead: + type: object + additionalProperties: false + required: [revision, expected_revision, expected_head] + properties: + revision: {$ref: '#/components/schemas/OpaqueId'} + expected_revision: {$ref: '#/components/schemas/NullableRevision'} + expected_head: {$ref: '#/components/schemas/NullableGeneration'} PublishSplit: type: object additionalProperties: false diff --git a/doc/design/access-server/dataset/spec-crowdb-dataset.md b/doc/design/access-server/dataset/spec-crowdb-dataset.md index 8ace93621..0f8b1dc5e 100644 --- a/doc/design/access-server/dataset/spec-crowdb-dataset.md +++ b/doc/design/access-server/dataset/spec-crowdb-dataset.md @@ -207,6 +207,12 @@ base MUST fail without rewinding currentSplit. It does not promise historical response replay. Unknown storage outcomes MUST propagate without a confirmation query. Existing legacy receipts remain readable; new Split attempts are absent from commit-status history. +Direct Split head updates MUST accept the complete resource ID and explicit +nullable expected revision and head generation without an operation/attempt ID or +draft lookup. A failed head condition MUST return the observed revision/generation, +including absence, without a confirmation query. The caller decides whether the +observed base is suitable for another update; the SDK MUST NOT rebase automatically +or fall back to a draft write when the direct-update capability is unavailable. ## 7. Selection, plans, and cursors diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 79abbc03e..4fe91d73f 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -161,10 +161,6 @@ CAS 不再满足只证明“本次条件不成立”，不能谎称“第一次 历史回复查询；Composition/Split 的 Committing 恢复与 P2/P3 一起删除，客户端保留完整输入。 旧记录只作为兼容解码/GC 输入，不让旧账本控制新请求。head 保留 expected base；同一 candidate 的当前结果可判断相同，已被后续 head 遮住时不保证历史请求结果重放。 -- [~] **R2.2b Split 发布直接接收资源与 precondition**：补无 draft 的 REST/SDK 入口， - 消除当前兼容路由的两次 draft 读取；冲突返回已观察的 revision/generation，供客户端审阅后 - 直接组成下一次条件请求，不为取得相同 head 再 GET。不得自动 rebase；旧服务能力缺失时 - 明确失败。验证缺省/null、鉴权、冲突与丢回复、Rust/Python SDK、无账本与实际读写成本。 - [ ] **R2 审查条件发布的回复丢失窗口**：用 H0→H1→H2 测试区分条件失败、相同结果与历史未知； 列出确实依赖历史精确结果的调用者，没有这种需求就移除相应 ledger/response replay。 @@ -431,6 +427,18 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 - [ ] **F1 请求级成本计数**：记录 GET/bounded GET/scan/CAS/PUT/delete、重复 key、冲突与 unknown、真实 routed RPC、串行阶段和复制字节；用测试 wrapper/已有 metrics，不加生产 每 RPC 日志或 profiling 锁。Files: Dataset/S3/Iceberg 测试 store、现有指标、`tools/sdk/`。 +- [ ] **F1.1 去掉 SDK 每次 control 调用的 capabilities GET**：R2.2b 抓到显式重试的请求 + 为 capabilities GET → PUT → capabilities GET → PUT；没有 head GET，但仍多一次 Access + 往返。能力是版本化路由/配置发现，不是鉴权或资源准入；研究按 client 复用已确认能力， + 保持服务端每次真实鉴权/条件检查、缺失能力不降级、重新发现/滚动升级的失效语义。 + 不增加热路径锁或 TTL 准入缓存。先写清语义，再补冷启动并发、服务失效和实际请求数。 + Files: SDK `client.rs`、capabilities、v2 `client.rs`、native Python dispatch 与协议 fixture。 + 已核实 `DatasetHttpService` 的能力由版本化路由和构造时的 uploads 配置决定，鉴权与资源 + 条件每个真实请求仍独立检查。拟在同一 client/clone 中以固定有界 atomic bitset 复用成功 + 发现的能力，不缓存拒绝/缺失、不保存服务端状态；显式 capabilities() 仍重新发现并替换 + 提示。冷启动并发可重复有界发现，避免 single-flight 锁。服务端重启/能力变化时真实路由 + 仍拒绝不支持的操作，没有 v1 fallback；预先 GET 本来也不能消除发现与写之间的重启窗口。 + 不将该提示用于跳过鉴权、删除准入或 expected-base 检查；未知传输结果不解释成能力撤销。 - [ ] **F2 复核 legacy/多源读取的重复准入**：现代 continuation 的 selection→HTTP/native delivery 已共用请求局部的 Snapshot/schema/membership，Split 同 Snapshot 也复用准入。 剩余复核 legacy `DatasetReadService`、多源/祖先元数据，独立入口和不同 Snapshot 仍各自准入； @@ -472,6 +480,9 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 无条件覆盖与 expected-base 重试测试。断言总 GET+写，不仅是零写。Files: Dataset `tests/*cost*`、`read_query*`、upload/Composition/Split/inspection；S3/Iceberg 对应 repository 测试。 - [ ] **V2 HTTP/native/SDK 集成**：各受影响 route 的鉴权、错误、协议/SDK 一致性；保留删除准入、 迟到写者、非空 namespace、共享 owner、held buffer、cancel/drop 和损坏内容测试。Files: Access/native tests、`sdk/tests/`、native Python consumer。 +- [~] **V2.2 Split 清理同源容器验收**：稳定树构建新镜像，运行全部五个 Dataset 容器用例， + 包括真实 Chunk-KV/存储重启、Rust 与安装后 Python SDK 的直接 Split head 更新/观察值。 + 构建期间不改任何非忽略文件，保留 source guard；记录镜像、客户端与 artifact 身份。 - [ ] **V3 工程门禁与文档**：每批执行适用 fmt/clippy/测试，全部经 `pixi run`；同步 owning design/spec/OpenAPI/SDK。只修改文档时检查 diff、引用和任务状态，不为此跑整套服务测试。 - [ ] **V4 同源真实负载验收**：重新构建当前源码的镜像，运行现有 HTTP/native Rust/native Python @@ -499,6 +510,23 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **R2.2b（2026-10-24）**：新增 `PUT snapshots/{snapshot}/split-head` 与 + `split-head-update` 能力，Rust/Python SDK 提供 `update_split_head`。仅携带完整 revision + 和两个必须显式提供的 nullable expected base，不携带 attempt ID，不查 draft，不保存过程。 + Native 共用 `SplitHeadUpdate`；旧 attempt 入口仅适配响应。条件失败携带 boxed head + observation，早期 base 不符和真实 CAS loser 都不回读 head；观测值可能随后过期，客户端 + 审阅后显式组成下一次请求，SDK 不自动 rebase。旧服务无能力时不回退到 draft/v1。 + Rust `SplitHead::from_conflict`、Python `ConflictError.observed_head` 完整保留观察值， + 旧错误无该字段仍可解码。新增两个 HTTP 用例实际删除 draft 身份后发布，断言首次 + 5G+1C、重复确认 4G+0C、冲突 4G+0C、基于返回值再提交 4G+1C；鉴权/缺省字段失败 + 不接触 storage，缺失最终资源 1G+0C。读计数包含 bounded GET，不能只看旧 reads 字段。 + Dataset 全 target、十一项 HTTP/Rust SDK、独立 SDK workspace、四 crate all-feature + clippy 和独立 SDK workspace clippy、fmt/diff/boundary 均通过。sdist 独立测试并打包的新 + Python wheel 安装到 `.crowdb-runtime/artifacts/sdk-split-head-20261024/consumer/`， + 显式运行安装后 Python HTTP consumer 验证更新与观察值通过。SDK 抓包同时发现每调用 + 都多 GET capabilities，已先登记 F1.1，未把当前两次 HTTP 往返称作单次。容器验证由 V2.2 + 继续，Snapshot 完整准入与 P2/P3/P4/R4 主线仍未完成。 + - **R2.2a（2026-10-24）**：新增直接接受完整 revision/显式 expected revision+generation 的 `publish_split_revision`，旧 draft 发布仅作适配；新发布只 CAS currentSplit，不再写 CA/CR/SRC、Committing/Published，也不帮助补写前一发布。相同当前结果返回当前事实， diff --git a/lib/crowdb-access-dataset/src/split.rs b/lib/crowdb-access-dataset/src/split.rs index 307da3e4e..08bebd1f1 100644 --- a/lib/crowdb-access-dataset/src/split.rs +++ b/lib/crowdb-access-dataset/src/split.rs @@ -52,7 +52,7 @@ pub use evaluation::SplitPreview; mod prepare; mod head; -pub use head::SplitReference; +pub use head::{SplitHeadUpdate, SplitReference}; mod publication; diff --git a/lib/crowdb-access-dataset/src/split/head.rs b/lib/crowdb-access-dataset/src/split/head.rs index 3a106eabc..9a6dd3b9c 100644 --- a/lib/crowdb-access-dataset/src/split/head.rs +++ b/lib/crowdb-access-dataset/src/split/head.rs @@ -13,6 +13,13 @@ pub struct SplitReference { pub revision: Option, pub generation: Option, } + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SplitHeadUpdate { + pub revision: SplitRevisionId, + pub expected_revision: Option, + pub expected_head: Option, +} #[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub(super) struct CommittedSplitHead { diff --git a/lib/crowdb-access-dataset/src/split/lifecycle.rs b/lib/crowdb-access-dataset/src/split/lifecycle.rs index 9d050aaed..6e97161ae 100644 --- a/lib/crowdb-access-dataset/src/split/lifecycle.rs +++ b/lib/crowdb-access-dataset/src/split/lifecycle.rs @@ -27,6 +27,11 @@ pub enum SplitLifecycleError { NotFound, #[error("Split draft or publication precondition changed")] Conflict, + #[error("Split head precondition changed")] + HeadConflict { + snapshot: SnapshotId, + observed: Box, + }, #[error("Split operation metadata is corrupt")] Corrupt, #[error("Split operation exceeded its work limit")] diff --git a/lib/crowdb-access-dataset/src/split/publication.rs b/lib/crowdb-access-dataset/src/split/publication.rs index 7a14118e6..ebf975f9c 100644 --- a/lib/crowdb-access-dataset/src/split/publication.rs +++ b/lib/crowdb-access-dataset/src/split/publication.rs @@ -1,10 +1,13 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{key, state::DraftState, SplitHead, SplitLifecycleError, SplitRevisionRecord}; +use super::{ + key, state::DraftState, SplitHead, SplitHeadUpdate, SplitLifecycleError, SplitReference, + SplitRevisionRecord, +}; use crate::{ composition::DraftProgress, CasOutcome, CommitAttempt, CommitGeneration, CommitTarget, DatasetAuthority, - DatasetIdentity, OperationId, SplitRevisionId, StoredValue, SuccessfulCommit, + DatasetIdentity, OperationId, SnapshotId, SplitRevisionId, StoredValue, SuccessfulCommit, }; impl DatasetAuthority { @@ -31,8 +34,13 @@ impl DatasetAuthority { if target != attempt.target { return Err(SplitLifecycleError::Conflict); } - self.commit_split_revision(identity, attempt, state.record.expected_revision, Some(operation)) - .await + let (snapshot, update) = attempt_update(attempt, state.record.expected_revision)?; + confirmed_commit( + attempt, + &self + .commit_split_revision(identity, snapshot, &update, Some(operation)) + .await?, + ) } /// Updates currentSplit from a complete immutable result and caller-owned preconditions. @@ -45,21 +53,37 @@ impl DatasetAuthority { attempt: &CommitAttempt, expected_revision: Option, ) -> Result { - self.commit_split_revision(identity, attempt, expected_revision, None) - .await + let (snapshot, update) = attempt_update(attempt, expected_revision)?; + confirmed_commit( + attempt, + &self.update_split_head(identity, snapshot, &update).await?, + ) + } + + /// Updates the resource pointer without a request identity or editing state. + /// # Errors + /// Rejects invalid or missing results, deletion admission, stale conditions and unknown outcomes. + pub async fn update_split_head( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + update: &SplitHeadUpdate, + ) -> Result { + self.commit_split_revision(identity, snapshot, update, None).await } async fn commit_split_revision( &self, identity: &DatasetIdentity, - attempt: &CommitAttempt, - expected_revision: Option, + snapshot: SnapshotId, + update: &SplitHeadUpdate, operation: Option, - ) -> Result { - attempt.validate()?; - let CommitTarget::SplitRevision { snapshot, revision } = attempt.target else { - return Err(SplitLifecycleError::Conflict); - }; + ) -> Result { + let revision = update.revision; + CommitTarget::SplitRevision { snapshot, revision }.validate()?; + if let Some(expected) = update.expected_revision { + SplitRevisionId::from_bytes(expected.as_bytes()).map_err(crate::AuthorityError::from)?; + } let stored = self .split_state_get( &key::revision(identity, snapshot, revision).map_err(crate::AuthorityError::from)?, @@ -84,10 +108,13 @@ impl DatasetAuthority { .resolve_split_reference(identity, current.as_ref(), snapshot) .await?; if observed.revision == Some(revision) && current.is_some() { - return current_success(attempt, current.as_ref()); + return current_reference(revision, current.as_ref()); } - if observed.generation != attempt.expected_head || observed.revision != expected_revision { - return Err(SplitLifecycleError::Conflict); + if observed.generation != update.expected_head || observed.revision != update.expected_revision { + return Err(SplitLifecycleError::HeadConflict { + snapshot, + observed: Box::new(observed), + }); } let bytes = SplitHead { version: 1, @@ -101,16 +128,19 @@ impl DatasetAuthority { .compare_exchange(&key, current.as_ref().map(|v| v.bytes.as_slice()), &bytes) .await? { - CasOutcome::Applied(_) => Ok(SuccessfulCommit { - version: 1, - attempt: attempt.clone(), - committed_head: CommitGeneration::from_head(&bytes), + CasOutcome::Applied(_) => Ok(SplitReference { + revision: Some(revision), + generation: Some(CommitGeneration::from_head(&bytes)), }), CasOutcome::Conflict(observed) => { - if super::head::reference(observed.as_ref(), snapshot)?.revision != Some(revision) { - return Err(SplitLifecycleError::Conflict); + let head = super::head::reference(observed.as_ref(), snapshot)?; + if head.revision != Some(revision) { + return Err(SplitLifecycleError::HeadConflict { + snapshot, + observed: Box::new(head), + }); } - current_success(attempt, observed.as_ref()) + current_reference(revision, observed.as_ref()) } } } @@ -145,14 +175,42 @@ impl DatasetAuthority { } } -fn current_success( - attempt: &CommitAttempt, +fn current_reference( + revision: SplitRevisionId, current: Option<&StoredValue>, -) -> Result { +) -> Result { let current = current.ok_or(SplitLifecycleError::Conflict)?; + Ok(SplitReference { + revision: Some(revision), + generation: Some(CommitGeneration::from_head(¤t.bytes)), + }) +} + +fn attempt_update( + attempt: &CommitAttempt, + expected_revision: Option, +) -> Result<(SnapshotId, SplitHeadUpdate), SplitLifecycleError> { + attempt.validate()?; + let CommitTarget::SplitRevision { snapshot, revision } = attempt.target else { + return Err(SplitLifecycleError::Conflict); + }; + Ok(( + snapshot, + SplitHeadUpdate { + revision, + expected_revision, + expected_head: attempt.expected_head, + }, + )) +} + +fn confirmed_commit( + attempt: &CommitAttempt, + head: &SplitReference, +) -> Result { Ok(SuccessfulCommit { version: 1, attempt: attempt.clone(), - committed_head: CommitGeneration::from_head(¤t.bytes), + committed_head: head.generation.ok_or(SplitLifecycleError::Corrupt)?, }) } diff --git a/lib/crowdb-access-dataset/src/split/recovery.rs b/lib/crowdb-access-dataset/src/split/recovery.rs index b18e9add2..438b12842 100644 --- a/lib/crowdb-access-dataset/src/split/recovery.rs +++ b/lib/crowdb-access-dataset/src/split/recovery.rs @@ -108,7 +108,9 @@ fn authority_error(error: SplitLifecycleError) -> AuthorityError { match error { SplitLifecycleError::Authority(error) => error, SplitLifecycleError::Store(error) => error.into(), - SplitLifecycleError::Conflict => AuthorityError::HeadConflict, + SplitLifecycleError::Conflict | SplitLifecycleError::HeadConflict { .. } => { + AuthorityError::HeadConflict + } SplitLifecycleError::NotFound => AuthorityError::SnapshotNotFound, SplitLifecycleError::Limit => crate::DatasetError::ReadLimitExceeded.into(), _ => AuthorityError::Corrupt, diff --git a/lib/crowdb-access-dataset/tests/common/upload.rs b/lib/crowdb-access-dataset/tests/common/upload.rs index d3e724da7..183826581 100644 --- a/lib/crowdb-access-dataset/tests/common/upload.rs +++ b/lib/crowdb-access-dataset/tests/common/upload.rs @@ -19,6 +19,7 @@ use std::{ pub struct TestStore { values: ArcSwap, StoredValue>>, pub reads: AtomicU64, + pub bounded_reads: AtomicU64, pub upload_reads: AtomicU64, pub admission_reads: AtomicU64, pub mutations: AtomicU64, @@ -41,6 +42,7 @@ impl DatasetStore for TestStore { budget: crowdb_access_dataset::MetadataScanBudget, cancel: &crowdb_access_dataset::ReadCancellation, ) -> Result, crowdb_access_dataset::StoreError> { + self.bounded_reads.fetch_add(1, Ordering::Relaxed); if key.starts_with(b"DS\0L\0") { self.admission_reads.fetch_add(1, Ordering::Relaxed); } diff --git a/lib/crowdb-access-dataset/tests/split_complete_result_test.rs b/lib/crowdb-access-dataset/tests/split_complete_result_test.rs index 869a26854..05b468c2d 100644 --- a/lib/crowdb-access-dataset/tests/split_complete_result_test.rs +++ b/lib/crowdb-access-dataset/tests/split_complete_result_test.rs @@ -37,7 +37,7 @@ async fn losing_pointer_cas_keeps_result_readable_without_split_proofs_or_writes authority .publish_split_draft(&identity, candidate.operation, &attempt) .await, - Err(SplitLifecycleError::Conflict) + Err(SplitLifecycleError::HeadConflict { .. }) )); let head = authority.inspect_split_head(&identity, snapshot).await.unwrap(); assert_eq!(head.revision, Some(winner.revision)); diff --git a/lib/crowdb-access-dataset/tests/split_publication_cost_test.rs b/lib/crowdb-access-dataset/tests/split_publication_cost_test.rs index 883c570be..e469265a8 100644 --- a/lib/crowdb-access-dataset/tests/split_publication_cost_test.rs +++ b/lib/crowdb-access-dataset/tests/split_publication_cost_test.rs @@ -78,7 +78,7 @@ async fn publication_uses_one_head_mutation_and_retries_current_facts_without_wr let before = counts(&store); assert!(matches!( authority.publish_split_revision(&identity, &attempt, None).await, - Err(SplitLifecycleError::Conflict) + Err(SplitLifecycleError::HeadConflict { .. }) )); assert_eq!(delta(&store, before), (4, 0)); } @@ -109,7 +109,7 @@ async fn definite_cas_conflicts_use_the_observation_and_unknown_outcomes_do_not_ .unwrap() ); } else { - assert!(matches!(result, Err(SplitLifecycleError::Conflict))); + assert!(matches!(result, Err(SplitLifecycleError::HeadConflict { .. }))); } } let (store, authority, identity, snapshot) = split_store::fixture().await; @@ -152,7 +152,7 @@ async fn lost_success_hidden_by_a_later_revision_is_a_conflict_without_replay() let before = counts(&store); assert!(matches!( authority.publish_split_revision(&identity, &attempt, None).await, - Err(SplitLifecycleError::Conflict) + Err(SplitLifecycleError::HeadConflict { .. }) )); assert_eq!(delta(&store, before), (4, 0)); assert_eq!( diff --git a/lib/crowdb-access-dataset/tests/split_publication_test.rs b/lib/crowdb-access-dataset/tests/split_publication_test.rs index 664bba37c..fcd41233f 100644 --- a/lib/crowdb-access-dataset/tests/split_publication_test.rs +++ b/lib/crowdb-access-dataset/tests/split_publication_test.rs @@ -24,7 +24,7 @@ async fn independent_revision_cas_and_historical_conflicts_never_move_data_head( authority .publish_split_draft(&identity, competing.operation, &stale) .await, - Err(SplitLifecycleError::Conflict) + Err(SplitLifecycleError::HeadConflict { .. }) )); assert!(matches!( authority @@ -51,7 +51,7 @@ async fn independent_revision_cas_and_historical_conflicts_never_move_data_head( authority .publish_split_draft(&identity, first.operation, &first_attempt) .await, - Err(SplitLifecycleError::Conflict) + Err(SplitLifecycleError::HeadConflict { .. }) )); assert_eq!( authority.inspect_split_head(&identity, snapshot).await.unwrap(), @@ -165,7 +165,7 @@ async fn replacing_current_split_does_not_write_or_help_receipts() { authority .publish_split_draft(&identity, first.operation, &first_attempt) .await, - Err(SplitLifecycleError::Conflict) + Err(SplitLifecycleError::HeadConflict { .. }) )); assert_eq!( authority diff --git a/sdk/README.md b/sdk/README.md index d299eee3e..cc9b3074f 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -211,6 +211,12 @@ Existing `DatasetClient` methods retain their v1 contracts. - Complete Split revisions can be read or used as frozen sources before updating currentSplit, after a head conflict, or after abandoning their draft. Each new read still checks Snapshot availability; incomplete revisions remain unavailable. +- `update_split_head(snapshot, request)` updates a complete revision directly with + `revision`, `expected_revision` and `expected_head`; both expected values must be + supplied, including null. It requires `split-head-update`, takes no attempt ID + and does not read a draft. Rust's `SplitHead::from_conflict` and Python's + `ConflictError.observed_head` return the failed condition's observation. Review + that base before retrying; the SDK does not rebase automatically. - `bind_read` pins Snapshot and optional Split revision without writes; `next_batch(cursor)` uses verified streaming framing. Save the next cursor after successful processing. The required capability is `stateless-reads`; old session diff --git a/sdk/crates/crowdb-sdk-core/src/error.rs b/sdk/crates/crowdb-sdk-core/src/error.rs index 8b91252fb..b6654a5b7 100644 --- a/sdk/crates/crowdb-sdk-core/src/error.rs +++ b/sdk/crates/crowdb-sdk-core/src/error.rs @@ -9,6 +9,9 @@ pub struct ServiceError { pub message: String, pub request_id: String, pub retryable: bool, + /// The bounded observation attached to a failed conditional mutation, when supported. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed_head: Option>, } /// Transport details deliberately omit request URLs and credentials. diff --git a/sdk/crates/crowdb-sdk-core/src/http.rs b/sdk/crates/crowdb-sdk-core/src/http.rs index 4d6c3f10d..62cb6c292 100644 --- a/sdk/crates/crowdb-sdk-core/src/http.rs +++ b/sdk/crates/crowdb-sdk-core/src/http.rs @@ -304,6 +304,7 @@ impl HttpClient { message: "service error body was not valid JSON".into(), request_id: String::new(), retryable: false, + observed_head: None, }); return Err(Error::Service { status: status.as_u16(), diff --git a/sdk/crates/crowdb-sdk-core/src/http/attempt.rs b/sdk/crates/crowdb-sdk-core/src/http/attempt.rs index a66ce6b2d..177031503 100644 --- a/sdk/crates/crowdb-sdk-core/src/http/attempt.rs +++ b/sdk/crates/crowdb-sdk-core/src/http/attempt.rs @@ -75,6 +75,7 @@ impl HttpClient { message: "invalid error response".into(), request_id: String::new(), retryable: false, + observed_head: None, }); Error::Service { status, error } } diff --git a/sdk/crates/crowdb-sdk-core/src/http/stream.rs b/sdk/crates/crowdb-sdk-core/src/http/stream.rs index 3c237daf7..c938c7d0a 100644 --- a/sdk/crates/crowdb-sdk-core/src/http/stream.rs +++ b/sdk/crates/crowdb-sdk-core/src/http/stream.rs @@ -63,6 +63,7 @@ impl HttpClient { message: "invalid error response".into(), request_id: String::new(), retryable: false, + observed_head: None, }); return Err(Error::Service { status: status.as_u16(), diff --git a/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs b/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs index edccfc24d..364e70928 100644 --- a/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs +++ b/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs @@ -253,16 +253,21 @@ async fn revise_splits( client.split_head(snapshot).await?.revision, Some(previous.clone()) ); - client - .publish_split( - &begin.operation, - &v2::PublishSplit { - attempt: v2::OpaqueId::random(), - revision: revision.revision.clone(), - expected_head: split_head.generation, - }, - ) - .await?; + let update = v2::UpdateSplitHead { + revision: revision.revision.clone(), + expected_revision: Some(previous.clone()), + expected_head: split_head.generation.clone(), + }; + client.update_split_head(snapshot, &update).await?; + let stale = v2::UpdateSplitHead { + revision: previous.clone(), + ..update + }; + let conflict = client.update_split_head(snapshot, &stale).await.unwrap_err(); + assert_eq!( + v2::SplitHead::from_conflict(&conflict).unwrap().revision, + Some(revision.revision.clone()) + ); assert_eq!(client.head().await?.generation, head.generation); assert_eq!( client.split_head(snapshot).await?.revision, diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2.rs b/sdk/crates/crowdb-sdk-dataset/src/v2.rs index fa6695119..3205b2849 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/v2.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/v2.rs @@ -12,6 +12,7 @@ mod paging; mod read; mod requests; mod split; +mod split_head; mod state; mod transfer; mod upload; @@ -23,6 +24,7 @@ pub use inspection::*; pub use paging::{PageCursor, PageKind}; pub use read::*; pub use split::*; +pub use split_head::UpdateSplitHead; pub use state::*; pub use upload::*; diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2/split_head.rs b/sdk/crates/crowdb-sdk-dataset/src/v2/split_head.rs new file mode 100644 index 000000000..cb44a7f3a --- /dev/null +++ b/sdk/crates/crowdb-sdk-dataset/src/v2/split_head.rs @@ -0,0 +1,48 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{required_option, ControlClient, Generation, OpaqueId, SplitHead}; +use crate::Error; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UpdateSplitHead { + pub revision: OpaqueId, + #[serde(deserialize_with = "required_option")] + pub expected_revision: Option, + #[serde(deserialize_with = "required_option")] + pub expected_head: Option, +} + +impl ControlClient { + /// Updates currentSplit using a complete result and explicit caller-owned preconditions. + /// Does not create a request ledger or depend on an editing draft. + /// # Errors + /// Returns capability, missing result, deletion admission, conditional or transport errors. + pub async fn update_split_head( + &self, + snapshot: &OpaqueId, + request: &UpdateSplitHead, + ) -> Result { + self.call( + "split-head-update", + "PUT", + &["snapshots", snapshot.as_str(), "split-head"], + Some(request), + ) + .await + .map_err(|error| error.with_identity(request.revision.as_str())) + } +} + +impl SplitHead { + /// Extracts the failed condition's observation; another writer may already have changed it. + #[must_use] + pub fn from_conflict(error: &Error) -> Option { + let Error::Service { status: 409, error } = error else { + return None; + }; + serde_json::from_value(error.observed_head.as_deref()?.clone()).ok() + } +} diff --git a/sdk/crates/crowdb-sdk-dataset/tests/split_head_test.rs b/sdk/crates/crowdb-sdk-dataset/tests/split_head_test.rs new file mode 100644 index 000000000..6d341147d --- /dev/null +++ b/sdk/crates/crowdb-sdk-dataset/tests/split_head_test.rs @@ -0,0 +1,97 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[allow(dead_code)] +#[path = "../../crowdb-sdk-core/tests/common/peer.rs"] +mod peer; + +use crowdb_sdk_dataset::{ + v2::{OpaqueId, SplitHead, UpdateSplitHead}, + DatasetClient, +}; +use serde_json::json; + +#[test] +fn conditions_are_required_nullable_values_and_older_errors_have_no_observation() { + let request = json!({"revision":"02".repeat(16),"expected_revision":null,"expected_head":null}); + serde_json::from_value::(request.clone()).unwrap(); + for field in ["expected_revision", "expected_head"] { + let mut invalid = request.clone(); + invalid.as_object_mut().unwrap().remove(field); + assert!(serde_json::from_value::(invalid).is_err()); + } + let error: crowdb_sdk_core::ServiceError = serde_json::from_value( + json!({"code":"conflict","message":"conflict","request_id":"test","retryable":false}), + ) + .unwrap(); + assert!(error.observed_head.is_none()); +} + +#[tokio::test] +async fn unsupported_head_updates_do_not_fall_back_to_draft_publication() { + let mut peer = + peer::TestPeer::start(r#"{"api_version":2,"capabilities":["split-revisions"]}"#, false).await; + let client = DatasetClient::new(&peer.endpoint, "test", None) + .unwrap() + .control(); + assert!(client + .update_split_head( + &OpaqueId::random(), + &UpdateSplitHead { + revision: OpaqueId::random(), + expected_revision: None, + expected_head: None + } + ) + .await + .is_err()); + assert!(peer + .requests + .recv() + .await + .unwrap() + .starts_with("GET /api/dataset/v2/capabilities ")); + assert!(peer.requests.try_recv().is_err()); +} + +#[tokio::test] +async fn a_conflict_can_supply_the_next_explicit_request_without_another_head_get() { + let mut peer = peer::TestPeer::start_statuses(r#"{"api_version":2,"capabilities":["split-head-update"],"revision":"02020202020202020202020202020202","generation":"0303030303030303030303030303030303030303030303030303030303030303","code":"conflict","message":"conflict","request_id":"test","retryable":false,"observed_head":{"snapshot":"01010101010101010101010101010101","revision":"04040404040404040404040404040404","generation":"0505050505050505050505050505050505050505050505050505050505050505"}}"#, false, vec![200, 409, 200, 200]).await; + let client = DatasetClient::new(&peer.endpoint, "test", None) + .unwrap() + .control(); + let snapshot = "01".repeat(16).parse().unwrap(); + let mut request = UpdateSplitHead { + revision: "02".repeat(16).parse().unwrap(), + expected_revision: None, + expected_head: None, + }; + let error = client.update_split_head(&snapshot, &request).await.unwrap_err(); + let current = SplitHead::from_conflict(&error).unwrap(); + assert_eq!(current.revision.as_ref().unwrap().as_str(), "04".repeat(16)); + request.expected_revision = current.revision; + request.expected_head = current.generation; + assert_eq!( + client + .update_split_head(&snapshot, &request) + .await + .unwrap() + .revision, + Some(request.revision) + ); + assert!(peer + .requests + .recv() + .await + .unwrap() + .starts_with("GET /api/dataset/v2/capabilities ")); + let first = peer.requests.recv().await.unwrap(); + let discovery = peer.requests.recv().await.unwrap(); + assert!(discovery.starts_with("GET /api/dataset/v2/capabilities ")); + let next = peer.requests.recv().await.unwrap(); + assert!(first.starts_with("PUT /api/dataset/v2/namespaces/dataset-ns/datasets/test/snapshots/")); + assert!(first.contains("\"expected_head\":null")); + assert!(next.contains(&format!("\"expected_head\":\"{}\"", "05".repeat(32)))); + assert!(next.contains(&format!("\"expected_revision\":\"{}\"", "04".repeat(16)))); + assert!(peer.requests.try_recv().is_err()); +} diff --git a/sdk/crates/crowdb-sdk-python/src/control.rs b/sdk/crates/crowdb-sdk-python/src/control.rs index 16393729f..cb6fa6cf1 100644 --- a/sdk/crates/crowdb-sdk-python/src/control.rs +++ b/sdk/crates/crowdb-sdk-python/src/control.rs @@ -75,6 +75,7 @@ async fn lifecycle(client: &v2::ControlClient, name: &str, value: &Value) -> Res "evaluate_split" => encode(&client.evaluate_split(&id()?, &request(value)?).await?), "prepare_split" => encode(&client.prepare_split(&id()?, &request(value)?).await?), "publish_split" => encode(&client.publish_split(&id()?, &request(value)?).await?), + "update_split_head" => encode(&client.update_split_head(&id()?, &request(value)?).await?), "abandon_split" => encode(&client.abandon_split(&id()?, &request(value)?).await?), "cleanup_split" => encode(&client.cleanup_split(&id()?).await?), "split_head" => encode(&client.split_head(&id()?).await?), diff --git a/sdk/crates/crowdb-sdk-python/src/lib.rs b/sdk/crates/crowdb-sdk-python/src/lib.rs index 502b73904..b8915bd53 100644 --- a/sdk/crates/crowdb-sdk-python/src/lib.rs +++ b/sdk/crates/crowdb-sdk-python/src/lib.rs @@ -274,7 +274,11 @@ fn failure(error: &Error) -> String { Error::Io(_) => ("io", 0, false), _ => ("invalid", 0, false), }; - serde_json::json!({"code":code,"status":status,"retryable":retryable,"message":error.to_string()}) + let observed_head = match error { + Error::Service { error, .. } | Error::Stream { error } => error.observed_head.as_deref(), + _ => None, + }; + serde_json::json!({"code":code,"status":status,"retryable":retryable,"message":error.to_string(),"observed_head":observed_head}) .to_string() } #[pymodule] diff --git a/sdk/crowdb_dataset/client.py b/sdk/crowdb_dataset/client.py index 9616da618..1f097b834 100644 --- a/sdk/crowdb_dataset/client.py +++ b/sdk/crowdb_dataset/client.py @@ -24,6 +24,7 @@ def __init__(self, record, identity=None): self.status = record.get("status", 0) self.retryable = record.get("retryable", False) self.identity = record.get("identity", identity) + self.observed_head = record.get("observed_head") class ConflictError(DatasetError): diff --git a/sdk/crowdb_dataset/control.py b/sdk/crowdb_dataset/control.py index 34f3b6c90..1d6edc062 100644 --- a/sdk/crowdb_dataset/control.py +++ b/sdk/crowdb_dataset/control.py @@ -79,6 +79,7 @@ def evaluate_split(self, operation, expected_revision, expected_page): def prepare_split(self, operation, expected_revision): return self._call("prepare_split", {"id": operation, "request": {"expected_revision": expected_revision}}) def publish_split(self, operation, request): return self._call("publish_split", {"id": operation, "request": request}) + def update_split_head(self, snapshot, request): return self._call("update_split_head", {"id": snapshot, "request": request}) def abandon_split(self, operation, expected_revision): return self._call("abandon_split", {"id": operation, "request": {"expected_revision": expected_revision}}) def cleanup_split(self, operation): return self._call("cleanup_split", {"id": operation}) diff --git a/sdk/tests/split_consumer.py b/sdk/tests/split_consumer.py index 0ce40d094..ba6492afb 100644 --- a/sdk/tests/split_consumer.py +++ b/sdk/tests/split_consumer.py @@ -8,6 +8,7 @@ import tempfile from pathlib import Path from crowdb_dataset import DatasetClient, AsyncDatasetClient, SplitSource, new_id, sample_id +from crowdb_dataset.client import ConflictError def prepare(client, operation, revision): @@ -122,7 +123,14 @@ def main(): page = preview["next_page"] else: raise AssertionError("Split evaluation budget") candidate = client.prepare_split(split_op, "1") - client.publish_split(split_op, {"attempt": new_id(), "revision": candidate["revision"], "expected_head": head["generation"]}) + update = {"revision": candidate["revision"], "expected_revision": revision, "expected_head": head["generation"]} + changed = client.update_split_head(snapshot, update) + try: + client.update_split_head(snapshot, dict(update, revision=revision)) + except ConflictError as error: + assert error.observed_head == dict(changed, snapshot=snapshot) + else: + raise AssertionError("stale Split update must conflict") assert client.head()["latest"] == snapshot source = SplitSource(snapshot, revision, "train/a b") assert next(client.pages("split_members", source=source))["entries"] == [sample] From 0c2b73fd8ed43e350392314348943e3a7640d0a6 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 18:38:23 +0800 Subject: [PATCH 122/258] Record split container validation and capability query follow-up --- doc/working/plan-access-simplification.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 4fe91d73f..1ce6132e4 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -427,7 +427,7 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 - [ ] **F1 请求级成本计数**：记录 GET/bounded GET/scan/CAS/PUT/delete、重复 key、冲突与 unknown、真实 routed RPC、串行阶段和复制字节；用测试 wrapper/已有 metrics，不加生产 每 RPC 日志或 profiling 锁。Files: Dataset/S3/Iceberg 测试 store、现有指标、`tools/sdk/`。 -- [ ] **F1.1 去掉 SDK 每次 control 调用的 capabilities GET**：R2.2b 抓到显式重试的请求 +- [~] **F1.1 去掉 SDK 每次 control 调用的 capabilities GET**：R2.2b 抓到显式重试的请求 为 capabilities GET → PUT → capabilities GET → PUT；没有 head GET，但仍多一次 Access 往返。能力是版本化路由/配置发现，不是鉴权或资源准入；研究按 client 复用已确认能力， 保持服务端每次真实鉴权/条件检查、缺失能力不降级、重新发现/滚动升级的失效语义。 @@ -480,9 +480,6 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 无条件覆盖与 expected-base 重试测试。断言总 GET+写，不仅是零写。Files: Dataset `tests/*cost*`、`read_query*`、upload/Composition/Split/inspection；S3/Iceberg 对应 repository 测试。 - [ ] **V2 HTTP/native/SDK 集成**：各受影响 route 的鉴权、错误、协议/SDK 一致性；保留删除准入、 迟到写者、非空 namespace、共享 owner、held buffer、cancel/drop 和损坏内容测试。Files: Access/native tests、`sdk/tests/`、native Python consumer。 -- [~] **V2.2 Split 清理同源容器验收**：稳定树构建新镜像，运行全部五个 Dataset 容器用例， - 包括真实 Chunk-KV/存储重启、Rust 与安装后 Python SDK 的直接 Split head 更新/观察值。 - 构建期间不改任何非忽略文件，保留 source guard；记录镜像、客户端与 artifact 身份。 - [ ] **V3 工程门禁与文档**：每批执行适用 fmt/clippy/测试，全部经 `pixi run`；同步 owning design/spec/OpenAPI/SDK。只修改文档时检查 diff、引用和任务状态，不为此跑整套服务测试。 - [ ] **V4 同源真实负载验收**：重新构建当前源码的镜像，运行现有 HTTP/native Rust/native Python @@ -510,6 +507,20 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **V2.2（2026-10-24）**：源码 `5f18bd58`、source tree + `1f22dcff0524d743893bc993824350827dc6049f` 稳定构建的新镜像 + `crowdb-node:access-cleanup-split-head-20261024`，OCI digest + `sha256:cd582599b79107dcd45ce814f642f0c949dfd30e68f431e0ffb23e91eb29724f`， + Docker identity `sha256:cee85d588e1c317e61d0bd462bc74b5cef65b058fe8e80dd7666c2d949b80c27`。 + 全部五个 Dataset 容器用例通过：发布/读取/重启、Split 完整候选和无新 receipt、鉴权、 + 实际 payload 重启、独立 Rust 与新安装 Python wheel 的直接更新/冲突观察值。 + 客户端同 revision；完整日志/身份/用时位于 + `.crowdb-runtime/artifacts/container-e2e/run-a8f9817b8c8b40958ff08325a52265d5/`， + status=pass，client build 33.35 s、SDK build 21.92 s、场景 54.58 s。 + build/import/验收均经 `pixi run`；首次 build 的 Git 快照写被沙箱只读 `.git` 拒绝， + 同命令提升权限后完成且保留 source guard；镜像只导入独立本地 tag。 + 此项是同主机 TCP 正确性/重启验收，不替代吞吐、HCA、独立 GC 发现或 Snapshot 主线清理。 + - **R2.2b（2026-10-24）**：新增 `PUT snapshots/{snapshot}/split-head` 与 `split-head-update` 能力，Rust/Python SDK 提供 `update_split_head`。仅携带完整 revision 和两个必须显式提供的 nullable expected base，不携带 attempt ID，不查 draft，不保存过程。 From becffcfbcf3c2462e34f74b214fefea0741b61f9 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 18:50:34 +0800 Subject: [PATCH 123/258] Reuse discovered SDK capabilities across client clones --- .../dataset/design-dataset-control.md | 11 + doc/working/plan-access-simplification.md | 25 +- sdk/README.md | 10 + .../crowdb-sdk-core/tests/common/peer.rs | 15 +- .../crowdb-sdk-dataset/src/capabilities.rs | 48 +++- sdk/crates/crowdb-sdk-dataset/src/client.rs | 4 + .../crowdb-sdk-dataset/src/v2/client.rs | 2 +- .../tests/capability_hint_test.rs | 219 ++++++++++++++++++ .../tests/split_head_test.rs | 4 +- .../tests/v2_contract_test.rs | 1 - sdk/tests/test_control.py | 19 ++ 11 files changed, 331 insertions(+), 27 deletions(-) create mode 100644 sdk/crates/crowdb-sdk-dataset/tests/capability_hint_test.rs diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index 1281a70d5..5f79ec322 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -223,6 +223,17 @@ cannot be omitted; u64s use decimal strings; binary IDs use canonical base64. Cl check capabilities before mutation and never downgrade a Split-aware operation to v1. Existing v1 routes, persisted read interpretation and binary framing remain available. Rust and Python control clients share transport/admission/retry/cancellation semantics. +SDK clones share a fixed atomic bitset of capabilities previously advertised by +the versioned endpoint. Positive hints remove discovery GETs from warm control +calls; absent capabilities are rediscovered, never downgraded. Explicit +`capabilities()` queries replace hints after a valid response and clear them for +an unsupported API version. Unknown transport outcomes do not imply capability +revocation. Concurrent cold calls may duplicate discovery under the existing +HTTP budget; there is no discovery lock or server session. Every actual operation +still passes server authentication, capacity, deletion admission and mutation +preconditions. Server redeployment can invalidate a hint and make the operation +fail; explicit discovery refreshes it. A preflight GET could not eliminate that +deployment window either. - **I1 — Immutable data:** committed Snapshot content never changes. - **I2 — Independent revision:** Split publication cannot move Dataset latest. diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 1ce6132e4..3eb39fc73 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -427,18 +427,6 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 - [ ] **F1 请求级成本计数**：记录 GET/bounded GET/scan/CAS/PUT/delete、重复 key、冲突与 unknown、真实 routed RPC、串行阶段和复制字节；用测试 wrapper/已有 metrics，不加生产 每 RPC 日志或 profiling 锁。Files: Dataset/S3/Iceberg 测试 store、现有指标、`tools/sdk/`。 -- [~] **F1.1 去掉 SDK 每次 control 调用的 capabilities GET**：R2.2b 抓到显式重试的请求 - 为 capabilities GET → PUT → capabilities GET → PUT；没有 head GET，但仍多一次 Access - 往返。能力是版本化路由/配置发现，不是鉴权或资源准入；研究按 client 复用已确认能力， - 保持服务端每次真实鉴权/条件检查、缺失能力不降级、重新发现/滚动升级的失效语义。 - 不增加热路径锁或 TTL 准入缓存。先写清语义，再补冷启动并发、服务失效和实际请求数。 - Files: SDK `client.rs`、capabilities、v2 `client.rs`、native Python dispatch 与协议 fixture。 - 已核实 `DatasetHttpService` 的能力由版本化路由和构造时的 uploads 配置决定，鉴权与资源 - 条件每个真实请求仍独立检查。拟在同一 client/clone 中以固定有界 atomic bitset 复用成功 - 发现的能力，不缓存拒绝/缺失、不保存服务端状态；显式 capabilities() 仍重新发现并替换 - 提示。冷启动并发可重复有界发现，避免 single-flight 锁。服务端重启/能力变化时真实路由 - 仍拒绝不支持的操作，没有 v1 fallback；预先 GET 本来也不能消除发现与写之间的重启窗口。 - 不将该提示用于跳过鉴权、删除准入或 expected-base 检查；未知传输结果不解释成能力撤销。 - [ ] **F2 复核 legacy/多源读取的重复准入**：现代 continuation 的 selection→HTTP/native delivery 已共用请求局部的 Snapshot/schema/membership，Split 同 Snapshot 也复用准入。 剩余复核 legacy `DatasetReadService`、多源/祖先元数据，独立入口和不同 Snapshot 仍各自准入； @@ -507,6 +495,19 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，F1.1**：同一 SDK client/clone/control wrapper 共享固定有界 atomic bitset， + 复用已成功发现的路由能力。连续五次 head 读取实测由十次 HTTP 请求降为六次；Split + 冲突显式重试由 capabilities GET → PUT → capabilities GET → PUT 降为 GET → PUT → PUT， + 使用冲突返回的 observed head，不补 head GET。每个业务请求仍独立取凭据并由服务端鉴权、 + 检查删除与 expected base；未新增锁、持久化、TTL 准入缓存或依赖。 + 显式 capabilities() 总是重新发现并替换提示；不缓存缺失能力；未知传输/解码结果不当成 + 撤销，已解码的不支持版本清空提示。冷启动并发允许有界重复发现，无 single-flight 状态； + 取消不安装提示，服务能力变化由真实版本化路由拒绝，不能降级写入。 + 独立 SDK workspace 的 29 项测试、all-target clippy、fmt、边界检查通过；sdist 在仓库外 + 独立测试并构建 wheel，安装后九项 Python 单元测试及 Rust/Python 两项真实 HTTP Split + consumer 通过。wheel 位于 `.crowdb-runtime/artifacts/sdk-capabilities-20261024`。 + SDK 变更不改变服务端镜像；此处是实际请求数与功能证据，没有声称 TPS 提升比例。 + - **V2.2（2026-10-24）**：源码 `5f18bd58`、source tree `1f22dcff0524d743893bc993824350827dc6049f` 稳定构建的新镜像 `crowdb-node:access-cleanup-split-head-20261024`，OCI digest diff --git a/sdk/README.md b/sdk/README.md index cc9b3074f..124c06bf5 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -189,6 +189,16 @@ HTTP pools, credentials and admission budgets. Every operation checks its server capability before sending a request; unsupported mutations never fall back to v1. Existing `DatasetClient` methods retain their v1 contracts. +Successful capability discovery supplies a bounded set of hints shared by client +clones and control wrappers. Warm calls send the business request directly; +authentication, deletion admission and conditional updates still run on the server. +Missing capabilities and failed discoveries are not remembered as denials. +`capabilities()` always queries the server and replaces the hints on a valid reply; +an unsupported API version clears them. Refresh discovery or create a new client +after changing server features. Concurrent cold calls may repeat discovery within +the existing HTTP admission budget. The hint has no TTL and is not a cached read +permission or promise that a redeployed endpoint still supports the operation. + - `OpaqueId` preserves operation/Snapshot/revision IDs; `Generation` is an opaque head precondition. `Decimal` encodes exact u64 values as JSON strings, and `SampleId` preserves binary IDs through canonical base64. diff --git a/sdk/crates/crowdb-sdk-core/tests/common/peer.rs b/sdk/crates/crowdb-sdk-core/tests/common/peer.rs index a090dfdc6..9631661d5 100644 --- a/sdk/crates/crowdb-sdk-core/tests/common/peer.rs +++ b/sdk/crates/crowdb-sdk-core/tests/common/peer.rs @@ -24,7 +24,11 @@ impl TestPeer { Self::start_statuses(body, gated, vec![200]).await } pub async fn start_statuses(body: &'static str, gated: bool, statuses: Vec) -> Self { - let statuses = Arc::new(statuses); + Self::start_responses(statuses.into_iter().map(|status| (status, body)).collect(), gated).await + } + pub async fn start_responses(responses: Vec<(u16, &'static str)>, gated: bool) -> Self { + assert!(!responses.is_empty()); + let responses = Arc::new(responses); let calls = Arc::new(AtomicUsize::new(0)); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let endpoint = format!("http://{}", listener.local_addr().unwrap()); @@ -38,7 +42,7 @@ impl TestPeer { tokio::select! { accepted=listener.accept()=> { let (socket,_)=accepted.unwrap(); counter.fetch_add(1,Ordering::AcqRel); - clients.spawn(serve(socket,tx.clone(),gate.clone(),body,statuses.clone(),calls.clone())); + clients.spawn(serve(socket,tx.clone(),gate.clone(),responses.clone(),calls.clone())); }, Some(_)=clients.join_next(),if !clients.is_empty()=>{}, } @@ -63,8 +67,7 @@ async fn serve( mut socket: TcpStream, tx: mpsc::Sender, mut gate: watch::Receiver, - body: &str, - statuses: Arc>, + responses: Arc>, calls: Arc, ) { loop { @@ -88,13 +91,13 @@ async fn serve( tx.send(format!("{header}{}", String::from_utf8_lossy(&data))) .await .unwrap(); + let index = calls.fetch_add(1, Ordering::AcqRel); + let (status, body) = responses[index.min(responses.len() - 1)]; while !*gate.borrow() { if gate.changed().await.is_err() { return; } } - let index = calls.fetch_add(1, Ordering::AcqRel); - let status = statuses[index.min(statuses.len() - 1)]; let response = format!( "HTTP/1.1 {status} Result\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len() diff --git a/sdk/crates/crowdb-sdk-dataset/src/capabilities.rs b/sdk/crates/crowdb-sdk-dataset/src/capabilities.rs index cc4c02a21..4cdef13da 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/capabilities.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/capabilities.rs @@ -5,6 +5,27 @@ use crate::{DatasetClient, Error}; use serde::{Deserialize, Serialize}; +use std::sync::atomic::Ordering; + +const KNOWN_CAPABILITIES: [&str; 10] = [ + "head-inspection", + "bounded-inspection", + "inspection-preparation", + "dataset-management", + "split-revisions", + "split-head-update", + "stateless-reads", + "uploads", + "composition", + "snapshot-reclaim", +]; + +fn capability_bit(capability: &str) -> u64 { + KNOWN_CAPABILITIES + .iter() + .position(|known| *known == capability) + .map_or(0, |index| 1 << index) +} #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct DatasetCapabilities { @@ -49,14 +70,15 @@ impl DatasetClient { /// # Errors /// Rejects unsupported capability, missing Dataset or invalid server responses. pub async fn head(&self) -> Result { - self.capabilities().await?.require(&["head-inspection"])?; + self.require_capability("head-inspection").await?; let mut path = self.path(&["head"]); path[2] = "v2"; self.http.json_idempotent::<(), _>("GET", &path, None).await } - /// Discovers capabilities without mutating a Dataset. A v1-only server returns - /// an explicit service error; the SDK never retries a v2 write through v1. + /// Always refreshes capabilities and replaces the positive hints shared by clones. + /// Hints never bypass server authentication, resource admission or mutation conditions. + /// A v1-only server returns an error; mutations never fall back to v1. /// # Errors /// Returns unsupported-version, service, decoding or transport errors. pub async fn capabilities(&self) -> Result { @@ -64,7 +86,25 @@ impl DatasetClient { .http .json_idempotent::<(), _>("GET", &["api", "dataset", "v2", "capabilities"], None) .await?; - capabilities.require(&[])?; + let valid = capabilities.require(&[]); + let bits = if valid.is_ok() { + capabilities + .capabilities + .iter() + .fold(0, |bits, capability| bits | capability_bit(capability)) + } else { + 0 + }; + self.capability_hints.store(bits, Ordering::Release); + valid?; Ok(capabilities) } + + pub(crate) async fn require_capability(&self, capability: &str) -> Result<(), Error> { + let bit = capability_bit(capability); + if bit != 0 && self.capability_hints.load(Ordering::Acquire) & bit != 0 { + return Ok(()); + } + self.capabilities().await?.require(&[capability]) + } } diff --git a/sdk/crates/crowdb-sdk-dataset/src/client.rs b/sdk/crates/crowdb-sdk-dataset/src/client.rs index 45dd3cc2b..bb98791e3 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/client.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/client.rs @@ -6,12 +6,14 @@ use crate::{ }; use crowdb_sdk_core::{HttpClient, HttpConfig}; use serde::{de::DeserializeOwned, Serialize}; +use std::sync::{atomic::AtomicU64, Arc}; #[derive(Clone)] pub struct DatasetClient { pub(crate) http: HttpClient, namespace: String, name: String, + pub(crate) capability_hints: Arc, } impl DatasetClient { @@ -34,6 +36,7 @@ impl DatasetClient { http: HttpClient::with_config(endpoint, bearer, config)?, namespace: "dataset-ns".into(), name: name.into(), + capability_hints: Arc::new(AtomicU64::new(0)), }) } @@ -50,6 +53,7 @@ impl DatasetClient { http: HttpClient::with_provider(endpoint, credentials, config)?, namespace: "dataset-ns".into(), name: name.into(), + capability_hints: Arc::new(AtomicU64::new(0)), }) } pub(crate) fn path<'a>(&'a self, suffix: &[&'a str]) -> Vec<&'a str> { diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2/client.rs b/sdk/crates/crowdb-sdk-dataset/src/v2/client.rs index 32cc28701..1ff77b142 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/v2/client.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/v2/client.rs @@ -26,7 +26,7 @@ impl ControlClient { path } pub(crate) async fn require(&self, capability: &str) -> Result<(), Error> { - self.inner.capabilities().await?.require(&[capability]) + self.inner.require_capability(capability).await } pub(crate) async fn call( &self, diff --git a/sdk/crates/crowdb-sdk-dataset/tests/capability_hint_test.rs b/sdk/crates/crowdb-sdk-dataset/tests/capability_hint_test.rs new file mode 100644 index 000000000..96d183fa0 --- /dev/null +++ b/sdk/crates/crowdb-sdk-dataset/tests/capability_hint_test.rs @@ -0,0 +1,219 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[allow(dead_code)] +#[path = "../../crowdb-sdk-core/tests/common/peer.rs"] +mod peer; + +use crowdb_sdk_core::{CredentialProvider, HttpConfig}; +use crowdb_sdk_dataset::{DatasetClient, Error}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +const SUPPORTED: &str = r#"{"api_version":2,"capabilities":["head-inspection","bounded-inspection"],"latest":null,"stable":null,"generation":null}"#; +const EMPTY: &str = r#"{"api_version":2,"capabilities":[]}"#; +const UNKNOWN: &str = r#"{"api_version":3,"capabilities":["head-inspection"]}"#; +const DENIED: &str = + r#"{"code":"unauthorized","message":"unauthorized","request_id":"test","retryable":false}"#; + +struct TestCredentials(AtomicUsize); +impl CredentialProvider for TestCredentials { + fn bearer(&self) -> Result, Error> { + Ok(Some(format!("token-{}", self.0.fetch_add(1, Ordering::Relaxed)))) + } +} + +#[tokio::test] +async fn warm_clones_reuse_discovery_and_each_request_gets_current_credentials() { + let mut peer = peer::TestPeer::start(SUPPORTED, false).await; + let credentials = Arc::new(TestCredentials(AtomicUsize::new(0))); + let client = DatasetClient::with_credential_provider( + &peer.endpoint, + "test", + credentials.clone(), + HttpConfig::default(), + ) + .unwrap(); + let credential_checks = credentials.0.load(Ordering::Relaxed); + client.head().await.unwrap(); + let mut tasks = tokio::task::JoinSet::new(); + for _ in 0..4 { + let clone = client.clone(); + tasks.spawn(async move { clone.control().head().await }); + } + while let Some(result) = tasks.join_next().await { + result.unwrap().unwrap(); + } + assert_eq!(credentials.0.load(Ordering::Relaxed) - credential_checks, 6); + let mut discovery = 0; + let mut head = 0; + for _ in 0..6 { + let request = peer.requests.recv().await.unwrap(); + assert!(request + .to_ascii_lowercase() + .contains("authorization: bearer token-")); + if request.starts_with("GET /api/dataset/v2/capabilities ") { + discovery += 1; + } else { + assert!(request.contains("/datasets/test/head ")); + head += 1; + } + } + assert_eq!((discovery, head), (1, 5)); + assert!(peer.requests.try_recv().is_err()); + client.capabilities().await.unwrap(); + client.control().head().await.unwrap(); + assert!(peer + .requests + .recv() + .await + .unwrap() + .starts_with("GET /api/dataset/v2/capabilities ")); + assert!(peer + .requests + .recv() + .await + .unwrap() + .contains("/datasets/test/head ")); + assert!(peer.requests.try_recv().is_err()); +} + +#[tokio::test] +async fn refresh_removes_lost_capabilities_and_missing_features_are_not_cached() { + let mut peer = peer::TestPeer::start_responses( + vec![ + (200, SUPPORTED), + (200, SUPPORTED), + (200, EMPTY), + (200, EMPTY), + (200, SUPPORTED), + (200, SUPPORTED), + ], + false, + ) + .await; + let client = DatasetClient::new(&peer.endpoint, "test", None).unwrap(); + client.head().await.unwrap(); + assert!(client.capabilities().await.unwrap().capabilities.is_empty()); + assert!(matches!(client.control().head().await, Err(Error::Invalid(_)))); + client.control().head().await.unwrap(); + let mut paths = Vec::new(); + for _ in 0..6 { + paths.push(peer.requests.recv().await.unwrap()); + } + assert!(paths[0].starts_with("GET /api/dataset/v2/capabilities ")); + assert!(paths[1].contains("/datasets/test/head ")); + for request in &paths[2..5] { + assert!(request.starts_with("GET /api/dataset/v2/capabilities ")); + } + assert!(paths[5].contains("/datasets/test/head ")); + assert!(peer.requests.try_recv().is_err()); +} + +#[tokio::test] +async fn authentication_and_unsupported_versions_still_fail_closed_after_discovery() { + let mut peer = peer::TestPeer::start_responses( + vec![(200, SUPPORTED), (401, DENIED), (200, UNKNOWN), (200, UNKNOWN)], + false, + ) + .await; + let client = DatasetClient::new(&peer.endpoint, "test", None).unwrap(); + client.capabilities().await.unwrap(); + assert!(matches!( + client.control().head().await, + Err(Error::Service { status: 401, .. }) + )); + assert!(matches!(client.capabilities().await, Err(Error::Invalid(_)))); + assert!(matches!(client.control().head().await, Err(Error::Invalid(_)))); + let first = peer.requests.recv().await.unwrap(); + let denied = peer.requests.recv().await.unwrap(); + assert!(first.starts_with("GET /api/dataset/v2/capabilities ")); + assert!(denied.contains("/datasets/test/head ")); + for _ in 0..2 { + assert!(peer + .requests + .recv() + .await + .unwrap() + .starts_with("GET /api/dataset/v2/capabilities ")); + } + assert!(peer.requests.try_recv().is_err()); +} + +#[tokio::test] +async fn cold_concurrent_calls_complete_without_global_serialization() { + let mut peer = peer::TestPeer::start(SUPPORTED, true).await; + let client = DatasetClient::new(&peer.endpoint, "test", None).unwrap(); + let mut tasks = tokio::task::JoinSet::new(); + for _ in 0..4 { + let clone = client.clone(); + tasks.spawn(async move { clone.control().head().await }); + } + let request = tokio::time::timeout(Duration::from_secs(2), peer.requests.recv()) + .await + .unwrap() + .unwrap(); + assert!(request.starts_with("GET /api/dataset/v2/capabilities ")); + peer.release.send(true).unwrap(); + while let Some(result) = tokio::time::timeout(Duration::from_secs(2), tasks.join_next()) + .await + .unwrap() + { + result.unwrap().unwrap(); + } + let mut discovery = 1; + let mut head = 0; + while let Ok(request) = peer.requests.try_recv() { + if request.starts_with("GET /api/dataset/v2/capabilities ") { + discovery += 1; + } else { + assert!(request.contains("/datasets/test/head ")); + head += 1; + } + } + assert!((1..=4).contains(&discovery)); + assert_eq!(head, 4); + client.control().head().await.unwrap(); + assert!(peer + .requests + .recv() + .await + .unwrap() + .contains("/datasets/test/head ")); + assert!(peer.requests.try_recv().is_err()); +} + +#[tokio::test] +async fn cancelling_discovery_does_not_publish_a_hint_or_leave_waiting_state() { + let mut peer = peer::TestPeer::start_responses(vec![(200, SUPPORTED), (200, EMPTY)], true).await; + let client = DatasetClient::new(&peer.endpoint, "test", None).unwrap(); + let clone = client.clone(); + let task = tokio::spawn(async move { clone.control().head().await }); + assert!(peer + .requests + .recv() + .await + .unwrap() + .starts_with("GET /api/dataset/v2/capabilities ")); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + peer.release.send(true).unwrap(); + assert!(matches!( + tokio::time::timeout(Duration::from_secs(2), client.control().head()) + .await + .unwrap(), + Err(Error::Invalid(_)) + )); + assert!(peer + .requests + .recv() + .await + .unwrap() + .starts_with("GET /api/dataset/v2/capabilities ")); + assert!(peer.requests.try_recv().is_err()); +} diff --git a/sdk/crates/crowdb-sdk-dataset/tests/split_head_test.rs b/sdk/crates/crowdb-sdk-dataset/tests/split_head_test.rs index 6d341147d..292b3276a 100644 --- a/sdk/crates/crowdb-sdk-dataset/tests/split_head_test.rs +++ b/sdk/crates/crowdb-sdk-dataset/tests/split_head_test.rs @@ -56,7 +56,7 @@ async fn unsupported_head_updates_do_not_fall_back_to_draft_publication() { #[tokio::test] async fn a_conflict_can_supply_the_next_explicit_request_without_another_head_get() { - let mut peer = peer::TestPeer::start_statuses(r#"{"api_version":2,"capabilities":["split-head-update"],"revision":"02020202020202020202020202020202","generation":"0303030303030303030303030303030303030303030303030303030303030303","code":"conflict","message":"conflict","request_id":"test","retryable":false,"observed_head":{"snapshot":"01010101010101010101010101010101","revision":"04040404040404040404040404040404","generation":"0505050505050505050505050505050505050505050505050505050505050505"}}"#, false, vec![200, 409, 200, 200]).await; + let mut peer = peer::TestPeer::start_statuses(r#"{"api_version":2,"capabilities":["split-head-update"],"revision":"02020202020202020202020202020202","generation":"0303030303030303030303030303030303030303030303030303030303030303","code":"conflict","message":"conflict","request_id":"test","retryable":false,"observed_head":{"snapshot":"01010101010101010101010101010101","revision":"04040404040404040404040404040404","generation":"0505050505050505050505050505050505050505050505050505050505050505"}}"#, false, vec![200, 409, 200]).await; let client = DatasetClient::new(&peer.endpoint, "test", None) .unwrap() .control(); @@ -86,8 +86,6 @@ async fn a_conflict_can_supply_the_next_explicit_request_without_another_head_ge .unwrap() .starts_with("GET /api/dataset/v2/capabilities ")); let first = peer.requests.recv().await.unwrap(); - let discovery = peer.requests.recv().await.unwrap(); - assert!(discovery.starts_with("GET /api/dataset/v2/capabilities ")); let next = peer.requests.recv().await.unwrap(); assert!(first.starts_with("PUT /api/dataset/v2/namespaces/dataset-ns/datasets/test/snapshots/")); assert!(first.contains("\"expected_head\":null")); diff --git a/sdk/crates/crowdb-sdk-dataset/tests/v2_contract_test.rs b/sdk/crates/crowdb-sdk-dataset/tests/v2_contract_test.rs index d8d96f41f..4aecc7c51 100644 --- a/sdk/crates/crowdb-sdk-dataset/tests/v2_contract_test.rs +++ b/sdk/crates/crowdb-sdk-dataset/tests/v2_contract_test.rs @@ -118,7 +118,6 @@ async fn pages_are_lazy_bounded_and_reject_a_repeated_continuation() { assert!(peer.requests.try_recv().is_err()); assert!(pages.next_page().await.is_err()); assert_eq!(pages.continuation(), Some("YQ==")); - peer.requests.recv().await.unwrap(); assert!(peer.requests.recv().await.unwrap().contains("after=YQ%3D%3D")); } diff --git a/sdk/tests/test_control.py b/sdk/tests/test_control.py index 3b6a99ec3..5b964ba72 100644 --- a/sdk/tests/test_control.py +++ b/sdk/tests/test_control.py @@ -48,6 +48,25 @@ def __exit__(self, *_): class TestControl(unittest.TestCase): + def test_successful_capabilities_are_reused_across_control_wrappers(self): + with TestControlPeer(["head-inspection"]) as peer, DatasetClient(peer.endpoint, "test") as owner: + owner.control().head() + owner.control().head() + paths = [peer.requests.get(timeout=10) for _ in range(3)] + self.assertEqual(sum(path.endswith("capabilities") for path in paths), 1) + self.assertTrue(peer.requests.empty()) + peer.capabilities = [] + owner.control().capabilities() + with self.assertRaises(DatasetError): owner.control().head() + self.assertTrue(peer.requests.get(timeout=10).endswith("capabilities")) + self.assertTrue(peer.requests.get(timeout=10).endswith("capabilities")) + self.assertTrue(peer.requests.empty()) + peer.capabilities = ["head-inspection"] + self.assertIsNone(owner.control().head()["latest"]) + self.assertTrue(peer.requests.get(timeout=10).endswith("capabilities")) + self.assertTrue(peer.requests.get(timeout=10).endswith("head")) + self.assertTrue(peer.requests.empty()) + def test_exact_integer_and_unavailable_mutation(self): self.assertEqual(decimal(2**64-1), "18446744073709551615") for value in (True, 1.0, -1, 2**64): From 80b9326d0d8504299dc0af6bb0ad428766ebcafe Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 19:11:54 +0800 Subject: [PATCH 124/258] Make complete snapshot results independent of head publication --- app/crowdb-access-server/src/dataset/api.rs | 2 +- .../src/dataset/api/publication.rs | 3 +- .../src/dataset/legacy.rs | 2 +- .../dataset_complete_result_contract_test.rs | 68 ++++ .../dataset_composition_contract_test.rs | 2 +- .../tests/dataset_contract_test.rs | 4 +- .../tests/dataset_inspection_contract_test.rs | 8 +- .../dataset/design-dataset-control.md | 32 +- .../dataset/spec-crowdb-dataset.md | 25 +- doc/working/plan-access-simplification.md | 24 +- lib/crowdb-access-dataset/src/authority.rs | 54 +-- .../src/authority/access.rs | 1 + .../src/authority/access/reclaim.rs | 44 ++- .../src/authority/publication.rs | 36 +- .../src/authority/publication/candidate.rs | 78 ++-- .../src/authority/publication/commit.rs | 5 +- .../src/authority/publication/complete.rs | 94 +++++ .../src/authority/publication/legacy.rs | 8 +- .../src/authority/publication/visibility.rs | 30 +- .../src/composition/candidates.rs | 2 +- .../src/composition/edit.rs | 2 +- .../src/composition/prepare.rs | 42 ++- .../src/composition/uploads/prepare.rs | 12 + .../src/composition/uploads/release.rs | 2 +- .../src/inspection/reclaim.rs | 9 +- .../src/split/cleanup.rs | 9 +- .../src/split/initial.rs | 25 +- lib/crowdb-access-dataset/src/transport.rs | 9 +- .../tests/candidate_publication_test.rs | 48 ++- .../tests/common/snapshot_complete.rs | 166 +++++++++ .../tests/composition_abandon_test.rs | 20 +- .../tests/composition_initial_test.rs | 6 +- .../tests/composition_upload_test.rs | 25 +- .../tests/inspection_index_test.rs | 8 + .../tests/inspection_pages_test.rs | 20 +- .../tests/inspection_visibility_test.rs | 12 +- .../tests/legacy_publication_test.rs | 33 ++ .../tests/legacy_read_cost_test.rs | 6 +- .../tests/projection_test.rs | 8 + .../tests/snapshot_branch_reclaim_test.rs | 7 +- .../tests/snapshot_complete_result_test.rs | 332 ++++++++++++++++++ 41 files changed, 1079 insertions(+), 244 deletions(-) create mode 100644 app/crowdb-access-server/tests/dataset_complete_result_contract_test.rs create mode 100644 lib/crowdb-access-dataset/src/authority/publication/complete.rs create mode 100644 lib/crowdb-access-dataset/tests/common/snapshot_complete.rs create mode 100644 lib/crowdb-access-dataset/tests/snapshot_complete_result_test.rs diff --git a/app/crowdb-access-server/src/dataset/api.rs b/app/crowdb-access-server/src/dataset/api.rs index bf9b2cf18..8609ba8a5 100644 --- a/app/crowdb-access-server/src/dataset/api.rs +++ b/app/crowdb-access-server/src/dataset/api.rs @@ -141,7 +141,7 @@ impl DatasetHttpService { json(&conversion::snapshot(record, authority.latest(&identity).await?)) } (Method::GET, ["snapshots", id, "manifest"]) => { - let manifest = authority.get_manifest(&identity, snapshot(id)?).await?; + let manifest = authority.read_manifest(&identity, snapshot(id)?).await?; json(&conversion::public_manifest(manifest, &identity)?) } (Method::PUT, ["names", "stable"]) => { diff --git a/app/crowdb-access-server/src/dataset/api/publication.rs b/app/crowdb-access-server/src/dataset/api/publication.rs index 12f488bca..11dd91980 100644 --- a/app/crowdb-access-server/src/dataset/api/publication.rs +++ b/app/crowdb-access-server/src/dataset/api/publication.rs @@ -56,7 +56,8 @@ impl DatasetHttpService { .publish_snapshot_with_operation(identity, parent, bytes, operation) .await }; - let _snapshot = result?; + let snapshot = result?; + authority.complete_snapshot_candidate(identity, snapshot).await?; json(&conversion::snapshot( authority.publication_status(identity, operation).await?, authority.latest(identity).await?, diff --git a/app/crowdb-access-server/src/dataset/legacy.rs b/app/crowdb-access-server/src/dataset/legacy.rs index e313122e4..a2f9f107f 100644 --- a/app/crowdb-access-server/src/dataset/legacy.rs +++ b/app/crowdb-access-server/src/dataset/legacy.rs @@ -198,7 +198,7 @@ impl DatasetHttpService { match self .reads .authority() - .get_manifest(self.reads.identity(), request.snapshot) + .read_manifest(self.reads.identity(), request.snapshot) .await { Ok(manifest) => encode(StatusCode::OK, &DatasetManifestResponse { manifest }), diff --git a/app/crowdb-access-server/tests/dataset_complete_result_contract_test.rs b/app/crowdb-access-server/tests/dataset_complete_result_contract_test.rs new file mode 100644 index 000000000..e455420ea --- /dev/null +++ b/app/crowdb-access-server/tests/dataset_complete_result_contract_test.rs @@ -0,0 +1,68 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/dataset.rs"] +mod dataset; +#[path = "../../../lib/crowdb-access-dataset/tests/common/publication.rs"] +mod publication; + +use crowdb_access_dataset::{ + DatasetAuthority, DatasetIdentity, DatasetSnapshotRequest, NamespacePath, OperationId, +}; +use dataset::{TestDatasetHttp, TestStore}; +use reqwest::Client; +use std::sync::Arc; + +async fn manifests( + client: &Client, + origin: &str, + snapshot: crowdb_access_dataset::SnapshotId, + expected: u16, +) { + let path = format!( + "{origin}/api/dataset/v1/namespaces/dataset-ns/datasets/legacy/snapshots/{snapshot}/manifest" + ); + let response = client.get(path).bearer_auth("secret").send().await.unwrap(); + let actual = response.status().as_u16(); + assert_eq!(actual, expected, "{}", response.text().await.unwrap()); + let response = client + .post(format!("{origin}/v1/dataset/manifest")) + .bearer_auth("secret") + .body(bincode::serialize(&DatasetSnapshotRequest { snapshot }).unwrap()) + .send() + .await + .unwrap(); + let actual = response.status().as_u16(); + let body = response.bytes().await.unwrap(); + assert_eq!(actual, expected, "{body:?}"); +} + +#[tokio::test] +async fn json_and_binary_manifest_reads_admit_only_complete_live_resources() { + let store = Arc::new(TestStore::default()); + let authority = DatasetAuthority::new(store.clone()); + let identity = DatasetIdentity::new(NamespacePath::root(), "legacy").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + let snapshot = authority + .prepare_snapshot_with_operation( + &identity, + None, + publication::publication_manifest(1), + OperationId::random(), + ) + .await + .unwrap(); + let test = TestDatasetHttp::start(store).await; + let client = Client::new(); + manifests(&client, &test.origin, snapshot, 404).await; + authority + .complete_snapshot_candidate(&identity, snapshot) + .await + .unwrap(); + assert_eq!(authority.latest(&identity).await.unwrap(), None); + manifests(&client, &test.origin, snapshot, 200).await; + authority.begin_reclaim(&identity, snapshot).await.unwrap(); + manifests(&client, &test.origin, snapshot, 409).await; + drop(client); + test.stop().await; +} diff --git a/app/crowdb-access-server/tests/dataset_composition_contract_test.rs b/app/crowdb-access-server/tests/dataset_composition_contract_test.rs index b52d2f237..e7cf73df0 100644 --- a/app/crowdb-access-server/tests/dataset_composition_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_composition_contract_test.rs @@ -50,7 +50,7 @@ async fn uploaded_composition_publishes_initial_splits_and_branches_from_a_froze response(client.delete(format!("{root}/uploads/{upload}")), 409).await; let prepared = composition::prepare(&client, &draft, "2", "1").await; let snapshot = prepared["preparation"]["snapshot"].as_str().unwrap(); - response(client.get(format!("{root}/snapshots/{snapshot}/summary")), 404).await; + response(client.get(format!("{root}/snapshots/{snapshot}/summary")), 200).await; let receipt = composition::publish( &client, &root, diff --git a/app/crowdb-access-server/tests/dataset_contract_test.rs b/app/crowdb-access-server/tests/dataset_contract_test.rs index d99aa28a0..e44e4dc57 100644 --- a/app/crowdb-access-server/tests/dataset_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_contract_test.rs @@ -63,7 +63,7 @@ async fn json_prepare_publish_retry_and_restart_preserve_one_candidate() { 200, ) .await; - assert_eq!(prepared["state"], "prepared"); + assert_eq!(prepared["state"], "published"); assert_eq!(prepared["is_latest"], false); let opened = call(&client, &test, Method::GET, ROOT, None, 200).await; assert_eq!(opened["latest"], Value::Null); @@ -73,7 +73,7 @@ async fn json_prepare_publish_retry_and_restart_preserve_one_candidate() { Method::GET, &format!("{ROOT}/snapshots/{}", prepared["id"].as_str().unwrap()), None, - 404, + 200, ) .await; drop(client); diff --git a/app/crowdb-access-server/tests/dataset_inspection_contract_test.rs b/app/crowdb-access-server/tests/dataset_inspection_contract_test.rs index d9da18ec6..4313bbf41 100644 --- a/app/crowdb-access-server/tests/dataset_inspection_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_inspection_contract_test.rs @@ -166,7 +166,7 @@ async fn catalog_continuations_do_not_cross_scope_or_repeat_entries() { } #[tokio::test] -async fn prepared_snapshot_is_hidden_from_metadata_and_preparation_status() { +async fn complete_prepared_snapshot_is_visible_without_updating_latest() { let test = TestDatasetHttp::start(Arc::new(TestStore::default())).await; let client = Client::new(); let (root, parent) = fixture(&client, &test).await; @@ -185,11 +185,11 @@ async fn prepared_snapshot_is_hidden_from_metadata_and_preparation_status() { .await; let id = prepared["id"].as_str().unwrap(); let inventory = response(client.get(format!("{root}/snapshot-inventory")), 200).await; - assert_eq!(inventory["entries"].as_array().unwrap().len(), 1); + assert_eq!(inventory["entries"].as_array().unwrap().len(), 2); for action in ["summary", "schema", "samples", "inspection"] { - response(client.get(format!("{root}/snapshots/{id}/{action}")), 404).await; + response(client.get(format!("{root}/snapshots/{id}/{action}")), 200).await; } - response(client.post(format!("{root}/snapshots/{id}/inspection")), 404).await; + response(client.post(format!("{root}/snapshots/{id}/inspection")), 200).await; drop(client); test.stop().await; } diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index 5f79ec322..c074e2424 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -22,9 +22,15 @@ bounded control operations. Payload delivery retains the existing binary framing ## 1. Identity and authority A Dataset has opaque latest/stable Snapshot references. A Snapshot may declare any -committed same-Dataset parent or no parent. Its parent is independent of the head +complete available same-Dataset parent or no parent. Its parent is independent of the head generation used for publication CAS. Multiple roots and off-latest branches remain -discoverable. Prepared records are not committed evidence. +discoverable. A preparation binding or a Published flag alone is not a complete result. +The final Snapshot resource root is installed after Manifest/index rows, the initial +Split revision and necessary payload ownership are ready. The current root uses +version 2 of the existing candidate binding; version 1 remains an incomplete legacy +preparation unless old successful-publication evidence exists. Complete results are +readable, enumerable, retainable and usable as parents/stable targets without winning +latest. ID reads consult this root rather than publication receipts or mutable heads. Publication has a candidate identity and a separate caller-owned attempt identity. Each attempt binds its target and exact expected head generation. Successful commit @@ -65,7 +71,8 @@ Evaluation advances revision-bound windows of at most twenty Samples. Durable redo windows precede row writes; CAS progress and content bindings prevent stale work from becoming the accepted result. Preparation validates the server-assembled Manifest, initial Split set and all candidate upload claims before publication. -The data head transition makes data and initial Splits visible together. +The complete resource root makes data and initial Splits available together; the +independent data head transition selects that resource as latest. ## 3. Split evaluation and publication @@ -115,10 +122,11 @@ automatic head query or a mutation downgrade. ## 4. Bounded observation and reads -Inspection projections contain committed summaries, Schema rows, Sample IDs, field +Inspection projections contain complete summaries, Schema rows, Sample IDs, field metadata and separate value bindings. Ordinary inspection and indexed read planning do not load a complete Manifest or chunk payload. Inventory and parent/child indexes -filter visibility through commit evidence, not merely candidate record state. +filter visibility through the complete resource root, with read-only compatibility +for old publication evidence, not merely candidate record state. Preparing an index overlaps at most sixteen immutable row writes, with at most 2 MiB of encoded keys and values queued. Encoding holds at most one additional @@ -175,6 +183,20 @@ request without a reader registry or remote close. ## 5. Ownership and cleanup +Incomplete-candidate cleanup and finalization conditionally replace the same +resource root. A cleanup seal prevents a delayed finalizer from exposing retired +rows. If the complete root wins, abandoning or superseding the draft preserves its +Manifest, initial Split members and payload owners; ordinary Snapshot reclamation +later retires them under deletion admission and grace rules. Completion transport +errors propagate without confirmation reads. Definite cleanup conflicts use the +returned value without reloading the root. + +The parent/child index is installed before the child's complete root. Finalization +rechecks parent availability after that index write. Parent deletion conservatively +protects any indexed child without terminal retirement, including an incomplete +child awaiting cleanup. Thus deletion either observes the child or the child's +last parent check rejects completion. Readers never register in this protocol. + Uploads are Dataset-scoped and content-verified. Composition registers staging and candidate upload IDs durably before claims. Publication is gated on complete ownership preparation, including unchanged shared parent uploads. Retiring one owner never diff --git a/doc/design/access-server/dataset/spec-crowdb-dataset.md b/doc/design/access-server/dataset/spec-crowdb-dataset.md index 0f8b1dc5e..bb6d6c94f 100644 --- a/doc/design/access-server/dataset/spec-crowdb-dataset.md +++ b/doc/design/access-server/dataset/spec-crowdb-dataset.md @@ -73,7 +73,7 @@ the resolved Snapshot ID and never stores a mutable name in its place. A Snapshot has zero or one declared parent. Parentage is explicit; ID ordering never implies ancestry. A child may add, replace, or tombstone samples and fields while inheriting all values not overridden by its visible parent chain. -Any committed same-Dataset Snapshot may be the parent, independently of latest. +Any complete available same-Dataset Snapshot may be the parent, independently of latest. Multiple parentless roots are permitted. Sample IDs are opaque bytes, represented as canonical padded base64 in JSON; they MUST NOT be normalized as text or paths. @@ -169,21 +169,29 @@ The authority MUST perform publication in this order: 2. Allocate or recover one candidate Snapshot ID for the operation. 3. Persist the prepared Snapshot record. 4. Persist every Manifest partition and the Snapshot binding. -5. Persist the published Snapshot record. -6. Advance `latest` atomically to the complete Snapshot. +5. Finish inspection metadata, initial Split membership and required payload owners. +6. Persist the published Snapshot record and install the final complete resource root. +7. Advance `latest` atomically to the complete Snapshot. Prepare completes the durable candidate without advancing latest. A failure -before step 5 leaves the candidate invisible and recoverable by the same -operation token. The Published record may exist before step 6 succeeds; an +before step 6 leaves the candidate invisible and recoverable by the same +operation token. The Published flag alone does not authorize reads. A complete root +allows ID reads and references before step 7 succeeds; an operation status alone is not proof that the head advanced. A successful Publish completes the head transition. Retry identical content to finish an ambiguous outcome; retries of historical publication MUST NOT rewind a newer head. A reader of `latest` MUST see either the previous complete Snapshot or the new complete Snapshot, never a prepared or partial Manifest. -`stable` may be set only to an existing published Snapshot. Publishing a new +`stable` may be set only to an existing complete available Snapshot. Publishing a new Snapshot never changes `stable` implicitly. A published Snapshot remains addressable by ID until reclamation removes its metadata under Section 9. +Complete results that never won latest, including head-CAS losers, MUST also be +readable by ID, enumerable and usable as parents or stable targets. Preparation +records alone MUST NOT authorize these operations. Draft abandonment or supersession +MUST preserve the complete result and its payload owners. Incomplete cleanup MUST +seal the same final resource key that completion conditionally installs, preventing +late finalization from exposing reclaimed data. Control v2 separates candidate creation from publication attempts. An attempt MUST bind the exact candidate and expected opaque head generation. Initial Split @@ -260,8 +268,11 @@ equivalent. ## 9. Retention and reclamation `retain` and `release` apply to an immutable Snapshot ID. The latest Snapshot, -stable Snapshot, retained Snapshots, parents of live committed Snapshots on any +stable Snapshot, retained Snapshots, parents of live complete Snapshots on any branch are protected from reclamation. Active reads do not prevent deletion. +The parent/child index MUST precede final child completion, which MUST recheck parent +availability. An indexed child protects its parent until terminal retirement even +while its own completion is pending; cleanup removes this conservative protection. Deletion first persists a terminal mark and a five-minute `reclaim_after` deadline. Cleanup MUST NOT remove snapshot-owned metadata or payload references before diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 3eb39fc73..fe0d8c0b2 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -152,11 +152,6 @@ CAS 不再满足只证明“本次条件不成立”，不能谎称“第一次 单独证明，不能仅凭本地超时回收。用户在 S0→S1/S2 竞争和回复丢失例子后明确选择此合同， 不采用“必须曾发布才能读”的备选。 -- [ ] **R2.1 统一完整资源准入**：只在 Manifest/inspection、初始 Split 与必要 payload owner - 已完整就绪后按 ID 开放；准备记录本身不等于结果完整。Snapshot/Split 按 ID、parent/stable、 - history/inventory、continuation 与 legacy 路径统一，删除状态仍拒绝新请求。 - 不能只将 SC/SRC lookup 改成 true：Composition 在 CB 之后还准备初始 Split、claim upload - owner，obsolete/abandon cleanup 和 child 保护也依赖旧“曾发布”区分，须同步调整。 - [ ] **R2.2 删除发布过程账本**：移除新 CA/CR/SC/SRC 写入、覆盖 head 前的帮助与按 attempt 历史回复查询；Composition/Split 的 Committing 恢复与 P2/P3 一起删除，客户端保留完整输入。 旧记录只作为兼容解码/GC 输入，不让旧账本控制新请求。head 保留 expected base；同一 @@ -495,6 +490,25 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，R2.1/R2.1b**：新 Snapshot 最后将已有 CB 条件替换为 version 2 完整根， + Manifest/index、初始 Split 与必要 upload owners 就绪后，才能按 ID 读取/引用、列入 + inventory/history、retain 或设 stable。版本 1 准备绑定和 Published 标志本身不够；旧成功 + 记录仍只读兼容。放弃/替换草稿保留完整结果及 owner；未完成清理先竞争同一 CB 封口， + 迟到完成失败。清理 CAS 观察到完成者后直接保留，无补 GET；完成回复未知直接传播， + 不确认查询。子索引先写，完成前再检查 parent；未终态退休的子索引保守保护 parent。 + 完整 ID summary 实测三次 bounded GET（删除标记、CB、summary）、零 mutation， + 不查 head/SC/CA/CR/LC。六项新测试覆盖完成丢回复、双方竞争、父删除两个顺序、 + 跨实例 continuation、retention 与删除。上传测试覆盖 owner 未就绪时 ID/原始发布均拒绝， + 放弃后仅释放 staging owner；初始 Split 的完整成员可保留并清理 scratch。 + JSON 与旧 bincode Manifest 入口新增统一只读准入，未完成 404、完成未发布 200、删除 409。 + legacy scan/空结果仍独立准入，移除与 Snapshot 读取重复的外层删除检查；单窗口仍一次 + 删除标记读取。修复 prepare→重启→publish 对完整结果误早退，以及当前旧 head 补 Manifest + 后漏安装最终根的问题，均有回归。 + Dataset 全目标与受影响追加回归、Access HTTP/安装 SDK、11 项 native 读取测试通过； + 三 crate all-target/all-feature clippy、fmt、diff 检查通过，均经 pixi。 + Snapshot CA/CR/SC/helper 尚未删除，R2.2 继续处理；未引用结果期限/独立孤儿 GC 仍属 R4。 + 同源码容器最终复验由 V2/V4 跟踪，此批不声称 TPS 或 GC 完成。 + - **2026-10-24，F1.1**：同一 SDK client/clone/control wrapper 共享固定有界 atomic bitset， 复用已成功发现的路由能力。连续五次 head 读取实测由十次 HTTP 请求降为六次；Split 冲突显式重试由 capabilities GET → PUT → capabilities GET → PUT 降为 GET → PUT → PUT， diff --git a/lib/crowdb-access-dataset/src/authority.rs b/lib/crowdb-access-dataset/src/authority.rs index f4a925bb3..d1592f179 100644 --- a/lib/crowdb-access-dataset/src/authority.rs +++ b/lib/crowdb-access-dataset/src/authority.rs @@ -1,4 +1,5 @@ mod access; +pub(crate) use access::CandidateRetirement; pub use access::{ MetadataCleanupProgress, SnapshotLifecycle, SnapshotReclaimError, SnapshotReclaimPhase, SnapshotReclaimStatus, @@ -164,16 +165,7 @@ impl DatasetAuthority { identity: &DatasetIdentity, snapshot: SnapshotId, ) -> Result<(), AuthorityError> { - let value = self - .store - .get(&key::snapshot(identity, snapshot)?) - .await? - .ok_or(AuthorityError::SnapshotNotFound)?; - let record: SnapshotRecord = decode(&value.bytes)?; - if record.publication.state != PublicationState::Published { - return Err(AuthorityError::SnapshotNotFound); - } - self.ensure_snapshot_live(identity, snapshot).await?; + self.get_snapshot(identity, snapshot).await?; let marker = encode(&snapshot)?; match self .store @@ -210,7 +202,6 @@ impl DatasetAuthority { snapshot: SnapshotId, ) -> Result<(), AuthorityError> { self.get_snapshot(identity, snapshot).await?; - self.require_committed_snapshot(identity, snapshot).await?; let key = key::head(identity)?; let current = self.store.get(&key).await?; if let Some(current) = ¤t { @@ -254,22 +245,14 @@ impl DatasetAuthority { if !seen.insert(id) { return Err(AuthorityError::AncestryCycle); } - let value = self - .store - .get(&key::snapshot(identity, id)?) - .await? - .ok_or(AuthorityError::SnapshotNotFound)?; - let record: SnapshotRecord = decode(&value.bytes)?; - if record.publication.state != PublicationState::Published { - return Err(AuthorityError::SnapshotNotFound); - } + let record = self.get_snapshot(identity, id).await?; current = record.publication.parent; chain.push(record); } Ok(chain) } - /// Opens one published Snapshot without changing the current head. + /// Opens one complete Snapshot without changing the current head. /// /// # Errors /// Returns a missing, unpublished, or malformed Snapshot. @@ -284,9 +267,13 @@ impl DatasetAuthority { .await? .ok_or(AuthorityError::SnapshotNotFound)?; let record: SnapshotRecord = decode(&value.bytes)?; + if record.publication.snapshot != snapshot { + return Err(AuthorityError::Corrupt); + } if record.publication.state != PublicationState::Published { return Err(AuthorityError::SnapshotNotFound); } + self.require_committed_snapshot(identity, snapshot).await?; Ok(record) } @@ -397,11 +384,15 @@ impl DatasetAuthority { CasOutcome::Applied(_) => {} CasOutcome::Conflict(_) => return Err(AuthorityError::HeadConflict), } + self.complete_snapshot_candidate(identity, snapshot).await?; self.advance_head(identity, None, snapshot_record.publication.parent, snapshot) .await?; } else if self.current_latest(identity).await? != Some(snapshot) { + self.complete_snapshot_candidate(identity, snapshot).await?; self.advance_head(identity, None, snapshot_record.publication.parent, snapshot) .await?; + } else { + self.complete_snapshot_candidate(identity, snapshot).await?; } Ok(decode::(&binding)?.partitions) } @@ -424,7 +415,8 @@ impl DatasetAuthority { decode(&value.bytes) } - /// Loads the lightweight manifest index for bounded planning and shuffle. + /// Loads stored Manifest partitions for internal planning and cleanup. + /// Public read handlers use `read_manifest` to enforce resource admission. /// /// # Errors /// Returns missing or malformed manifest partitions and storage failures. @@ -475,6 +467,18 @@ impl DatasetAuthority { Ok(manifest) } + /// Reads the complete Manifest of an available Snapshot without changing any state. + /// # Errors + /// Rejects incomplete or deleted resources, corrupt content and storage failures. + pub async fn read_manifest( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + ) -> Result { + self.require_committed_snapshot(identity, snapshot).await?; + self.get_manifest(identity, snapshot).await + } + /// Plans a scan directly from persisted manifest partitions. Only the /// selected IDs and their small locator metadata are retained, bounded by /// `limits`; payloads are never opened during planning. @@ -734,6 +738,9 @@ impl DatasetAuthority { if sample_ids.len() > crate::MAX_BATCH_SIZE { return Err(DatasetError::InvalidManifest.into()); } + if sample_ids.is_empty() || fields.is_empty() { + self.get_snapshot(identity, snapshot).await?; + } let mut batch = Vec::with_capacity(sample_ids.len()); for sample_id in sample_ids { batch.push(SampleView { @@ -787,6 +794,9 @@ impl DatasetAuthority { if cancel.is_cancelled() { return Err(DatasetError::ReadCancelled.into()); } + if sample_ids.is_empty() || fields.is_empty() { + self.get_snapshot(identity, snapshot).await?; + } let mut batch = Vec::with_capacity(sample_ids.len()); for sample_id in sample_ids { batch.push(SampleView { diff --git a/lib/crowdb-access-dataset/src/authority/access.rs b/lib/crowdb-access-dataset/src/authority/access.rs index e6f99bc1f..47d7383f0 100644 --- a/lib/crowdb-access-dataset/src/authority/access.rs +++ b/lib/crowdb-access-dataset/src/authority/access.rs @@ -8,6 +8,7 @@ mod observation; pub use lifecycle::{SnapshotReclaimError, SnapshotReclaimPhase, SnapshotReclaimStatus}; pub use observation::SnapshotLifecycle; mod reclaim; +pub(crate) use reclaim::CandidateRetirement; mod sweep; use crate::{key, AuthorityError, CasOutcome, DatasetIdentity, OperationId, SnapshotId}; pub use cleanup::MetadataCleanupProgress; diff --git a/lib/crowdb-access-dataset/src/authority/access/reclaim.rs b/lib/crowdb-access-dataset/src/authority/access/reclaim.rs index 10993c231..952a84f5e 100644 --- a/lib/crowdb-access-dataset/src/authority/access/reclaim.rs +++ b/lib/crowdb-access-dataset/src/authority/access/reclaim.rs @@ -5,6 +5,17 @@ use super::DatasetAuthority; use crate::authority::{decode, decode_head, encode_head}; use crate::{key, AuthorityError, CasOutcome, DatasetIdentity, HeadRecord, OperationId, SnapshotId}; +pub(crate) enum CandidateRetirement { + Retired, + Complete, + LegacyCommitted(crate::SuccessfulCommit), +} +impl CandidateRetirement { + pub(crate) fn retained(&self) -> bool { + !matches!(self, Self::Retired) + } +} + impl DatasetAuthority { pub(super) async fn fence_reclaim( &self, @@ -183,39 +194,24 @@ impl DatasetAuthority { &self, identity: &DatasetIdentity, snapshot: SnapshotId, - ) -> Result, AuthorityError> { - if self.snapshot_candidate(identity, snapshot).await?.is_none() { - let value = self - .store - .get_bounded( - &key::snapshot(identity, snapshot)?, - crate::MetadataScanBudget { - items: 1, - bytes: 9 * 1024 * 1024, - timeout: std::time::Duration::from_secs(2), - }, - &crate::ReadCancellation::new(), - ) - .await? - .ok_or(AuthorityError::SnapshotNotFound)?; - let record: crate::SnapshotRecord = decode(&value.bytes)?; - if record.publication.state != crate::PublicationState::Prepared - && record.publication.state != crate::PublicationState::Aborted - { - return Err(AuthorityError::SnapshotNotFound); - } + ) -> Result { + if let Some(receipt) = self.snapshot_commit(identity, snapshot).await? { + return Ok(CandidateRetirement::LegacyCommitted(receipt)); + } + if self.seal_incomplete_snapshot(identity, snapshot).await? { + return Ok(CandidateRetirement::Complete); } if let Some(owner) = self.fence_reclaim(identity, snapshot).await? { if let Some(receipt) = self.snapshot_commit(identity, snapshot).await? { self.unfence_reclaim(identity, snapshot, owner).await?; - return Ok(Some(receipt)); + return Ok(CandidateRetirement::LegacyCommitted(receipt)); } self.begin_terminal_retirement(identity, snapshot, owner, 0) .await?; } else if let Some(receipt) = self.snapshot_commit(identity, snapshot).await? { - return Ok(Some(receipt)); + return Ok(CandidateRetirement::LegacyCommitted(receipt)); } self.retire_snapshot_record(identity, snapshot).await?; - Ok(None) + Ok(CandidateRetirement::Retired) } } diff --git a/lib/crowdb-access-dataset/src/authority/publication.rs b/lib/crowdb-access-dataset/src/authority/publication.rs index 09a290844..34ed01b99 100644 --- a/lib/crowdb-access-dataset/src/authority/publication.rs +++ b/lib/crowdb-access-dataset/src/authority/publication.rs @@ -3,6 +3,7 @@ mod candidate; mod commit; +mod complete; mod legacy; mod legacy_reclaim; mod opaque; @@ -199,8 +200,15 @@ impl DatasetAuthority { return Err(AuthorityError::AlreadyExists); } if existing.publication.state == PublicationState::Published - && self.snapshot_commit(identity, snapshot).await?.is_some() + && (self + .snapshot_candidate(identity, snapshot) + .await? + .is_some_and(|c| c.version == 2) + || self.snapshot_commit(identity, snapshot).await?.is_some()) { + if matches!(mode, PublicationMode::LegacyPublish) { + return self.advance_head(identity, None, parent, snapshot).await; + } return Ok(snapshot); } if existing.publication.state == PublicationState::Published @@ -245,31 +253,7 @@ impl DatasetAuthority { if !matches!(mode, PublicationMode::LegacyPublish) { return Ok(snapshot); } - let published = SnapshotPublication { - state: PublicationState::Published, - ..prepared - }; - let published_bytes = encode(&SnapshotRecord { - publication: published, - })?; - match self.store.get(&snapshot_key).await? { - Some(value) => { - let existing: SnapshotRecord = decode(&value.bytes)?; - if existing.publication.state == PublicationState::Prepared { - match self - .store - .compare_exchange(&snapshot_key, Some(&value.bytes), &published_bytes) - .await? - { - CasOutcome::Applied(_) => {} - CasOutcome::Conflict(_) => return Err(AuthorityError::HeadConflict), - } - } else if existing.publication.state != PublicationState::Published { - return Err(AuthorityError::Invalid(DatasetError::InvalidPublicationState)); - } - } - None => return Err(AuthorityError::Corrupt), - } + self.complete_snapshot_candidate(identity, snapshot).await?; self.advance_head(identity, None, parent, snapshot).await } diff --git a/lib/crowdb-access-dataset/src/authority/publication/candidate.rs b/lib/crowdb-access-dataset/src/authority/publication/candidate.rs index f143a7faa..d953f9648 100644 --- a/lib/crowdb-access-dataset/src/authority/publication/candidate.rs +++ b/lib/crowdb-access-dataset/src/authority/publication/candidate.rs @@ -21,7 +21,8 @@ pub(crate) struct CandidateBinding { } impl DatasetAuthority { - /// Prepares immutable content from an arbitrary committed parent or a new root. + /// Prepares complete immutable content from an available parent or a new root. + /// Callers supplying external chunk locators must acquire their payload owners first. /// The expected Dataset head belongs to a later commit attempt. /// # Errors /// Rejects invalid content, unavailable parents and divergent operation retries. @@ -39,14 +40,17 @@ impl DatasetAuthority { if let Some(parent) = parent { self.require_committed_snapshot(identity, parent).await?; } - self.write_publication( - identity, - parent, - manifest, - operation, - PublicationMode::Candidate(None), - ) - .await + let snapshot = self + .write_publication( + identity, + parent, + manifest, + operation, + PublicationMode::Candidate(None), + ) + .await?; + self.complete_snapshot_candidate(identity, snapshot).await?; + Ok(snapshot) } pub(crate) async fn prepare_candidate_with_split( @@ -55,7 +59,7 @@ impl DatasetAuthority { parent: Option, manifest: Vec, operation: OperationId, - initial: SplitRevisionId, + initial: Option, ) -> Result { if manifest.len() > 8 * 1024 * 1024 { return Err(DatasetError::ReadLimitExceeded.into()); @@ -69,7 +73,7 @@ impl DatasetAuthority { parent, manifest, operation, - PublicationMode::Candidate(Some(initial)), + PublicationMode::Candidate(initial), ) .await } @@ -84,6 +88,15 @@ impl DatasetAuthority { match self.store.compare_exchange(&key, None, &bytes).await? { CasOutcome::Applied(_) => Ok(()), CasOutcome::Conflict(Some(value)) if value.bytes == bytes => Ok(()), + CasOutcome::Conflict(Some(value)) => { + let mut existing = decode_candidate(&value.bytes, binding.snapshot)?; + existing.version = binding.version; + if existing == *binding { + Ok(()) + } else { + Err(AuthorityError::AlreadyExists) + } + } CasOutcome::Conflict(_) => Err(AuthorityError::AlreadyExists), } } @@ -96,24 +109,33 @@ impl DatasetAuthority { let Some(value) = self.store.get(&candidate_key(identity, snapshot)?).await? else { return Ok(None); }; - if value.bytes.len() > 4096 { - return Err(AuthorityError::Corrupt); - } - let binding: CandidateBinding = - serde_json::from_slice(&value.bytes).map_err(|_| AuthorityError::Corrupt)?; - if binding.version != 1 || binding.snapshot != snapshot || binding.parent == Some(snapshot) { - return Err(AuthorityError::Corrupt); - } - SnapshotId::from_bytes(binding.snapshot.as_bytes())?; - OperationId::from_bytes(binding.operation.as_bytes())?; - if let Some(parent) = binding.parent { - SnapshotId::from_bytes(parent.as_bytes())?; - } - if let Some(revision) = binding.initial_split { - SplitRevisionId::from_bytes(revision.as_bytes())?; - } - Ok(Some(binding)) + decode_candidate(&value.bytes, snapshot).map(Some) + } +} + +pub(crate) fn decode_candidate( + bytes: &[u8], + snapshot: SnapshotId, +) -> Result { + if bytes == super::complete::RETIRED { + return Err(AuthorityError::SnapshotProtected); + } + if bytes.len() > 4096 { + return Err(AuthorityError::Corrupt); + } + let binding: CandidateBinding = serde_json::from_slice(bytes).map_err(|_| AuthorityError::Corrupt)?; + if !matches!(binding.version, 1 | 2) || binding.snapshot != snapshot || binding.parent == Some(snapshot) { + return Err(AuthorityError::Corrupt); + } + SnapshotId::from_bytes(binding.snapshot.as_bytes())?; + OperationId::from_bytes(binding.operation.as_bytes())?; + if let Some(parent) = binding.parent { + SnapshotId::from_bytes(parent.as_bytes())?; + } + if let Some(revision) = binding.initial_split { + SplitRevisionId::from_bytes(revision.as_bytes())?; } + Ok(binding) } pub(in crate::authority) fn binding( diff --git a/lib/crowdb-access-dataset/src/authority/publication/commit.rs b/lib/crowdb-access-dataset/src/authority/publication/commit.rs index e507e7cd4..0b83d0c88 100644 --- a/lib/crowdb-access-dataset/src/authority/publication/commit.rs +++ b/lib/crowdb-access-dataset/src/authority/publication/commit.rs @@ -44,6 +44,9 @@ impl DatasetAuthority { .snapshot_candidate(identity, snapshot) .await? .ok_or(AuthorityError::SnapshotNotFound)?; + if candidate.version != 2 { + return Err(AuthorityError::SnapshotNotFound); + } self.persist_attempt(identity, attempt).await?; if let Some(receipt) = self.snapshot_commit(identity, snapshot).await? { return Ok(receipt); @@ -106,7 +109,7 @@ impl DatasetAuthority { } } - async fn mark_candidate_published( + pub(super) async fn mark_candidate_published( &self, identity: &DatasetIdentity, candidate: &super::candidate::CandidateBinding, diff --git a/lib/crowdb-access-dataset/src/authority/publication/complete.rs b/lib/crowdb-access-dataset/src/authority/publication/complete.rs new file mode 100644 index 000000000..1f16064a3 --- /dev/null +++ b/lib/crowdb-access-dataset/src/authority/publication/complete.rs @@ -0,0 +1,94 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::candidate::{candidate_key, decode_candidate}; +use crate::{ + AuthorityError, CasOutcome, DatasetAuthority, DatasetIdentity, MetadataScanBudget, ReadCancellation, + SnapshotId, +}; +use std::time::Duration; + +pub(crate) const RETIRED: &[u8] = b"retired-snapshot-content-v1"; + +impl DatasetAuthority { + /// Installs the complete immutable resource root after payload ownership is acquired. + /// It does not change latest/stable or create a publication receipt. + /// # Errors + /// Rejects incomplete content, unavailable parents, deletion and cleanup races. + pub async fn complete_snapshot_candidate( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + ) -> Result<(), AuthorityError> { + let key = candidate_key(identity, snapshot)?; + let stored = self + .store + .get_bounded( + &key, + MetadataScanBudget { + items: 1, + bytes: 4096, + timeout: Duration::from_secs(2), + }, + &ReadCancellation::new(), + ) + .await? + .ok_or(AuthorityError::SnapshotNotFound)?; + let mut candidate = decode_candidate(&stored.bytes, snapshot)?; + self.ensure_snapshot_live(identity, snapshot).await?; + if candidate.version == 2 { + return Ok(()); + } + if let Some(parent) = candidate.parent { + self.require_committed_snapshot(identity, parent).await?; + } + self.validate_initial_split(identity, snapshot, candidate.initial_split) + .await?; + self.mark_candidate_published(identity, &candidate).await?; + candidate.version = 2; + let complete = serde_json::to_vec(&candidate).map_err(|_| AuthorityError::Corrupt)?; + match self + .store + .compare_exchange(&key, Some(&stored.bytes), &complete) + .await? + { + CasOutcome::Applied(_) => Ok(()), + CasOutcome::Conflict(Some(observed)) + if decode_candidate(&observed.bytes, snapshot)? == candidate => + { + Ok(()) + } + CasOutcome::Conflict(_) => Err(AuthorityError::HeadConflict), + } + } + + /// Competes with finalization on the resource root before retiring any content. + /// Returns true when the complete resource must survive draft cleanup. + pub(crate) async fn seal_incomplete_snapshot( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + ) -> Result { + let key = candidate_key(identity, snapshot)?; + let mut observed = self.store.get(&key).await?; + for _ in 0..64 { + if let Some(value) = &observed { + if value.bytes == RETIRED { + return Ok(false); + } + if decode_candidate(&value.bytes, snapshot)?.version == 2 { + return Ok(true); + } + } + match self + .store + .compare_exchange(&key, observed.as_ref().map(|v| v.bytes.as_slice()), RETIRED) + .await? + { + CasOutcome::Applied(_) => return Ok(false), + CasOutcome::Conflict(value) => observed = value, + } + } + Err(AuthorityError::HeadConflict) + } +} diff --git a/lib/crowdb-access-dataset/src/authority/publication/legacy.rs b/lib/crowdb-access-dataset/src/authority/publication/legacy.rs index f2e7a495f..df9eb4f1a 100644 --- a/lib/crowdb-access-dataset/src/authority/publication/legacy.rs +++ b/lib/crowdb-access-dataset/src/authority/publication/legacy.rs @@ -60,7 +60,7 @@ impl DatasetAuthority { Ok(true) } - /// Checks live publication proof, resolving bounded legacy ancestry when needed. + /// Checks a live complete result, resolving bounded legacy publication evidence when needed. /// # Errors /// Returns unavailable or retired snapshots, corrupt proof or storage limits. pub async fn require_committed_snapshot( @@ -69,12 +69,16 @@ impl DatasetAuthority { snapshot: SnapshotId, ) -> Result<(), AuthorityError> { self.ensure_snapshot_live(identity, snapshot).await?; + let candidate = self.snapshot_candidate(identity, snapshot).await?; + if candidate.as_ref().is_some_and(|c| c.version == 2) { + return Ok(()); + } if self.snapshot_commit(identity, snapshot).await?.is_some() || self.legacy_commit_exists(identity, snapshot).await? { return Ok(()); } - if self.snapshot_candidate(identity, snapshot).await?.is_some() { + if candidate.is_some() { return Err(AuthorityError::SnapshotNotFound); } let cancel = ReadCancellation::new(); diff --git a/lib/crowdb-access-dataset/src/authority/publication/visibility.rs b/lib/crowdb-access-dataset/src/authority/publication/visibility.rs index 3ab2425db..2d9bbce93 100644 --- a/lib/crowdb-access-dataset/src/authority/publication/visibility.rs +++ b/lib/crowdb-access-dataset/src/authority/publication/visibility.rs @@ -35,6 +35,20 @@ impl DatasetAuthority { work: &mut InspectionWork<'_>, ) -> Result { use crate::AuthorityError; + let candidate = work + .get( + &super::candidate::candidate_key(identity, snapshot).map_err(AuthorityError::from)?, + 4096, + ) + .await?; + if let Some(candidate) = &candidate { + if candidate.bytes == super::complete::RETIRED { + return Ok(false); + } + if super::candidate::decode_candidate(&candidate.bytes, snapshot)?.version == 2 { + return Ok(true); + } + } if let Some(value) = work .get( &snapshot_commit_key(identity, snapshot).map_err(AuthorityError::from)?, @@ -69,14 +83,7 @@ impl DatasetAuthority { } // A prepared modern candidate is not a legacy publication, even when // a malformed old head happens to name it. - if work - .get( - &super::candidate::candidate_key(identity, snapshot).map_err(AuthorityError::from)?, - 4096, - ) - .await? - .is_some() - { + if candidate.is_some() { return Ok(false); } Ok(self @@ -101,11 +108,10 @@ impl DatasetAuthority { else { return Ok(None); }; - let binding: super::candidate::CandidateBinding = - serde_json::from_slice(&value.bytes).map_err(|_| InspectionError::Corrupt)?; - if binding.version != 1 || binding.snapshot != snapshot { - return Err(InspectionError::Corrupt); + if value.bytes == super::complete::RETIRED { + return Ok(None); } + let binding = super::candidate::decode_candidate(&value.bytes, snapshot)?; Ok(Some(binding.operation)) } } diff --git a/lib/crowdb-access-dataset/src/composition/candidates.rs b/lib/crowdb-access-dataset/src/composition/candidates.rs index a8c56f106..b3751caa0 100644 --- a/lib/crowdb-access-dataset/src/composition/candidates.rs +++ b/lib/crowdb-access-dataset/src/composition/candidates.rs @@ -107,7 +107,7 @@ impl DatasetAuthority { if self .abort_uncommitted_candidate(identity, record.publication.snapshot) .await? - .is_some() + .retained() { return Ok(true); } diff --git a/lib/crowdb-access-dataset/src/composition/edit.rs b/lib/crowdb-access-dataset/src/composition/edit.rs index eb493d4ac..b630786b1 100644 --- a/lib/crowdb-access-dataset/src/composition/edit.rs +++ b/lib/crowdb-access-dataset/src/composition/edit.rs @@ -123,7 +123,7 @@ impl DatasetAuthority { next.edit = None; self.replace_composition(identity, &old, &state, &next).await?; if let Some(preparation) = &next.draft.preparation { - if let Some(receipt) = self + if let crate::authority::CandidateRetirement::LegacyCommitted(receipt) = self .abort_uncommitted_candidate(identity, preparation.candidate) .await? { diff --git a/lib/crowdb-access-dataset/src/composition/prepare.rs b/lib/crowdb-access-dataset/src/composition/prepare.rs index 831442034..926edb011 100644 --- a/lib/crowdb-access-dataset/src/composition/prepare.rs +++ b/lib/crowdb-access-dataset/src/composition/prepare.rs @@ -28,6 +28,18 @@ impl DatasetAuthority { return Err(CompositionError::Conflict); } if matches!(state.draft.record.progress, DraftProgress::Prepared { .. }) { + if state.uploads.is_none() { + self.complete_snapshot_candidate( + identity, + state + .draft + .preparation + .as_ref() + .ok_or(CompositionError::Corrupt)? + .candidate, + ) + .await?; + } return Ok(state.draft); } if !matches!(state.draft.record.progress, DraftProgress::Evaluated { .. }) { @@ -77,6 +89,17 @@ impl DatasetAuthority { } next.draft.preparation = Some(preparation); self.replace_composition(identity, &old, &state, &next).await?; + if complete && next.uploads.is_none() { + self.complete_snapshot_candidate( + identity, + next.draft + .preparation + .as_ref() + .ok_or(CompositionError::Corrupt)? + .candidate, + ) + .await?; + } Ok(next.draft) } async fn create_composition_candidate( @@ -96,19 +119,12 @@ impl DatasetAuthority { } else { Some(candidate_operation(operation, 1)?) }; - let snapshot = if let Some(split_operation) = split_operation { - self.prepare_candidate_with_split( - identity, - state.draft.record.parent, - bytes, - operation, - crate::split::candidate_revision(split_operation, 1)?, - ) - .await? - } else { - self.prepare_snapshot_candidate(identity, state.draft.record.parent, bytes, operation) - .await? - }; + let initial = split_operation + .map(|operation| crate::split::candidate_revision(operation, 1)) + .transpose()?; + let snapshot = self + .prepare_candidate_with_split(identity, state.draft.record.parent, bytes, operation, initial) + .await?; Ok(CompositionPreparation { candidate: snapshot, split_operation, diff --git a/lib/crowdb-access-dataset/src/composition/uploads/prepare.rs b/lib/crowdb-access-dataset/src/composition/uploads/prepare.rs index 172016a11..d23687df4 100644 --- a/lib/crowdb-access-dataset/src/composition/uploads/prepare.rs +++ b/lib/crowdb-access-dataset/src/composition/uploads/prepare.rs @@ -58,6 +58,8 @@ impl DatasetAuthority { return Err(CompositionError::Corrupt); } if candidate.complete { + self.complete_snapshot_candidate(identity, candidate.snapshot) + .await?; return super::state::status(&state); } let page = self @@ -96,6 +98,16 @@ impl DatasetAuthority { candidate.after = page.after; self.replace_composition(identity, &stored, ¤t, &completed) .await?; + if completed + .uploads + .as_ref() + .ok_or(CompositionError::Corrupt)? + .candidates[position] + .complete + { + self.complete_snapshot_candidate(identity, preparation.candidate) + .await?; + } super::state::status(&completed) } } diff --git a/lib/crowdb-access-dataset/src/composition/uploads/release.rs b/lib/crowdb-access-dataset/src/composition/uploads/release.rs index 12261362a..7d76027d8 100644 --- a/lib/crowdb-access-dataset/src/composition/uploads/release.rs +++ b/lib/crowdb-access-dataset/src/composition/uploads/release.rs @@ -76,7 +76,7 @@ impl DatasetAuthority { if self .abort_uncommitted_candidate(identity, candidate.snapshot) .await? - .is_some() + .retained() { candidate.keep_committed = true; continue; diff --git a/lib/crowdb-access-dataset/src/inspection/reclaim.rs b/lib/crowdb-access-dataset/src/inspection/reclaim.rs index 7ad258cf0..53d480e79 100644 --- a/lib/crowdb-access-dataset/src/inspection/reclaim.rs +++ b/lib/crowdb-access-dataset/src/inspection/reclaim.rs @@ -65,12 +65,9 @@ impl DatasetAuthority { } continue; } - if self - .snapshot_has_commit_proof(identity, summary.snapshot, &mut work) - .await? - { - return Ok(true); - } + // The child index precedes its complete root. Protecting it until + // terminal retirement closes the parent-delete/finalization race. + return Ok(true); } after = page.after; if after.is_none() { diff --git a/lib/crowdb-access-dataset/src/split/cleanup.rs b/lib/crowdb-access-dataset/src/split/cleanup.rs index 5c3818348..df78aa177 100644 --- a/lib/crowdb-access-dataset/src/split/cleanup.rs +++ b/lib/crowdb-access-dataset/src/split/cleanup.rs @@ -73,14 +73,19 @@ impl DatasetAuthority { let retired = self .inspection_snapshot_retired(identity, state.record.snapshot, &mut work) .await?; - if state.initial && !retired { + let complete_initial = state.initial + && !retired + && self + .snapshot_has_commit_proof(identity, state.record.snapshot, &mut work) + .await?; + if state.initial && !retired && !complete_initial { return Err(SplitLifecycleError::Frozen); } let end = state .record .revision .checked_add(u64::from( - retired || matches!(state.record.progress, DraftProgress::Abandoned), + retired || complete_initial || matches!(state.record.progress, DraftProgress::Abandoned), )) .ok_or(SplitLifecycleError::Limit)?; let cursor_key = state_key(b"DS\0SCG\0", identity, operation, None)?; diff --git a/lib/crowdb-access-dataset/src/split/initial.rs b/lib/crowdb-access-dataset/src/split/initial.rs index 1974ff7ed..49e5523b4 100644 --- a/lib/crowdb-access-dataset/src/split/initial.rs +++ b/lib/crowdb-access-dataset/src/split/initial.rs @@ -167,22 +167,6 @@ impl DatasetAuthority { if record.snapshot != snapshot || record.revision != revision { return Err(AuthorityError::Corrupt); } - let (_, state) = - self.load_split_draft(identity, record.operation) - .await - .map_err(|error| match error { - SplitLifecycleError::Authority(error) => error, - SplitLifecycleError::Store(error) => error.into(), - _ => AuthorityError::Corrupt, - })?; - if !state.initial - || state.record.progress - != (DraftProgress::Prepared { - target: crate::CommitTarget::SplitRevision { snapshot, revision }, - }) - { - return Err(AuthorityError::SnapshotNotFound); - } Ok(()) } pub(crate) async fn initial_split_revision( @@ -190,9 +174,10 @@ impl DatasetAuthority { identity: &DatasetIdentity, snapshot: SnapshotId, ) -> Result, AuthorityError> { - Ok(self - .snapshot_candidate(identity, snapshot) - .await? - .and_then(|candidate| candidate.initial_split)) + match self.snapshot_candidate(identity, snapshot).await { + Ok(candidate) => Ok(candidate.and_then(|candidate| candidate.initial_split)), + Err(AuthorityError::SnapshotProtected) => Ok(None), + Err(error) => Err(error), + } } } diff --git a/lib/crowdb-access-dataset/src/transport.rs b/lib/crowdb-access-dataset/src/transport.rs index 707c9d69b..68621006c 100644 --- a/lib/crowdb-access-dataset/src/transport.rs +++ b/lib/crowdb-access-dataset/src/transport.rs @@ -66,9 +66,6 @@ impl DatasetReadService { return Err(ReadTransportError::WindowFull); } let permit = WindowPermit(Arc::clone(&self.window)); - self.authority - .check_snapshot_readable(&self.identity, request.snapshot) - .await?; let fields: Vec<&str> = request.fields.iter().map(String::as_str).collect(); let samples = self .authority @@ -106,9 +103,6 @@ impl DatasetReadService { return Err(ReadTransportError::WindowFull); } let permit = WindowPermit(Arc::clone(&self.window)); - self.authority - .check_snapshot_readable(&self.identity, request.plan.snapshot) - .await?; let cursor = request.cursor.unwrap_or_else(|| ReadCursor::start(&request.plan)); cursor.validate_for(&request.plan)?; let planned = match request.shuffle.as_ref() { @@ -146,6 +140,9 @@ impl DatasetReadService { let ids: Vec<_> = planned.into_iter().skip(range.start).take(range.len()).collect(); let fields: Vec<&str> = request.plan.projection.iter().map(String::as_str).collect(); let samples = if ids.is_empty() { + self.authority + .get_snapshot(&self.identity, request.plan.snapshot) + .await?; Vec::new() } else { self.authority diff --git a/lib/crowdb-access-dataset/tests/candidate_publication_test.rs b/lib/crowdb-access-dataset/tests/candidate_publication_test.rs index ca249ea6d..90204f019 100644 --- a/lib/crowdb-access-dataset/tests/candidate_publication_test.rs +++ b/lib/crowdb-access-dataset/tests/candidate_publication_test.rs @@ -230,20 +230,20 @@ async fn head_mutators_help_pending_success_before_replacing_its_marker() { assert_eq!(authority.inspect_head(&identity).await.unwrap(), changed); } #[tokio::test] -async fn uncommitted_parent_and_divergent_attempt_cannot_publish() { +async fn complete_unpublished_parent_is_usable_and_divergent_attempt_is_rejected() { let (_, authority, identity) = fixture().await; let parent = prepare(&authority, &identity, None, 7).await; - assert!(matches!( + let child = prepare(&authority, &identity, Some(parent), 8).await; + assert_eq!( authority - .prepare_snapshot_candidate( - &identity, - Some(parent), - publication_manifest(8), - OperationId::random() - ) - .await, - Err(AuthorityError::SnapshotNotFound) - )); + .get_snapshot(&identity, child) + .await + .unwrap() + .publication + .parent, + Some(parent) + ); + assert_eq!(authority.latest(&identity).await.unwrap(), None); let committed = attempt(&authority, &identity, parent).await; authority .publish_snapshot_candidate(&identity, &committed) @@ -262,7 +262,7 @@ async fn uncommitted_parent_and_divergent_attempt_cannot_publish() { } #[tokio::test] -async fn lost_cas_candidate_is_not_a_committed_parent_or_stable_target() { +async fn lost_cas_candidate_remains_a_complete_parent_and_stable_target() { let (store, authority, identity) = fixture().await; let snapshot = prepare(&authority, &identity, None, 10).await; let stale = attempt(&authority, &identity, snapshot).await; @@ -285,21 +285,17 @@ async fn lost_cas_candidate_is_not_a_committed_parent_or_stable_target() { .await .unwrap() .is_none()); - assert!(matches!( - authority.set_stable(&identity, snapshot).await, - Err(AuthorityError::SnapshotNotFound) - )); - assert!(matches!( + authority.set_stable(&identity, snapshot).await.unwrap(); + let child = prepare(&authority, &identity, Some(snapshot), 11).await; + assert_eq!( authority - .prepare_snapshot_candidate( - &identity, - Some(snapshot), - publication_manifest(11), - OperationId::random() - ) - .await, - Err(AuthorityError::SnapshotNotFound) - )); + .get_snapshot(&identity, child) + .await + .unwrap() + .publication + .parent, + Some(snapshot) + ); let retry = attempt(&authority, &identity, snapshot).await; authority .publish_snapshot_candidate(&identity, &retry) diff --git a/lib/crowdb-access-dataset/tests/common/snapshot_complete.rs b/lib/crowdb-access-dataset/tests/common/snapshot_complete.rs new file mode 100644 index 000000000..3cf9fe3e3 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/common/snapshot_complete.rs @@ -0,0 +1,166 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "dataset_store.rs"] +#[allow(dead_code)] +mod dataset_store; + +use crowdb_access_dataset::*; +use sha2::{Digest, Sha256}; +use std::sync::{ + atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering}, + Arc, +}; +use tokio::sync::Notify; + +pub const RETIRED: &[u8] = b"retired-snapshot-content-v1"; + +#[derive(Default)] +pub struct Store { + inner: dataset_store::TestDatasetStore, + pub gate: AtomicU8, + pub started: Notify, + pub release: Notify, + pub lose_final_reply: AtomicBool, + pub unknown: AtomicBool, + pub reads_after_unknown: AtomicUsize, + pub seal_winner: arc_swap::ArcSwapOption>, + pub seal_conflicted: AtomicBool, + pub cb_reads_after_conflict: AtomicUsize, + pub reads_only: AtomicBool, + pub bounded_reads: AtomicUsize, +} + +impl Store { + pub fn storage(&self) -> &dyn DatasetStore { + &self.inner + } + fn check_read(&self, key: &[u8]) { + if self.unknown.load(Ordering::Relaxed) { + self.reads_after_unknown.fetch_add(1, Ordering::Relaxed); + } + if key.starts_with(b"DS\0CB\0") && self.seal_conflicted.load(Ordering::Relaxed) { + self.cb_reads_after_conflict.fetch_add(1, Ordering::Relaxed); + } + if self.reads_only.load(Ordering::Relaxed) { + assert!( + ![ + b"DS\0SC\0".as_slice(), + b"DS\0CA\0", + b"DS\0CR\0", + b"DS\0LC\0", + b"DS\0H\0" + ] + .iter() + .any(|prefix| key.starts_with(prefix)), + "complete ID read queried publication evidence" + ); + } + } +} + +#[async_trait::async_trait] +impl DatasetStore for Store { + async fn get(&self, key: &[u8]) -> Result, StoreError> { + self.check_read(key); + self.inner.get(key).await + } + async fn get_bounded( + &self, + key: &[u8], + budget: MetadataScanBudget, + cancel: &ReadCancellation, + ) -> Result, StoreError> { + self.check_read(key); + self.bounded_reads.fetch_add(1, Ordering::Relaxed); + self.inner.get_bounded(key, budget, cancel).await + } + async fn scan_page( + &self, + request: &MetadataScan, + cancel: &ReadCancellation, + ) -> Result { + self.inner.scan_page(request, cancel).await + } + async fn compare_exchange( + &self, + key: &[u8], + expected: Option<&[u8]>, + value: &[u8], + ) -> Result { + assert!( + !self.reads_only.load(Ordering::Relaxed), + "read installed metadata" + ); + let final_root = key.starts_with(b"DS\0CB\0") + && value.first() == Some(&b'{') + && serde_json::from_slice::(value).unwrap()["version"] == 2; + let gate = self.gate.load(Ordering::Relaxed); + if (gate == 1 && final_root) || (gate == 2 && key.starts_with(b"DS\0IC\0")) { + self.gate.store(0, Ordering::Relaxed); + self.started.notify_one(); + self.release.notified().await; + } + if key.starts_with(b"DS\0CB\0") && value == RETIRED { + if let Some(winner) = self.seal_winner.swap(None) { + assert!(matches!( + self.inner.compare_exchange(key, expected, &winner).await?, + CasOutcome::Applied(_) + )); + self.seal_conflicted.store(true, Ordering::Relaxed); + } + } + let result = self.inner.compare_exchange(key, expected, value).await?; + if final_root + && matches!(result, CasOutcome::Applied(_)) + && self.lose_final_reply.swap(false, Ordering::Relaxed) + { + self.unknown.store(true, Ordering::Relaxed); + return Err(StoreError::Rejected); + } + Ok(result) + } + async fn delete(&self, key: &[u8]) -> Result<(), StoreError> { + assert!(!self.reads_only.load(Ordering::Relaxed)); + self.inner.delete(key).await + } +} + +pub fn key(prefix: &[u8], identity: &DatasetIdentity, snapshot: SnapshotId) -> Vec { + [prefix, identity.key().unwrap().as_slice(), snapshot.as_bytes()].concat() +} +pub fn candidate(operation: OperationId) -> OperationId { + let mut hash = Sha256::new(); + hash.update(b"crowdb-composition-candidate-v1\0"); + hash.update(operation.as_bytes()); + hash.update(1u64.to_be_bytes()); + OperationId::from_bytes(&hash.finalize()[..16]).unwrap() +} +pub async fn fixture() -> (Arc, Arc, DatasetIdentity, SnapshotId) { + let store = Arc::new(Store::default()); + let authority = Arc::new(DatasetAuthority::new(store.clone())); + let identity = DatasetIdentity::new(NamespacePath::root(), "complete-snapshots").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + let parent = authority + .publish_snapshot(&identity, None, crate::publication::publication_manifest(1)) + .await + .unwrap(); + (store, authority, identity, parent) +} +pub async fn finish_cleanup( + authority: &DatasetAuthority, + identity: &DatasetIdentity, + operation: OperationId, +) { + for _ in 0..100 { + if authority + .cleanup_composition_candidates_step(identity, operation, &ReadCancellation::new()) + .await + .unwrap() + .complete + { + return; + } + } + panic!("candidate cleanup did not finish"); +} diff --git a/lib/crowdb-access-dataset/tests/composition_abandon_test.rs b/lib/crowdb-access-dataset/tests/composition_abandon_test.rs index 20520284b..7b74ada64 100644 --- a/lib/crowdb-access-dataset/tests/composition_abandon_test.rs +++ b/lib/crowdb-access-dataset/tests/composition_abandon_test.rs @@ -10,7 +10,7 @@ mod split_store; use crowdb_access_dataset::{composition::*, *}; #[tokio::test] -async fn abandoned_prepared_composition_fences_raw_candidate_and_initial_split() { +async fn abandoned_complete_composition_preserves_raw_candidate_and_initial_split() { let (_, authority, identity, parent) = split_store::fixture().await; let operation = OperationId::random(); common::begin_evaluated( @@ -35,14 +35,10 @@ async fn abandoned_prepared_composition_fences_raw_candidate_and_initial_split() .progress, DraftProgress::Abandoned ); - assert!(authority - .publish_snapshot_candidate(&identity, &attempt) - .await - .is_err()); assert!(authority .prepare_split_draft(&identity, preparation.split_operation.unwrap(), 1) .await - .is_err()); + .is_ok()); assert_eq!( authority.inspect_head(&identity).await.unwrap().latest, Some(parent) @@ -66,7 +62,17 @@ async fn abandoned_prepared_composition_fences_raw_candidate_and_initial_split() } assert!(split_cleaned); authority - .reclaim_snapshot_metadata(&identity, preparation.candidate) + .get_snapshot(&identity, preparation.candidate) + .await + .unwrap(); + assert!(authority + .inspect_split_head(&identity, preparation.candidate) + .await + .unwrap() + .revision + .is_some()); + authority + .publish_snapshot_candidate(&identity, &attempt) .await .unwrap(); } diff --git a/lib/crowdb-access-dataset/tests/composition_initial_test.rs b/lib/crowdb-access-dataset/tests/composition_initial_test.rs index 62d31bb4a..835f24c85 100644 --- a/lib/crowdb-access-dataset/tests/composition_initial_test.rs +++ b/lib/crowdb-access-dataset/tests/composition_initial_test.rs @@ -35,13 +35,13 @@ async fn initial_split_and_data_share_visibility_and_later_revision_is_independe assert!(authority.inspect_split_head(&identity, snapshot).await.is_err()); let prepared = common::prepare(&authority, &identity, operation, 1).await; assert!(matches!(prepared.record.progress, DraftProgress::Prepared { .. })); - assert!(authority.inspect_split_head(&identity, snapshot).await.is_err()); + let initial = authority.inspect_split_head(&identity, snapshot).await.unwrap(); + assert!(initial.revision.is_some()); + assert_eq!(authority.latest(&identity).await.unwrap(), Some(parent)); let receipt = authority .publish_composition(&identity, operation, &attempt) .await .unwrap(); - let initial = authority.inspect_split_head(&identity, snapshot).await.unwrap(); - assert!(initial.revision.is_some()); assert_eq!(initial.generation, None); let record = authority .get_split_revision( diff --git a/lib/crowdb-access-dataset/tests/composition_upload_test.rs b/lib/crowdb-access-dataset/tests/composition_upload_test.rs index f5ef40b6c..80487875c 100644 --- a/lib/crowdb-access-dataset/tests/composition_upload_test.rs +++ b/lib/crowdb-access-dataset/tests/composition_upload_test.rs @@ -117,6 +117,16 @@ async fn publication_requires_candidate_owners_and_preserves_payload_after_stagi )); let snapshot = f.prepare().await; let attempt = common::attempt(&f.authority, &f.identity, snapshot).await; + assert!(matches!( + f.authority.get_snapshot(&f.identity, snapshot).await, + Err(AuthorityError::SnapshotNotFound) + )); + assert!(matches!( + f.authority + .publish_snapshot_candidate(&f.identity, &attempt) + .await, + Err(AuthorityError::SnapshotNotFound) + )); assert!(matches!( f.authority .publish_composition(&f.identity, f.operation, &attempt) @@ -124,6 +134,7 @@ async fn publication_requires_candidate_owners_and_preserves_payload_after_stagi Err(CompositionError::Conflict) )); f.own().await; + assert!(f.authority.get_snapshot(&f.identity, snapshot).await.is_ok()); f.authority .publish_composition(&f.identity, f.operation, &attempt) .await @@ -161,9 +172,9 @@ async fn publication_requires_candidate_owners_and_preserves_payload_after_stagi } #[tokio::test] -async fn cancelling_prepared_composition_retires_both_owners_and_is_restartable() { +async fn cancelling_complete_composition_retires_staging_and_preserves_resource_owner() { let f = TestComposition::new().await; - f.prepare().await; + let snapshot = f.prepare().await; f.own().await; f.authority .abandon_composition(&f.identity, f.operation, 2) @@ -171,9 +182,13 @@ async fn cancelling_prepared_composition_retires_both_owners_and_is_restartable( .unwrap(); let cleanup = f.cleanup().await; assert!(cleanup.complete); - assert_eq!(cleanup.retired_owners, 2); + assert_eq!(cleanup.retired_owners, 1); assert_eq!(f.cleanup().await, cleanup); - assert!(f.uploads.abort(&f.location).await.unwrap().cleanup_complete); + assert!(matches!( + f.uploads.abort(&f.location).await, + Err(UploadError::Conflict) + )); + assert!(f.authority.get_snapshot(&f.identity, snapshot).await.is_ok()); assert!(f .authority .prepare_composition_uploads(&f.identity, f.operation, &f.uploads, &ReadCancellation::new()) @@ -234,7 +249,7 @@ async fn child_candidate_claims_inherited_upload_and_cancellation_preserves_pare .await .unwrap(); assert!(cleanup.complete); - assert_eq!(cleanup.retired_owners, 1); + assert_eq!(cleanup.retired_owners, 0); assert!(matches!( f.uploads.abort(&f.location).await, Err(UploadError::Conflict) diff --git a/lib/crowdb-access-dataset/tests/inspection_index_test.rs b/lib/crowdb-access-dataset/tests/inspection_index_test.rs index fda1be00d..69626cc07 100644 --- a/lib/crowdb-access-dataset/tests/inspection_index_test.rs +++ b/lib/crowdb-access-dataset/tests/inspection_index_test.rs @@ -137,6 +137,14 @@ struct TestInterruptedStore { } #[async_trait::async_trait] impl DatasetStore for TestInterruptedStore { + async fn get_bounded( + &self, + key: &[u8], + budget: crowdb_access_dataset::MetadataScanBudget, + cancel: &crowdb_access_dataset::ReadCancellation, + ) -> Result, crowdb_access_dataset::StoreError> { + self.inner.get_bounded(key, budget, cancel).await + } async fn get( &self, key: &[u8], diff --git a/lib/crowdb-access-dataset/tests/inspection_pages_test.rs b/lib/crowdb-access-dataset/tests/inspection_pages_test.rs index 42a744edd..ad213e014 100644 --- a/lib/crowdb-access-dataset/tests/inspection_pages_test.rs +++ b/lib/crowdb-access-dataset/tests/inspection_pages_test.rs @@ -10,7 +10,7 @@ mod publication; use publication::publication_manifest; #[tokio::test] -async fn paged_forest_excludes_prepared_candidates_without_manifest_or_payload_reads() { +async fn paged_forest_includes_complete_unpublished_results_and_excludes_staging() { let (store, authority, identity) = fixture().await; let root = prepare(&authority, &identity, None, 1).await; publish(&authority, &identity, root).await; @@ -19,6 +19,15 @@ async fn paged_forest_excludes_prepared_candidates_without_manifest_or_payload_r let new_root = prepare(&authority, &identity, None, 3).await; publish(&authority, &identity, new_root).await; let prepared = prepare(&authority, &identity, Some(root), 4).await; + let incomplete = authority + .prepare_snapshot_with_operation( + &identity, + Some(new_root), + publication_manifest(5), + OperationId::random(), + ) + .await + .unwrap(); store.inspecting.store(true, Ordering::Relaxed); let cancel = ReadCancellation::new(); let mut after = None; @@ -35,15 +44,16 @@ async fn paged_forest_excludes_prepared_candidates_without_manifest_or_payload_r } } visible.sort(); - let mut expected = vec![root, child, new_root]; + let mut expected = vec![root, child, new_root, prepared]; expected.sort(); assert_eq!(visible, expected); let children = authority .snapshot_children_page(&identity, Some(root), None, budget(100), &cancel) .await .unwrap(); - assert_eq!(children.entries.len(), 1); - assert_eq!(children.entries[0].snapshot, child); + assert_eq!(children.entries.len(), 2); + assert!(children.entries.iter().any(|entry| entry.snapshot == child)); + assert!(children.entries.iter().any(|entry| entry.snapshot == prepared)); let roots = authority .snapshot_children_page(&identity, None, None, budget(100), &cancel) .await @@ -51,7 +61,7 @@ async fn paged_forest_excludes_prepared_candidates_without_manifest_or_payload_r assert_eq!(roots.entries.len(), 2); assert!(matches!( authority - .snapshot_inspection_summary(&identity, prepared, budget(1), &cancel) + .snapshot_inspection_summary(&identity, incomplete, budget(1), &cancel) .await, Err(InspectionError::Authority(AuthorityError::SnapshotNotFound)) )); diff --git a/lib/crowdb-access-dataset/tests/inspection_visibility_test.rs b/lib/crowdb-access-dataset/tests/inspection_visibility_test.rs index b1d579376..7f8988a5c 100644 --- a/lib/crowdb-access-dataset/tests/inspection_visibility_test.rs +++ b/lib/crowdb-access-dataset/tests/inspection_visibility_test.rs @@ -10,11 +10,19 @@ use inspection::{budget, fixture, prepare, publish}; use std::sync::atomic::Ordering; #[tokio::test] -async fn visibility_uses_cas_proof_and_never_published_state_alone() { +async fn visibility_uses_complete_roots_and_never_published_state_alone() { let (store, authority, identity) = fixture().await; let committed = prepare(&authority, &identity, None, 1).await; publish(&authority, &identity, committed).await; - let uncommitted = prepare(&authority, &identity, None, 2).await; + let uncommitted = authority + .prepare_snapshot_with_operation( + &identity, + Some(committed), + publication::publication_manifest(2), + OperationId::random(), + ) + .await + .unwrap(); let record_key = [ b"DS\0S\0".as_slice(), identity.key().unwrap().as_slice(), diff --git a/lib/crowdb-access-dataset/tests/legacy_publication_test.rs b/lib/crowdb-access-dataset/tests/legacy_publication_test.rs index 93b43320c..51c4f7286 100644 --- a/lib/crowdb-access-dataset/tests/legacy_publication_test.rs +++ b/lib/crowdb-access-dataset/tests/legacy_publication_test.rs @@ -132,3 +132,36 @@ async fn unindexed_legacy_child_protects_parent_after_latest_moves_to_new_root() authority.begin_reclaim(&identity, child).await.unwrap(); authority.begin_reclaim(&identity, root).await.unwrap(); } + +#[tokio::test] +async fn completing_an_already_current_legacy_manifest_installs_a_readable_resource_root() { + let store = Arc::new(dataset_store::TestDatasetStore::default()); + let authority = DatasetAuthority::new(store.clone()); + let identity = DatasetIdentity::new(NamespacePath::root(), "legacy-complete").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + let snapshot = authority + .publish_snapshot(&identity, None, publication_manifest(1)) + .await + .unwrap(); + let manifest = authority.get_manifest(&identity, snapshot).await.unwrap(); + legacy::emulate_legacy(store.as_ref(), &authority, &identity, &[snapshot], snapshot).await; + let head = authority.inspect_head(&identity).await.unwrap(); + authority + .persist_manifest(&identity, snapshot, &manifest, 1024) + .await + .unwrap(); + assert_eq!(authority.inspect_head(&identity).await.unwrap(), head); + let cb = [ + b"DS\0CB\0".as_slice(), + &identity.key().unwrap(), + snapshot.as_bytes(), + ] + .concat(); + let root: serde_json::Value = + serde_json::from_slice(&store.get(&cb).await.unwrap().unwrap().bytes).unwrap(); + assert_eq!(root["version"], 2); + assert!(DatasetAuthority::new(store) + .read_manifest(&identity, snapshot) + .await + .is_ok()); +} diff --git a/lib/crowdb-access-dataset/tests/legacy_read_cost_test.rs b/lib/crowdb-access-dataset/tests/legacy_read_cost_test.rs index 62755e21a..7abd5b6ab 100644 --- a/lib/crowdb-access-dataset/tests/legacy_read_cost_test.rs +++ b/lib/crowdb-access-dataset/tests/legacy_read_cost_test.rs @@ -152,12 +152,12 @@ async fn legacy_inspection_fallback_obeys_aggregate_budget_and_cancellation() { #[tokio::test] async fn unpublished_modern_candidate_cannot_fall_back_to_legacy_visibility() { - let (store, identity, _, _) = fixture().await; + let (store, identity, _, child) = fixture().await; let authority = DatasetAuthority::new(store.clone()); let snapshot = authority - .prepare_snapshot_candidate( + .prepare_snapshot_with_operation( &identity, - None, + Some(child), publication::publication_manifest(3), OperationId::random(), ) diff --git a/lib/crowdb-access-dataset/tests/projection_test.rs b/lib/crowdb-access-dataset/tests/projection_test.rs index 40cc2a509..5f7df2116 100644 --- a/lib/crowdb-access-dataset/tests/projection_test.rs +++ b/lib/crowdb-access-dataset/tests/projection_test.rs @@ -27,6 +27,14 @@ impl DatasetStore for TestReadStore { } self.inner.get(key).await } + async fn get_bounded( + &self, + key: &[u8], + budget: crowdb_access_dataset::MetadataScanBudget, + cancel: &crowdb_access_dataset::ReadCancellation, + ) -> Result, StoreError> { + self.inner.get_bounded(key, budget, cancel).await + } async fn compare_exchange( &self, key: &[u8], diff --git a/lib/crowdb-access-dataset/tests/snapshot_branch_reclaim_test.rs b/lib/crowdb-access-dataset/tests/snapshot_branch_reclaim_test.rs index d52216fcc..eb2dc0323 100644 --- a/lib/crowdb-access-dataset/tests/snapshot_branch_reclaim_test.rs +++ b/lib/crowdb-access-dataset/tests/snapshot_branch_reclaim_test.rs @@ -101,7 +101,7 @@ async fn temporary_child_reclaim_fence_is_not_terminal_retirement() { } #[tokio::test] -async fn prepared_uncommitted_children_do_not_pin_unrelated_history() { +async fn complete_unpublished_children_protect_parent_until_their_retirement() { let (_, authority, identity, parent) = split_store::fixture().await; let candidate = authority .prepare_snapshot_candidate( @@ -125,6 +125,11 @@ async fn prepared_uncommitted_children_do_not_pin_unrelated_history() { .publish_snapshot_candidate(&identity, &common::attempt(&authority, &identity, root).await) .await .unwrap(); + assert!(matches!( + authority.begin_reclaim(&identity, parent).await, + Err(AuthorityError::SnapshotProtected) + )); + authority.begin_reclaim(&identity, candidate).await.unwrap(); authority.begin_reclaim(&identity, parent).await.unwrap(); assert!(authority .publish_snapshot_candidate( diff --git a/lib/crowdb-access-dataset/tests/snapshot_complete_result_test.rs b/lib/crowdb-access-dataset/tests/snapshot_complete_result_test.rs new file mode 100644 index 000000000..a087c6ac1 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/snapshot_complete_result_test.rs @@ -0,0 +1,332 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/snapshot_complete.rs"] +mod complete; +#[path = "common/composition.rs"] +#[allow(dead_code)] +mod composition; +#[path = "common/publication.rs"] +mod publication; + +use complete::{candidate, finish_cleanup, fixture, key}; +use crowdb_access_dataset::*; +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; + +#[tokio::test] +async fn lost_final_reply_keeps_the_complete_result_without_confirmation_or_draft_pointer() { + let (store, authority, identity, parent) = fixture().await; + let operation = OperationId::random(); + authority + .begin_composition(&identity, operation, &composition::input(parent)) + .await + .unwrap(); + store.lose_final_reply.store(true, Ordering::Relaxed); + assert!(matches!( + authority + .prepare_snapshot_candidate( + &identity, + Some(parent), + publication::publication_manifest(2), + candidate(operation) + ) + .await, + Err(AuthorityError::Store(StoreError::Rejected)) + )); + assert_eq!(store.reads_after_unknown.load(Ordering::Relaxed), 0); + store.unknown.store(false, Ordering::Relaxed); + let snapshot = authority + .publication_status(&identity, candidate(operation)) + .await + .unwrap() + .publication + .snapshot; + authority + .abandon_composition(&identity, operation, 1) + .await + .unwrap(); + finish_cleanup(&authority, &identity, operation).await; + let restarted = DatasetAuthority::new(store.clone()); + store.reads_only.store(true, Ordering::Relaxed); + let before = store.bounded_reads.load(Ordering::Relaxed); + let summary = restarted + .snapshot_inspection_summary( + &identity, + snapshot, + composition::budget(), + &ReadCancellation::new(), + ) + .await + .unwrap(); + assert_eq!(summary.snapshot, snapshot); + assert_eq!(store.bounded_reads.load(Ordering::Relaxed) - before, 3); + let record = restarted.get_snapshot(&identity, snapshot).await.unwrap(); + assert_eq!(record.publication.parent, Some(parent)); + assert_eq!( + restarted + .read_batch(&identity, snapshot, &[b"sample-2".to_vec()], &["marker"]) + .await + .unwrap()[0] + .values["marker"], + [2] + ); +} + +#[tokio::test] +async fn cleanup_seals_the_same_root_before_a_delayed_finalizer_can_publish() { + let (store, authority, identity, parent) = fixture().await; + let operation = OperationId::random(); + authority + .begin_composition(&identity, operation, &composition::input(parent)) + .await + .unwrap(); + store.gate.store(1, Ordering::Relaxed); + let client = authority.clone(); + let scope = identity.clone(); + let pending = tokio::spawn(async move { + client + .prepare_snapshot_candidate( + &scope, + Some(parent), + publication::publication_manifest(2), + candidate(operation), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(2), store.started.notified()) + .await + .unwrap(); + let snapshot = authority + .publication_status(&identity, candidate(operation)) + .await + .unwrap() + .publication + .snapshot; + assert!(authority.get_snapshot(&identity, snapshot).await.is_err()); + authority + .abandon_composition(&identity, operation, 1) + .await + .unwrap(); + finish_cleanup(&authority, &identity, operation).await; + store.release.notify_one(); + assert!(matches!( + pending.await.unwrap(), + Err(AuthorityError::SnapshotProtected) + )); + assert_eq!( + store + .storage() + .get(&key(b"DS\0CB\0", &identity, snapshot)) + .await + .unwrap() + .unwrap() + .bytes, + complete::RETIRED + ); + let restarted = DatasetAuthority::new(store); + assert!(restarted.get_snapshot(&identity, snapshot).await.is_err()); + assert!(restarted + .complete_snapshot_candidate(&identity, snapshot) + .await + .is_err()); +} + +#[tokio::test] +async fn cleanup_uses_a_completed_cas_winner_without_reloading_or_retiring_content() { + let (store, authority, identity, parent) = fixture().await; + let operation = OperationId::random(); + authority + .begin_composition(&identity, operation, &composition::input(parent)) + .await + .unwrap(); + let snapshot = authority + .prepare_snapshot_candidate( + &identity, + Some(parent), + publication::publication_manifest(2), + candidate(operation), + ) + .await + .unwrap(); + let cb = key(b"DS\0CB\0", &identity, snapshot); + let complete = store.storage().get(&cb).await.unwrap().unwrap(); + let mut staged: serde_json::Value = serde_json::from_slice(&complete.bytes).unwrap(); + staged["version"] = 1.into(); + assert!(matches!( + store + .storage() + .compare_exchange(&cb, Some(&complete.bytes), &serde_json::to_vec(&staged).unwrap()) + .await + .unwrap(), + CasOutcome::Applied(_) + )); + store.seal_winner.store(Some(Arc::new(complete.bytes))); + authority + .abandon_composition(&identity, operation, 1) + .await + .unwrap(); + finish_cleanup(&authority, &identity, operation).await; + assert!(store.seal_conflicted.load(Ordering::Relaxed)); + assert_eq!(store.cb_reads_after_conflict.load(Ordering::Relaxed), 0); + assert!(authority.get_snapshot(&identity, snapshot).await.is_ok()); + assert!(authority.get_manifest(&identity, snapshot).await.is_ok()); +} + +async fn off_head_parent(authority: &DatasetAuthority, identity: &DatasetIdentity) { + let root = authority + .prepare_snapshot_candidate( + identity, + None, + publication::publication_manifest(3), + OperationId::random(), + ) + .await + .unwrap(); + authority + .publish_snapshot_candidate(identity, &composition::attempt(authority, identity, root).await) + .await + .unwrap(); +} + +#[tokio::test] +async fn indexed_child_protects_its_parent_while_finalization_is_pending() { + let (store, authority, identity, parent) = fixture().await; + off_head_parent(&authority, &identity).await; + store.gate.store(1, Ordering::Relaxed); + let client = authority.clone(); + let scope = identity.clone(); + let pending = tokio::spawn(async move { + client + .prepare_snapshot_candidate( + &scope, + Some(parent), + publication::publication_manifest(2), + OperationId::random(), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(2), store.started.notified()) + .await + .unwrap(); + assert!(matches!( + authority.begin_reclaim(&identity, parent).await, + Err(AuthorityError::SnapshotProtected) + )); + store.release.notify_one(); + let child = pending.await.unwrap().unwrap(); + assert_eq!( + authority + .get_snapshot(&identity, child) + .await + .unwrap() + .publication + .parent, + Some(parent) + ); +} + +#[tokio::test] +async fn child_cannot_complete_when_parent_retirement_precedes_its_index() { + let (store, authority, identity, parent) = fixture().await; + off_head_parent(&authority, &identity).await; + let operation = OperationId::random(); + store.gate.store(2, Ordering::Relaxed); + let client = authority.clone(); + let scope = identity.clone(); + let pending = tokio::spawn(async move { + client + .prepare_snapshot_candidate( + &scope, + Some(parent), + publication::publication_manifest(2), + operation, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(2), store.started.notified()) + .await + .unwrap(); + authority.begin_reclaim(&identity, parent).await.unwrap(); + store.release.notify_one(); + assert!(pending.await.unwrap().is_err()); + let snapshot = authority + .publication_status(&identity, operation) + .await + .unwrap() + .publication + .snapshot; + assert!(authority.get_snapshot(&identity, snapshot).await.is_err()); + assert!(authority + .publish_snapshot_candidate( + &identity, + &composition::attempt(&authority, &identity, snapshot).await + ) + .await + .is_err()); +} + +#[tokio::test] +async fn unpublished_continuations_cross_instances_and_still_obey_deletion() { + let (store, authority, identity, parent) = fixture().await; + let snapshot = authority + .prepare_snapshot_candidate( + &identity, + Some(parent), + publication::publication_manifest(2), + OperationId::random(), + ) + .await + .unwrap(); + let bound = DatasetReads::new(authority.clone()) + .bind( + &identity, + ReadSpec { + snapshot: snapshot.to_string(), + selection: Selection::Prefix(b"sample-".to_vec()), + fields: vec!["marker".into()], + batch_size: 1, + partition: ReadPartition { + worker: 0, + workers: 1, + rank: 0, + world_size: 1, + }, + shuffle: None, + split: None, + }, + ) + .await + .unwrap(); + off_head_parent(&authority, &identity).await; + let restarted = Arc::new(DatasetAuthority::new(store)); + let reads = DatasetReads::new(restarted.clone()); + let page = reads + .next(&identity, &bound.cursor, &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(page.ids, [b"sample-2".to_vec()]); + restarted.retain_snapshot(&identity, snapshot).await.unwrap(); + assert!(matches!( + restarted.begin_reclaim(&identity, snapshot).await, + Err(AuthorityError::SnapshotProtected) + )); + restarted.release_snapshot(&identity, snapshot).await.unwrap(); + restarted.begin_reclaim(&identity, snapshot).await.unwrap(); + assert!(reads + .next(&identity, &bound.cursor, &ReadCancellation::new()) + .await + .is_err()); + assert!(restarted.read_manifest(&identity, snapshot).await.is_err()); + assert!(DatasetReadService::new(restarted, identity, 1) + .unwrap() + .read(DatasetReadRequest { + snapshot, + sample_ids: vec![b"sample-2".to_vec()], + fields: vec!["marker".into()], + }) + .await + .is_err()); +} From 520dfb4c6f289e802247fa974d3947b78bf3b66e Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 19:55:52 +0800 Subject: [PATCH 125/258] Publish snapshot pointers without request ledgers or duplicate preparation --- app/crowdb-access-server/src/dataset.rs | 1 + .../src/dataset/api/conversion.rs | 23 +- .../src/dataset/api/error.rs | 7 + .../src/dataset/api/publication.rs | 43 +-- .../src/dataset/api/v2/compositions.rs | 2 +- .../dataset_composition_contract_test.rs | 4 +- .../dataset_composition_recovery_test.rs | 8 +- .../tests/dataset_publication_cost_test.rs | 93 ++++++ .../dataset/design-dataset-control.md | 31 +- .../dataset/openapi-crowdb-dataset-v2.yaml | 28 +- .../dataset/spec-crowdb-dataset.md | 11 +- doc/working/plan-access-simplification.md | 28 ++ e2e-test/tests/access_dataset_split.rs | 2 +- lib/crowdb-access-dataset/src/authority.rs | 21 +- .../src/authority/access.rs | 1 - .../src/authority/access/reclaim.rs | 28 +- .../src/authority/head.rs | 42 +-- .../src/authority/publication.rs | 48 +-- .../src/authority/publication/candidate.rs | 6 +- .../src/authority/publication/commit.rs | 137 ++++---- .../src/authority/publication/complete.rs | 8 +- .../src/authority/publication/legacy.rs | 19 +- .../authority/publication/legacy_candidate.rs | 96 ++++++ .../src/authority/publication/receipt.rs | 57 +--- .../src/authority/publication/visibility.rs | 16 +- .../src/composition/candidates.rs | 1 - .../src/composition/cleanup.rs | 2 +- .../src/composition/edit.rs | 11 +- .../src/composition/edit_cleanup.rs | 2 +- .../src/composition/publication.rs | 66 +--- .../src/composition/uploads/release.rs | 22 +- lib/crowdb-access-dataset/src/lib.rs | 4 +- .../tests/candidate_publication_test.rs | 303 +++--------------- .../tests/common/legacy.rs | 8 +- .../tests/common/reclaim_observed.rs | 6 + .../tests/common/split_store.rs | 31 +- .../tests/composition_abandon_test.rs | 13 +- .../tests/composition_publication_test.rs | 34 +- .../tests/composition_upload_test.rs | 7 +- lib/crowdb-access-dataset/tests/core_test.rs | 2 +- .../tests/legacy_commit_observation_test.rs | 222 +++++++++++++ .../tests/legacy_publication_test.rs | 46 +++ .../tests/reclaim_observed_test.rs | 10 +- .../tests/snapshot_complete_result_test.rs | 2 +- .../tests/snapshot_publication_cost_test.rs | 185 +++++++++++ sdk/README.md | 14 +- .../examples/split_control.rs | 5 +- .../crowdb-sdk-dataset/src/v2/client.rs | 2 +- sdk/tests/split_consumer.py | 2 +- 49 files changed, 1102 insertions(+), 658 deletions(-) create mode 100644 app/crowdb-access-server/tests/dataset_publication_cost_test.rs create mode 100644 lib/crowdb-access-dataset/src/authority/publication/legacy_candidate.rs create mode 100644 lib/crowdb-access-dataset/tests/legacy_commit_observation_test.rs create mode 100644 lib/crowdb-access-dataset/tests/snapshot_publication_cost_test.rs diff --git a/app/crowdb-access-server/src/dataset.rs b/app/crowdb-access-server/src/dataset.rs index 81b9d3c42..889b4d5c7 100644 --- a/app/crowdb-access-server/src/dataset.rs +++ b/app/crowdb-access-server/src/dataset.rs @@ -93,6 +93,7 @@ fn status_for_authority(error: &AuthorityError) -> StatusCode { AuthorityError::AlreadyExists | AuthorityError::ParentConflict | AuthorityError::HeadConflict + | AuthorityError::SnapshotHeadConflict { .. } | AuthorityError::CursorConflict | AuthorityError::ReclaimPending { .. } | AuthorityError::SnapshotProtected => StatusCode::CONFLICT, diff --git a/app/crowdb-access-server/src/dataset/api/conversion.rs b/app/crowdb-access-server/src/dataset/api/conversion.rs index ebb01ee69..67acd9fcd 100644 --- a/app/crowdb-access-server/src/dataset/api/conversion.rs +++ b/app/crowdb-access-server/src/dataset/api/conversion.rs @@ -4,8 +4,8 @@ use super::{error::ApiError, models}; use base64::{engine::general_purpose::STANDARD, Engine}; use crowdb_access_dataset::{ - DatasetRecord, FieldLocator, FieldRecord, ManifestRecord, PublicationState, SampleRecord, SchemaRecord, - SnapshotId, SnapshotRecord, + DatasetRecord, FieldLocator, FieldRecord, ManifestRecord, OperationId, PublicationState, SampleRecord, + SchemaRecord, SnapshotId, SnapshotRecord, }; use md5::{Digest, Md5}; use std::collections::BTreeMap; @@ -21,17 +21,26 @@ pub(super) fn dataset(value: &DatasetRecord) -> models::DatasetInfo { } pub(super) fn snapshot(value: SnapshotRecord, latest: Option) -> models::Snapshot { let value = value.publication; + snapshot_metadata(value.snapshot, value.parent, value.operation, value.state, latest) +} +pub(super) fn snapshot_metadata( + snapshot: SnapshotId, + parent: Option, + operation: OperationId, + state: PublicationState, + latest: Option, +) -> models::Snapshot { models::Snapshot { - id: value.snapshot.to_string(), - parent: value.parent.map(|v| v.to_string()), - operation: value.operation.to_string(), - state: match value.state { + id: snapshot.to_string(), + parent: parent.map(|v| v.to_string()), + operation: operation.to_string(), + state: match state { PublicationState::Prepared => "prepared", PublicationState::Published => "published", PublicationState::Aborted => "aborted", } .into(), - is_latest: latest == Some(value.snapshot), + is_latest: latest == Some(snapshot), } } pub(super) fn manifest( diff --git a/app/crowdb-access-server/src/dataset/api/error.rs b/app/crowdb-access-server/src/dataset/api/error.rs index 6e6871693..8c721fc70 100644 --- a/app/crowdb-access-server/src/dataset/api/error.rs +++ b/app/crowdb-access-server/src/dataset/api/error.rs @@ -96,6 +96,13 @@ impl From for ApiError { } impl From for ApiError { fn from(error: AuthorityError) -> Self { + if let AuthorityError::SnapshotHeadConflict { observed } = error { + return Self::conflict().with_observed_head(serde_json::json!({ + "latest":observed.latest.map(|id|id.to_string()), + "stable":observed.stable.map(|id|id.to_string()), + "generation":observed.generation.map(|generation|hex::encode(generation.as_bytes())), + })); + } if matches!( &error, AuthorityError::Invalid(crowdb_access_dataset::DatasetError::ReadCancelled) diff --git a/app/crowdb-access-server/src/dataset/api/publication.rs b/app/crowdb-access-server/src/dataset/api/publication.rs index 11dd91980..554c07589 100644 --- a/app/crowdb-access-server/src/dataset/api/publication.rs +++ b/app/crowdb-access-server/src/dataset/api/publication.rs @@ -3,7 +3,7 @@ use super::{conversion, error::ApiError, json, models, operation, read_json, snapshot, ApiResult}; use crate::dataset::{unix_seconds, DatasetHttpService}; -use crowdb_access_dataset::{AuthorityError, DatasetIdentity, FieldLocator, ManifestRecord, UploadState}; +use crowdb_access_dataset::{DatasetIdentity, FieldLocator, ManifestRecord, PublicationState, UploadState}; use hyper::{body::Incoming, Request}; use std::collections::BTreeSet; @@ -21,25 +21,10 @@ impl DatasetHttpService { let locations = self.resolve_payloads(&mut manifest).await?; let bytes = bincode::serialize(&manifest).map_err(|_| ApiError::integrity())?; let authority = self.reads.authority(); - match authority.publication_status(identity, operation).await { - Ok(record) if record.publication.parent != parent || record.publication.manifest != bytes => { - return Err(ApiError::conflict()) - } - Ok(_) => {} - Err(AuthorityError::SnapshotNotFound) => { - if authority.latest(identity).await? != parent { - return Err(ApiError::conflict()); - } - if let Some(parent) = parent { - authority.get_snapshot(identity, parent).await?; - } - } - Err(error) => return Err(error.into()), - } // The candidate is durable before ownership is acquired, so failed or // disconnected requests always have a recoverable publication identity. - authority - .prepare_snapshot_with_operation(identity, parent, bytes.clone(), operation) + let snapshot = authority + .prepare_snapshot_with_operation(identity, parent, bytes, operation) .await?; if !locations.is_empty() { let uploads = self.uploads.as_ref().ok_or_else(ApiError::unsupported)?; @@ -47,19 +32,17 @@ impl DatasetHttpService { uploads.claim(location, operation, unix_seconds()).await?; } } - let result = if action == "prepare" { - authority - .prepare_snapshot_with_operation(identity, parent, bytes, operation) - .await - } else { - authority - .publish_snapshot_with_operation(identity, parent, bytes, operation) - .await - }; - let snapshot = result?; authority.complete_snapshot_candidate(identity, snapshot).await?; - json(&conversion::snapshot( - authority.publication_status(identity, operation).await?, + if action == "publish" { + authority + .publish_prepared_snapshot(identity, snapshot, parent) + .await?; + } + json(&conversion::snapshot_metadata( + snapshot, + parent, + operation, + PublicationState::Published, authority.latest(identity).await?, )) } diff --git a/app/crowdb-access-server/src/dataset/api/v2/compositions.rs b/app/crowdb-access-server/src/dataset/api/v2/compositions.rs index d315a09fc..cd964fad7 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/compositions.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/compositions.rs @@ -145,7 +145,7 @@ impl DatasetHttpService { let draft = authority.composition_status(identity, operation).await?; let edits = if matches!( draft.record.progress, - DraftProgress::Published { .. } | DraftProgress::Abandoned + DraftProgress::Prepared { .. } | DraftProgress::Published { .. } | DraftProgress::Abandoned ) { Some( authority diff --git a/app/crowdb-access-server/tests/dataset_composition_contract_test.rs b/app/crowdb-access-server/tests/dataset_composition_contract_test.rs index e7cf73df0..8df99605e 100644 --- a/app/crowdb-access-server/tests/dataset_composition_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_composition_contract_test.rs @@ -51,7 +51,7 @@ async fn uploaded_composition_publishes_initial_splits_and_branches_from_a_froze let prepared = composition::prepare(&client, &draft, "2", "1").await; let snapshot = prepared["preparation"]["snapshot"].as_str().unwrap(); response(client.get(format!("{root}/snapshots/{snapshot}/summary")), 200).await; - let receipt = composition::publish( + composition::publish( &client, &root, &draft, @@ -65,7 +65,7 @@ async fn uploaded_composition_publishes_initial_splits_and_branches_from_a_froze 200 ) .await["receipt"], - receipt + Value::Null ); let head = response(client.get(format!("{root}/head")), 200).await; assert_eq!(head["latest"], snapshot); diff --git a/app/crowdb-access-server/tests/dataset_composition_recovery_test.rs b/app/crowdb-access-server/tests/dataset_composition_recovery_test.rs index 82afeb9fa..a3102eb8a 100644 --- a/app/crowdb-access-server/tests/dataset_composition_recovery_test.rs +++ b/app/crowdb-access-server/tests/dataset_composition_recovery_test.rs @@ -15,7 +15,7 @@ use serde_json::{json, Value}; use std::sync::Arc; #[tokio::test] -async fn restart_and_head_conflict_preserve_candidates_and_receipts() { +async fn restart_and_head_conflict_preserve_candidates_without_receipt_writes() { let store = Arc::new(TestStore::default()); let first = TestDatasetHttp::start_payload(store.clone()).await; let client = Client::new(); @@ -46,13 +46,15 @@ async fn restart_and_head_conflict_preserve_candidates_and_receipts() { .await; let attempt = "13131313131313131313131313131313"; let mut publish = json!({"attempt":attempt,"snapshot":prepared["preparation"]["snapshot"],"expected_head":old_head["generation"]}); - response(client.post(format!("{draft}/publish")).json(&publish), 409).await; + let conflict = response(client.post(format!("{draft}/publish")).json(&publish), 409).await; + assert_eq!(conflict["observed_head"]["latest"], parent); + assert_eq!(conflict["observed_head"]["stable"], parent); assert_eq!( response(client.get(&draft), 200).await["progress"]["state"], "prepared" ); publish["attempt"] = json!("14141414141414141414141414141414"); - publish["expected_head"] = response(client.get(format!("{root}/head")), 200).await["generation"].clone(); + publish["expected_head"] = conflict["observed_head"]["generation"].clone(); let receipt = response(client.post(format!("{draft}/publish")).json(&publish), 200).await; composition::cleanup(&client, &draft).await; assert_eq!( diff --git a/app/crowdb-access-server/tests/dataset_publication_cost_test.rs b/app/crowdb-access-server/tests/dataset_publication_cost_test.rs new file mode 100644 index 000000000..0c9683a53 --- /dev/null +++ b/app/crowdb-access-server/tests/dataset_publication_cost_test.rs @@ -0,0 +1,93 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/dataset.rs"] +mod dataset; +use base64::{engine::general_purpose::STANDARD, Engine}; +use dataset::{TestDatasetHttp, TestStore}; +use reqwest::Client; +use serde_json::Value; +use std::sync::{atomic::Ordering, Arc}; + +fn counts(store: &TestStore) -> (u64, u64) { + ( + store.reads.load(Ordering::Relaxed) + store.bounded_reads.load(Ordering::Relaxed), + store.mutations.load(Ordering::Relaxed), + ) +} +async fn call(client: &Client, url: &str, input: &Value) -> Value { + let response = client + .post(url) + .bearer_auth("secret") + .json(input) + .send() + .await + .unwrap(); + let status = response.status().as_u16(); + let body: Value = response.json().await.unwrap(); + assert_eq!(status, 200, "{body}"); + body +} + +#[tokio::test] +async fn inline_publication_and_identical_replay_have_bounded_metadata_cost() { + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start(store.clone()).await; + let client = Client::new(); + for samples in [1u8, 40] { + let root = format!( + "{}/api/dataset/v1/namespaces/dataset-ns/datasets/publication-cost-{samples}", + test.origin + ); + assert_eq!( + client + .put(&root) + .bearer_auth("secret") + .send() + .await + .unwrap() + .status() + .as_u16(), + 200 + ); + let mut input: Value = serde_json::from_str(include_str!( + "../../../sdk/contract-fixtures/dataset-v1/publication.json" + )) + .unwrap(); + let template = input["manifest"]["samples"][0].clone(); + input["manifest"]["samples"] = (0..samples) + .map(|id| { + let mut sample = template.clone(); + sample["id"] = STANDARD.encode([id]).into(); + sample + }) + .collect(); + let before = counts(&store); + let result = call(&client, &format!("{root}/snapshots/publish"), &input).await; + let after = counts(&store); + assert_eq!( + (after.0 - before.0, after.1 - before.1), + (12, 12 + 3 * u64::from(samples)) + ); + println!( + "cold publication: {} reads, {} mutations", + after.0 - before.0, + after.1 - before.1 + ); + assert_eq!(result["is_latest"], true); + let before = after; + assert_eq!( + call(&client, &format!("{root}/snapshots/publish"), &input).await, + result + ); + let after = counts(&store); + assert_eq!((after.0 - before.0, after.1 - before.1), (11, 0)); + println!( + "identical replay: {} reads, {} mutations", + after.0 - before.0, + after.1 - before.1 + ); + } + drop(client); + test.stop().await; +} diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index c074e2424..7bac27c66 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -27,17 +27,32 @@ generation used for publication CAS. Multiple roots and off-latest branches rema discoverable. A preparation binding or a Published flag alone is not a complete result. The final Snapshot resource root is installed after Manifest/index rows, the initial Split revision and necessary payload ownership are ready. The current root uses -version 2 of the existing candidate binding; version 1 remains an incomplete legacy -preparation unless old successful-publication evidence exists. Complete results are +version 2 of the existing candidate binding; new staging uses version 3 and is not +readable. Old version 1 bindings use existing publication evidence, or read-only +validation of their Published content identity/digest and complete initial Split. +They never acquire new publication proofs during reads or head replacement. Complete results are readable, enumerable, retainable and usable as parents/stable targets without winning latest. ID reads consult this root rather than publication receipts or mutable heads. -Publication has a candidate identity and a separate caller-owned attempt identity. -Each attempt binds its target and exact expected head generation. Successful commit -receipts remain durable and help incomplete publication finalization after restart. -Retrying a successful historical attempt returns its receipt without rewinding a -later head. On a head conflict the candidate remains available; an explicit new -attempt can use a newly observed head without changing its parent or contents. +Publication selects a complete candidate with an exact expected head generation. +It writes only the fenced head, without attempt/response records, success proofs +or Composition committing/published progress. Compatibility attempt IDs are echoed +in the immediate response and do not create request history. Retrying the same +current latest confirms its current effect; after another Snapshot replaces it, +the stale condition returns the observed latest/stable/generation without rewinding +the head or querying again. Unknown storage outcomes propagate unchanged. The +caller decides how to use an observed base; the server never rebases content. +Existing receipts and attempt-bearing heads are decoded read-only. Before replacing +an old head without complete resource roots, the existing bounded legacy ancestry +anchor preserves those resources; new version 4 heads need no receipt helping or +anchor lookup. Stable-only legacy roots are included in that compatibility index. + +The v1 JSON adapter prepares content once before acquiring payload owners. It then +completes that resource and selects it using the existing latest-parent condition; +it does not rerun Manifest/index preparation. Authority validation replaces the +old operation/head preflight. Responses use the known immutable metadata plus one +latest observation, without rereading operation/Snapshot records or copying the +Manifest for a second preparation. Each Snapshot owns immutable Split-set revisions and an independently fenced currentSplit pointer. Names are scoped to Dataset/Snapshot/revision. Publishing a diff --git a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml index 08a798899..68aa7a0c7 100644 --- a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml +++ b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml @@ -81,7 +81,7 @@ paths: /namespaces/dataset-ns/datasets/{dataset}/snapshot-inventory: get: operationId: snapshotInventory - summary: List committed Snapshot inventory + summary: List complete Snapshot inventory parameters: - {$ref: '#/components/parameters/dataset'} - {$ref: '#/components/parameters/after'} @@ -97,7 +97,7 @@ paths: /namespaces/dataset-ns/datasets/{dataset}/roots: get: operationId: snapshotRoots - summary: List committed parentless Snapshot roots + summary: List complete parentless Snapshot roots parameters: - {$ref: '#/components/parameters/dataset'} - {$ref: '#/components/parameters/after'} @@ -113,7 +113,7 @@ paths: /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/children: get: operationId: snapshotChildren - summary: List committed children of a concrete Snapshot + summary: List complete children of a concrete Snapshot parameters: - {$ref: '#/components/parameters/dataset'} - {$ref: '#/components/parameters/snapshot'} @@ -130,7 +130,7 @@ paths: /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/summary: get: operationId: snapshotSummary - summary: Read bounded committed Snapshot metadata + summary: Read bounded complete Snapshot metadata parameters: - {$ref: '#/components/parameters/dataset'} - {$ref: '#/components/parameters/snapshot'} @@ -775,8 +775,8 @@ paths: /namespaces/dataset-ns/datasets/{dataset}/commits/{attempt}: get: operationId: commitStatusV2 - summary: Recover a successful publication receipt - description: Returns Snapshot or legacy publication receipts. New independent Split publication does not create historical receipts; retry the same conditional request to confirm its current effect. + summary: Read a legacy publication receipt + description: Returns existing legacy receipts without backfill. New Snapshot and Split publication creates no historical receipts; retry the same conditional request to confirm its current effect. parameters: - $ref: '#/components/parameters/dataset' - in: path @@ -1082,9 +1082,10 @@ paths: $ref: '#/components/schemas/CommitReceipt' default: $ref: '#/components/responses/InspectionError' - description: Successful publication advances latest only. Null expected_head is explicit; omission fails. Keep - the exact attempt and request on unknown outcome. A head conflict preserves the prepared candidate; refresh - the head and use a new attempt. + description: Successful publication advances latest only, leaving the draft Prepared and creating no attempt, + receipt or success-proof record. Null expected_head is explicit; omission fails. Retry the exact conditional + request on unknown outcome. A conflict preserves the complete candidate and returns observed_head for caller + review, without a confirmation query. A later publication prevents historical replay from rewinding latest. requestBody: required: true content: @@ -1110,8 +1111,8 @@ paths: $ref: '#/components/schemas/CompositionCleanup' default: $ref: '#/components/responses/InspectionError' - description: Repeat until complete. Active drafts preserve current work; terminal drafts also retire staging - owners and edit payloads. Successful publication receipts remain available. + description: Repeat until complete. Prepared resources allow scratch and edit payload cleanup; completed + candidate ownership permits staging-owner retirement. Final resource owners remain until Snapshot reclamation. components: parameters: dataset: @@ -1150,7 +1151,10 @@ components: message: {type: string} request_id: {type: string} retryable: {type: boolean} - observed_head: {$ref: '#/components/schemas/SplitHead'} + observed_head: + oneOf: + - {$ref: '#/components/schemas/SplitHead'} + - {$ref: '#/components/schemas/Head'} securitySchemes: bearerAuth: type: http diff --git a/doc/design/access-server/dataset/spec-crowdb-dataset.md b/doc/design/access-server/dataset/spec-crowdb-dataset.md index bb6d6c94f..4ea408738 100644 --- a/doc/design/access-server/dataset/spec-crowdb-dataset.md +++ b/doc/design/access-server/dataset/spec-crowdb-dataset.md @@ -196,9 +196,14 @@ late finalization from exposing reclaimed data. Control v2 separates candidate creation from publication attempts. An attempt MUST bind the exact candidate and expected opaque head generation. Initial Split membership and candidate upload ownership MUST be ready before the head transition. -A successful Snapshot attempt MUST have a durable recoverable receipt. Retry of a historical -success MUST NOT rewind latest. A head conflict preserves the candidate and parent; -a reviewed retry uses a new attempt with a newly observed generation. +New Snapshot publication MUST NOT persist attempt/response ledgers, success proofs +or draft publication progress. A response may echo the compatibility attempt ID; +it does not establish historical response replay. Retrying the same current target +can confirm its effect; a stale request after a later publication MUST NOT rewind +latest. A head conflict MUST return the observed latest/stable/generation without +a confirmation query. A reviewed retry uses that observation with caller-chosen +content; the server MUST NOT rebase. Unknown storage outcomes MUST propagate. +Existing legacy receipts remain readable without backfill. Publishing an independent Split revision MUST update only its Snapshot's currentSplit pointer with an expected revision/head CAS. It MUST NOT create a data Snapshot or diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index fe0d8c0b2..36ab6bacf 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -156,6 +156,10 @@ CAS 不再满足只证明“本次条件不成立”，不能谎称“第一次 历史回复查询；Composition/Split 的 Committing 恢复与 P2/P3 一起删除，客户端保留完整输入。 旧记录只作为兼容解码/GC 输入，不让旧账本控制新请求。head 保留 expected base；同一 candidate 的当前结果可判断相同，已被后续 head 遮住时不保证历史请求结果重放。 +- [ ] **R2.2d Snapshot 直接 REST/SDK 发布入口**：提供只接收完整 Snapshot ID 与显式 nullable + expected head 的 head 更新 API、能力和 Rust/Python SDK；不读草稿或使用 attempt ID。 + SDK 提供 typed observed head，正常/冲突/unknown 请求计数和真实 installed consumer 验证， + 迁移示例与容器流程；保留父内容不可隐式 rebase、鉴权、删除与不支持时无降级。 - [ ] **R2 审查条件发布的回复丢失窗口**：用 H0→H1→H2 测试区分条件失败、相同结果与历史未知； 列出确实依赖历史精确结果的调用者，没有这种需求就移除相应 ledger/response replay。 @@ -490,6 +494,30 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，R2.2c/R1.1**：Snapshot 新发布只写 version 4 fenced head，删除 CA/CR/SC + 写入、head 成功补写和新 Composition Committing/Published 转换。当前同目标确认效果， + 后续目标遮住的旧条件返回 observed latest/stable/generation，真实 CAS 失败直接解码返回值， + unknown 不确认 GET。原生 `SnapshotHeadUpdate` 不含 attempt ID；compatibility response + 仅回显调用者输入。正常/真实冲突各 3G + 1C，当前目标重试/过期条件各 3G + 0C； + 四次真实冲突后客户端直接复用返回代际，共五次请求 15G + 5C，没有单独 head 查询。 + Composition adapter 只多一次草稿 GET（4G + 1C），Prepared 状态保持不变。 + cleanup 在最终 owner 就绪后可释放 staging owner，Prepared 计算 scratch/edit payload + 也可回收，最终数据及初始 Split 仍保留。旧 Committing 只读取既有结果后复位；旧回执 + 和 version 3 attempt head 只读，不补写。旧 CB v1 Published 通过有界身份/digest/初始 + Split 校验保持可读，新暂存绑定用 version 3，不能旁路最终根。旧无 CB 资源仅保留已有 + bounded legacy ancestry anchor；stable-only 旧根也覆盖，现代 head 不做该查询/写入。 + 同时删除 JSON 发布的重复预检、整次二次 prepare、完成回查和响应 operation/Snapshot + 回读；获取 owner 前先保留候选可发现性。单 inline 请求真实 listener 冷发布从 28G + 25 + mutations 降为 12G + 15；同内容重放从 24G 降为 11G、零写。40 个 sample 实测 12G + + 132 mutations，重放仍 11G、零写，immutable 行只准备一遍；去掉一份 Manifest Vec 复制。 + 上述均为逻辑元数据调用，未等同于 routed 网络 RPC 或 TPS/CPU 实测。 + Dataset 全目标与追加 cost/legacy/ownership 回归通过；HTTP restart/冲突观察值重试、 + Rust/installed Python SDK 与 payload consumer、11 项 native 读取测试通过。三 crate + all-target/all-feature clippy、独立 SDK 与 e2e-client clippy、两个 workspace fmt、diff + 检查通过；OpenAPI 解析及 333 个本地引用验证通过。均经 pixi。 + R2.2d 跟进直接 REST/SDK head API；P2/P3/R4 继续删除计算过程与补全孤儿 GC，V2/V4 + 仍需新源码镜像复验，不能把此批算作全部清理完成。 + - **2026-10-24，R2.1/R2.1b**：新 Snapshot 最后将已有 CB 条件替换为 version 2 完整根， Manifest/index、初始 Split 与必要 upload owners 就绪后，才能按 ID 读取/引用、列入 inventory/history、retain 或设 stable。版本 1 准备绑定和 Published 标志本身不够；旧成功 diff --git a/e2e-test/tests/access_dataset_split.rs b/e2e-test/tests/access_dataset_split.rs index 45b565263..b80021390 100644 --- a/e2e-test/tests/access_dataset_split.rs +++ b/e2e-test/tests/access_dataset_split.rs @@ -93,7 +93,7 @@ async fn split_commit_restart_preserves_candidates_receipts_and_historical_membe 200 ) .await["receipt"], - receipt + Value::Null ); revise(&client, snapshot).await; assert_eq!( diff --git a/lib/crowdb-access-dataset/src/authority.rs b/lib/crowdb-access-dataset/src/authority.rs index d1592f179..202e65470 100644 --- a/lib/crowdb-access-dataset/src/authority.rs +++ b/lib/crowdb-access-dataset/src/authority.rs @@ -1,5 +1,4 @@ mod access; -pub(crate) use access::CandidateRetirement; pub use access::{ MetadataCleanupProgress, SnapshotLifecycle, SnapshotReclaimError, SnapshotReclaimPhase, SnapshotReclaimStatus, @@ -8,8 +7,8 @@ mod discovery; mod head; mod projection; -pub use head::DatasetHead; use head::{decode_head, encode_head}; +pub use head::{DatasetHead, SnapshotHeadUpdate}; mod publication; use std::collections::{BTreeMap, HashSet}; @@ -54,6 +53,8 @@ pub enum AuthorityError { ParentConflict, #[error("publication lost its head race")] HeadConflict, + #[error("Snapshot publication condition no longer matches the head")] + SnapshotHeadConflict { observed: Box }, #[error("stored dataset record is malformed")] Corrupt, #[error("snapshot was not found")] @@ -204,9 +205,6 @@ impl DatasetAuthority { self.get_snapshot(identity, snapshot).await?; let key = key::head(identity)?; let current = self.store.get(&key).await?; - if let Some(current) = ¤t { - self.help_head_commit(identity, current).await?; - } let mut head = current .as_ref() .map(|value| decode_head(&value.bytes)) @@ -219,12 +217,23 @@ impl DatasetAuthority { return Ok(()); } self.ensure_snapshot_live(identity, snapshot).await?; + if let Some(current) = ¤t { + self.preserve_legacy_head(identity, current).await?; + } head.stable = Some(snapshot); let bytes = encode_head(head)?; let expected = current.as_ref().map(|value| value.bytes.as_slice()); match self.store.compare_exchange(&key, expected, &bytes).await? { CasOutcome::Applied(_) => Ok(()), - CasOutcome::Conflict(_) => Err(AuthorityError::HeadConflict), + CasOutcome::Conflict(value) => { + let observed = head::reference(value.as_ref())?; + if observed.stable == Some(snapshot) { + return Ok(()); + } + Err(AuthorityError::SnapshotHeadConflict { + observed: Box::new(observed), + }) + } } } diff --git a/lib/crowdb-access-dataset/src/authority/access.rs b/lib/crowdb-access-dataset/src/authority/access.rs index 47d7383f0..e6f99bc1f 100644 --- a/lib/crowdb-access-dataset/src/authority/access.rs +++ b/lib/crowdb-access-dataset/src/authority/access.rs @@ -8,7 +8,6 @@ mod observation; pub use lifecycle::{SnapshotReclaimError, SnapshotReclaimPhase, SnapshotReclaimStatus}; pub use observation::SnapshotLifecycle; mod reclaim; -pub(crate) use reclaim::CandidateRetirement; mod sweep; use crate::{key, AuthorityError, CasOutcome, DatasetIdentity, OperationId, SnapshotId}; pub use cleanup::MetadataCleanupProgress; diff --git a/lib/crowdb-access-dataset/src/authority/access/reclaim.rs b/lib/crowdb-access-dataset/src/authority/access/reclaim.rs index 952a84f5e..f0a571ad2 100644 --- a/lib/crowdb-access-dataset/src/authority/access/reclaim.rs +++ b/lib/crowdb-access-dataset/src/authority/access/reclaim.rs @@ -5,17 +5,6 @@ use super::DatasetAuthority; use crate::authority::{decode, decode_head, encode_head}; use crate::{key, AuthorityError, CasOutcome, DatasetIdentity, HeadRecord, OperationId, SnapshotId}; -pub(crate) enum CandidateRetirement { - Retired, - Complete, - LegacyCommitted(crate::SuccessfulCommit), -} -impl CandidateRetirement { - pub(crate) fn retained(&self) -> bool { - !matches!(self, Self::Retired) - } -} - impl DatasetAuthority { pub(super) async fn fence_reclaim( &self, @@ -43,7 +32,7 @@ impl DatasetAuthority { let mut old = self.store.get(&key).await?; for _ in 0..64 { if let Some(old) = &old { - self.help_head_commit(identity, old).await?; + self.preserve_legacy_head(identity, old).await?; } let head = old .as_ref() @@ -194,24 +183,15 @@ impl DatasetAuthority { &self, identity: &DatasetIdentity, snapshot: SnapshotId, - ) -> Result { - if let Some(receipt) = self.snapshot_commit(identity, snapshot).await? { - return Ok(CandidateRetirement::LegacyCommitted(receipt)); - } + ) -> Result { if self.seal_incomplete_snapshot(identity, snapshot).await? { - return Ok(CandidateRetirement::Complete); + return Ok(true); } if let Some(owner) = self.fence_reclaim(identity, snapshot).await? { - if let Some(receipt) = self.snapshot_commit(identity, snapshot).await? { - self.unfence_reclaim(identity, snapshot, owner).await?; - return Ok(CandidateRetirement::LegacyCommitted(receipt)); - } self.begin_terminal_retirement(identity, snapshot, owner, 0) .await?; - } else if let Some(receipt) = self.snapshot_commit(identity, snapshot).await? { - return Ok(CandidateRetirement::LegacyCommitted(receipt)); } self.retire_snapshot_record(identity, snapshot).await?; - Ok(CandidateRetirement::Retired) + Ok(false) } } diff --git a/lib/crowdb-access-dataset/src/authority/head.rs b/lib/crowdb-access-dataset/src/authority/head.rs index e1bc2a4ed..a5077d52d 100644 --- a/lib/crowdb-access-dataset/src/authority/head.rs +++ b/lib/crowdb-access-dataset/src/authority/head.rs @@ -4,6 +4,7 @@ use super::{decode, AuthorityError, VersionedRecord}; use crate::{ key, CommitAttempt, CommitGeneration, DatasetAuthority, DatasetIdentity, HeadRecord, OperationId, + SnapshotId, StoredValue, }; use serde::{Deserialize, Serialize}; @@ -28,6 +29,27 @@ pub struct DatasetHead { pub generation: Option, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SnapshotHeadUpdate { + pub snapshot: SnapshotId, + pub expected_head: Option, +} + +pub(crate) fn reference(stored: Option<&StoredValue>) -> Result { + let head = stored + .map(|value| decode_head(&value.bytes)) + .transpose()? + .unwrap_or(HeadRecord { + latest: None, + stable: None, + }); + Ok(DatasetHead { + latest: head.latest, + stable: head.stable, + generation: stored.map(|value| CommitGeneration::from_head(&value.bytes)), + }) +} + impl DatasetAuthority { /// # Errors /// Rejects missing Datasets and malformed or unavailable head records. @@ -61,14 +83,14 @@ pub(super) fn decode_head(bytes: &[u8]) -> Result { } let record: VersionedRecord = bincode::deserialize(bytes).map_err(|_| AuthorityError::Corrupt)?; - if record.version != 2 { + if !matches!(record.version, 2 | 4) { return Err(AuthorityError::Corrupt); } Ok(record.payload.head) } pub(super) fn encode_head(head: HeadRecord) -> Result, AuthorityError> { bincode::serialize(&VersionedRecord { - version: 2, + version: 4, payload: HeadEnvelope { head, fence: OperationId::random(), @@ -87,22 +109,6 @@ pub(super) fn head_attempt(bytes: &[u8]) -> Result, Author validate_commit_head(&record.payload)?; Ok(Some(record.payload.attempt)) } -pub(super) fn encode_commit_head( - head: HeadRecord, - attempt: CommitAttempt, -) -> Result, AuthorityError> { - attempt.validate()?; - bincode::serialize(&VersionedRecord { - version: 3, - payload: CommitHeadEnvelope { - head, - fence: OperationId::random(), - attempt, - }, - }) - .map_err(|_| AuthorityError::Corrupt) -} - fn validate_commit_head(head: &CommitHeadEnvelope) -> Result<(), AuthorityError> { head.attempt.validate()?; match head.attempt.target { diff --git a/lib/crowdb-access-dataset/src/authority/publication.rs b/lib/crowdb-access-dataset/src/authority/publication.rs index 34ed01b99..395410fa0 100644 --- a/lib/crowdb-access-dataset/src/authority/publication.rs +++ b/lib/crowdb-access-dataset/src/authority/publication.rs @@ -5,6 +5,7 @@ mod candidate; mod commit; mod complete; mod legacy; +mod legacy_candidate; mod legacy_reclaim; mod opaque; mod receipt; @@ -86,6 +87,19 @@ impl DatasetAuthority { .await } + /// Compatibility selection of pre-built content using the v1 latest-parent condition. + /// The Snapshot's declared parent is immutable and is not implicitly rebased. + /// # Errors + /// Rejects unavailable content, stale conditions and unknown storage outcomes. + pub async fn publish_prepared_snapshot( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + expected_parent: Option, + ) -> Result { + self.advance_head(identity, None, expected_parent, snapshot).await + } + /// Returns the candidate associated with a durable publication token. /// /// # Errors @@ -158,9 +172,14 @@ impl DatasetAuthority { } SnapshotId::from_bytes(&value.bytes)? } else { - let current_latest = self.current_latest(identity).await?; - if !matches!(mode, PublicationMode::Candidate(_)) && current_latest != parent { - return Err(AuthorityError::ParentConflict); + if !matches!(mode, PublicationMode::Candidate(_)) { + let current = self.store.get(&key::head(identity)?).await?; + let observed = super::head::reference(current.as_ref())?; + if observed.latest != parent { + return Err(AuthorityError::SnapshotHeadConflict { + observed: Box::new(observed), + }); + } } let snapshot = SnapshotId::random(); match self @@ -336,32 +355,25 @@ impl DatasetAuthority { expected_parent: Option, snapshot: SnapshotId, ) -> Result { - if self.snapshot_commit(identity, snapshot).await?.is_some() - || self.legacy_commit_exists(identity, snapshot).await? - { - return Ok(snapshot); - } let current_head = self.store.get(&key::head(identity)?).await?; - if let Some(current) = ¤t_head { - self.help_head_commit(identity, current).await?; - } let current_latest = current_head .as_ref() .map(|value| decode_head(&value.bytes)) .transpose()? .and_then(|head| head.latest); - if current_latest != expected_parent { - return Err(AuthorityError::ParentConflict); + if current_latest != expected_parent && current_latest != Some(snapshot) { + return Err(AuthorityError::SnapshotHeadConflict { + observed: Box::new(super::head::reference(current_head.as_ref())?), + }); } - let attempt = crate::CommitAttempt { - version: 1, - id: OperationId::random(), - target: crate::CommitTarget::Snapshot { snapshot }, + let update = crate::SnapshotHeadUpdate { + snapshot, expected_head: current_head .as_ref() .map(|value| crate::CommitGeneration::from_head(&value.bytes)), }; - self.commit_snapshot_attempt(identity, &attempt, stable).await?; + self.commit_snapshot_head(identity, &update, stable, current_head) + .await?; Ok(snapshot) } } diff --git a/lib/crowdb-access-dataset/src/authority/publication/candidate.rs b/lib/crowdb-access-dataset/src/authority/publication/candidate.rs index d953f9648..00b34d19b 100644 --- a/lib/crowdb-access-dataset/src/authority/publication/candidate.rs +++ b/lib/crowdb-access-dataset/src/authority/publication/candidate.rs @@ -124,7 +124,7 @@ pub(crate) fn decode_candidate( return Err(AuthorityError::Corrupt); } let binding: CandidateBinding = serde_json::from_slice(bytes).map_err(|_| AuthorityError::Corrupt)?; - if !matches!(binding.version, 1 | 2) || binding.snapshot != snapshot || binding.parent == Some(snapshot) { + if !matches!(binding.version, 1..=3) || binding.snapshot != snapshot || binding.parent == Some(snapshot) { return Err(AuthorityError::Corrupt); } SnapshotId::from_bytes(binding.snapshot.as_bytes())?; @@ -146,7 +146,7 @@ pub(in crate::authority) fn binding( initial_split: Option, ) -> CandidateBinding { CandidateBinding { - version: 1, + version: 3, snapshot, parent, operation, @@ -175,7 +175,7 @@ impl DatasetAuthority { use crate::{key, PublicationState}; let snapshot = snapshot_record.publication.snapshot; if let Some(head) = self.store.get(&key::head(identity)?).await? { - self.help_head_commit(identity, &head).await?; + self.preserve_legacy_head(identity, &head).await?; } if snapshot_record.publication.state == PublicationState::Published && self.snapshot_candidate(identity, snapshot).await?.is_none() diff --git a/lib/crowdb-access-dataset/src/authority/publication/commit.rs b/lib/crowdb-access-dataset/src/authority/publication/commit.rs index 0b83d0c88..466d5bd79 100644 --- a/lib/crowdb-access-dataset/src/authority/publication/commit.rs +++ b/lib/crowdb-access-dataset/src/authority/publication/commit.rs @@ -3,17 +3,17 @@ use super::super::{ decode, encode, - head::{decode_head, encode_commit_head}, + head::{encode_head, reference}, }; use crate::{ key, AuthorityError, CasOutcome, CommitAttempt, CommitGeneration, CommitTarget, DatasetAuthority, - DatasetError, DatasetIdentity, HeadRecord, PublicationState, SnapshotId, SnapshotRecord, - SuccessfulCommit, + DatasetError, DatasetHead, DatasetIdentity, HeadRecord, PublicationState, SnapshotHeadUpdate, SnapshotId, + SnapshotRecord, StoredValue, SuccessfulCommit, }; use sha2::{Digest, Sha256}; impl DatasetAuthority { - /// Commits one immutable candidate using an explicit, independently retryable attempt. + /// Compatibility adapter for caller-owned expected-base publication inputs. /// # Errors /// Rejects divergent attempt IDs, stale heads, unavailable parents and incomplete candidates. pub async fn publish_snapshot_candidate( @@ -21,69 +21,73 @@ impl DatasetAuthority { identity: &DatasetIdentity, attempt: &CommitAttempt, ) -> Result { - self.commit_snapshot_attempt(identity, attempt, None).await + attempt.validate()?; + let CommitTarget::Snapshot { snapshot } = attempt.target else { + return Err(DatasetError::InvalidPublicationState.into()); + }; + let head = self + .update_snapshot_head( + identity, + &SnapshotHeadUpdate { + snapshot, + expected_head: attempt.expected_head, + }, + ) + .await?; + Ok(SuccessfulCommit { + version: 1, + attempt: attempt.clone(), + committed_head: head.generation.ok_or(AuthorityError::Corrupt)?, + }) } - pub(super) async fn commit_snapshot_attempt( + /// Conditionally selects one complete Snapshot without request or response persistence. + /// # Errors + /// Returns incomplete/deleted resources, the observed conflicting head or unknown storage outcomes. + pub async fn update_snapshot_head( &self, identity: &DatasetIdentity, - attempt: &CommitAttempt, + update: &SnapshotHeadUpdate, + ) -> Result { + SnapshotId::from_bytes(update.snapshot.as_bytes())?; + let current = self.store.get(&key::head(identity)?).await?; + self.commit_snapshot_head(identity, update, None, current).await + } + + pub(super) async fn commit_snapshot_head( + &self, + identity: &DatasetIdentity, + update: &SnapshotHeadUpdate, stable: Option, - ) -> Result { - attempt.validate()?; - let CommitTarget::Snapshot { snapshot } = attempt.target else { - return Err(DatasetError::InvalidPublicationState.into()); - }; - if let Some(receipt) = self.commit_status(identity, attempt.id).await? { - if receipt.attempt != *attempt { - return Err(AuthorityError::AlreadyExists); - } - return Ok(receipt); - } + current: Option, + ) -> Result { + let snapshot = update.snapshot; + SnapshotId::from_bytes(snapshot.as_bytes())?; let candidate = self .snapshot_candidate(identity, snapshot) .await? .ok_or(AuthorityError::SnapshotNotFound)?; - if candidate.version != 2 { + if candidate.version != 2 && !self.legacy_candidate_ready(identity, &candidate).await? { return Err(AuthorityError::SnapshotNotFound); } - self.persist_attempt(identity, attempt).await?; - if let Some(receipt) = self.snapshot_commit(identity, snapshot).await? { - return Ok(receipt); - } - let current = self.store.get(&key::head(identity)?).await?; - if let Some(current) = ¤t { - self.help_head_commit(identity, current).await?; - } - if current - .as_ref() - .map(|value| CommitGeneration::from_head(&value.bytes)) - != attempt.expected_head + self.ensure_snapshot_live(identity, snapshot).await?; + let observed = reference(current.as_ref())?; + if observed.latest == Some(snapshot) && stable.map_or(true, |stable| observed.stable == Some(stable)) { - return Err(AuthorityError::HeadConflict); + return Ok(observed); } - if let Some(parent) = candidate.parent { - self.require_committed_snapshot(identity, parent).await?; - } - self.ensure_snapshot_live(identity, snapshot).await?; - self.validate_initial_split(identity, snapshot, candidate.initial_split) - .await?; - self.mark_candidate_published(identity, &candidate).await?; - let old_head = current - .as_ref() - .map(|value| decode_head(&value.bytes)) - .transpose()? - .unwrap_or(HeadRecord { - latest: None, - stable: None, + if observed.generation != update.expected_head { + return Err(AuthorityError::SnapshotHeadConflict { + observed: Box::new(observed), }); - let bytes = encode_commit_head( - HeadRecord { - latest: Some(snapshot), - stable: stable.or(old_head.stable), - }, - attempt.clone(), - )?; + } + if let Some(current) = ¤t { + self.preserve_legacy_head(identity, current).await?; + } + let bytes = encode_head(HeadRecord { + latest: Some(snapshot), + stable: stable.or(observed.stable), + })?; match self .store .compare_exchange( @@ -93,19 +97,22 @@ impl DatasetAuthority { ) .await? { - CasOutcome::Applied(_) => { - let receipt = SuccessfulCommit { - version: 1, - attempt: attempt.clone(), - committed_head: CommitGeneration::from_head(&bytes), - }; - self.persist_commit_receipt(identity, &receipt).await?; - Ok(receipt) + CasOutcome::Applied(_) => Ok(DatasetHead { + latest: Some(snapshot), + stable: stable.or(observed.stable), + generation: Some(CommitGeneration::from_head(&bytes)), + }), + CasOutcome::Conflict(value) => { + let observed = reference(value.as_ref())?; + if observed.latest == Some(snapshot) + && stable.map_or(true, |stable| observed.stable == Some(stable)) + { + return Ok(observed); + } + Err(AuthorityError::SnapshotHeadConflict { + observed: Box::new(observed), + }) } - CasOutcome::Conflict(_) => self - .snapshot_commit(identity, snapshot) - .await? - .ok_or(AuthorityError::HeadConflict), } } diff --git a/lib/crowdb-access-dataset/src/authority/publication/complete.rs b/lib/crowdb-access-dataset/src/authority/publication/complete.rs index 1f16064a3..dbc1525b1 100644 --- a/lib/crowdb-access-dataset/src/authority/publication/complete.rs +++ b/lib/crowdb-access-dataset/src/authority/publication/complete.rs @@ -76,9 +76,15 @@ impl DatasetAuthority { if value.bytes == RETIRED { return Ok(false); } - if decode_candidate(&value.bytes, snapshot)?.version == 2 { + let candidate = decode_candidate(&value.bytes, snapshot)?; + if candidate.version == 2 { return Ok(true); } + if self.legacy_candidate_ready(identity, &candidate).await? { + return Ok(true); + } + } else if self.snapshot_commit(identity, snapshot).await?.is_some() { + return Ok(true); } match self .store diff --git a/lib/crowdb-access-dataset/src/authority/publication/legacy.rs b/lib/crowdb-access-dataset/src/authority/publication/legacy.rs index df9eb4f1a..7b726019b 100644 --- a/lib/crowdb-access-dataset/src/authority/publication/legacy.rs +++ b/lib/crowdb-access-dataset/src/authority/publication/legacy.rs @@ -29,10 +29,11 @@ impl DatasetAuthority { stored: &StoredValue, ) -> Result<(), AuthorityError> { let head = decode_head(&stored.bytes)?; - let Some(snapshot) = head.latest else { - return Ok(()); - }; - if self.snapshot_candidate(identity, snapshot).await?.is_some() { + let mut legacy = false; + for snapshot in [head.latest, head.stable].into_iter().flatten() { + legacy |= self.snapshot_candidate(identity, snapshot).await?.is_none(); + } + if !legacy { return Ok(()); } let key = anchor_key(identity)?; @@ -73,12 +74,18 @@ impl DatasetAuthority { if candidate.as_ref().is_some_and(|c| c.version == 2) { return Ok(()); } + if candidate.as_ref().is_some_and(|c| c.version == 3) { + return Err(AuthorityError::SnapshotNotFound); + } if self.snapshot_commit(identity, snapshot).await?.is_some() || self.legacy_commit_exists(identity, snapshot).await? { return Ok(()); } - if candidate.is_some() { + if let Some(candidate) = candidate { + if self.legacy_candidate_ready(identity, &candidate).await? { + return Ok(()); + } return Err(AuthorityError::SnapshotNotFound); } let cancel = ReadCancellation::new(); @@ -204,7 +211,7 @@ pub(super) fn validate_visibility(bytes: &[u8], snapshot: SnapshotId) -> Result< Ok(()) } -fn legacy_read_error(error: InspectionError) -> AuthorityError { +pub(super) fn legacy_read_error(error: InspectionError) -> AuthorityError { match error { InspectionError::Authority(error) => error, InspectionError::Store(error) => error.into(), diff --git a/lib/crowdb-access-dataset/src/authority/publication/legacy_candidate.rs b/lib/crowdb-access-dataset/src/authority/publication/legacy_candidate.rs new file mode 100644 index 000000000..8dcfcda65 --- /dev/null +++ b/lib/crowdb-access-dataset/src/authority/publication/legacy_candidate.rs @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::candidate::CandidateBinding; +use crate::{ + authority::VersionedRecord, + inspection::{work::InspectionWork, InspectionError}, + key, + split::SplitRevisionRecord, + AuthorityError, DatasetAuthority, DatasetIdentity, PublicationState, SnapshotRecord, +}; +use bincode::Options; +use sha2::{Digest, Sha256}; + +impl DatasetAuthority { + pub(super) async fn legacy_candidate_ready( + &self, + identity: &DatasetIdentity, + candidate: &CandidateBinding, + ) -> Result { + if candidate.version != 1 { + return Ok(false); + } + let cancel = crate::ReadCancellation::new(); + let mut work = InspectionWork::new( + self.store.as_ref(), + &cancel, + crate::MetadataScanBudget { + items: 1, + bytes: 16 * 1024 * 1024, + timeout: std::time::Duration::from_secs(2), + }, + ) + .map_err(super::legacy::legacy_read_error)?; + self.legacy_candidate_available(identity, candidate, &mut work) + .await + .map_err(super::legacy::legacy_read_error) + } + /// Old publishers marked content Published only after its initial Split and owners were ready. + /// New staged bindings use another version and cannot take this compatibility path. + pub(crate) async fn legacy_candidate_available( + &self, + identity: &DatasetIdentity, + candidate: &CandidateBinding, + work: &mut InspectionWork<'_>, + ) -> Result { + if candidate.version != 1 { + return Ok(false); + } + let Some(value) = work + .get( + &key::snapshot(identity, candidate.snapshot).map_err(AuthorityError::from)?, + 9 * 1024 * 1024, + ) + .await? + else { + return Ok(false); + }; + let stored: VersionedRecord = bincode::DefaultOptions::new() + .with_fixint_encoding() + .with_limit(value.bytes.len() as u64) + .reject_trailing_bytes() + .deserialize(&value.bytes) + .map_err(|_| InspectionError::Corrupt)?; + let record = stored.payload.publication; + let digest: [u8; 32] = Sha256::digest(&record.manifest).into(); + if stored.version != 1 + || record.snapshot != candidate.snapshot + || record.parent != candidate.parent + || record.operation != candidate.operation + || digest != candidate.manifest_sha256 + { + return Err(InspectionError::Corrupt); + } + if record.state != PublicationState::Published { + return Ok(false); + } + if let Some(revision) = candidate.initial_split { + let Some(value) = work + .get( + &crate::split::key::revision(identity, candidate.snapshot, revision) + .map_err(AuthorityError::from)?, + 1024 * 1024, + ) + .await? + else { + return Ok(false); + }; + let record = SplitRevisionRecord::decode(&value.bytes).map_err(|_| InspectionError::Corrupt)?; + if record.snapshot != candidate.snapshot || record.revision != revision { + return Err(InspectionError::Corrupt); + } + } + Ok(true) + } +} diff --git a/lib/crowdb-access-dataset/src/authority/publication/receipt.rs b/lib/crowdb-access-dataset/src/authority/publication/receipt.rs index e6f1dbc90..edcff0624 100644 --- a/lib/crowdb-access-dataset/src/authority/publication/receipt.rs +++ b/lib/crowdb-access-dataset/src/authority/publication/receipt.rs @@ -3,54 +3,20 @@ use super::super::head::head_attempt; use crate::{ - AuthorityError, CasOutcome, CommitAttempt, CommitGeneration, CommitTarget, DatasetAuthority, - DatasetError, DatasetIdentity, OperationId, SnapshotId, StoredValue, SuccessfulCommit, + AuthorityError, CommitAttempt, CommitGeneration, CommitTarget, DatasetAuthority, DatasetError, + DatasetIdentity, OperationId, SnapshotId, StoredValue, SuccessfulCommit, }; impl DatasetAuthority { - pub(in crate::authority) async fn help_head_commit( + pub(in crate::authority) async fn preserve_legacy_head( &self, identity: &DatasetIdentity, head: &StoredValue, ) -> Result<(), AuthorityError> { - let Some(attempt) = head_attempt(&head.bytes)? else { - return self.capture_legacy_head(identity, head).await; - }; - let receipt = SuccessfulCommit { - version: 1, - attempt, - committed_head: CommitGeneration::from_head(&head.bytes), - }; - self.persist_commit_receipt(identity, &receipt).await - } - - pub(crate) async fn persist_commit_receipt( - &self, - identity: &DatasetIdentity, - receipt: &SuccessfulCommit, - ) -> Result<(), AuthorityError> { - let bytes = receipt.encode()?; - self.put_commit_record(&receipt_key(identity, receipt.attempt.id)?, &bytes) - .await?; - let key = match receipt.attempt.target { - CommitTarget::Snapshot { snapshot } => snapshot_commit_key(identity, snapshot)?, - CommitTarget::SplitRevision { snapshot, revision } => { - crate::split::key::committed_revision(identity, snapshot, revision)? - } - }; - self.put_commit_record(&key, &bytes).await - } - - pub(in crate::authority) async fn put_commit_record( - &self, - key: &[u8], - bytes: &[u8], - ) -> Result<(), AuthorityError> { - match self.store.compare_exchange(key, None, bytes).await? { - CasOutcome::Applied(_) => Ok(()), - CasOutcome::Conflict(Some(value)) if value.bytes == bytes => Ok(()), - CasOutcome::Conflict(_) => Err(AuthorityError::AlreadyExists), + if head.bytes.get(..2) == Some(4u16.to_le_bytes().as_slice()) { + return Ok(()); } + self.capture_legacy_head(identity, head).await } /// Recovers success from durable evidence or the current head's commit marker. @@ -70,7 +36,6 @@ impl DatasetAuthority { } if let Some(head) = self.store.get(&crate::key::head(identity)?).await? { if head_attempt(&head.bytes)?.is_some_and(|pending| pending.id == attempt) { - self.help_head_commit(identity, &head).await?; let receipt = SuccessfulCommit { version: 1, attempt: head_attempt(&head.bytes)?.ok_or(AuthorityError::Corrupt)?, @@ -109,7 +74,6 @@ impl DatasetAuthority { if let Some(head) = self.store.get(&crate::key::head(identity)?).await? { if let Some(attempt) = head_attempt(&head.bytes)? { if attempt.target == (CommitTarget::Snapshot { snapshot }) { - self.help_head_commit(identity, &head).await?; return Ok(Some(SuccessfulCommit { version: 1, attempt, @@ -138,15 +102,6 @@ impl DatasetAuthority { }) .transpose() } - - pub(crate) async fn persist_attempt( - &self, - identity: &DatasetIdentity, - attempt: &CommitAttempt, - ) -> Result<(), AuthorityError> { - self.put_commit_record(&attempt_key(identity, attempt.id)?, &attempt.encode()?) - .await - } } fn scoped(prefix: &[u8], identity: &DatasetIdentity, id: &[u8]) -> Result, DatasetError> { let mut key = prefix.to_vec(); diff --git a/lib/crowdb-access-dataset/src/authority/publication/visibility.rs b/lib/crowdb-access-dataset/src/authority/publication/visibility.rs index 2d9bbce93..ae0770422 100644 --- a/lib/crowdb-access-dataset/src/authority/publication/visibility.rs +++ b/lib/crowdb-access-dataset/src/authority/publication/visibility.rs @@ -45,9 +45,13 @@ impl DatasetAuthority { if candidate.bytes == super::complete::RETIRED { return Ok(false); } - if super::candidate::decode_candidate(&candidate.bytes, snapshot)?.version == 2 { + let candidate = super::candidate::decode_candidate(&candidate.bytes, snapshot)?; + if candidate.version == 2 { return Ok(true); } + if candidate.version == 3 { + return Ok(false); + } } if let Some(value) = work .get( @@ -83,8 +87,14 @@ impl DatasetAuthority { } // A prepared modern candidate is not a legacy publication, even when // a malformed old head happens to name it. - if candidate.is_some() { - return Ok(false); + if let Some(candidate) = candidate { + return self + .legacy_candidate_available( + identity, + &super::candidate::decode_candidate(&candidate.bytes, snapshot)?, + work, + ) + .await; } Ok(self .legacy_snapshot_visible(identity, snapshot, head.as_ref(), work) diff --git a/lib/crowdb-access-dataset/src/composition/candidates.rs b/lib/crowdb-access-dataset/src/composition/candidates.rs index b3751caa0..d7c001762 100644 --- a/lib/crowdb-access-dataset/src/composition/candidates.rs +++ b/lib/crowdb-access-dataset/src/composition/candidates.rs @@ -107,7 +107,6 @@ impl DatasetAuthority { if self .abort_uncommitted_candidate(identity, record.publication.snapshot) .await? - .retained() { return Ok(true); } diff --git a/lib/crowdb-access-dataset/src/composition/cleanup.rs b/lib/crowdb-access-dataset/src/composition/cleanup.rs index b45abad08..17901d9ed 100644 --- a/lib/crowdb-access-dataset/src/composition/cleanup.rs +++ b/lib/crowdb-access-dataset/src/composition/cleanup.rs @@ -57,7 +57,7 @@ impl DatasetAuthority { let end = state.draft.record.revision + u64::from(matches!( state.draft.record.progress, - DraftProgress::Published { .. } | DraftProgress::Abandoned + DraftProgress::Prepared { .. } | DraftProgress::Published { .. } | DraftProgress::Abandoned )); let mut work = InspectionWork::new( self.store.as_ref(), diff --git a/lib/crowdb-access-dataset/src/composition/edit.rs b/lib/crowdb-access-dataset/src/composition/edit.rs index b630786b1..5d38f54c6 100644 --- a/lib/crowdb-access-dataset/src/composition/edit.rs +++ b/lib/crowdb-access-dataset/src/composition/edit.rs @@ -123,15 +123,8 @@ impl DatasetAuthority { next.edit = None; self.replace_composition(identity, &old, &state, &next).await?; if let Some(preparation) = &next.draft.preparation { - if let crate::authority::CandidateRetirement::LegacyCommitted(receipt) = self - .abort_uncommitted_candidate(identity, preparation.candidate) - .await? - { - let (stored, current) = self.load_composition(identity, operation).await?; - self.finish_composition_publication(identity, &stored, ¤t, &receipt) - .await?; - return Err(CompositionError::Frozen); - } + self.abort_uncommitted_candidate(identity, preparation.candidate) + .await?; } Ok(next.draft) } diff --git a/lib/crowdb-access-dataset/src/composition/edit_cleanup.rs b/lib/crowdb-access-dataset/src/composition/edit_cleanup.rs index 10df391ab..05664a5e8 100644 --- a/lib/crowdb-access-dataset/src/composition/edit_cleanup.rs +++ b/lib/crowdb-access-dataset/src/composition/edit_cleanup.rs @@ -69,7 +69,7 @@ impl DatasetAuthority { let (_, state) = self.load_composition(identity, operation).await?; if !matches!( state.draft.record.progress, - DraftProgress::Published { .. } | DraftProgress::Abandoned + DraftProgress::Prepared { .. } | DraftProgress::Published { .. } | DraftProgress::Abandoned ) { return Err(CompositionError::Frozen); } diff --git a/lib/crowdb-access-dataset/src/composition/publication.rs b/lib/crowdb-access-dataset/src/composition/publication.rs index b116a3b9a..c3b50f8e0 100644 --- a/lib/crowdb-access-dataset/src/composition/publication.rs +++ b/lib/crowdb-access-dataset/src/composition/publication.rs @@ -2,13 +2,10 @@ // Licensed under the Apache License, Version 2.0. use super::{state::CompositionState, CompositionDraft, CompositionError, DraftProgress}; -use crate::{ - AuthorityError, CommitAttempt, DatasetAuthority, DatasetIdentity, OperationId, StoredValue, - SuccessfulCommit, -}; +use crate::{CommitAttempt, DatasetAuthority, DatasetIdentity, OperationId, StoredValue, SuccessfulCommit}; impl DatasetAuthority { - /// Publishes a complete immutable composition with one data-head CAS and durable receipt. + /// Compatibility adapter for publishing a prepared immutable result without process writes. /// # Errors /// Rejects stale heads, divergent attempts, abandoned drafts and incomplete preparation. pub async fn publish_composition( @@ -18,7 +15,7 @@ impl DatasetAuthority { attempt: &CommitAttempt, ) -> Result { attempt.validate()?; - let (mut old, mut state) = self.load_composition(identity, operation).await?; + let (_, state) = self.load_composition(identity, operation).await?; let (DraftProgress::Prepared { target } | DraftProgress::Committing { target, .. } | DraftProgress::Published { target, .. }) = state.draft.record.progress @@ -28,48 +25,10 @@ impl DatasetAuthority { if target != attempt.target { return Err(CompositionError::Conflict); } - if let Some(receipt) = self.commit_status(identity, attempt.id).await? { - if receipt.attempt != *attempt { - return Err(CompositionError::Conflict); - } - self.finish_composition_publication(identity, &old, &state, &receipt) - .await?; - return Ok(receipt); - } if state.uploads.is_some() && !super::uploads::publication_ready(&state)? { return Err(CompositionError::Conflict); } - self.persist_attempt(identity, attempt).await?; - match state.draft.record.progress { - DraftProgress::Prepared { .. } => { - self.check_composition_source(identity, &state.draft).await?; - let mut next = state.clone(); - next.edit = None; - next.draft.record.progress = DraftProgress::Committing { - target, - attempt: attempt.id, - }; - self.replace_composition(identity, &old, &state, &next).await?; - (old, state) = self.load_composition(identity, operation).await?; - } - DraftProgress::Committing { attempt: pending, .. } if pending == attempt.id => {} - DraftProgress::Published { .. } => {} - _ => return Err(CompositionError::Conflict), - } - match self.publish_snapshot_candidate(identity, attempt).await { - Ok(receipt) => { - self.finish_composition_publication(identity, &old, &state, &receipt) - .await?; - Ok(receipt) - } - Err(AuthorityError::HeadConflict) => { - let mut next = state.clone(); - next.draft.record.progress = DraftProgress::Prepared { target }; - self.replace_composition(identity, &old, &state, &next).await?; - Err(CompositionError::Conflict) - } - Err(error) => Err(error.into()), - } + Ok(self.publish_snapshot_candidate(identity, attempt).await?) } pub(super) async fn reconcile_composition( &self, @@ -87,23 +46,18 @@ impl DatasetAuthority { return Err(CompositionError::Corrupt); } if let Some(receipt) = self.commit_status(identity, attempt).await? { - self.finish_composition_publication(identity, &old, &state, &receipt) + self.finish_legacy_composition_publication(identity, &old, &state, &receipt) .await?; - } else if self.inspect_head(identity).await?.generation != pending.expected_head { - if let Some(receipt) = self.commit_status(identity, attempt).await? { - self.finish_composition_publication(identity, &old, &state, &receipt) - .await?; - } else { - let mut next = state.clone(); - next.draft.record.progress = DraftProgress::Prepared { target }; - self.replace_composition(identity, &old, &state, &next).await?; - } + } else { + let mut next = state.clone(); + next.draft.record.progress = DraftProgress::Prepared { target }; + self.replace_composition(identity, &old, &state, &next).await?; } return Ok(self.load_composition(identity, operation).await?.1.draft); } Ok(state.draft) } - pub(super) async fn finish_composition_publication( + pub(super) async fn finish_legacy_composition_publication( &self, identity: &DatasetIdentity, old: &StoredValue, diff --git a/lib/crowdb-access-dataset/src/composition/uploads/release.rs b/lib/crowdb-access-dataset/src/composition/uploads/release.rs index 7d76027d8..1afc1dd88 100644 --- a/lib/crowdb-access-dataset/src/composition/uploads/release.rs +++ b/lib/crowdb-access-dataset/src/composition/uploads/release.rs @@ -47,13 +47,10 @@ impl DatasetAuthority { let (old, state) = self.load_composition(identity, operation).await?; let mut next = state.clone(); next.edit = None; - let terminal = matches!( - state.draft.record.progress, - DraftProgress::Abandoned | DraftProgress::Published { .. } - ); + let release_staged = staging_releasable(&state)?; let ownership = next.uploads.as_mut().ok_or(CompositionError::Conflict)?; let mut remaining = 20; - if terminal { + if release_staged { let owner = super::staging_owner(operation)?; while remaining > 0 && ownership.staged_retired < ownership.staged.len() { let id = ownership.staged[ownership.staged_retired]; @@ -76,7 +73,6 @@ impl DatasetAuthority { if self .abort_uncommitted_candidate(identity, candidate.snapshot) .await? - .retained() { candidate.keep_committed = true; continue; @@ -105,14 +101,11 @@ fn obsolete(state: &CompositionState, revision: u64, snapshot: crate::SnapshotId } fn cleanup_status(state: &CompositionState) -> Result { let ownership = state.uploads.as_ref().ok_or(CompositionError::Conflict)?; - let terminal = matches!( - state.draft.record.progress, - DraftProgress::Published { .. } | DraftProgress::Abandoned - ); + let release_staged = staging_releasable(state)?; Ok(CompositionUploadCleanup { retired_owners: ownership.staged_retired as u64 + ownership.candidates.iter().map(|c| c.retired as u64).sum::(), - complete: (!terminal || ownership.staged_retired == ownership.staged.len()) + complete: (!release_staged || ownership.staged_retired == ownership.staged.len()) && ownership.candidates.iter().all(|c| { !obsolete(state, c.input_revision, c.snapshot) || c.keep_committed @@ -120,3 +113,10 @@ fn cleanup_status(state: &CompositionState) -> Result Result { + Ok(matches!( + state.draft.record.progress, + DraftProgress::Abandoned | DraftProgress::Published { .. } + ) || (matches!(state.draft.record.progress, DraftProgress::Prepared { .. }) + && super::publication_ready(state)?)) +} diff --git a/lib/crowdb-access-dataset/src/lib.rs b/lib/crowdb-access-dataset/src/lib.rs index 03106e9df..70ef896e1 100644 --- a/lib/crowdb-access-dataset/src/lib.rs +++ b/lib/crowdb-access-dataset/src/lib.rs @@ -68,8 +68,8 @@ pub use wire::{ pub use adapter::{BoundedPrefetch, WorkerPartition}; pub use authority::{ - AuthorityError, DatasetAuthority, DatasetHead, MetadataCleanupProgress, SampleView, SnapshotLifecycle, - SnapshotReclaimError, SnapshotReclaimPhase, SnapshotReclaimStatus, + AuthorityError, DatasetAuthority, DatasetHead, MetadataCleanupProgress, SampleView, SnapshotHeadUpdate, + SnapshotLifecycle, SnapshotReclaimError, SnapshotReclaimPhase, SnapshotReclaimStatus, }; pub use chunk_store::{ unavailable_chunk_reader, ChunkKvDatasetStore, ChunkReadError, ChunkReader, ReadCancellation, diff --git a/lib/crowdb-access-dataset/tests/candidate_publication_test.rs b/lib/crowdb-access-dataset/tests/candidate_publication_test.rs index 90204f019..c214c2200 100644 --- a/lib/crowdb-access-dataset/tests/candidate_publication_test.rs +++ b/lib/crowdb-access-dataset/tests/candidate_publication_test.rs @@ -1,92 +1,15 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -#[path = "common/dataset_store.rs"] +#[path = "common/composition.rs"] #[allow(dead_code)] -mod dataset_store; +mod common; #[path = "common/publication.rs"] mod publication; +#[path = "common/split_store.rs"] +mod split_store; use crowdb_access_dataset::*; -use publication::publication_manifest; -use std::sync::{ - atomic::{AtomicBool, Ordering}, - Arc, -}; -struct TestStore { - inner: dataset_store::TestDatasetStore, - lose_head_reply: AtomicBool, - reject_receipts: AtomicBool, - race_head: AtomicBool, -} -#[async_trait::async_trait] -impl DatasetStore for TestStore { - async fn get(&self, key: &[u8]) -> Result, StoreError> { - self.inner.get(key).await - } - async fn get_bounded( - &self, - key: &[u8], - budget: MetadataScanBudget, - cancel: &ReadCancellation, - ) -> Result, StoreError> { - self.inner.get_bounded(key, budget, cancel).await - } - async fn scan_page( - &self, - request: &MetadataScan, - cancel: &ReadCancellation, - ) -> Result { - self.inner.scan_page(request, cancel).await - } - async fn compare_exchange( - &self, - key: &[u8], - expected: Option<&[u8]>, - value: &[u8], - ) -> Result { - if key.starts_with(b"DS\0CR\0") - && self.reject_receipts.load(Ordering::Relaxed) - && self.inner.get(key).await?.is_none() - { - return Err(StoreError::Rejected); - } - if key.starts_with(b"DS\0H\0") && self.race_head.swap(false, Ordering::Relaxed) { - let competing = bincode::serialize(&( - 1u16, - HeadRecord { - latest: None, - stable: None, - }, - )) - .unwrap(); - self.inner.compare_exchange(key, expected, &competing).await?; - } - let result = self.inner.compare_exchange(key, expected, value).await?; - if key.starts_with(b"DS\0H\0") - && matches!(result, CasOutcome::Applied(_)) - && self.lose_head_reply.swap(false, Ordering::Relaxed) - { - return Err(StoreError::Rejected); - } - Ok(result) - } - async fn delete(&self, key: &[u8]) -> Result<(), StoreError> { - self.inner.delete(key).await - } -} -async fn fixture() -> (Arc, DatasetAuthority, DatasetIdentity) { - let store = Arc::new(TestStore { - inner: dataset_store::TestDatasetStore::default(), - lose_head_reply: false.into(), - reject_receipts: false.into(), - race_head: false.into(), - }); - let authority = DatasetAuthority::new(store.clone()); - let identity = DatasetIdentity::new(NamespacePath::root(), "candidates").unwrap(); - authority.create_dataset(identity.clone()).await.unwrap(); - (store, authority, identity) -} async fn prepare( authority: &DatasetAuthority, identity: &DatasetIdentity, @@ -97,50 +20,36 @@ async fn prepare( .prepare_snapshot_candidate( identity, parent, - publication_manifest(marker), + publication::publication_manifest(marker), OperationId::random(), ) .await .unwrap() } -async fn attempt( - authority: &DatasetAuthority, - identity: &DatasetIdentity, - snapshot: SnapshotId, -) -> CommitAttempt { - CommitAttempt { - version: 1, - id: OperationId::random(), - target: CommitTarget::Snapshot { snapshot }, - expected_head: authority.inspect_head(identity).await.unwrap().generation, - } -} + #[tokio::test] async fn historical_parent_and_new_roots_use_independent_head_preconditions() { - let (_, authority, identity) = fixture().await; - let root = prepare(&authority, &identity, None, 1).await; - authority - .publish_snapshot_candidate(&identity, &attempt(&authority, &identity, root).await) - .await - .unwrap(); + let (_, authority, identity, root) = split_store::fixture().await; let first = prepare(&authority, &identity, Some(root), 2).await; let second = prepare(&authority, &identity, Some(root), 3).await; - let first_attempt = attempt(&authority, &identity, first).await; - let stale = attempt(&authority, &identity, second).await; + let first_attempt = common::attempt(&authority, &identity, first).await; + let stale = common::attempt(&authority, &identity, second).await; authority .publish_snapshot_candidate(&identity, &first_attempt) .await .unwrap(); - assert!(matches!( - authority.publish_snapshot_candidate(&identity, &stale).await, - Err(AuthorityError::HeadConflict) - )); - assert!(authority - .snapshot_commit(&identity, second) + let error = authority + .publish_snapshot_candidate(&identity, &stale) .await - .unwrap() - .is_none()); - let retry = attempt(&authority, &identity, second).await; + .unwrap_err(); + let AuthorityError::SnapshotHeadConflict { observed } = error else { + panic!("{error:?}") + }; + assert_eq!(observed.latest, Some(first)); + let retry = CommitAttempt { + expected_head: observed.generation, + ..stale + }; authority .publish_snapshot_candidate(&identity, &retry) .await @@ -156,184 +65,56 @@ async fn historical_parent_and_new_roots_use_independent_head_preconditions() { ); let new_root = prepare(&authority, &identity, None, 4).await; authority - .publish_snapshot_candidate(&identity, &attempt(&authority, &identity, new_root).await) + .publish_snapshot_candidate(&identity, &common::attempt(&authority, &identity, new_root).await) .await .unwrap(); let head = authority.inspect_head(&identity).await.unwrap(); - assert_eq!( + assert!(matches!( authority .publish_snapshot_candidate(&identity, &first_attempt) - .await - .unwrap() - .attempt, - first_attempt - ); + .await, + Err(AuthorityError::SnapshotHeadConflict { .. }) + )); assert_eq!(authority.inspect_head(&identity).await.unwrap(), head); assert_eq!(head.latest, Some(new_root)); + assert!(authority.get_snapshot(&identity, first).await.is_ok()); assert!(authority - .commit_status(&identity, stale.id) + .commit_status(&identity, first_attempt.id) .await .unwrap() .is_none()); } + #[tokio::test] -async fn lost_head_reply_recovers_receipt_without_republishing() { - let (store, authority, identity) = fixture().await; - let snapshot = prepare(&authority, &identity, None, 5).await; - let attempt = attempt(&authority, &identity, snapshot).await; - store.lose_head_reply.store(true, Ordering::Relaxed); - assert!(authority - .publish_snapshot_candidate(&identity, &attempt) - .await - .is_err()); - let committed_head = authority.inspect_head(&identity).await.unwrap(); - assert_eq!(committed_head.latest, Some(snapshot)); - let restarted = DatasetAuthority::new(store); - let receipt = restarted - .publish_snapshot_candidate(&identity, &attempt) - .await - .unwrap(); - assert_eq!(receipt.committed_head, committed_head.generation.unwrap()); - assert_eq!(restarted.inspect_head(&identity).await.unwrap(), committed_head); - assert_eq!( - restarted.commit_status(&identity, attempt.id).await.unwrap(), - Some(receipt) - ); -} -#[tokio::test] -async fn head_mutators_help_pending_success_before_replacing_its_marker() { - let (store, authority, identity) = fixture().await; - let snapshot = prepare(&authority, &identity, None, 6).await; - let committed = attempt(&authority, &identity, snapshot).await; - store.reject_receipts.store(true, Ordering::Relaxed); - assert!(authority - .publish_snapshot_candidate(&identity, &committed) - .await - .is_err()); - let head = authority.inspect_head(&identity).await.unwrap(); - assert!(authority.set_stable(&identity, snapshot).await.is_err()); - assert_eq!(authority.inspect_head(&identity).await.unwrap(), head); - store.reject_receipts.store(false, Ordering::Relaxed); - authority.set_stable(&identity, snapshot).await.unwrap(); - assert!(authority - .commit_status(&identity, committed.id) - .await - .unwrap() - .is_some()); - let changed = authority.inspect_head(&identity).await.unwrap(); - assert_ne!(changed.generation, head.generation); - assert_eq!(changed.stable, Some(snapshot)); - authority - .publish_snapshot_candidate(&identity, &committed) - .await - .unwrap(); - assert_eq!(authority.inspect_head(&identity).await.unwrap(), changed); -} -#[tokio::test] -async fn complete_unpublished_parent_is_usable_and_divergent_attempt_is_rejected() { - let (_, authority, identity) = fixture().await; +async fn complete_unpublished_results_are_parents_and_stable_targets_without_attempt_identity_state() { + let (_, authority, identity, _) = split_store::fixture().await; let parent = prepare(&authority, &identity, None, 7).await; let child = prepare(&authority, &identity, Some(parent), 8).await; - assert_eq!( - authority - .get_snapshot(&identity, child) - .await - .unwrap() - .publication - .parent, - Some(parent) - ); - assert_eq!(authority.latest(&identity).await.unwrap(), None); - let committed = attempt(&authority, &identity, parent).await; + authority.set_stable(&identity, parent).await.unwrap(); + let attempt = common::attempt(&authority, &identity, parent).await; authority - .publish_snapshot_candidate(&identity, &committed) + .publish_snapshot_candidate(&identity, &attempt) .await .unwrap(); - let child = prepare(&authority, &identity, Some(parent), 9).await; - let divergent = CommitAttempt { + let current = authority.inspect_head(&identity).await.unwrap(); + let reviewed = CommitAttempt { target: CommitTarget::Snapshot { snapshot: child }, - ..committed + expected_head: current.generation, + ..attempt }; - assert!(matches!( - authority.publish_snapshot_candidate(&identity, &divergent).await, - Err(AuthorityError::AlreadyExists) - )); - assert_eq!(authority.latest(&identity).await.unwrap(), Some(parent)); -} - -#[tokio::test] -async fn lost_cas_candidate_remains_a_complete_parent_and_stable_target() { - let (store, authority, identity) = fixture().await; - let snapshot = prepare(&authority, &identity, None, 10).await; - let stale = attempt(&authority, &identity, snapshot).await; - store.race_head.store(true, Ordering::Relaxed); - assert!(matches!( - authority.publish_snapshot_candidate(&identity, &stale).await, - Err(AuthorityError::HeadConflict) - )); - assert_eq!( - authority - .get_snapshot(&identity, snapshot) - .await - .unwrap() - .publication - .state, - PublicationState::Published - ); - assert!(authority - .snapshot_commit(&identity, snapshot) - .await - .unwrap() - .is_none()); - authority.set_stable(&identity, snapshot).await.unwrap(); - let child = prepare(&authority, &identity, Some(snapshot), 11).await; - assert_eq!( - authority - .get_snapshot(&identity, child) - .await - .unwrap() - .publication - .parent, - Some(snapshot) - ); - let retry = attempt(&authority, &identity, snapshot).await; authority - .publish_snapshot_candidate(&identity, &retry) + .publish_snapshot_candidate(&identity, &reviewed) .await .unwrap(); - assert_eq!(authority.latest(&identity).await.unwrap(), Some(snapshot)); + assert_eq!(authority.latest(&identity).await.unwrap(), Some(child)); assert!(authority - .commit_status(&identity, stale.id) + .commit_status(&identity, attempt.id) .await .unwrap() .is_none()); -} - -#[tokio::test] -async fn reclaim_head_fence_preserves_pending_commit_evidence() { - let (store, authority, identity) = fixture().await; - let old = prepare(&authority, &identity, None, 12).await; - authority - .publish_snapshot_candidate(&identity, &attempt(&authority, &identity, old).await) - .await - .unwrap(); - let latest = prepare(&authority, &identity, None, 13).await; - let publication = attempt(&authority, &identity, latest).await; - store.reject_receipts.store(true, Ordering::Relaxed); assert!(authority - .publish_snapshot_candidate(&identity, &publication) - .await - .is_err()); - let head = authority.inspect_head(&identity).await.unwrap(); - assert!(authority.begin_reclaim(&identity, old).await.is_err()); - assert_eq!(authority.inspect_head(&identity).await.unwrap(), head); - store.reject_receipts.store(false, Ordering::Relaxed); - authority.begin_reclaim(&identity, old).await.unwrap(); - let receipt = authority - .commit_status(&identity, publication.id) + .snapshot_commit(&identity, parent) .await .unwrap() - .unwrap(); - assert_eq!(receipt.committed_head, head.generation.unwrap()); - assert_eq!(authority.latest(&identity).await.unwrap(), Some(latest)); + .is_none()); } diff --git a/lib/crowdb-access-dataset/tests/common/legacy.rs b/lib/crowdb-access-dataset/tests/common/legacy.rs index 841668bd7..c63449252 100644 --- a/lib/crowdb-access-dataset/tests/common/legacy.rs +++ b/lib/crowdb-access-dataset/tests/common/legacy.rs @@ -4,22 +4,16 @@ use crowdb_access_dataset::*; pub(super) async fn emulate_legacy( store: &dyn DatasetStore, - authority: &DatasetAuthority, + _authority: &DatasetAuthority, identity: &DatasetIdentity, snapshots: &[SnapshotId], latest: SnapshotId, ) { for snapshot in snapshots { - let receipt = authority - .snapshot_commit(identity, *snapshot) - .await - .unwrap() - .unwrap(); for (prefix, suffix) in [ (b"DS\0CB\0".as_slice(), snapshot.as_bytes()), (b"DS\0SC\0", snapshot.as_bytes()), (b"DS\0LC\0", snapshot.as_bytes()), - (b"DS\0CR\0", receipt.attempt.id.as_bytes()), ] { store .delete(&[prefix, identity.key().unwrap().as_slice(), suffix].concat()) diff --git a/lib/crowdb-access-dataset/tests/common/reclaim_observed.rs b/lib/crowdb-access-dataset/tests/common/reclaim_observed.rs index a834881a6..828bfeb01 100644 --- a/lib/crowdb-access-dataset/tests/common/reclaim_observed.rs +++ b/lib/crowdb-access-dataset/tests/common/reclaim_observed.rs @@ -81,6 +81,12 @@ impl DatasetStore for TestReclaimObserved { expected: Option<&[u8]>, value: &[u8], ) -> Result { + assert!( + ![b"DS\0CA\0".as_slice(), b"DS\0CR\0", b"DS\0SC\0"] + .iter() + .any(|prefix| key.starts_with(prefix)), + "reclaim wrote a publication ledger" + ); if key.starts_with(b"DS\0H\0") && self .head_conflicts diff --git a/lib/crowdb-access-dataset/tests/common/split_store.rs b/lib/crowdb-access-dataset/tests/common/split_store.rs index 0d8956a4e..3f313f3b6 100644 --- a/lib/crowdb-access-dataset/tests/common/split_store.rs +++ b/lib/crowdb-access-dataset/tests/common/split_store.rs @@ -23,6 +23,10 @@ pub(super) struct TestDraftStore { pub mutations: AtomicUsize, pub split_head_reads: AtomicUsize, pub candidate_reads: AtomicUsize, + pub data_head_reads: AtomicUsize, + pub data_head_mutations: AtomicUsize, + pub data_head_conflicts: AtomicUsize, + pub keep_data_conflict_target: AtomicBool, pub split_head_mutations: AtomicUsize, pub split_head_conflicts: AtomicUsize, pub clear_split_conflict_target: AtomicBool, @@ -35,6 +39,9 @@ pub(super) struct TestDraftStore { impl DatasetStore for TestDraftStore { async fn get(&self, key: &[u8]) -> Result, StoreError> { self.unbounded_reads.fetch_add(1, Ordering::Relaxed); + if key.starts_with(b"DS\0H\0") { + self.data_head_reads.fetch_add(1, Ordering::Relaxed); + } if key.starts_with(b"DS\0CB\0") { self.candidate_reads.fetch_add(1, Ordering::Relaxed); } @@ -60,6 +67,9 @@ impl DatasetStore for TestDraftStore { cancel: &ReadCancellation, ) -> Result, StoreError> { self.bounded_reads.fetch_add(1, Ordering::Relaxed); + if key.starts_with(b"DS\0H\0") { + self.data_head_reads.fetch_add(1, Ordering::Relaxed); + } if key.starts_with(b"DS\0SH\0") { self.split_head_reads.fetch_add(1, Ordering::Relaxed); } @@ -105,12 +115,31 @@ impl DatasetStore for TestDraftStore { self.mutations.fetch_add(1, Ordering::Relaxed); if self.reject_publishing_ledgers.load(Ordering::Relaxed) { assert!( - ![b"DS\0CA\0".as_slice(), b"DS\0CR\0", b"DS\0SRC\0"] + ![b"DS\0CA\0".as_slice(), b"DS\0CR\0", b"DS\0SC\0", b"DS\0SRC\0"] .iter() .any(|prefix| key.starts_with(prefix)), "Split publication wrote a request ledger" ); } + if key.starts_with(b"DS\0H\0") { + self.data_head_mutations.fetch_add(1, Ordering::Relaxed); + if self + .data_head_conflicts + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |n| n.checked_sub(1)) + .is_ok() + { + let (version, (mut head, _)): (u16, (HeadRecord, OperationId)) = + bincode::deserialize(value).unwrap(); + if !self.keep_data_conflict_target.load(Ordering::Relaxed) { + head.latest = None; + } + let competing = bincode::serialize(&(version, (head, OperationId::random()))).unwrap(); + assert!(matches!( + self.inner.compare_exchange(key, expected, &competing).await?, + CasOutcome::Applied(_) + )); + } + } if key.starts_with(b"DS\0SH\0") { self.split_head_mutations.fetch_add(1, Ordering::Relaxed); if self diff --git a/lib/crowdb-access-dataset/tests/composition_abandon_test.rs b/lib/crowdb-access-dataset/tests/composition_abandon_test.rs index 7b74ada64..0fea20888 100644 --- a/lib/crowdb-access-dataset/tests/composition_abandon_test.rs +++ b/lib/crowdb-access-dataset/tests/composition_abandon_test.rs @@ -102,10 +102,10 @@ async fn abandon_recognizes_a_successful_historical_candidate_without_deleting_i .publish_snapshot_candidate(&identity, &common::attempt(&authority, &identity, root).await) .await .unwrap(); - assert!(matches!( - authority.abandon_composition(&identity, operation, 1).await, - Err(CompositionError::Frozen) - )); + authority + .abandon_composition(&identity, operation, 1) + .await + .unwrap(); assert_eq!( authority .composition_status(&identity, operation) @@ -113,10 +113,7 @@ async fn abandon_recognizes_a_successful_historical_candidate_without_deleting_i .unwrap() .record .progress, - DraftProgress::Published { - target: attempt.target, - attempt: attempt.id - } + DraftProgress::Abandoned ); assert!(authority .snapshot_inspection_summary(&identity, snapshot, common::budget(), &ReadCancellation::new()) diff --git a/lib/crowdb-access-dataset/tests/composition_publication_test.rs b/lib/crowdb-access-dataset/tests/composition_publication_test.rs index f8e9f6d51..f9e221124 100644 --- a/lib/crowdb-access-dataset/tests/composition_publication_test.rs +++ b/lib/crowdb-access-dataset/tests/composition_publication_test.rs @@ -11,7 +11,7 @@ use crowdb_access_dataset::{composition::*, *}; use std::sync::atomic::Ordering; #[tokio::test] -async fn composition_commit_reconciles_lost_head_reply_and_prevents_abandonment() { +async fn composition_retry_observes_current_effect_without_published_process_state() { let (store, authority, identity, parent) = split_store::fixture().await; let operation = OperationId::random(); authority @@ -27,17 +27,12 @@ async fn composition_commit_reconciles_lost_head_reply_and_prevents_abandonment( .publish_composition(&identity, operation, &attempt) .await .is_err()); - assert!(authority - .abandon_composition(&identity, operation, 1) - .await - .is_err()); let restarted = DatasetAuthority::new(store); let status = restarted.composition_status(&identity, operation).await.unwrap(); assert_eq!( status.record.progress, - DraftProgress::Published { - target: attempt.target, - attempt: attempt.id + DraftProgress::Prepared { + target: attempt.target } ); assert_eq!( @@ -48,6 +43,11 @@ async fn composition_commit_reconciles_lost_head_reply_and_prevents_abandonment( .attempt, attempt ); + restarted + .abandon_composition(&identity, operation, 1) + .await + .unwrap(); + assert!(restarted.get_snapshot(&identity, snapshot).await.is_ok()); } #[tokio::test] @@ -66,10 +66,14 @@ async fn composition_head_conflict_preserves_candidate_for_explicit_new_attempt( .publish_snapshot(&identity, Some(parent), publication::publication_manifest(2)) .await .unwrap(); - assert!(matches!( - authority.publish_composition(&identity, operation, &stale).await, - Err(CompositionError::Conflict) - )); + let error = authority + .publish_composition(&identity, operation, &stale) + .await + .unwrap_err(); + let CompositionError::Authority(AuthorityError::SnapshotHeadConflict { observed }) = error else { + panic!("{error:?}") + }; + assert_eq!(observed.latest, Some(other)); assert_eq!( authority .composition_status(&identity, operation) @@ -83,7 +87,11 @@ async fn composition_head_conflict_preserves_candidate_for_explicit_new_attempt( authority.inspect_head(&identity).await.unwrap().latest, Some(other) ); - let next = common::attempt(&authority, &identity, snapshot).await; + let next = CommitAttempt { + id: OperationId::random(), + expected_head: observed.generation, + ..stale + }; authority .publish_composition(&identity, operation, &next) .await diff --git a/lib/crowdb-access-dataset/tests/composition_upload_test.rs b/lib/crowdb-access-dataset/tests/composition_upload_test.rs index 80487875c..79c79a90b 100644 --- a/lib/crowdb-access-dataset/tests/composition_upload_test.rs +++ b/lib/crowdb-access-dataset/tests/composition_upload_test.rs @@ -135,13 +135,14 @@ async fn publication_requires_candidate_owners_and_preserves_payload_after_stagi )); f.own().await; assert!(f.authority.get_snapshot(&f.identity, snapshot).await.is_ok()); + let cleanup = f.cleanup().await; + assert!(cleanup.complete); + assert_eq!(cleanup.retired_owners, 1); f.authority .publish_composition(&f.identity, f.operation, &attempt) .await .unwrap(); - let cleanup = f.cleanup().await; - assert!(cleanup.complete); - assert_eq!(cleanup.retired_owners, 1); + assert_eq!(f.cleanup().await, cleanup); assert!(matches!( f.uploads.abort(&f.location).await, Err(UploadError::Conflict) diff --git a/lib/crowdb-access-dataset/tests/core_test.rs b/lib/crowdb-access-dataset/tests/core_test.rs index 5458eef05..b70619db1 100644 --- a/lib/crowdb-access-dataset/tests/core_test.rs +++ b/lib/crowdb-access-dataset/tests/core_test.rs @@ -667,7 +667,7 @@ async fn authority_requires_default_namespace_and_publishes_complete_head() { authority .publish_snapshot(&identity, Some(snapshot), publication_manifest(3)) .await, - Err(AuthorityError::ParentConflict) + Err(AuthorityError::SnapshotHeadConflict { observed }) if observed.latest == Some(first) )); assert!(matches!( authority.create_dataset(identity.clone()).await, diff --git a/lib/crowdb-access-dataset/tests/legacy_commit_observation_test.rs b/lib/crowdb-access-dataset/tests/legacy_commit_observation_test.rs new file mode 100644 index 000000000..b7c6e5622 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/legacy_commit_observation_test.rs @@ -0,0 +1,222 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/composition.rs"] +#[allow(dead_code)] +mod common; +#[path = "common/publication.rs"] +mod publication; +#[path = "common/split_store.rs"] +mod split_store; +use crowdb_access_dataset::*; +use std::sync::atomic::Ordering; + +fn key(prefix: &[u8], identity: &DatasetIdentity, suffix: &[u8]) -> Vec { + [prefix, identity.key().unwrap().as_slice(), suffix].concat() +} +async fn legacy_binding(store: &dyn DatasetStore, identity: &DatasetIdentity, snapshot: SnapshotId) { + let key = key(b"DS\0CB\0", identity, snapshot.as_bytes()); + let stored = store.get(&key).await.unwrap().unwrap(); + let mut binding: serde_json::Value = serde_json::from_slice(&stored.bytes).unwrap(); + binding["version"] = 1.into(); + assert!(matches!( + store + .compare_exchange(&key, Some(&stored.bytes), &serde_json::to_vec(&binding).unwrap()) + .await + .unwrap(), + CasOutcome::Applied(_) + )); +} + +#[tokio::test] +async fn stored_legacy_receipts_remain_readable_but_do_not_control_new_publication() { + let (store, authority, identity, snapshot) = split_store::fixture().await; + let old = common::attempt(&authority, &identity, snapshot).await; + let receipt = SuccessfulCommit { + version: 1, + attempt: old.clone(), + committed_head: old.expected_head.unwrap(), + }; + for (prefix, suffix) in [ + (b"DS\0CR\0".as_slice(), old.id.as_bytes()), + (b"DS\0SC\0", snapshot.as_bytes()), + ] { + store + .compare_exchange(&key(prefix, &identity, suffix), None, &receipt.encode().unwrap()) + .await + .unwrap(); + } + store.reject_publishing_ledgers.store(true, Ordering::Relaxed); + let writes = store.mutations.load(Ordering::Relaxed); + assert_eq!( + authority.commit_status(&identity, old.id).await.unwrap(), + Some(receipt.clone()) + ); + assert_eq!( + authority.snapshot_commit(&identity, snapshot).await.unwrap(), + Some(receipt) + ); + assert_eq!(store.mutations.load(Ordering::Relaxed), writes); + let next = authority + .prepare_snapshot_candidate( + &identity, + None, + publication::publication_manifest(2), + OperationId::random(), + ) + .await + .unwrap(); + authority + .publish_snapshot_candidate(&identity, &common::attempt(&authority, &identity, next).await) + .await + .unwrap(); + assert!(matches!( + authority.publish_snapshot_candidate(&identity, &old).await, + Err(AuthorityError::SnapshotHeadConflict { .. }) + )); + assert_eq!(authority.latest(&identity).await.unwrap(), Some(next)); +} + +#[tokio::test] +async fn a_legacy_head_marker_is_observed_without_backfill_and_its_complete_resource_survives_replacement() { + let (store, authority, identity, snapshot) = split_store::fixture().await; + legacy_binding(store.as_ref(), &identity, snapshot).await; + let old = CommitAttempt { + version: 1, + id: OperationId::random(), + target: CommitTarget::Snapshot { snapshot }, + expected_head: None, + }; + let head_key = key(b"DS\0H\0", &identity, &[]); + let current = store.get(&head_key).await.unwrap().unwrap(); + let bytes = bincode::serialize(&( + 3u16, + ( + HeadRecord { + latest: Some(snapshot), + stable: None, + }, + OperationId::random(), + old.clone(), + ), + )) + .unwrap(); + store + .compare_exchange(&head_key, Some(¤t.bytes), &bytes) + .await + .unwrap(); + let generation = CommitGeneration::from_head(&bytes); + store.reject_publishing_ledgers.store(true, Ordering::Relaxed); + let writes = store.mutations.load(Ordering::Relaxed); + assert_eq!( + authority + .commit_status(&identity, old.id) + .await + .unwrap() + .unwrap() + .committed_head, + generation + ); + assert_eq!( + authority + .snapshot_commit(&identity, snapshot) + .await + .unwrap() + .unwrap() + .attempt, + old + ); + assert_eq!(store.mutations.load(Ordering::Relaxed), writes); + let next = authority + .prepare_snapshot_candidate( + &identity, + None, + publication::publication_manifest(2), + OperationId::random(), + ) + .await + .unwrap(); + authority + .update_snapshot_head( + &identity, + &SnapshotHeadUpdate { + snapshot: next, + expected_head: Some(generation), + }, + ) + .await + .unwrap(); + let writes = store.mutations.load(Ordering::Relaxed); + assert!(authority.get_snapshot(&identity, snapshot).await.is_ok()); + assert!(authority + .snapshot_inspection_summary(&identity, snapshot, common::budget(), &ReadCancellation::new()) + .await + .is_ok()); + assert!(authority + .commit_status(&identity, old.id) + .await + .unwrap() + .is_none()); + assert_eq!(store.mutations.load(Ordering::Relaxed), writes); + for (prefix, suffix) in [ + (b"DS\0CR\0".as_slice(), old.id.as_bytes()), + (b"DS\0SC\0", snapshot.as_bytes()), + ] { + assert!(store + .get(&key(prefix, &identity, suffix)) + .await + .unwrap() + .is_none()); + } + authority.set_stable(&identity, snapshot).await.unwrap(); + assert_eq!(store.mutations.load(Ordering::Relaxed), writes + 1); +} + +#[tokio::test] +async fn legacy_preparation_and_missing_initial_split_are_not_complete_resources() { + let (store, authority, identity, parent) = split_store::fixture().await; + let incomplete = authority + .prepare_snapshot_with_operation( + &identity, + Some(parent), + publication::publication_manifest(2), + OperationId::random(), + ) + .await + .unwrap(); + legacy_binding(store.as_ref(), &identity, incomplete).await; + assert!(matches!( + authority.require_committed_snapshot(&identity, incomplete).await, + Err(AuthorityError::SnapshotNotFound) + )); + let binding_key = key(b"DS\0CB\0", &identity, parent.as_bytes()); + let stored = store.get(&binding_key).await.unwrap().unwrap(); + let mut binding: serde_json::Value = serde_json::from_slice(&stored.bytes).unwrap(); + binding["version"] = 1.into(); + binding["initial_split"] = + serde_json::to_value(SplitRevisionId::from_bytes(OperationId::random().as_bytes()).unwrap()).unwrap(); + store + .compare_exchange( + &binding_key, + Some(&stored.bytes), + &serde_json::to_vec(&binding).unwrap(), + ) + .await + .unwrap(); + assert!(matches!( + authority.require_committed_snapshot(&identity, parent).await, + Err(AuthorityError::SnapshotNotFound) + )); + assert!(matches!( + authority + .update_snapshot_head( + &identity, + &SnapshotHeadUpdate { + snapshot: parent, + expected_head: authority.inspect_head(&identity).await.unwrap().generation + } + ) + .await, + Err(AuthorityError::SnapshotNotFound) + )); +} diff --git a/lib/crowdb-access-dataset/tests/legacy_publication_test.rs b/lib/crowdb-access-dataset/tests/legacy_publication_test.rs index 51c4f7286..a67a6374c 100644 --- a/lib/crowdb-access-dataset/tests/legacy_publication_test.rs +++ b/lib/crowdb-access-dataset/tests/legacy_publication_test.rs @@ -165,3 +165,49 @@ async fn completing_an_already_current_legacy_manifest_installs_a_readable_resou .await .is_ok()); } + +#[tokio::test] +async fn a_stable_only_legacy_root_remains_readable_after_both_names_move() { + let store = Arc::new(dataset_store::TestDatasetStore::default()); + let authority = DatasetAuthority::new(store.clone()); + let identity = DatasetIdentity::new(NamespacePath::root(), "legacy-stable").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + let old = authority + .publish_snapshot(&identity, None, publication_manifest(1)) + .await + .unwrap(); + legacy::emulate_legacy(store.as_ref(), &authority, &identity, &[old], old).await; + let head_key = [b"DS\0H\0".as_slice(), &identity.key().unwrap()].concat(); + let stored = store.get(&head_key).await.unwrap().unwrap(); + let head = bincode::serialize(&( + 1u16, + HeadRecord { + latest: None, + stable: Some(old), + }, + )) + .unwrap(); + store + .compare_exchange(&head_key, Some(&stored.bytes), &head) + .await + .unwrap(); + let next = authority + .prepare_snapshot_candidate(&identity, None, publication_manifest(2), OperationId::random()) + .await + .unwrap(); + authority + .update_snapshot_head( + &identity, + &SnapshotHeadUpdate { + snapshot: next, + expected_head: Some(CommitGeneration::from_head(&head)), + }, + ) + .await + .unwrap(); + authority.set_stable(&identity, next).await.unwrap(); + assert!(DatasetAuthority::new(store) + .get_snapshot(&identity, old) + .await + .is_ok()); +} diff --git a/lib/crowdb-access-dataset/tests/reclaim_observed_test.rs b/lib/crowdb-access-dataset/tests/reclaim_observed_test.rs index dacf86969..a16f5a37f 100644 --- a/lib/crowdb-access-dataset/tests/reclaim_observed_test.rs +++ b/lib/crowdb-access-dataset/tests/reclaim_observed_test.rs @@ -23,12 +23,9 @@ async fn reclaim_reuses_four_head_observations_and_a_terminal_snapshot_without_c assert!(!fixture.held.samples.is_empty()); let head_key = scoped(b"DS\0H\0", &fixture, &[]); let head_value = fixture.store.get(&head_key).await.unwrap().unwrap(); - let (version, head, _, attempt): (u16, HeadRecord, OperationId, CommitAttempt) = - bincode::deserialize(&head_value.bytes).unwrap(); - assert_eq!(version, 3); - let receipt = scoped(b"DS\0CR\0", &fixture, attempt.id.as_bytes()); + let (version, head, _): (u16, HeadRecord, OperationId) = bincode::deserialize(&head_value.bytes).unwrap(); + assert_eq!(version, 4); let snapshot_receipt = scoped(b"DS\0SC\0", &fixture, fixture.next.as_bytes()); - fixture.store.delete(&receipt).await.unwrap(); fixture.store.delete(&snapshot_receipt).await.unwrap(); let snapshot = scoped(b"DS\0S\0", &fixture, fixture.snapshot.as_bytes()); let store = Arc::new(observed::TestReclaimObserved::new( @@ -52,8 +49,7 @@ async fn reclaim_reuses_four_head_observations_and_a_terminal_snapshot_without_c current.generation.unwrap(), CommitGeneration::from_head(&head_value.bytes) ); - assert!(fixture.store.get(&receipt).await.unwrap().is_some()); - assert!(fixture.store.get(&snapshot_receipt).await.unwrap().is_some()); + assert!(fixture.store.get(&snapshot_receipt).await.unwrap().is_none()); } #[tokio::test] diff --git a/lib/crowdb-access-dataset/tests/snapshot_complete_result_test.rs b/lib/crowdb-access-dataset/tests/snapshot_complete_result_test.rs index a087c6ac1..6cc672e41 100644 --- a/lib/crowdb-access-dataset/tests/snapshot_complete_result_test.rs +++ b/lib/crowdb-access-dataset/tests/snapshot_complete_result_test.rs @@ -154,7 +154,7 @@ async fn cleanup_uses_a_completed_cas_winner_without_reloading_or_retiring_conte let cb = key(b"DS\0CB\0", &identity, snapshot); let complete = store.storage().get(&cb).await.unwrap().unwrap(); let mut staged: serde_json::Value = serde_json::from_slice(&complete.bytes).unwrap(); - staged["version"] = 1.into(); + staged["version"] = 3.into(); assert!(matches!( store .storage() diff --git a/lib/crowdb-access-dataset/tests/snapshot_publication_cost_test.rs b/lib/crowdb-access-dataset/tests/snapshot_publication_cost_test.rs new file mode 100644 index 000000000..deee89c33 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/snapshot_publication_cost_test.rs @@ -0,0 +1,185 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/composition.rs"] +#[allow(dead_code)] +mod common; +#[path = "common/publication.rs"] +mod publication; +#[path = "common/split_store.rs"] +mod split_store; +use crowdb_access_dataset::*; +use std::sync::atomic::Ordering; + +fn counts(store: &split_store::TestDraftStore) -> (usize, usize, usize) { + ( + store.unbounded_reads.load(Ordering::Relaxed) + store.bounded_reads.load(Ordering::Relaxed), + store.mutations.load(Ordering::Relaxed), + store.data_head_reads.load(Ordering::Relaxed), + ) +} +fn delta(store: &split_store::TestDraftStore, before: (usize, usize, usize)) -> (usize, usize, usize) { + let now = counts(store); + (now.0 - before.0, now.1 - before.1, now.2 - before.2) +} +async fn prepare(authority: &DatasetAuthority, identity: &DatasetIdentity, marker: u8) -> SnapshotId { + authority + .prepare_snapshot_candidate( + identity, + None, + publication::publication_manifest(marker), + OperationId::random(), + ) + .await + .unwrap() +} + +#[tokio::test] +async fn publication_writes_only_the_pointer_and_retries_current_effect_without_a_ledger() { + let (store, authority, identity, parent) = split_store::fixture().await; + let first = prepare(&authority, &identity, 2).await; + let attempt = common::attempt(&authority, &identity, first).await; + store.reject_publishing_ledgers.store(true, Ordering::Relaxed); + let before = counts(&store); + let receipt = authority + .publish_snapshot_candidate(&identity, &attempt) + .await + .unwrap(); + assert_eq!(delta(&store, before), (3, 1, 1)); + let before = counts(&store); + assert_eq!( + authority + .publish_snapshot_candidate(&identity, &attempt) + .await + .unwrap(), + receipt + ); + assert_eq!(delta(&store, before), (3, 0, 1)); + let second = prepare(&authority, &identity, 3).await; + let next = common::attempt(&authority, &identity, second).await; + authority + .publish_snapshot_candidate(&identity, &next) + .await + .unwrap(); + let before = counts(&store); + let error = authority + .publish_snapshot_candidate(&identity, &attempt) + .await + .unwrap_err(); + let AuthorityError::SnapshotHeadConflict { observed } = error else { + panic!("{error:?}") + }; + assert_eq!(observed.latest, Some(second)); + assert_eq!(delta(&store, before), (3, 0, 1)); + assert!(authority + .commit_status(&identity, attempt.id) + .await + .unwrap() + .is_none()); + assert!(authority + .snapshot_commit(&identity, first) + .await + .unwrap() + .is_none()); + assert!(authority.get_snapshot(&identity, first).await.is_ok()); + assert!(authority.get_snapshot(&identity, parent).await.is_ok()); +} + +#[tokio::test] +async fn real_cas_conflicts_return_the_observation_and_unknown_outcomes_do_not_query() { + for same in [false, true] { + let (store, authority, identity, _) = split_store::fixture().await; + let snapshot = prepare(&authority, &identity, 2).await; + let attempt = common::attempt(&authority, &identity, snapshot).await; + store.keep_data_conflict_target.store(same, Ordering::Relaxed); + store.data_head_conflicts.store(1, Ordering::Relaxed); + store.reject_publishing_ledgers.store(true, Ordering::Relaxed); + let before = counts(&store); + let result = authority.publish_snapshot_candidate(&identity, &attempt).await; + if same { + assert_eq!(result.unwrap().attempt, attempt); + } else { + let AuthorityError::SnapshotHeadConflict { observed } = result.unwrap_err() else { + panic!("missing CAS observation") + }; + assert_eq!(observed.latest, None); + assert!(observed.generation.is_some()); + } + assert_eq!(delta(&store, before), (3, 1, 1)); + } + let (store, authority, identity, _) = split_store::fixture().await; + let snapshot = prepare(&authority, &identity, 2).await; + let attempt = common::attempt(&authority, &identity, snapshot).await; + store.lose_data_reply.store(true, Ordering::Relaxed); + let before = counts(&store); + assert!(matches!( + authority.publish_snapshot_candidate(&identity, &attempt).await, + Err(AuthorityError::Store(StoreError::Rejected)) + )); + assert_eq!(delta(&store, before), (3, 1, 1)); + let before = counts(&store); + authority + .publish_snapshot_candidate(&identity, &attempt) + .await + .unwrap(); + assert_eq!(delta(&store, before), (3, 0, 1)); +} + +#[tokio::test] +async fn four_explicit_retries_use_each_returned_generation_without_a_head_query() { + let (store, authority, identity, _) = split_store::fixture().await; + let snapshot = prepare(&authority, &identity, 2).await; + let mut update = SnapshotHeadUpdate { + snapshot, + expected_head: authority.inspect_head(&identity).await.unwrap().generation, + }; + store.data_head_conflicts.store(4, Ordering::Relaxed); + store.reject_publishing_ledgers.store(true, Ordering::Relaxed); + let before = counts(&store); + for _ in 0..4 { + let error = authority + .update_snapshot_head(&identity, &update) + .await + .unwrap_err(); + let AuthorityError::SnapshotHeadConflict { observed } = error else { + panic!("{error:?}") + }; + assert_ne!(observed.generation, update.expected_head); + update.expected_head = observed.generation; + } + assert_eq!( + authority + .update_snapshot_head(&identity, &update) + .await + .unwrap() + .latest, + Some(snapshot) + ); + assert_eq!(delta(&store, before), (15, 5, 5)); +} + +#[tokio::test] +async fn composition_publication_keeps_prepared_state_and_adds_only_its_draft_read() { + let (store, authority, identity, parent) = split_store::fixture().await; + let operation = OperationId::random(); + common::begin_evaluated(&authority, &identity, operation, &common::input(parent)).await; + let draft = common::prepare(&authority, &identity, operation, 1).await; + let snapshot = draft.preparation.as_ref().unwrap().candidate; + let attempt = common::attempt(&authority, &identity, snapshot).await; + store.reject_publishing_ledgers.store(true, Ordering::Relaxed); + let before = counts(&store); + authority + .publish_composition(&identity, operation, &attempt) + .await + .unwrap(); + assert_eq!(delta(&store, before), (4, 1, 1)); + assert_eq!( + authority.composition_status(&identity, operation).await.unwrap(), + draft + ); + authority + .abandon_composition(&identity, operation, 1) + .await + .unwrap(); + assert!(authority.get_snapshot(&identity, snapshot).await.is_ok()); +} diff --git a/sdk/README.md b/sdk/README.md index 124c06bf5..dbb043283 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -208,12 +208,14 @@ permission or promise that a redeployed endpoint still supports the operation. `uploads.publication_ready` is true. Initial Splits publish with the Snapshot. - `publish_composition` advances latest; `publish_split` updates only currentSplit on its owning Snapshot. Null expected heads/revisions are explicit. On an - unknown outcome, retry the exact request. `commit_status` exposes Snapshot and - legacy receipts; new independent Split publication saves no request ledger. - A Split retry confirms the same current revision or returns a condition conflict - after a later revision replaces it. A deliberate retry after a head conflict - uses newly reviewed inputs and an observed generation. Publishing a Split does - not change its compatibility draft from Prepared to Published. + unknown outcome, retry the exact request. `commit_status` exposes only legacy + receipts; new Snapshot/Split publication saves no request ledger. A retry + confirms the same current target or returns a condition conflict after a later + target replaces it. `observed_head` supplies the failed condition's observation, + without a confirmation GET. A deliberate retry uses reviewed inputs and that + generation; content is never silently rebased. New publication leaves the + compatibility draft Prepared. Completed resources are readable and retain + payload ownership even before publication or after draft abandonment. - `split_members`, `split_history`, `inventory`, `roots`, `children`, `schema` and `samples` return lazy `PageCursor`s. Each `next_page` fetches at most 100 items; it never eagerly collects the Dataset. Persist `continuation()` for scoped diff --git a/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs b/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs index 364e70928..73703a622 100644 --- a/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs +++ b/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs @@ -117,10 +117,7 @@ async fn import(client: &v2::ControlClient) -> Result<(v2::OpaqueId, v2::OpaqueI assert_eq!(lifecycle.snapshot, snapshot); assert!(lifecycle.latest); assert!(!lifecycle.retained); - assert_eq!( - client.commit_status(&request.attempt).await?.unwrap().attempt, - receipt.attempt - ); + assert!(client.commit_status(&request.attempt).await?.is_none()); assert_eq!( client.publish_composition(&operation, &request).await?.attempt, receipt.attempt diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2/client.rs b/sdk/crates/crowdb-sdk-dataset/src/v2/client.rs index 1ff77b142..93981982f 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/v2/client.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/v2/client.rs @@ -116,7 +116,7 @@ impl ControlClient { ) .await } - /// Resolves an unknown publication outcome without issuing a new attempt. + /// Reads legacy publication receipts. Current pointer updates create no receipt history. /// # Errors /// Returns capability, transport, invalid response or service errors. pub async fn commit_status(&self, attempt: &OpaqueId) -> Result, Error> { diff --git a/sdk/tests/split_consumer.py b/sdk/tests/split_consumer.py index ba6492afb..698443faa 100644 --- a/sdk/tests/split_consumer.py +++ b/sdk/tests/split_consumer.py @@ -34,7 +34,7 @@ def publish(client, operation, snapshot): request = {"attempt": new_id(), "snapshot": snapshot, "expected_head": client.head()["generation"]} receipt = client.publish_composition(operation, request) assert client.publish_composition(operation, request) == receipt - assert client.commit_status(request["attempt"]) == receipt + assert client.commit_status(request["attempt"]) is None return receipt From d8d322d0cb542f7dba1dbc6adfd0f69999937b9f Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 20:21:37 +0800 Subject: [PATCH 126/258] Expose direct snapshot head updates to portable SDKs --- .../src/dataset/api/v2.rs | 30 +--- .../src/dataset/api/v2/heads.rs | 65 +++++++ .../dataset_snapshot_head_contract_test.rs | 162 ++++++++++++++++++ .../tests/dataset_v2_contract_test.rs | 1 + .../dataset/design-dataset-control.md | 9 + .../dataset/openapi-crowdb-dataset-v2.yaml | 37 +++- .../dataset/spec-crowdb-dataset.md | 5 + doc/working/plan-access-simplification.md | 19 +- .../src/authority/head.rs | 26 ++- sdk/README.md | 8 + .../examples/split_control.rs | 36 ++-- .../crowdb-sdk-dataset/src/capabilities.rs | 3 +- sdk/crates/crowdb-sdk-dataset/src/v2.rs | 2 + .../src/v2/snapshot_head.rs | 34 ++++ .../tests/snapshot_head_test.rs | 85 +++++++++ sdk/crates/crowdb-sdk-python/src/control.rs | 1 + sdk/crowdb_dataset/control.py | 3 +- sdk/tests/split_consumer.py | 29 +++- sdk/tests/test_control.py | 33 +++- 19 files changed, 516 insertions(+), 72 deletions(-) create mode 100644 app/crowdb-access-server/src/dataset/api/v2/heads.rs create mode 100644 app/crowdb-access-server/tests/dataset_snapshot_head_contract_test.rs create mode 100644 sdk/crates/crowdb-sdk-dataset/src/v2/snapshot_head.rs create mode 100644 sdk/crates/crowdb-sdk-dataset/tests/snapshot_head_test.rs diff --git a/app/crowdb-access-server/src/dataset/api/v2.rs b/app/crowdb-access-server/src/dataset/api/v2.rs index 5c58045af..f009c0e59 100644 --- a/app/crowdb-access-server/src/dataset/api/v2.rs +++ b/app/crowdb-access-server/src/dataset/api/v2.rs @@ -8,6 +8,7 @@ mod composition_fields; mod composition_models; mod composition_responses; mod compositions; +mod heads; mod inspection; mod inspection_models; mod predicate; @@ -32,13 +33,6 @@ struct Capabilities { capabilities: Vec<&'static str>, } -#[derive(Serialize)] -struct Head { - latest: Option, - stable: Option, - generation: Option, -} - impl DatasetHttpService { pub(super) async fn route_v2(&self, parts: &[String], request: Request) -> ApiResult { if parts == ["api", "dataset", "v2", "namespaces", "dataset-ns", "datasets"] { @@ -49,6 +43,7 @@ impl DatasetHttpService { read_body(request, false).await?; let mut capabilities = vec![ "head-inspection", + "snapshot-head-update", "bounded-inspection", "inspection-preparation", "dataset-management", @@ -64,28 +59,13 @@ impl DatasetHttpService { capabilities, }); } - if parts.len() == 8 - && parts[..6] == ["api", "dataset", "v2", "namespaces", "dataset-ns", "datasets"] - && parts[7] == "head" - && request.method() == Method::GET - { - query::Query::parse(request.uri().query(), &[])?; - read_body(request, false).await?; - let identity = - DatasetIdentity::new(NamespacePath::root(), &parts[6]).map_err(|_| ApiError::invalid())?; - let head = self.reads.authority().inspect_head(&identity).await?; - return json(&Head { - latest: head.latest.map(|id| id.to_string()), - stable: head.stable.map(|id| id.to_string()), - generation: head - .generation - .map(|generation| hex::encode(generation.as_bytes())), - }); - } if parts.len() >= 7 && parts[..6] == ["api", "dataset", "v2", "namespaces", "dataset-ns", "datasets"] { let identity = DatasetIdentity::new(NamespacePath::root(), &parts[6]).map_err(|_| ApiError::invalid())?; + if parts.len() == 8 && parts[7] == "head" { + return self.v2_head(&identity, request).await; + } let tail: Vec<_> = parts[7..].iter().map(String::as_str).collect(); if tail.first() == Some(&"uploads") { return self.v2_uploads(&identity, &tail, request).await; diff --git a/app/crowdb-access-server/src/dataset/api/v2/heads.rs b/app/crowdb-access-server/src/dataset/api/v2/heads.rs new file mode 100644 index 000000000..0e37499a4 --- /dev/null +++ b/app/crowdb-access-server/src/dataset/api/v2/heads.rs @@ -0,0 +1,65 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + super::{read_body_limit, snapshot}, + json, + query::Query, + read_body, + split_models::required_option, + split_requests::generation, + ApiError, ApiResult, DatasetHttpService, +}; +use crowdb_access_dataset::{DatasetHead, DatasetIdentity, SnapshotHeadUpdate}; +use hyper::{body::Incoming, Method, Request}; +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct UpdateHead { + snapshot: String, + #[serde(deserialize_with = "required_option")] + expected_head: Option, +} +#[derive(Serialize)] +struct Head { + latest: Option, + stable: Option, + generation: Option, +} +impl From for Head { + fn from(head: DatasetHead) -> Self { + Self { + latest: head.latest.map(|id| id.to_string()), + stable: head.stable.map(|id| id.to_string()), + generation: head.generation.map(|value| hex::encode(value.as_bytes())), + } + } +} +impl DatasetHttpService { + pub(super) async fn v2_head(&self, identity: &DatasetIdentity, request: Request) -> ApiResult { + Query::parse(request.uri().query(), &[])?; + let authority = self.reads.authority(); + let head = match *request.method() { + Method::GET => { + read_body(request, false).await?; + authority.inspect_head(identity).await? + } + Method::PUT => { + let body = read_body_limit(request, true, 4096).await?; + let input: UpdateHead = serde_json::from_slice(&body).map_err(|_| ApiError::invalid())?; + authority + .update_snapshot_head( + identity, + &SnapshotHeadUpdate { + snapshot: snapshot(&input.snapshot)?, + expected_head: generation(input.expected_head.as_deref())?, + }, + ) + .await? + } + _ => return Err(ApiError::not_found()), + }; + json(&Head::from(head)) + } +} diff --git a/app/crowdb-access-server/tests/dataset_snapshot_head_contract_test.rs b/app/crowdb-access-server/tests/dataset_snapshot_head_contract_test.rs new file mode 100644 index 000000000..5aa83a517 --- /dev/null +++ b/app/crowdb-access-server/tests/dataset_snapshot_head_contract_test.rs @@ -0,0 +1,162 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/dataset_v2.rs"] +#[allow(dead_code)] +mod common; +#[path = "common/dataset.rs"] +mod dataset; +#[path = "../../../lib/crowdb-access-dataset/tests/common/publication.rs"] +mod publication; +use crowdb_access_dataset::*; +use dataset::{TestDatasetHttp, TestStore}; +use reqwest::Client; +use serde_json::json; +use std::sync::{atomic::Ordering, Arc}; + +fn counts(store: &TestStore) -> (u64, u64) { + ( + store.reads.load(Ordering::Relaxed) + store.bounded_reads.load(Ordering::Relaxed), + store.mutations.load(Ordering::Relaxed), + ) +} +fn delta(store: &TestStore, before: (u64, u64)) -> (u64, u64) { + let now = counts(store); + (now.0 - before.0, now.1 - before.1) +} +async fn fixture() -> ( + Arc, + DatasetAuthority, + DatasetIdentity, + TestDatasetHttp, + String, + Vec, +) { + let store = Arc::new(TestStore::default()); + let authority = DatasetAuthority::new(store.clone()); + let identity = DatasetIdentity::new(NamespacePath::root(), "snapshot-head").unwrap(); + authority.create_dataset(identity.clone()).await.unwrap(); + let mut snapshots = Vec::new(); + for marker in [1, 2] { + snapshots.push( + authority + .prepare_snapshot_candidate( + &identity, + None, + publication::publication_manifest(marker), + OperationId::random(), + ) + .await + .unwrap(), + ); + } + let test = TestDatasetHttp::start(store.clone()).await; + let root = format!( + "{}/api/dataset/v2/namespaces/dataset-ns/datasets/snapshot-head", + test.origin + ); + (store, authority, identity, test, root, snapshots) +} + +#[tokio::test] +async fn head_updates_use_complete_resource_ids_and_return_the_failed_condition() { + let (store, authority, identity, test, root, snapshots) = fixture().await; + let client = Client::new(); + let url = format!("{root}/head"); + let before = counts(&store); + assert_eq!( + common::response(client.get(&url), 200).await, + json!({"latest":null,"stable":null,"generation":null}) + ); + assert_eq!(delta(&store, before), (2, 0)); + let first = json!({"snapshot":snapshots[0].to_string(),"expected_head":null}); + let before = counts(&store); + let head = common::response(client.put(&url).json(&first), 200).await; + assert_eq!(delta(&store, before), (3, 1)); + assert_eq!(head["latest"], snapshots[0].to_string()); + let before = counts(&store); + assert_eq!(common::response(client.put(&url).json(&first), 200).await, head); + assert_eq!(delta(&store, before), (3, 0)); + let mut next = json!({"snapshot":snapshots[1].to_string(),"expected_head":null}); + let before = counts(&store); + let conflict = common::response(client.put(&url).json(&next), 409).await; + assert_eq!(conflict["observed_head"], head); + assert_eq!(delta(&store, before), (3, 0)); + next["expected_head"] = conflict["observed_head"]["generation"].clone(); + let before = counts(&store); + let second = common::response(client.put(&url).json(&next), 200).await; + assert_eq!(delta(&store, before), (3, 1)); + let before = counts(&store); + assert_eq!( + common::response(client.put(&url).json(&first), 409).await["observed_head"], + second + ); + assert_eq!(delta(&store, before), (3, 0)); + authority.begin_reclaim(&identity, snapshots[0]).await.unwrap(); + let before = counts(&store); + common::response(client.put(&url).json(&first), 409).await; + assert_eq!(delta(&store, before), (3, 0)); + drop(client); + test.stop().await; +} + +#[tokio::test] +async fn invalid_conditions_and_auth_fail_before_storage_without_a_draft_fallback() { + let (store, _, _, test, root, snapshots) = fixture().await; + let client = Client::new(); + let url = format!("{root}/head"); + let valid = json!({"snapshot":snapshots[0].to_string(),"expected_head":null}); + let before = counts(&store); + assert_eq!( + client + .put(&url) + .json(&valid) + .send() + .await + .unwrap() + .status() + .as_u16(), + 401 + ); + for invalid in [ + json!({"snapshot":snapshots[0].to_string()}), + json!({"snapshot":null,"expected_head":null}), + json!({"snapshot":"bad","expected_head":null}), + json!({"snapshot":snapshots[0].to_string(),"expected_head":4}), + json!({"snapshot":snapshots[0].to_string(),"expected_head":"A".repeat(64)}), + json!({"snapshot":snapshots[0].to_string(),"expected_head":null,"attempt":"11".repeat(16)}), + ] { + common::response(client.put(&url).json(&invalid), 400).await; + } + common::response(client.put(&url).body(valid.to_string()), 400).await; + common::response(client.put(format!("{url}?unknown=1")).json(&valid), 400).await; + common::response( + client + .put(&url) + .header("content-type", "application/json") + .body(" ".repeat(4097)), + 413, + ) + .await; + assert_eq!(delta(&store, before), (0, 0)); + let before = counts(&store); + common::response( + client + .put(&url) + .json(&json!({"snapshot":"11".repeat(16),"expected_head":null})), + 404, + ) + .await; + assert_eq!(delta(&store, before), (2, 0)); + let caps = common::response( + client.get(format!("{}/api/dataset/v2/capabilities", test.origin)), + 200, + ) + .await; + assert!(caps["capabilities"] + .as_array() + .unwrap() + .contains(&json!("snapshot-head-update"))); + drop(client); + test.stop().await; +} diff --git a/app/crowdb-access-server/tests/dataset_v2_contract_test.rs b/app/crowdb-access-server/tests/dataset_v2_contract_test.rs index 96d04a5fc..709116b6f 100644 --- a/app/crowdb-access-server/tests/dataset_v2_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_v2_contract_test.rs @@ -34,6 +34,7 @@ async fn capabilities_use_existing_auth_and_match_configured_operations() { value["capabilities"], serde_json::json!([ "head-inspection", + "snapshot-head-update", "bounded-inspection", "inspection-preparation", "dataset-management", diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index 7bac27c66..8889bb39d 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -54,6 +54,15 @@ old operation/head preflight. Responses use the known immutable metadata plus on latest observation, without rereading operation/Snapshot records or copying the Manifest for a second preparation. +`PUT .../datasets/{dataset}/head` selects an existing complete Snapshot from +`snapshot` and explicit nullable `expected_head`, under `snapshot-head-update`. +It reads no draft or attempt and returns the same latest/stable/generation shape +as GET head. Stable is preserved; failed conditions include `observed_head` in that +shape. GET head reads Dataset identity and head once each. Rust exposes +`ControlClient::update_snapshot_head` and `Head::from_conflict`; Python exposes +`update_snapshot_head` and `ConflictError.observed_head` for synchronous and async +clients. Missing capability never invokes a composition or v1 publication fallback. + Each Snapshot owns immutable Split-set revisions and an independently fenced currentSplit pointer. Names are scoped to Dataset/Snapshot/revision. Publishing a new revision replaces the complete set, including rename/removal; old revisions diff --git a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml index 68aa7a0c7..4d13e0251 100644 --- a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml +++ b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml @@ -12,7 +12,8 @@ info: available under their original routes. Metadata GET routes require bounded-inspection; explicit POST inspection preparation requires inspection-preparation. Dataset creation, stable/retention changes and receipts - require dataset-management; Split lifecycle requires split-revisions; direct Split head updates require split-head-update; versioned + require dataset-management; direct Snapshot head updates require snapshot-head-update; + Split lifecycle requires split-revisions; direct Split head updates require split-head-update; versioned stateless reads require stateless-reads. Upload, composition and Snapshot retirement require uploads, composition and snapshot-reclaim respectively, advertised only when the upload service is configured. Query parameters are strict and duplicate @@ -44,6 +45,31 @@ paths: schema: {$ref: '#/components/schemas/Head'} '404': description: Dataset or namespace is unavailable + put: + operationId: updateSnapshotHead + summary: Select a complete Snapshot using an explicit head condition + description: >- + Requires snapshot-head-update. The input contains only the complete resource ID + and explicit nullable expected_head; no draft or attempt identity is read. + Stable is preserved. Same-current-target retries confirm the current effect; + stale conditions after another publication return observed_head without + a confirmation query or implicit content rebase. Unknown outcomes do not + create response history. The JSON body is limited to 4096 bytes. + parameters: + - {$ref: '#/components/parameters/dataset'} + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/UpdateSnapshotHead'} + responses: + '200': + description: Current effect confirmed without a historical response receipt + content: + application/json: + schema: {$ref: '#/components/schemas/Head'} + default: + $ref: '#/components/responses/InspectionError' /capabilities: get: operationId: datasetCapabilities @@ -1173,6 +1199,15 @@ components: generation: type: [string, 'null'] pattern: '^[0-9a-f]{64}$' + UpdateSnapshotHead: + type: object + additionalProperties: false + required: [snapshot, expected_head] + properties: + snapshot: {$ref: '#/components/schemas/OpaqueId'} + expected_head: + type: [string, 'null'] + pattern: '^[0-9a-f]{64}$' Capabilities: type: object required: [api_version, capabilities] diff --git a/doc/design/access-server/dataset/spec-crowdb-dataset.md b/doc/design/access-server/dataset/spec-crowdb-dataset.md index 4ea408738..d2df8505e 100644 --- a/doc/design/access-server/dataset/spec-crowdb-dataset.md +++ b/doc/design/access-server/dataset/spec-crowdb-dataset.md @@ -204,6 +204,11 @@ latest. A head conflict MUST return the observed latest/stable/generation withou a confirmation query. A reviewed retry uses that observation with caller-chosen content; the server MUST NOT rebase. Unknown storage outcomes MUST propagate. Existing legacy receipts remain readable without backfill. +Direct Snapshot head updates MUST accept the complete Snapshot ID and explicit +nullable expected generation without an operation or draft identity. They MUST +preserve stable and require the `snapshot-head-update` capability in portable SDKs. +Their response and conflict observation use the latest/stable/generation head +shape. Unsupported direct updates MUST NOT fall back to composition or v1 writes. Publishing an independent Split revision MUST update only its Snapshot's currentSplit pointer with an expected revision/head CAS. It MUST NOT create a data Snapshot or diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 36ab6bacf..c849d3e3c 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -156,10 +156,6 @@ CAS 不再满足只证明“本次条件不成立”，不能谎称“第一次 历史回复查询；Composition/Split 的 Committing 恢复与 P2/P3 一起删除，客户端保留完整输入。 旧记录只作为兼容解码/GC 输入，不让旧账本控制新请求。head 保留 expected base；同一 candidate 的当前结果可判断相同，已被后续 head 遮住时不保证历史请求结果重放。 -- [ ] **R2.2d Snapshot 直接 REST/SDK 发布入口**：提供只接收完整 Snapshot ID 与显式 nullable - expected head 的 head 更新 API、能力和 Rust/Python SDK；不读草稿或使用 attempt ID。 - SDK 提供 typed observed head，正常/冲突/unknown 请求计数和真实 installed consumer 验证， - 迁移示例与容器流程；保留父内容不可隐式 rebase、鉴权、删除与不支持时无降级。 - [ ] **R2 审查条件发布的回复丢失窗口**：用 H0→H1→H2 测试区分条件失败、相同结果与历史未知； 列出确实依赖历史精确结果的调用者，没有这种需求就移除相应 ledger/response replay。 @@ -494,6 +490,21 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，R2.2d**：新增 `PUT .../datasets/{dataset}/head`、 + `snapshot-head-update` 能力、Rust/Python `update_snapshot_head`；只接收完整 Snapshot ID + 和必填 nullable expected head，body 上限 4096 字节。不读取草稿、使用 attempt 或写请求 + 账本；不支持能力时 SDK 拒绝写入，不降级。409 提供 typed observed head，Python 同步和 + 异步均保留 Snapshot identity，重试由调用者明确决定。GET head 删除重复读取，由 3G + 降至 2G；PUT 正常 3G + 1C，相同当前结果/过期条件 3G + 0C。真实 listener 覆盖显式 + 复用观察值、删除资源、缺失、鉴权、错误条件和超限 body，非法输入零存储调用。 + 示例在开始计算前固定 head 条件，新 Dataset 直接使用已知 absent；不会计算后刷新 + base 强行覆盖并发结果。13 项 native Snapshot/legacy/cost 测试、6 项 listener/consumer + 测试、独立 SDK 32 项测试和 installed wheel 10 项 Python 测试通过；真实 Rust 和 Python + Split consumer 均已改为直接 Snapshot 发布，包含同步/异步冲突。最终 sdist/wheel 位于 + `.crowdb-runtime/artifacts/sdk-snapshot-head-20261024`。三 crate all-target/all-feature + clippy、SDK all-target clippy、两个 workspace fmt、SDK boundary、diff 检查通过，均经 + pixi。OpenAPI 解析及 338 个本地引用校验通过。新源码容器复验仍由 V2/V4 跟踪。 + - **2026-10-24，R2.2c/R1.1**：Snapshot 新发布只写 version 4 fenced head，删除 CA/CR/SC 写入、head 成功补写和新 Composition Committing/Published 转换。当前同目标确认效果， 后续目标遮住的旧条件返回 observed latest/stable/generation，真实 CAS 失败直接解码返回值， diff --git a/lib/crowdb-access-dataset/src/authority/head.rs b/lib/crowdb-access-dataset/src/authority/head.rs index a5077d52d..537070648 100644 --- a/lib/crowdb-access-dataset/src/authority/head.rs +++ b/lib/crowdb-access-dataset/src/authority/head.rs @@ -3,8 +3,8 @@ use super::{decode, AuthorityError, VersionedRecord}; use crate::{ - key, CommitAttempt, CommitGeneration, DatasetAuthority, DatasetIdentity, HeadRecord, OperationId, - SnapshotId, StoredValue, + key, CommitAttempt, CommitGeneration, DatasetAuthority, DatasetIdentity, DatasetRecord, HeadRecord, + OperationId, SnapshotId, StoredValue, }; use serde::{Deserialize, Serialize}; @@ -54,20 +54,14 @@ impl DatasetAuthority { /// # Errors /// Rejects missing Datasets and malformed or unavailable head records. pub async fn inspect_head(&self, identity: &DatasetIdentity) -> Result { - self.get_dataset(identity).await?; - let Some(stored) = self.store.get(&key::head(identity)?).await? else { - return Ok(DatasetHead { - latest: None, - stable: None, - generation: None, - }); - }; - let head = decode_head(&stored.bytes)?; - Ok(DatasetHead { - latest: head.latest, - stable: head.stable, - generation: Some(CommitGeneration::from_head(&stored.bytes)), - }) + let dataset = self + .store + .get(&key::dataset(identity)?) + .await? + .ok_or(AuthorityError::NotFound)?; + let _: DatasetRecord = decode(&dataset.bytes)?; + let stored = self.store.get(&key::head(identity)?).await?; + reference(stored.as_ref()) } } pub(super) fn decode_head(bytes: &[u8]) -> Result { diff --git a/sdk/README.md b/sdk/README.md index dbb043283..df8b4fc61 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -229,6 +229,14 @@ permission or promise that a redeployed endpoint still supports the operation. and does not read a draft. Rust's `SplitHead::from_conflict` and Python's `ConflictError.observed_head` return the failed condition's observation. Review that base before retrying; the SDK does not rebase automatically. +- `update_snapshot_head(request)` selects an existing complete Snapshot with + `snapshot` and explicit nullable `expected_head`; it requires + `snapshot-head-update`, preserves stable and reads no draft or attempt ID. + Rust uses `UpdateSnapshotHead` and `Head::from_conflict`; synchronous and async + Python clients expose `ConflictError.observed_head`. A conflict supplies the + observed latest/stable/generation without an extra head GET. The caller reviews + the new base and may need to rebuild content from it; the SDK never rebases or + falls back to a legacy publication route. - `bind_read` pins Snapshot and optional Split revision without writes; `next_batch(cursor)` uses verified streaming framing. Save the next cursor after successful processing. The required capability is `stateless-reads`; old session diff --git a/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs b/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs index 73703a622..c929f8905 100644 --- a/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs +++ b/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs @@ -107,20 +107,18 @@ async fn import(client: &v2::ControlClient) -> Result<(v2::OpaqueId, v2::OpaqueI ); let prepared = prepare(client, &operation, edited.revision).await?; let snapshot = prepared.preparation.unwrap().snapshot; - let request = v2::PublishComposition { - attempt: v2::OpaqueId::random(), + let request = v2::UpdateSnapshotHead { snapshot: snapshot.clone(), - expected_head: client.head().await?.generation, + expected_head: None, }; - let receipt = client.publish_composition(&operation, &request).await?; + let head = client.update_snapshot_head(&request).await?; let lifecycle = client.lifecycle(&snapshot).await?; assert_eq!(lifecycle.snapshot, snapshot); assert!(lifecycle.latest); assert!(!lifecycle.retained); - assert!(client.commit_status(&request.attempt).await?.is_none()); assert_eq!( - client.publish_composition(&operation, &request).await?.attempt, - receipt.attempt + client.update_snapshot_head(&request).await?.generation, + head.generation ); cleanup(client, &operation).await?; let revision = client.split_head(&snapshot).await?.revision.unwrap(); @@ -284,6 +282,7 @@ async fn revise_splits( Ok(()) } async fn branch(client: &v2::ControlClient, source: v2::SplitSource) -> Result { + let expected_head = client.head().await?.generation; let operation = v2::OpaqueId::random(); client .begin_composition(&v2::BeginComposition { @@ -306,15 +305,22 @@ async fn branch(client: &v2::ControlClient, source: v2::SplitSource) -> Result u64 { diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2.rs b/sdk/crates/crowdb-sdk-dataset/src/v2.rs index 3205b2849..9a9321e18 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/v2.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/v2.rs @@ -11,6 +11,7 @@ mod lifecycle; mod paging; mod read; mod requests; +mod snapshot_head; mod split; mod split_head; mod state; @@ -23,6 +24,7 @@ pub use identity::{Decimal, Generation, OpaqueId, SampleId}; pub use inspection::*; pub use paging::{PageCursor, PageKind}; pub use read::*; +pub use snapshot_head::UpdateSnapshotHead; pub use split::*; pub use split_head::UpdateSplitHead; pub use state::*; diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2/snapshot_head.rs b/sdk/crates/crowdb-sdk-dataset/src/v2/snapshot_head.rs new file mode 100644 index 000000000..9523cd0e2 --- /dev/null +++ b/sdk/crates/crowdb-sdk-dataset/src/v2/snapshot_head.rs @@ -0,0 +1,34 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{required_option, ControlClient, Generation, Head, OpaqueId}; +use crate::Error; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UpdateSnapshotHead { + pub snapshot: OpaqueId, + #[serde(deserialize_with = "required_option")] + pub expected_head: Option, +} +impl ControlClient { + /// Selects a complete Snapshot with an explicit condition, without a draft or attempt ID. + /// # Errors + /// Returns capability, unavailable resource, conditional and transport errors. + pub async fn update_snapshot_head(&self, request: &UpdateSnapshotHead) -> Result { + self.call("snapshot-head-update", "PUT", &["head"], Some(request)) + .await + .map_err(|error| error.with_identity(request.snapshot.as_str())) + } +} +impl Head { + /// Extracts the failed condition's observation; a retry requires caller review of that base. + #[must_use] + pub fn from_conflict(error: &Error) -> Option { + let Error::Service { status: 409, error } = error else { + return None; + }; + serde_json::from_value(error.observed_head.as_deref()?.clone()).ok() + } +} diff --git a/sdk/crates/crowdb-sdk-dataset/tests/snapshot_head_test.rs b/sdk/crates/crowdb-sdk-dataset/tests/snapshot_head_test.rs new file mode 100644 index 000000000..20d60d3ff --- /dev/null +++ b/sdk/crates/crowdb-sdk-dataset/tests/snapshot_head_test.rs @@ -0,0 +1,85 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +#[allow(dead_code)] +#[path = "../../crowdb-sdk-core/tests/common/peer.rs"] +mod peer; +use crowdb_sdk_dataset::{ + v2::{Head, OpaqueId, UpdateSnapshotHead}, + DatasetClient, +}; +use serde_json::json; + +#[test] +fn expected_head_is_required_and_nullable_and_legacy_conflicts_remain_decodable() { + let input = json!({"snapshot":"01".repeat(16),"expected_head":null}); + serde_json::from_value::(input).unwrap(); + assert!(serde_json::from_value::(json!({"snapshot":"01".repeat(16)})).is_err()); + assert!(serde_json::from_value::( + json!({"snapshot":"01".repeat(16),"expected_head":null,"attempt":"02".repeat(16)}) + ) + .is_err()); + let error = crowdb_sdk_dataset::Error::Service { + status: 409, + error: serde_json::from_value( + json!({"code":"conflict","message":"conflict","request_id":"test","retryable":false}), + ) + .unwrap(), + }; + assert!(Head::from_conflict(&error).is_none()); +} + +#[tokio::test] +async fn unsupported_updates_do_not_fall_back_to_a_composition_or_v1_write() { + let mut peer = peer::TestPeer::start(r#"{"api_version":2,"capabilities":["composition"]}"#, false).await; + let client = DatasetClient::new(&peer.endpoint, "test", None) + .unwrap() + .control(); + assert!(client + .update_snapshot_head(&UpdateSnapshotHead { + snapshot: OpaqueId::random(), + expected_head: None + }) + .await + .is_err()); + assert!(peer + .requests + .recv() + .await + .unwrap() + .starts_with("GET /api/dataset/v2/capabilities ")); + assert!(peer.requests.try_recv().is_err()); +} + +#[tokio::test] +async fn conflict_observations_supply_the_next_explicit_update_without_a_head_get() { + let mut peer = peer::TestPeer::start_statuses(r#"{"api_version":2,"capabilities":["snapshot-head-update"],"latest":"01010101010101010101010101010101","stable":null,"generation":"0202020202020202020202020202020202020202020202020202020202020202","code":"conflict","message":"conflict","request_id":"test","retryable":false,"observed_head":{"latest":"03030303030303030303030303030303","stable":null,"generation":"0404040404040404040404040404040404040404040404040404040404040404"}}"#, false, vec![200,409,200]).await; + let client = DatasetClient::new(&peer.endpoint, "test", None) + .unwrap() + .control(); + let mut update = UpdateSnapshotHead { + snapshot: "01".repeat(16).parse().unwrap(), + expected_head: None, + }; + let error = client.update_snapshot_head(&update).await.unwrap_err(); + let observed = Head::from_conflict(&error).unwrap(); + assert_eq!(observed.latest.unwrap().as_str(), "03".repeat(16)); + update.expected_head = observed.generation; + assert_eq!( + client.update_snapshot_head(&update).await.unwrap().latest, + Some(update.snapshot.clone()) + ); + assert!(peer + .requests + .recv() + .await + .unwrap() + .starts_with("GET /api/dataset/v2/capabilities ")); + for expected in [None, update.expected_head.clone()] { + let request = peer.requests.recv().await.unwrap(); + assert!(request.starts_with("PUT /api/dataset/v2/namespaces/dataset-ns/datasets/test/head ")); + let body: serde_json::Value = + serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!(body, json!({"snapshot":update.snapshot,"expected_head":expected})); + } + assert!(peer.requests.try_recv().is_err()); +} diff --git a/sdk/crates/crowdb-sdk-python/src/control.rs b/sdk/crates/crowdb-sdk-python/src/control.rs index cb6fa6cf1..24ab11ed6 100644 --- a/sdk/crates/crowdb-sdk-python/src/control.rs +++ b/sdk/crates/crowdb-sdk-python/src/control.rs @@ -24,6 +24,7 @@ pub(super) async fn dispatch(client: DatasetClient, name: &str, args: &str) -> R "create" => encode(&client.create().await?), "open" => encode(&client.open().await?), "head" => encode(&client.head().await?), + "update_snapshot_head" => encode(&client.update_snapshot_head(&request(&value)?).await?), "commit_status" => encode(&client.commit_status(&id()?).await?), "summary" => encode(&client.summary(&id()?).await?), "retain" => encode(&client.retain(&id()?).await?), diff --git a/sdk/crowdb_dataset/control.py b/sdk/crowdb_dataset/control.py index 1d6edc062..e5e22fa80 100644 --- a/sdk/crowdb_dataset/control.py +++ b/sdk/crowdb_dataset/control.py @@ -26,7 +26,7 @@ def record(self): def _identity(arguments): request = arguments.get("request") or {} - return request.get("attempt", request.get("edit", request.get("operation", request.get("id", arguments.get("id"))))) + return request.get("attempt", request.get("edit", request.get("operation", request.get("id", request.get("snapshot", arguments.get("id")))))) class ControlClient: @@ -44,6 +44,7 @@ def capabilities(self): return self._call("capabilities", {}) def create(self): return self._call("create", {}) def open(self): return self._call("open", {}) def head(self): return self._call("head", {}) + def update_snapshot_head(self, request): return self._call("update_snapshot_head", {"request": request}) def commit_status(self, attempt): return self._call("commit_status", {"id": attempt}) def summary(self, snapshot): return self._call("summary", {"id": snapshot}) def retain(self, snapshot): return self._call("retain", {"id": snapshot}) diff --git a/sdk/tests/split_consumer.py b/sdk/tests/split_consumer.py index 698443faa..72c8f3612 100644 --- a/sdk/tests/split_consumer.py +++ b/sdk/tests/split_consumer.py @@ -30,12 +30,12 @@ def cleanup(client, operation): raise AssertionError("cleanup budget") -def publish(client, operation, snapshot): - request = {"attempt": new_id(), "snapshot": snapshot, "expected_head": client.head()["generation"]} - receipt = client.publish_composition(operation, request) - assert client.publish_composition(operation, request) == receipt - assert client.commit_status(request["attempt"]) is None - return receipt +def publish(client, snapshot, expected_head): + request = {"snapshot": snapshot, "expected_head": expected_head} + head = client.update_snapshot_head(request) + assert head["latest"] == snapshot + assert client.update_snapshot_head(request) == head + return head def upload_source(client, source, upload_id, length, md5, part_size=1024*1024): @@ -65,6 +65,13 @@ async def verify_async(endpoint, name, token, snapshot, revision): async with AsyncDatasetClient(endpoint, name, token) as owner: client = owner.control() assert (await client.lifecycle(snapshot))["snapshot"] == snapshot + try: + await client.update_snapshot_head({"snapshot": snapshot, "expected_head": None}) + except ConflictError as error: + assert error.observed_head["latest"] != snapshot + assert error.identity == snapshot + else: + raise AssertionError("async stale Snapshot update must conflict") plan = {"snapshot": snapshot, "selection": {"kind": "all"}, "fields": ["value"], "batch_size": 1, "partition": {"worker": 0, "workers": 1, "rank": 0, "world_size": 1}, @@ -105,7 +112,7 @@ def main(): client.edit_composition(operation, {"edit": new_id(), "expected_revision": "1", "schema": None, "samples": [{"kind": "upsert", "id": sample, "fields": [{"name": "value", "value": {"kind": "upload", "id": upload}}]}], "splits": None, "policy": None}) snapshot = prepare(client, operation, "2") - publish(client, operation, snapshot) + published_head = publish(client, snapshot, None) lifecycle = client.lifecycle(snapshot) assert lifecycle["snapshot"] == snapshot and lifecycle["latest"] and not lifecycle["retained"] cleanup(client, operation) @@ -139,7 +146,13 @@ def main(): "source": {"kind": "split", "source": source.record(), "selection": None}, "schema": None, "inherit_parent_splits": True, "parent_split_revision": revision, "splits": [], "policy": None}) child_snapshot = prepare(client, child, "1") - publish(client, child, child_snapshot) + child_head = publish(client, child_snapshot, published_head["generation"]) + try: + client.update_snapshot_head({"snapshot": snapshot, "expected_head": None}) + except ConflictError as error: + assert error.observed_head == child_head + else: + raise AssertionError("stale Snapshot update must conflict") cleanup(client, child) assert client.summary(child_snapshot)["parent"] == snapshot asyncio.run(verify_async(endpoint, name, token, snapshot, revision)) diff --git a/sdk/tests/test_control.py b/sdk/tests/test_control.py index 5b964ba72..2c7f0396e 100644 --- a/sdk/tests/test_control.py +++ b/sdk/tests/test_control.py @@ -7,16 +7,29 @@ import threading import unittest from crowdb_dataset import DatasetClient, AsyncDatasetClient, DatasetError, decimal, new_id +from crowdb_dataset.client import ConflictError class TestControlPeer: - def __init__(self, capabilities=(), gated=False): + def __init__(self, capabilities=(), gated=False, head_replies=()): self.capabilities, self.gated = list(capabilities), gated self.requests, self.release = queue.Queue(), threading.Event() + self.head_replies = queue.Queue() + for reply in head_replies: self.head_replies.put(reply) def __enter__(self): owner = self class Handler(http.server.BaseHTTPRequestHandler): + def do_PUT(self): + request = json.loads(self.rfile.read(int(self.headers["content-length"]))) + owner.requests.put((self.path, request)) + status, value = owner.head_replies.get(timeout=10) + body = json.dumps(value).encode() + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) def do_GET(self): owner.requests.put(self.path) if self.path.endswith("capabilities"): @@ -48,6 +61,24 @@ def __exit__(self, *_): class TestControl(unittest.TestCase): + def test_snapshot_head_conflict_uses_the_observation_without_a_head_get(self): + snapshot = new_id() + observed = {"latest": new_id(), "stable": None, "generation": "11" * 32} + selected = dict(observed, latest=snapshot, generation="22" * 32) + error = {"code": "conflict", "message": "conflict", "request_id": "test", "retryable": False, "observed_head": observed} + with TestControlPeer(["snapshot-head-update"], head_replies=[(409, error), (200, selected)]) as peer, DatasetClient(peer.endpoint, "test") as owner: + control = owner.control() + request = {"snapshot": snapshot, "expected_head": None} + with self.assertRaises(ConflictError) as failure: control.update_snapshot_head(request) + self.assertEqual(failure.exception.identity, snapshot) + self.assertEqual(failure.exception.observed_head, observed) + reviewed = dict(request, expected_head=observed["generation"]) + self.assertEqual(control.update_snapshot_head(reviewed), selected) + self.assertTrue(peer.requests.get(timeout=10).endswith("/capabilities")) + self.assertEqual(peer.requests.get(timeout=10), ("/api/dataset/v2/namespaces/dataset-ns/datasets/test/head", request)) + self.assertEqual(peer.requests.get(timeout=10), ("/api/dataset/v2/namespaces/dataset-ns/datasets/test/head", reviewed)) + self.assertTrue(peer.requests.empty()) + def test_successful_capabilities_are_reused_across_control_wrappers(self): with TestControlPeer(["head-inspection"]) as peer, DatasetClient(peer.endpoint, "test") as owner: owner.control().head() From b16e00eb802f9df429136ea2c5254b08c56d8f8e Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 20:36:12 +0800 Subject: [PATCH 127/258] Validate complete snapshot results across container restarts --- doc/working/plan-access-simplification.md | 30 +++++++++++++++++ e2e-test/tests/access_dataset_split.rs | 39 +++++++++++++++++------ 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index c849d3e3c..ccddc3b3b 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -360,6 +360,21 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 ### P6 — Iceberg 旧 multipart / 全局 credits +**补充 review（2026-10-24）**：`reserve_for_tests`/`initialize_for_tests` 仅由兼容 fixture +调用，生产新上传没有全局硬配额入口。仍活跃的 `MultipartRecovery` 却在每页扫描前调用 +`MultipartAdmission::load`（context 前后检查加 policy，共 3G），旧终态 upload 又通过 +`release` 写 session snapshot、共享 pending 和 released receipt。其作用只是结算已停止 +使用的计数器，不决定 part、已发布 FileId 或 GC 的物理引用资格。 + +移除方案：删生产 reserve/release/settle driver 和恢复中的 policy 步骤；已存在的 session +继续按自身期限、part/publish 资源事实处理，忽略 legacy credit。仅有 pending Reserve、 +还没有 session 的旧记录不再凭 after payload 重新创建 upload：旧 reserve 的顺序是在 +session 准入前，没有该 session 就没有合法 part 写入。pending Release 的 after 只修改 +credit/revision，不改变已发布文件，故也无需回放。旧 policy、payload、credit 字段保留 +只读格式识别，不将旧计数器归零冒充业务数据清理；其旧 key 的清理归 P13 的 legacy +metadata 扫描。没有活跃硬配额需求，不触发 D4。测试覆盖遗留/损坏 policy 对空页与已存在 +上传零影响、过期与终态资源不丢失，恢复总读取由空页 5G 降至 2G，零 policy GET/写。 + - [ ] **P6.2 删除确证无用的协调**：没有调用/恢复义务的代码可删；有旧记录则给出迁移/结算方案。 只有仍有全局硬配额需求才触发 D4。Files: `file/multipart_admission.rs`、`file/multipart_credits/settle.rs` 及恢复测试。 @@ -490,6 +505,21 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，V2.3**：当前 Snapshot 源码镜像 + `crowdb-node:access-cleanup-snapshot-head-20261024` 的 Dataset 全部五项容器用例通过， + 包含 JSON 发布/持久化重启、鉴权、Split 历史、真实 Rust/installed Python SDK 数据与 + 直接 head API。首轮四项通过，Split fixture 仍要求完整但未发布 Snapshot 返回 404； + 按已确认的合同改为 summary 200、latest 仍 absent，重启后完整内容相同，再发布。 + 第二轮五项全通过，artifact 为 + `.crowdb-runtime/artifacts/container-e2e/run-542c939cb0b24685b432f4c1e5412dc6`； + 失败首轮 `run-248a6a5222644cca983da2ca5bf7f5de` 保留，未掩盖失败。 + image/client revision `ed9fa01a54b92447749f7a77f9c2a2b42a58b7ce`、 + source tree `b5571cd807daed143af91ff0a724e9c60efb59a2`，server_sources_match=true； + OCI digest `sha256:1f632c08a0cb09a9dcfb0c17fb1d50ead4672b85cb857d3cb07b4a8320979e30`， + Docker identity `sha256:50fbad326aa9764cf0a3b8f0b717cc2780532c99e65f586a8e6909a196c45fa2`。 + 仅客户端断言调整，服务端源码保持同源；e2e-client all-target clippy、fmt、diff 通过。 + 此为本机 TCP 功能/重启验证，未测 TPS/HCA 或完整孤儿 GC，V4 和 R4 仍未完成。 + - **2026-10-24，R2.2d**：新增 `PUT .../datasets/{dataset}/head`、 `snapshot-head-update` 能力、Rust/Python `update_snapshot_head`；只接收完整 Snapshot ID 和必填 nullable expected head，body 上限 4096 字节。不读取草稿、使用 attempt 或写请求 diff --git a/e2e-test/tests/access_dataset_split.rs b/e2e-test/tests/access_dataset_split.rs index b80021390..1ee0316f0 100644 --- a/e2e-test/tests/access_dataset_split.rs +++ b/e2e-test/tests/access_dataset_split.rs @@ -53,15 +53,7 @@ async fn split_commit_restart_preserves_candidates_receipts_and_historical_membe ); let prepared = prepare(&client, &draft).await; let snapshot = prepared["preparation"]["snapshot"].as_str().unwrap(); - call( - &client, - Method::GET, - &format!("/snapshots/{snapshot}/summary"), - None, - 404, - ) - .await; - restart().await; + complete_result_survives_restart(&client, snapshot).await; let publish = json!({"attempt":"23232323232323232323232323232323","snapshot":snapshot,"expected_head":null}); let receipt = call( @@ -120,6 +112,35 @@ async fn split_commit_restart_preserves_candidates_receipts_and_historical_membe ); } +async fn complete_result_survives_restart(client: &Client, snapshot: &str) { + let summary = call( + client, + Method::GET, + &format!("/snapshots/{snapshot}/summary"), + None, + 200, + ) + .await; + assert_eq!(summary["sample_count"], "1"); + assert_eq!(summary["inspection"], "ready"); + assert_eq!( + call(client, Method::GET, "/head", None, 200).await["latest"], + Value::Null + ); + restart().await; + assert_eq!( + call( + client, + Method::GET, + &format!("/snapshots/{snapshot}/summary"), + None, + 200 + ) + .await, + summary + ); +} + async fn prepare(client: &Client, draft: &str) -> Value { let mut page = json!("0"); let mut done = false; From 9c7ee4f0e38052006f89a66ecad3c628c7a97fcb Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 20:41:45 +0800 Subject: [PATCH 128/258] Remove legacy multipart quota coordination and recovery --- .../common/iceberg_file_recovery/component.rs | 50 +-- .../iceberge/design-crowdb-iceberg.md | 21 +- doc/working/plan-access-simplification.md | 21 +- .../access_iceberg/native_files/worker.rs | 29 +- lib/crowdb-access-iceberg/src/file.rs | 2 - .../src/file/multipart_admission.rs | 33 +- .../src/file/multipart_credits.rs | 268 ----------- .../src/file/multipart_credits/settle.rs | 129 ------ .../src/file/multipart_recovery.rs | 47 +- .../src/file/multipart_repository.rs | 1 + .../tests/common/store.rs | 19 + .../tests/multipart_admission_record_test.rs | 70 +-- .../tests/multipart_admission_test.rs | 423 ++++++++---------- 13 files changed, 307 insertions(+), 806 deletions(-) delete mode 100644 lib/crowdb-access-iceberg/src/file/multipart_credits.rs delete mode 100644 lib/crowdb-access-iceberg/src/file/multipart_credits/settle.rs diff --git a/app/crowdb-access-server/tests/common/iceberg_file_recovery/component.rs b/app/crowdb-access-server/tests/common/iceberg_file_recovery/component.rs index 8b18738a0..272187e3e 100644 --- a/app/crowdb-access-server/tests/common/iceberg_file_recovery/component.rs +++ b/app/crowdb-access-server/tests/common/iceberg_file_recovery/component.rs @@ -3,14 +3,10 @@ use std::collections::BTreeSet; use std::path::Path; -use std::time::Duration; use crowdb_access_iceberg::{ - catalog::ClearBounds, - file::{ - FileKind, FileRepository, MultipartAdmission, MultipartPhase, MultipartRepository, MultipartSession, - TableLocation, - }, + catalog::{CatalogStore, ClearBounds}, + file::{FileKind, FileRepository, MultipartPhase, MultipartRepository, MultipartSession, TableLocation}, }; use reqwest::Method; use serde_json::Value; @@ -127,42 +123,20 @@ async fn interrupt( .unwrap(); assert_eq!(session.phase, MultipartPhase::Published); assert_eq!(session.published, Some(record.file)); - released_by_recovery(stack, &session).await; + no_global_admission(stack, &session).await; } label } -async fn released_by_recovery(stack: &TestCommitFixture, published: &MultipartSession) { - let store = stack.store(); - let sessions = MultipartRepository::new(store.clone()); - let admission = MultipartAdmission::new(store); - if published.credit.is_none() { - // Current sessions have per-session limits and no catalog-wide credits. - // Recovery must not recreate the legacy admission journal. - assert!(admission.load(published.context).await.unwrap().is_none()); - return; - } - tokio::time::timeout(Duration::from_secs(5), async { - loop { - let current = sessions - .load(published.context, published.upload) - .await - .unwrap() - .unwrap(); - assert_eq!(current.phase, MultipartPhase::Published); - assert_eq!(current.published, published.published); - let policy = admission.load(published.context).await.unwrap().unwrap(); - assert!(policy.sessions <= 1); - assert!(policy.reserved_bytes <= published.limits.max_staged_bytes); - if current.credit.unwrap().released && policy.pending.is_none() { - assert_eq!((policy.sessions, policy.reserved_bytes), (0, 0)); - break; - } - tokio::time::sleep(Duration::from_millis(20)).await; - } - }) - .await - .expect("background recovery must settle terminal credits without another Complete request"); +async fn no_global_admission(stack: &TestCommitFixture, published: &MultipartSession) { + use crowdb_access_iceberg::key::{CatalogScope, IcebergKey}; + assert!(published.credit.is_none()); + let key = IcebergKey::Catalog { + catalog: published.context.catalog, + scope: CatalogScope::MultipartAdmission, + suffix: Vec::new(), + }; + assert!(stack.store().get(&key.encode().unwrap()).await.unwrap().is_none()); } struct TestFileCase { diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg.md b/doc/design/access-server/iceberge/design-crowdb-iceberg.md index a9309d9c6..b637bc489 100644 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg.md +++ b/doc/design/access-server/iceberge/design-crowdb-iceberg.md @@ -560,10 +560,15 @@ separate catalog key scopes, retaining only bounded checkpoint references and current-part digest state. Unknown phases and invalid revisions fail closed. New multipart sessions publish their own authority without a catalog-wide credit reserve or release. Persisted per-session limits bound staged bytes and -part count. Legacy admission journals and credit receipts remain recoverable in -the background; ordinary new sessions never contend on those shared records. -Creating or reserving a legacy global credit policy is available only through -explicit compatibility test hooks. Production admission exposes no such path. +part count. Global credit reserve/release/journal drivers have been removed, +including compatibility write hooks. New sessions reject legacy credit input; +existing session and policy formats remain decodable. Recovery never reads or +settles a global policy. A legacy pending reservation without a session is ignored: +session admission preceded any permitted part writes. A pending credit release +does not change the published FileId or part references and is not replayed. +Existing sessions still expire or finish from their own resource facts; obsolete +policy/payload metadata remains available to legacy metadata cleanup without +keeping accounting alive or creating another upload. The legacy tree multipart repository reserves one part mutation in the session before changing its part authority. A bounded before/after snapshot and monotonically increasing revisions make the write and fence release recoverable across servers. @@ -613,10 +618,10 @@ context/session CAS checks remain authoritative. An observation does not retain file bytes or survive a restart. It resets its cursor when the active context changes and bounds each session by the persisted catalog request deadline. Timeout defers only that session, allowing later entries in the page to progress. A separate outer budget bounds the -whole page and context/scan work. One separately bounded admission-journal recovery -step runs before scanning, including a reservation whose session is not yet present. -Legacy terminal sessions settle retained credits on a later recovery visit; -new sessions have no global credit release. +whole page and context/scan work. There is no admission-journal recovery step or +global credit release. Empty and terminal-only sweeps perform the two context +checks and the bounded session scan, with no policy GET or mutation; legacy +credit fields do not delay expiry or modify complete file resources. The HTTP driver composes this durable state machine with physical sealing; recovery remains the authority for abandoned or uncertain work. diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index ccddc3b3b..3002a967a 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -375,8 +375,7 @@ credit/revision，不改变已发布文件，故也无需回放。旧 policy、p metadata 扫描。没有活跃硬配额需求，不触发 D4。测试覆盖遗留/损坏 policy 对空页与已存在 上传零影响、过期与终态资源不丢失，恢复总读取由空页 5G 降至 2G，零 policy GET/写。 -- [ ] **P6.2 删除确证无用的协调**：没有调用/恢复义务的代码可删；有旧记录则给出迁移/结算方案。 - 只有仍有全局硬配额需求才触发 D4。Files: `file/multipart_admission.rs`、`file/multipart_credits/settle.rs` 及恢复测试。 +P6.2 已删除全局配额 driver；迁移与验证见记录。旧格式仅用于解码，不再参与生产恢复。 ### P7 — Iceberg 物理写入（执行 D2；必要安全例外先 review） @@ -424,6 +423,8 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 - [ ] **P13.2 审查 cleanup 持久化**：将 claim/phase/cursor/tombstone 分为可重做进度与不可丢的 义务/代际；即使重扫昂贵也先优化发现/预算；新增持久化信息必须 review，用途和额外前台写均写明。Files: Dataset `authority/access/`、Split/Composition cleanup、Iceberg `gc/`、S3 terminal cleanup。 + 包含已停止执行的 legacy MultipartAdmission policy 和其 before/after payload：从已存在的 + session/最终文件事实判定可删，不能为清这些 key 恢复全局计数器或重建上传。 - [ ] **P13.3 量化回收滞后**：稀疏过期、大 upload、多 cleaner、频繁重启、损坏记录下测 backlog、 前台 p99 和孤儿收敛；如需确定完成时限再讨论 D5。Files: 相关 cleanup 测试、负载工具。 @@ -478,6 +479,9 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 无条件覆盖与 expected-base 重试测试。断言总 GET+写，不仅是零写。Files: Dataset `tests/*cost*`、`read_query*`、upload/Composition/Split/inspection；S3/Iceberg 对应 repository 测试。 - [ ] **V2 HTTP/native/SDK 集成**：各受影响 route 的鉴权、错误、协议/SDK 一致性；保留删除准入、 迟到写者、非空 namespace、共享 owner、held buffer、cancel/drop 和损坏内容测试。Files: Access/native tests、`sdk/tests/`、native Python consumer。 +- [~] **V2.4 Iceberg 配额清理后容器复验**：构建本批源码镜像，运行 + `access_iceberg::native_files::native_file_tree_publication_and_ranges_survive_catalog_storage_restart`， + 覆盖普通上传的后台过期、part 保留和文件重启读取；记录来源与结果，不能复用清理前镜像。 - [ ] **V3 工程门禁与文档**：每批执行适用 fmt/clippy/测试，全部经 `pixi run`；同步 owning design/spec/OpenAPI/SDK。只修改文档时检查 diff、引用和任务状态，不为此跑整套服务测试。 - [ ] **V4 同源真实负载验收**：重新构建当前源码的镜像，运行现有 HTTP/native Rust/native Python @@ -505,6 +509,19 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，P6.2**：删除 `MultipartAdmission`、reserve/release/settle driver、journal + proposal 和相关 test-util 写入口。后台每页不再读全局 policy，终态 upload 不再写 snapshot、 + pending 或 released receipt；新 begin 拒绝 legacy credit 输入。旧记录仍可解码；已存在的 + 上传按自身 part/publish 事实继续处理。未创建 session 的旧 pending reserve 不重建上传， + pending release 不回放；旧计数器已无业务使用者，保留物理文件和其正式引用不依赖它。 + 用真实 legacy pending Reserve payload、缺失/损坏 policy、过期旧 session、已发布文件、 + 新请求拒绝旧 credit 验证迁移；空页和终态页均 2G、零 mutation、零 policy 访问，原空页 + 为 5G。30 项 metadata/part/发布/恢复/预算测试及 HTTP 跨进程全 durable-write 切断矩阵 + 通过；Iceberg/Access all-target/all-feature clippy、e2e-client all-target clippy、fmt 和 + diff 检查通过，均经 pixi。容器 worker fixture 改用普通 upload，不再人为创建 global + policy；它的当前源码容器复验由 V2.4 跟进。旧 policy/payload key 清理归 P13.2，未宣称 + 独立物理孤儿 GC 或全部 multipart 中间状态已清理。 + - **2026-10-24，V2.3**：当前 Snapshot 源码镜像 `crowdb-node:access-cleanup-snapshot-head-20261024` 的 Dataset 全部五项容器用例通过， 包含 JSON 发布/持久化重启、鉴权、Split 历史、真实 Rust/installed Python SDK 数据与 diff --git a/e2e-test/tests/access_iceberg/native_files/worker.rs b/e2e-test/tests/access_iceberg/native_files/worker.rs index ae8a56177..ad5c4152f 100644 --- a/e2e-test/tests/access_iceberg/native_files/worker.rs +++ b/e2e-test/tests/access_iceberg/native_files/worker.rs @@ -5,8 +5,8 @@ use std::time::Duration; use crowdb_access_iceberg::catalog::CatalogContext; use crowdb_access_iceberg::file::{ - FileIdentity, FileTree, MultipartAdmission, MultipartAdmissionLimits, MultipartLimits, MultipartPart, - MultipartPhase, MultipartRepository, MultipartSession, TableLocation, + FileIdentity, FileTree, MultipartLimits, MultipartPart, MultipartPhase, MultipartRepository, + MultipartSession, TableLocation, }; use crowdb_access_iceberg::key::{FileId, OperationId}; use sha2::{Digest, Sha256}; @@ -43,20 +43,8 @@ pub async fn verify(stack: &TestNativeStorage, context: CatalogContext, table: T pending: None, credit: None, }; - let admission = MultipartAdmission::new(store.clone()); - let policy = admission - .initialize_for_tests( - context, - MultipartAdmissionLimits { - max_sessions: 2, - max_reserved_bytes: 200, - }, - ) - .await - .unwrap(); - assert!(admission.reserve_for_tests(&policy, &initial, 1).await.unwrap()); let repository = MultipartRepository::new(store); - let admitted = repository.load(context, initial.upload).await.unwrap().unwrap(); + let admitted = repository.begin(&initial, 1).await.unwrap(); let part = MultipartPart { upload: initial.upload, number: 1, @@ -82,18 +70,11 @@ pub async fn verify(stack: &TestNativeStorage, context: CatalogContext, table: T "Iceberg worker exited" ); let current = repository.load(context, initial.upload).await.unwrap().unwrap(); - if current.phase == MultipartPhase::Aborted - && current.credit.is_some_and(|credit| credit.released) - { + if current.phase == MultipartPhase::Aborted { + assert!(current.credit.is_none()); assert!(current.pending.is_none()); assert_eq!((current.part_count, current.staged_bytes), (1, 0)); assert_eq!(repository.part(¤t, 1).await.unwrap().as_ref(), Some(&part)); - let policy = admission.load(context).await.unwrap().unwrap(); - if policy.pending.is_some() { - tokio::time::sleep(Duration::from_millis(20)).await; - continue; - } - assert_eq!((policy.sessions, policy.reserved_bytes), (0, 0)); break; } tokio::time::sleep(Duration::from_millis(20)).await; diff --git a/lib/crowdb-access-iceberg/src/file.rs b/lib/crowdb-access-iceberg/src/file.rs index d2b45ad19..37a19afca 100644 --- a/lib/crowdb-access-iceberg/src/file.rs +++ b/lib/crowdb-access-iceberg/src/file.rs @@ -19,7 +19,6 @@ mod location; pub use listing::{FileListPage, FileListRequest, FileListTokens, FileLocationScan, ListedFile}; mod multipart; mod multipart_admission; -mod multipart_credits; mod multipart_list; mod multipart_recovery; mod multipart_repository; @@ -76,7 +75,6 @@ pub use multipart_admission::{ MultipartAdmissionLimits, MultipartAdmissionRecord, MultipartCredit, MultipartCreditAction, MultipartCreditMutation, }; -pub use multipart_credits::MultipartAdmission; pub use multipart_list::{MultipartLister, MultipartPartPage, MultipartPartScan, MultipartPartStore}; pub use multipart_recovery::{ MultipartRecovery, MultipartRecoveryObservation, MultipartRecoveryPage, MultipartRecoveryScan, diff --git a/lib/crowdb-access-iceberg/src/file/multipart_admission.rs b/lib/crowdb-access-iceberg/src/file/multipart_admission.rs index 31de41a4c..8570d3cb7 100644 --- a/lib/crowdb-access-iceberg/src/file/multipart_admission.rs +++ b/lib/crowdb-access-iceberg/src/file/multipart_admission.rs @@ -1,3 +1,5 @@ +//! Legacy global multipart credit formats, retained only for metadata decoding. + use crate::catalog::CatalogContext; use crate::error::ValidationError; use crate::key::{CatalogScope, IcebergKey, OperationId}; @@ -89,37 +91,6 @@ impl MultipartAdmissionRecord { } Ok(()) } - - /// Constructs one bounded journal proposal; the caller must CAS it before child mutation. - /// # Errors - /// Rejects occupied journals, exhausted credits, counter underflow and revision overflow. - pub fn proposed(&self, mutation: MultipartCreditMutation) -> Result { - self.validate()?; - if self.pending.is_some() { - return Err(ValidationError::Record); - } - let mut next = self.clone(); - next.revision = self.revision.checked_add(1).ok_or(ValidationError::Record)?; - match mutation.action { - MultipartCreditAction::Reserve => { - next.sessions = self.sessions.checked_add(1).ok_or(ValidationError::Record)?; - next.reserved_bytes = self - .reserved_bytes - .checked_add(mutation.reservation_bytes) - .ok_or(ValidationError::Record)?; - } - MultipartCreditAction::Release => { - next.sessions = self.sessions.checked_sub(1).ok_or(ValidationError::Record)?; - next.reserved_bytes = self - .reserved_bytes - .checked_sub(mutation.reservation_bytes) - .ok_or(ValidationError::Record)?; - } - } - next.pending = Some(mutation); - next.validate()?; - Ok(next) - } } impl MultipartCreditMutation { diff --git a/lib/crowdb-access-iceberg/src/file/multipart_credits.rs b/lib/crowdb-access-iceberg/src/file/multipart_credits.rs deleted file mode 100644 index f931f4ddd..000000000 --- a/lib/crowdb-access-iceberg/src/file/multipart_credits.rs +++ /dev/null @@ -1,268 +0,0 @@ -use std::sync::Arc; - -use crate::catalog::{check_context, CasOutcome, CatalogContext, CatalogError, CatalogStore}; -use crate::error::ValidationError; -#[cfg(feature = "test-util")] -use crate::key::OperationId; -use crate::key::{CatalogScope, IcebergKey}; -use crate::operation::{mutation_identity, PayloadStore}; -use crate::record::StorageRecord; - -#[cfg(feature = "test-util")] -use super::MultipartAdmissionLimits; -use super::{ - MultipartAdmissionRecord, MultipartCredit, MultipartCreditAction, MultipartCreditMutation, - MultipartPhase, MultipartRepository, MultipartSession, -}; - -mod settle; - -pub struct MultipartAdmission { - store: Arc, - sessions: MultipartRepository, -} - -impl MultipartAdmission { - #[must_use] - pub fn new(store: Arc) -> Self { - Self { - sessions: MultipartRepository::new(store.clone()), - store, - } - } - - /// Reproduces a legacy policy for compatibility and recovery tests. - /// # Errors - /// Rejects invalid bounds, conflicting policy, retired contexts and storage failures. - #[cfg(feature = "test-util")] - pub async fn initialize_for_tests( - &self, - context: CatalogContext, - limits: MultipartAdmissionLimits, - ) -> Result { - context.validate()?; - limits.validate()?; - if let Some(record) = self.load(context).await? { - return if record.limits == limits { - Ok(record) - } else { - Err(CatalogError::Conflict) - }; - } - let record = MultipartAdmissionRecord { - context, - policy: OperationId::random(), - revision: 1, - limits, - sessions: 0, - reserved_bytes: 0, - pending: None, - }; - let key = record.key().encode()?; - let bytes = encode(&record)?; - check_context(self.store.as_ref(), context).await?; - self.store - .compare_exchange(&key, None, &bytes, mutation_identity(&key, None, &bytes)) - .await?; - let actual = self.load(context).await?.ok_or(ValidationError::Record)?; - if actual.limits != limits { - return Err(CatalogError::Conflict); - } - Ok(actual) - } - - /// # Errors - /// Rejects retired contexts and corrupt or foreign policy records. - pub async fn load( - &self, - context: CatalogContext, - ) -> Result, CatalogError> { - check_context(self.store.as_ref(), context).await?; - let key = IcebergKey::Catalog { - catalog: context.catalog, - scope: CatalogScope::MultipartAdmission, - suffix: Vec::new(), - }; - let value = self.store.get(&key.encode()?).await?; - let record = value - .map(|value| { - let StorageRecord::MultipartAdmission(record) = StorageRecord::decode(&key, &value.bytes)? - else { - return Err(ValidationError::Record); - }; - if record.context != context { - return Err(ValidationError::IdentityMismatch); - } - Ok(*record) - }) - .transpose()?; - check_context(self.store.as_ref(), context).await?; - Ok(record) - } - - /// Reproduces legacy reservation writes, including interrupted journals, for recovery tests. - /// # Errors - /// Rejects stale or occupied policies, expired requests, exhausted credits and identity reuse. - #[cfg(feature = "test-util")] - pub async fn reserve_for_tests( - &self, - record: &MultipartAdmissionRecord, - session: &MultipartSession, - now_ms: u64, - ) -> Result { - ready(record)?; - initial(session)?; - if session.context != record.context || session.credit.is_some() { - return Err(ValidationError::IdentityMismatch.into()); - } - if let Some(existing) = self.sessions.load(session.context, session.upload).await? { - let existing = super::multipart_repository::matching_request(session, existing)?; - return if existing - .credit - .is_some_and(|credit| credit.policy == record.policy) - { - Ok(true) - } else { - Err(CatalogError::Conflict) - }; - } - if now_ms < session.created_ms || now_ms >= session.expires_ms { - return Err(CatalogError::Conflict); - } - let mut after = session.clone(); - after.credit = Some(MultipartCredit { - policy: record.policy, - sequence: record.revision + 1, - released: false, - }); - let mutation = MultipartCreditMutation { - action: MultipartCreditAction::Reserve, - upload: session.upload, - reservation_bytes: session.limits.max_staged_bytes, - before: None, - after: self.snapshot(&after).await?, - }; - let next = record.proposed(mutation)?; - if !self.exchange(record, &next).await? { - return Ok(false); - } - self.settle(&next).await - } - - /// Returns credits only for a terminal session, retaining a durable released receipt. - /// # Errors - /// Rejects live sessions, foreign receipts, occupied journals and storage failures. - pub async fn release( - &self, - record: &MultipartAdmissionRecord, - session: &MultipartSession, - ) -> Result { - ready(record)?; - session.validate()?; - let credit = session.credit.ok_or(ValidationError::Record)?; - if session.context != record.context - || credit.policy != record.policy - || credit.sequence > record.revision - { - return Err(ValidationError::IdentityMismatch.into()); - } - if !terminal(session) { - return Err(CatalogError::Conflict); - } - let Some(current) = self.sessions.load(session.context, session.upload).await? else { - return Err(ValidationError::Record.into()); - }; - if current.credit.is_some_and(|value| value.released) { - return Ok(current.credit - == Some(MultipartCredit { - released: true, - ..credit - })); - } - if current != *session { - return Ok(false); - } - let mut after = session.clone(); - after.revision = session.revision.checked_add(1).ok_or(ValidationError::Record)?; - after.credit = Some(MultipartCredit { - released: true, - ..credit - }); - let mutation = MultipartCreditMutation { - action: MultipartCreditAction::Release, - upload: session.upload, - reservation_bytes: session.limits.max_staged_bytes, - before: Some(self.snapshot(session).await?), - after: self.snapshot(&after).await?, - }; - let next = record.proposed(mutation)?; - if !self.exchange(record, &next).await? { - return Ok(false); - } - self.settle(&next).await - } - - async fn snapshot( - &self, - session: &MultipartSession, - ) -> Result { - let bytes = StorageRecord::MultipartSession(Box::new(session.clone())).encode()?; - PayloadStore::new(self.store.clone()) - .put(session.context.catalog, session.upload, &bytes) - .await - } - - async fn exchange( - &self, - before: &MultipartAdmissionRecord, - after: &MultipartAdmissionRecord, - ) -> Result { - let key = before.key().encode()?; - let expected = encode(before)?; - let value = encode(after)?; - check_context(self.store.as_ref(), before.context).await?; - let outcome = self - .store - .compare_exchange( - &key, - Some(&expected), - &value, - mutation_identity(&key, Some(&expected), &value), - ) - .await?; - check_context(self.store.as_ref(), before.context).await?; - Ok(matches!(outcome, CasOutcome::Applied(_))) - } -} - -fn ready(record: &MultipartAdmissionRecord) -> Result<(), CatalogError> { - record.validate()?; - if record.pending.is_some() { - return Err(CatalogError::Busy); - } - record.revision.checked_add(2).ok_or(ValidationError::Record)?; - Ok(()) -} - -fn initial(session: &MultipartSession) -> Result<(), CatalogError> { - session.validate()?; - if session.phase != MultipartPhase::Open - || session.revision != 1 - || session.part_count != 0 - || session.pending.is_some() - { - return Err(ValidationError::Record.into()); - } - Ok(()) -} - -fn terminal(session: &MultipartSession) -> bool { - matches!( - session.phase, - MultipartPhase::Published | MultipartPhase::Aborted | MultipartPhase::Conflicted - ) -} - -fn encode(record: &MultipartAdmissionRecord) -> Result, ValidationError> { - StorageRecord::MultipartAdmission(Box::new(record.clone())).encode() -} diff --git a/lib/crowdb-access-iceberg/src/file/multipart_credits/settle.rs b/lib/crowdb-access-iceberg/src/file/multipart_credits/settle.rs deleted file mode 100644 index 569637e94..000000000 --- a/lib/crowdb-access-iceberg/src/file/multipart_credits/settle.rs +++ /dev/null @@ -1,129 +0,0 @@ -use crate::catalog::{CasOutcome, CatalogError}; -use crate::error::ValidationError; -use crate::operation::{mutation_identity, PayloadReference, PayloadStore}; -use crate::record::StorageRecord; - -use super::{ - MultipartAdmission, MultipartAdmissionRecord, MultipartCredit, MultipartCreditAction, MultipartSession, -}; - -impl MultipartAdmission { - /// Completes one durable admission journal without repeating its counter change. - /// # Errors - /// Rejects corrupt snapshots, foreign receipts, retired contexts and storage failures. - pub async fn settle(&self, record: &MultipartAdmissionRecord) -> Result { - record.validate()?; - let mutation = record.pending.as_ref().ok_or(ValidationError::Record)?; - let after = self.read_snapshot(record, &mutation.after).await?; - let before = match &mutation.before { - Some(reference) => Some(self.read_snapshot(record, reference).await?), - None => None, - }; - validate_transition(record, before.as_ref(), &after)?; - if self.load(record.context).await?.as_ref() != Some(record) { - return Ok(false); - } - let key = after.key().encode()?; - let expected = before - .map(|session| StorageRecord::MultipartSession(Box::new(session)).encode()) - .transpose()?; - let value = StorageRecord::MultipartSession(Box::new(after.clone())).encode()?; - let outcome = self - .store - .compare_exchange( - &key, - expected.as_deref(), - &value, - mutation_identity(&key, expected.as_deref(), &value), - ) - .await?; - if let CasOutcome::Conflict(observed) = outcome { - let Some(observed) = observed else { - return Err(ValidationError::Record.into()); - }; - let StorageRecord::MultipartSession(current) = - StorageRecord::decode(&after.key(), &observed.bytes)? - else { - return Err(ValidationError::Record.into()); - }; - super::super::multipart_repository::matching_request(&after, (*current).clone())?; - if current.credit != after.credit || current.revision < after.revision { - if self.load(record.context).await?.as_ref() != Some(record) { - return Ok(false); - } - return Err(ValidationError::Record.into()); - } - } - let mut next = record.clone(); - next.revision = record.revision.checked_add(1).ok_or(ValidationError::Record)?; - next.pending = None; - self.exchange(record, &next).await - } - - async fn read_snapshot( - &self, - record: &MultipartAdmissionRecord, - reference: &PayloadReference, - ) -> Result { - let bytes = PayloadStore::new(self.store.clone()).get(reference).await?; - let key = crate::key::IcebergKey::Catalog { - catalog: record.context.catalog, - scope: crate::key::CatalogScope::MultipartSession, - suffix: reference.operation.as_bytes().to_vec(), - }; - let StorageRecord::MultipartSession(session) = StorageRecord::decode(&key, &bytes)? else { - return Err(ValidationError::Record.into()); - }; - if session.context != record.context { - return Err(ValidationError::IdentityMismatch.into()); - } - Ok(*session) - } -} - -fn validate_transition( - record: &MultipartAdmissionRecord, - before: Option<&MultipartSession>, - after: &MultipartSession, -) -> Result<(), CatalogError> { - let mutation = record.pending.as_ref().ok_or(ValidationError::Record)?; - if after.upload != mutation.upload || after.limits.max_staged_bytes != mutation.reservation_bytes { - return Err(ValidationError::Record.into()); - } - match mutation.action { - MultipartCreditAction::Reserve => { - super::initial(after)?; - if before.is_some() - || after.credit - != Some(MultipartCredit { - policy: record.policy, - sequence: record.revision, - released: false, - }) - { - return Err(ValidationError::Record.into()); - } - } - MultipartCreditAction::Release => { - let before = before.ok_or(ValidationError::Record)?; - let credit = before.credit.ok_or(ValidationError::Record)?; - if !super::terminal(before) - || credit.released - || credit.policy != record.policy - || credit.sequence >= record.revision - { - return Err(ValidationError::Record.into()); - } - let mut expected = before.clone(); - expected.revision = before.revision.checked_add(1).ok_or(ValidationError::Record)?; - expected.credit = Some(MultipartCredit { - released: true, - ..credit - }); - if expected != *after { - return Err(ValidationError::Record.into()); - } - } - } - Ok(()) -} diff --git a/lib/crowdb-access-iceberg/src/file/multipart_recovery.rs b/lib/crowdb-access-iceberg/src/file/multipart_recovery.rs index b916cc691..43a7f1f53 100644 --- a/lib/crowdb-access-iceberg/src/file/multipart_recovery.rs +++ b/lib/crowdb-access-iceberg/src/file/multipart_recovery.rs @@ -9,8 +9,8 @@ use crate::key::{IcebergKey, OperationId}; use crate::record::{StorageRecord, MAX_RECORD_BYTES}; use super::{ - FileBlockStore, MultipartAdmission, MultipartPhase, MultipartRepository, MultipartSession, - MultipartWorkError, MAX_FILE_BLOCK_BYTES, + FileBlockStore, MultipartPhase, MultipartRepository, MultipartSession, MultipartWorkError, + MAX_FILE_BLOCK_BYTES, }; mod observation; @@ -20,7 +20,6 @@ pub use scan::{MultipartRecoveryScan, MultipartRecoveryStore}; pub struct MultipartRecovery { repository: MultipartRepository, - admission: MultipartAdmission, store: Arc, blocks: Arc, step_bytes: usize, @@ -56,7 +55,6 @@ impl MultipartRecovery { } Ok(Self { repository: MultipartRepository::new(store.clone()), - admission: MultipartAdmission::new(store.clone()), store, blocks, step_bytes, @@ -96,14 +94,6 @@ impl MultipartRecovery { observation: Option<&MultipartRecoveryObservation>, ) -> Result { self.repository.check_context(context).await?; - let work = self.recover_admission(context); - let admission = if let Some(timeout) = self.session_timeout { - tokio::time::timeout(timeout, work) - .await - .unwrap_or(Ok(RecoveryAction::Deferred))? - } else { - work.await? - }; let scan = MultipartRecoveryScan { catalog: context.catalog, continuation, @@ -113,8 +103,8 @@ impl MultipartRecovery { let sessions = validate_page(context, &scan, &page)?; let mut report = MultipartRecoveryPage { continuation: page.continuation, - progressed: usize::from(matches!(admission, RecoveryAction::Progressed)), - deferred: usize::from(matches!(admission, RecoveryAction::Deferred)), + progressed: 0, + deferred: 0, retained: 0, awaiting_seal: Vec::new(), failures: Vec::new(), @@ -178,21 +168,6 @@ impl MultipartRecovery { .await? } else if session.phase == MultipartPhase::Publishing { self.repository.publish(session).await?.is_some() - } else if session.credit.is_some_and(|credit| !credit.released) - && matches!( - session.phase, - MultipartPhase::Published | MultipartPhase::Aborted | MultipartPhase::Conflicted - ) - { - let record = self - .admission - .load(session.context) - .await? - .ok_or(ValidationError::Record)?; - if record.pending.is_some() { - return Ok(RecoveryAction::Deferred); - } - self.admission.release(&record, session).await? } else { return Ok(RecoveryAction::Retained); }; @@ -202,20 +177,6 @@ impl MultipartRecovery { RecoveryAction::Deferred }) } - - async fn recover_admission(&self, context: CatalogContext) -> Result { - let Some(record) = self.admission.load(context).await? else { - return Ok(RecoveryAction::Retained); - }; - if record.pending.is_none() { - return Ok(RecoveryAction::Retained); - } - Ok(if self.admission.settle(&record).await? { - RecoveryAction::Progressed - } else { - RecoveryAction::Deferred - }) - } } enum RecoveryAction { diff --git a/lib/crowdb-access-iceberg/src/file/multipart_repository.rs b/lib/crowdb-access-iceberg/src/file/multipart_repository.rs index 230ef5bb5..9bd986dcd 100644 --- a/lib/crowdb-access-iceberg/src/file/multipart_repository.rs +++ b/lib/crowdb-access-iceberg/src/file/multipart_repository.rs @@ -42,6 +42,7 @@ impl MultipartRepository { || session.revision != 1 || session.part_count != 0 || session.pending.is_some() + || session.credit.is_some() { return Err(ValidationError::Record.into()); } diff --git a/lib/crowdb-access-iceberg/tests/common/store.rs b/lib/crowdb-access-iceberg/tests/common/store.rs index 24f878ddb..2423d3356 100644 --- a/lib/crowdb-access-iceberg/tests/common/store.rs +++ b/lib/crowdb-access-iceberg/tests/common/store.rs @@ -24,6 +24,7 @@ pub struct TestStore { pub reads: AtomicUsize, pub cas_attempts: AtomicUsize, pub projection_reads: AtomicUsize, + pub legacy_admission_calls: AtomicUsize, pub file_scans: AtomicUsize, pub fencing_delay_ms: AtomicUsize, pub fencing_barrier: Option>, @@ -228,6 +229,15 @@ impl CatalogStore for TestStore { tokio::task::yield_now().await; } self.reads.fetch_add(1, Ordering::SeqCst); + if matches!( + IcebergKey::decode(key), + Ok(IcebergKey::Catalog { + scope: CatalogScope::MultipartAdmission, + .. + }) + ) { + self.legacy_admission_calls.fetch_add(1, Ordering::SeqCst); + } if matches!( StorageKey::decode(key), Ok(StorageKey::Legacy(IcebergKey::Catalog { @@ -251,6 +261,15 @@ impl CatalogStore for TestStore { identity: ClientRequestId, ) -> Result { self.cas_attempts.fetch_add(1, Ordering::SeqCst); + if matches!( + IcebergKey::decode(key), + Ok(IcebergKey::Catalog { + scope: CatalogScope::MultipartAdmission, + .. + }) + ) { + self.legacy_admission_calls.fetch_add(1, Ordering::SeqCst); + } identity.validate().unwrap(); if self.yield_operations { tokio::task::yield_now().await; diff --git a/lib/crowdb-access-iceberg/tests/multipart_admission_record_test.rs b/lib/crowdb-access-iceberg/tests/multipart_admission_record_test.rs index 2a3e1f7b6..d3b89d0b3 100644 --- a/lib/crowdb-access-iceberg/tests/multipart_admission_record_test.rs +++ b/lib/crowdb-access-iceberg/tests/multipart_admission_record_test.rs @@ -41,18 +41,7 @@ fn mutation(record: &MultipartAdmissionRecord, bytes: u64) -> MultipartCreditMut } #[test] -fn admission_limits_independently_bound_sessions_and_reserved_bytes() { - let initial = record(); - let mut first = initial.proposed(mutation(&initial, 60)).unwrap(); - assert_eq!((first.sessions, first.reserved_bytes), (1, 60)); - assert!(first.proposed(mutation(&first, 1)).is_err()); - first.pending = None; - assert!(first.proposed(mutation(&first, 41)).is_err()); - let mut second = first.proposed(mutation(&first, 40)).unwrap(); - second.pending = None; - assert!(second.proposed(mutation(&second, 1)).is_err()); - second.reserved_bytes = 2; - assert!(second.proposed(mutation(&second, 1)).is_err()); +fn legacy_limits_and_counter_records_reject_invalid_bounds() { for limits in [ MultipartAdmissionLimits { max_sessions: 0, @@ -73,18 +62,32 @@ fn admission_limits_independently_bound_sessions_and_reserved_bytes() { ] { assert!(limits.validate().is_err()); } + for (sessions, bytes) in [(3, 3), (1, 101), (1, 0), (0, 1)] { + let mut value = record(); + value.sessions = sessions; + value.reserved_bytes = bytes; + assert!(value.validate().is_err()); + } } #[test] fn bounded_credit_journals_round_trip_and_reject_foreign_keys() { let initial = record(); - let mut reserved = initial.proposed(mutation(&initial, 100)).unwrap(); + let mut reserved = initial.clone(); + reserved.revision = 2; + reserved.sessions = 1; + reserved.reserved_bytes = 100; + reserved.pending = Some(mutation(&initial, 100)); let mut release = reserved.pending.clone().unwrap(); release.action = MultipartCreditAction::Release; release.before = Some(release.after.clone()); release.after.digest = [2; 32]; reserved.pending = None; - let released = reserved.proposed(release).unwrap(); + let mut released = reserved.clone(); + released.revision += 1; + released.sessions = 0; + released.reserved_bytes = 0; + released.pending = Some(release); assert_eq!((released.sessions, released.reserved_bytes), (0, 0)); for value in [initial, reserved, released] { let key = value.key(); @@ -109,23 +112,24 @@ fn bounded_credit_journals_round_trip_and_reject_foreign_keys() { } #[test] -fn malformed_credit_snapshots_and_counter_underflow_fail_closed() { - let initial = record(); - let valid = mutation(&initial, 1); - let mut invalid = valid.clone(); - invalid.after.catalog = CatalogId::random(); - assert!(initial.proposed(invalid).is_err()); - let mut invalid = valid.clone(); - invalid.after.operation = OperationId::random(); - assert!(initial.proposed(invalid).is_err()); - let mut invalid = valid.clone(); - invalid.after.length = 0; - assert!(initial.proposed(invalid).is_err()); - let mut invalid = valid.clone(); - invalid.action = MultipartCreditAction::Release; - invalid.before = Some(invalid.after.clone()); - assert!(initial.proposed(invalid).is_err()); - let mut overflow = initial; - overflow.revision = u64::MAX; - assert!(overflow.proposed(valid).is_err()); +fn malformed_legacy_credit_payloads_fail_closed() { + let mut valid = record(); + valid.revision = 2; + valid.sessions = 1; + valid.reserved_bytes = 1; + valid.pending = Some(mutation(&valid, 1)); + valid.validate().unwrap(); + for case in 0..5 { + let mut invalid = valid.clone(); + let pending = invalid.pending.as_mut().unwrap(); + match case { + 0 => pending.after.catalog = CatalogId::random(), + 1 => pending.after.operation = OperationId::random(), + 2 => pending.after.length = 0, + 3 => pending.before = Some(pending.after.clone()), + 4 => invalid.revision = u64::MAX, + _ => unreachable!(), + } + assert!(invalid.validate().is_err()); + } } diff --git a/lib/crowdb-access-iceberg/tests/multipart_admission_test.rs b/lib/crowdb-access-iceberg/tests/multipart_admission_test.rs index 2d3b88f31..cab4f0c02 100644 --- a/lib/crowdb-access-iceberg/tests/multipart_admission_test.rs +++ b/lib/crowdb-access-iceberg/tests/multipart_admission_test.rs @@ -9,266 +9,233 @@ mod fixtures; #[path = "common/multipart_recovery_store.rs"] mod scan; -use crowdb_access_iceberg::catalog::CatalogError; +use crowdb_access_iceberg::catalog::StoredValue; use crowdb_access_iceberg::file::{ - MultipartAdmission, MultipartAdmissionLimits, MultipartAdmissionRecord, MultipartPhase, - MultipartRecovery, MultipartRepository, MultipartSession, + MultipartCredit, MultipartPhase, MultipartRecovery, MultipartRepository, MultipartSession, }; -use crowdb_access_iceberg::key::OperationId; +use crowdb_access_iceberg::key::{CatalogScope, IcebergKey, OperationId}; +use crowdb_access_iceberg::record::StorageRecord; use std::sync::{atomic::Ordering, Arc}; -async fn setup() -> (file::TestFile, MultipartSession, MultipartAdmissionRecord) { +async fn setup() -> (file::TestFile, MultipartSession) { let fixture = file::TestFile::new(common::TestStore::default()).await; - let mut initial = fixtures::session(); - initial.context = fixture.context; - initial.owner.table = fixture.table; - initial.owner.file = fixture.record("file", b"{}").file; - initial.location = fixture.table.file("file").unwrap(); - let record = MultipartAdmission::new(fixture.store.clone()) - .initialize_for_tests( - fixture.context, - MultipartAdmissionLimits { - max_sessions: 2, - max_reserved_bytes: 3000, - }, - ) - .await - .unwrap(); - (fixture, initial, record) + let mut session = fixtures::session(); + session.context = fixture.context; + session.owner.table = fixture.table; + session.owner.file = fixture.record("file", b"{}").file; + session.location = fixture.table.file("file").unwrap(); + (fixture, session) } - -async fn policy(admission: &MultipartAdmission, initial: &MultipartSession) -> MultipartAdmissionRecord { - admission.load(initial.context).await.unwrap().unwrap() -} - -async fn session(repository: &MultipartRepository, initial: &MultipartSession) -> MultipartSession { - repository - .load(initial.context, initial.upload) - .await - .unwrap() - .unwrap() +fn legacy_credit() -> MultipartCredit { + MultipartCredit { + policy: OperationId::random(), + sequence: 2, + released: false, + } } - -#[tokio::test] -async fn reservation_reply_loss_recovers_once_across_instances_at_every_write() { - for lost in 1..=4 { - let (fixture, initial, record) = setup().await; - let admission = MultipartAdmission::new(fixture.store.clone()); - fixture.store.fail_after.store( - fixture.store.writes.load(Ordering::SeqCst) + lost, - Ordering::SeqCst, - ); - assert!(admission.reserve_for_tests(&record, &initial, 100).await.is_err()); - fixture.store.fail_after.store(0, Ordering::SeqCst); - let admission = MultipartAdmission::new(fixture.store.clone()); - let record = policy(&admission, &initial).await; - if record.pending.is_some() { - assert!(admission.settle(&record).await.unwrap()); - } - let record = policy(&admission, &initial).await; - assert!(admission.reserve_for_tests(&record, &initial, 100).await.unwrap()); - let record = policy(&admission, &initial).await; - assert_eq!((record.sessions, record.reserved_bytes), (1, 1500)); - assert!(record.pending.is_none()); - let repository = MultipartRepository::new(fixture.store.clone()); - let admitted = session(&repository, &initial).await; - assert_eq!(admitted.credit.unwrap().policy, record.policy); - assert!(!admitted.credit.unwrap().released); - let writes = fixture.store.writes.load(Ordering::SeqCst); - assert!(admission - .reserve_for_tests(&record, &initial, 5000) - .await - .unwrap()); - assert_eq!(fixture.store.writes.load(Ordering::SeqCst), writes); +fn poison_policy(fixture: &file::TestFile) -> Vec { + let key = IcebergKey::Catalog { + catalog: fixture.context.catalog, + scope: CatalogScope::MultipartAdmission, + suffix: Vec::new(), } + .encode() + .unwrap(); + let mut values = (**fixture.store.values.load()).clone(); + values.insert( + key.clone(), + StoredValue { + bytes: b"retired policy must never be decoded".to_vec(), + revision: 1, + }, + ); + fixture.store.values.store(Arc::new(values)); + key +} +fn install_session(fixture: &file::TestFile, session: &MultipartSession) { + let mut values = (**fixture.store.values.load()).clone(); + values.insert( + session.key().encode().unwrap(), + StoredValue { + bytes: StorageRecord::MultipartSession(Box::new(session.clone())) + .encode() + .unwrap(), + revision: session.revision, + }, + ); + fixture.store.values.store(Arc::new(values)); +} +fn recovery(fixture: &file::TestFile) -> MultipartRecovery { + MultipartRecovery::new( + fixture.store.clone(), + Arc::new(blocks::TestBlocks::default()), + 8, + 32, + ) + .unwrap() } #[tokio::test] -async fn terminal_release_reply_loss_never_returns_credits_twice() { - for lost in 1..=5 { - let (fixture, initial, record) = setup().await; - let admission = MultipartAdmission::new(fixture.store.clone()); - let repository = MultipartRepository::new(fixture.store.clone()); - assert!(admission.reserve_for_tests(&record, &initial, 100).await.unwrap()); - let active = session(&repository, &initial).await; - let record = policy(&admission, &initial).await; - assert!(matches!( - admission.release(&record, &active).await, - Err(CatalogError::Conflict) - )); - assert!(repository.abort(&active).await.unwrap()); - let terminal = session(&repository, &initial).await; - fixture.store.fail_after.store( - fixture.store.writes.load(Ordering::SeqCst) + lost, - Ordering::SeqCst, - ); - assert!(admission.release(&record, &terminal).await.is_err()); - fixture.store.fail_after.store(0, Ordering::SeqCst); - let admission = MultipartAdmission::new(fixture.store.clone()); - let current = policy(&admission, &initial).await; - if current.pending.is_some() { - assert!(admission.settle(¤t).await.unwrap()); - } - let current = policy(&admission, &initial).await; - assert!(admission.release(¤t, &terminal).await.unwrap()); - let current = policy(&admission, &initial).await; - assert_eq!((current.sessions, current.reserved_bytes), (0, 0)); - let retained = session(&repository, &initial).await; - assert!(retained.credit.unwrap().released); - let writes = fixture.store.writes.load(Ordering::SeqCst); - assert!(admission.release(¤t, &terminal).await.unwrap()); - assert!(admission.release(¤t, &retained).await.unwrap()); - assert_eq!(fixture.store.writes.load(Ordering::SeqCst), writes); - } +async fn empty_sweeps_only_check_context_and_never_read_or_recreate_a_legacy_policy() { + let (fixture, session) = setup().await; + poison_policy(&fixture); + let before = fixture.store.values.load_full(); + let reads = fixture.store.reads.load(Ordering::SeqCst); + let writes = fixture.store.writes.load(Ordering::SeqCst); + let report = recovery(&fixture) + .recover_page(fixture.context, None, session.expires_ms) + .await + .unwrap(); + assert_eq!((report.progressed, report.retained, report.deferred), (0, 0, 0)); + assert!(report.failures.is_empty()); + assert_eq!(fixture.store.reads.load(Ordering::SeqCst) - reads, 2); + assert_eq!(fixture.store.writes.load(Ordering::SeqCst), writes); + assert_eq!(*fixture.store.values.load_full(), *before); + assert_eq!(fixture.store.legacy_admission_calls.load(Ordering::SeqCst), 0); } #[tokio::test] -async fn admission_fences_competing_reservations_and_persisted_policy_changes() { - let (fixture, initial, record) = setup().await; - let first = MultipartAdmission::new(fixture.store.clone()); - let second = MultipartAdmission::new(fixture.store.clone()); - let mut other = initial.clone(); - other.upload = OperationId::random(); - let (left, right) = tokio::join!( - first.reserve_for_tests(&record, &initial, 100), - second.reserve_for_tests(&record, &other, 100) - ); - assert_ne!(left.unwrap(), right.unwrap()); - let current = policy(&first, &initial).await; - assert_eq!(current.sessions, 1); - assert_eq!( - first - .initialize_for_tests(initial.context, record.limits) - .await - .unwrap(), - current - ); - assert!(matches!( - first - .initialize_for_tests( - initial.context, - MultipartAdmissionLimits { - max_sessions: 3, - ..record.limits - } - ) - .await, - Err(CatalogError::Conflict) - )); - let repository = MultipartRepository::new(fixture.store.clone()); - let missing = if repository - .load(initial.context, initial.upload) - .await - .unwrap() - .is_none() - { - &initial - } else { - &other +async fn a_pending_legacy_reservation_cannot_recreate_an_upload_from_its_journal() { + use crowdb_access_iceberg::file::{ + MultipartAdmissionLimits, MultipartAdmissionRecord, MultipartCreditAction, MultipartCreditMutation, }; - assert!(first.reserve_for_tests(¤t, missing, 100).await.unwrap()); - let mut overflow = initial.clone(); - overflow.upload = OperationId::random(); - assert!(first - .reserve_for_tests(&policy(&first, &initial).await, &overflow, 100) + use crowdb_access_iceberg::operation::PayloadStore; + let (fixture, mut session) = setup().await; + let credit = legacy_credit(); + session.credit = Some(credit); + let after = PayloadStore::new(fixture.store.clone()) + .put( + fixture.context.catalog, + session.upload, + &StorageRecord::MultipartSession(Box::new(session.clone())) + .encode() + .unwrap(), + ) .await - .is_err()); - assert!(repository - .load(initial.context, overflow.upload) + .unwrap(); + let record = MultipartAdmissionRecord { + context: fixture.context, + policy: credit.policy, + revision: credit.sequence, + limits: MultipartAdmissionLimits { + max_sessions: 2, + max_reserved_bytes: 3000, + }, + sessions: 1, + reserved_bytes: session.limits.max_staged_bytes, + pending: Some(MultipartCreditMutation { + action: MultipartCreditAction::Reserve, + upload: session.upload, + reservation_bytes: session.limits.max_staged_bytes, + before: None, + after, + }), + }; + let mut values = (**fixture.store.values.load()).clone(); + values.insert( + record.key().encode().unwrap(), + StoredValue { + bytes: StorageRecord::MultipartAdmission(Box::new(record)) + .encode() + .unwrap(), + revision: 2, + }, + ); + fixture.store.values.store(Arc::new(values)); + let before = fixture.store.values.load_full(); + let report = recovery(&fixture) + .recover_page(fixture.context, None, session.expires_ms) .await - .unwrap() - .is_none()); + .unwrap(); + assert_eq!((report.progressed, report.retained, report.deferred), (0, 0, 0)); + assert!(report.failures.is_empty()); + assert_eq!(*fixture.store.values.load_full(), *before); + assert_eq!(fixture.store.legacy_admission_calls.load(Ordering::SeqCst), 0); + assert!(!fixture + .store + .values + .load() + .contains_key(&session.key().encode().unwrap())); } #[tokio::test] -async fn sweep_recovers_precreation_credit_then_expires_and_releases_without_deletion() { - let (fixture, initial, record) = setup().await; - let admission = MultipartAdmission::new(fixture.store.clone()); - fixture - .store - .fail_after - .store(fixture.store.writes.load(Ordering::SeqCst) + 2, Ordering::SeqCst); - assert!(admission.reserve_for_tests(&record, &initial, 100).await.is_err()); - fixture.store.fail_after.store(0, Ordering::SeqCst); - let repository = MultipartRepository::new(fixture.store.clone()); - assert!(repository - .load(initial.context, initial.upload) +async fn legacy_open_upload_expires_without_global_credits_and_terminal_sweeps_do_not_write() { + let (fixture, mut session) = setup().await; + session.credit = Some(legacy_credit()); + install_session(&fixture, &session); + let policy = poison_policy(&fixture); + let before = fixture.store.values.load().get(&policy).unwrap().clone(); + let report = recovery(&fixture) + .recover_page(fixture.context, None, session.expires_ms) + .await + .unwrap(); + assert_eq!(report.progressed, 1); + assert!(report.failures.is_empty()); + let terminal = MultipartRepository::new(fixture.store.clone()) + .load(fixture.context, session.upload) .await .unwrap() - .is_none()); - let recovery = MultipartRecovery::new( - fixture.store.clone(), - Arc::new(blocks::TestBlocks::default()), - 8, - 32, - ) - .unwrap(); - let first = recovery - .recover_page(initial.context, None, initial.expires_ms) + .unwrap(); + assert_eq!(terminal.phase, MultipartPhase::Aborted); + assert_eq!(terminal.credit, session.credit); + assert_eq!(terminal.owner, session.owner); + let writes = fixture.store.writes.load(Ordering::SeqCst); + let reads = fixture.store.reads.load(Ordering::SeqCst); + let report = recovery(&fixture) + .recover_page(fixture.context, None, session.expires_ms) .await .unwrap(); - assert_eq!(first.progressed, 2); - assert!(first.failures.is_empty()); - assert_eq!( - session(&repository, &initial).await.phase, - MultipartPhase::Aborted - ); - let second = recovery - .recover_page(initial.context, None, initial.expires_ms) + assert_eq!((report.progressed, report.retained, report.deferred), (0, 1, 0)); + assert!(report.failures.is_empty()); + assert_eq!(fixture.store.writes.load(Ordering::SeqCst), writes); + assert_eq!(fixture.store.reads.load(Ordering::SeqCst) - reads, 2); + assert_eq!(fixture.store.legacy_admission_calls.load(Ordering::SeqCst), 0); + assert_eq!(fixture.store.values.load().get(&policy), Some(&before)); +} + +#[tokio::test] +async fn published_legacy_resources_remain_unchanged_even_when_the_credit_policy_is_missing() { + let (fixture, mut session) = setup().await; + let files = crowdb_access_iceberg::file::FileRepository::new(fixture.store.clone()); + let record = files + .publish(fixture.context, &fixture.record("file", b"{}")) + .await + .unwrap(); + session.owner.file = record.file; + session.credit = Some(legacy_credit()); + session.phase = MultipartPhase::Published; + session.part_count = 1; + session.completion = Some(fixtures::completion(&session)); + let completion = session.completion.as_mut().unwrap(); + completion.progress.next_part = 1; + completion.progress.completed_bytes = record.length; + completion.publication = Some(completion.selection.clone()); + session.published = Some(session.owner.file); + install_session(&fixture, &session); + let before = fixture.store.values.load_full(); + let report = recovery(&fixture) + .recover_page(fixture.context, None, session.expires_ms) .await .unwrap(); - assert_eq!(second.progressed, 1); - assert!(second.failures.is_empty()); - assert!(session(&repository, &initial).await.credit.unwrap().released); - let current = policy(&admission, &initial).await; - assert_eq!((current.sessions, current.reserved_bytes), (0, 0)); + assert_eq!((report.progressed, report.retained, report.deferred), (0, 1, 0)); + assert!(report.failures.is_empty()); + assert_eq!(*fixture.store.values.load_full(), *before); + assert_eq!(fixture.store.legacy_admission_calls.load(Ordering::SeqCst), 0); assert_eq!( - recovery - .recover_page(initial.context, None, initial.expires_ms) - .await - .unwrap() - .retained, - 1 + files.load(fixture.context, &record.location).await.unwrap(), + Some(record) ); } #[tokio::test] -async fn superseded_helpers_cannot_recreate_released_sessions_or_change_new_credits() { - let (fixture, initial, record) = setup().await; - let admission = MultipartAdmission::new(fixture.store.clone()); - fixture - .store - .fail_after - .store(fixture.store.writes.load(Ordering::SeqCst) + 2, Ordering::SeqCst); - assert!(admission.reserve_for_tests(&record, &initial, 100).await.is_err()); - fixture.store.fail_after.store(0, Ordering::SeqCst); - let reserved = policy(&admission, &initial).await; - assert!(admission.settle(&reserved).await.unwrap()); - let repository = MultipartRepository::new(fixture.store.clone()); - assert!(repository - .abort(&session(&repository, &initial).await) - .await - .unwrap()); - let terminal = session(&repository, &initial).await; - let current = policy(&admission, &initial).await; - fixture - .store - .fail_after - .store(fixture.store.writes.load(Ordering::SeqCst) + 3, Ordering::SeqCst); - assert!(admission.release(¤t, &terminal).await.is_err()); - fixture.store.fail_after.store(0, Ordering::SeqCst); - let releasing = policy(&admission, &initial).await; - assert!(admission.settle(&releasing).await.unwrap()); - let mut other = initial.clone(); - other.upload = OperationId::random(); - let current = policy(&admission, &initial).await; - assert!(admission.reserve_for_tests(¤t, &other, 100).await.unwrap()); - let current = policy(&admission, &initial).await; +async fn new_uploads_cannot_request_legacy_global_credit_state() { + let (fixture, mut session) = setup().await; + session.credit = Some(legacy_credit()); let writes = fixture.store.writes.load(Ordering::SeqCst); - assert!(!admission.settle(&reserved).await.unwrap()); - assert!(!admission.settle(&releasing).await.unwrap()); - assert!(admission.release(¤t, &terminal).await.unwrap()); - assert_eq!(policy(&admission, &initial).await, current); + assert!(MultipartRepository::new(fixture.store.clone()) + .begin(&session, 100) + .await + .is_err()); assert_eq!(fixture.store.writes.load(Ordering::SeqCst), writes); } From 9d01f4a9ae361de7c03dd650bbe2cc1c3aa72a3c Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 20:49:49 +0800 Subject: [PATCH 129/258] Record quota-free worker container validation --- doc/working/plan-access-simplification.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 3002a967a..7accfdfe4 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -156,6 +156,13 @@ CAS 不再满足只证明“本次条件不成立”，不能谎称“第一次 历史回复查询；Composition/Split 的 Committing 恢复与 P2/P3 一起删除，客户端保留完整输入。 旧记录只作为兼容解码/GC 输入，不让旧账本控制新请求。head 保留 expected base；同一 candidate 的当前结果可判断相同，已被后续 head 遮住时不保证历史请求结果重放。 +- [ ] **R2.2e 旧草稿 status 不恢复发布过程**：Composition status 目前仍 help edit receipt， + Committing 还读 CA/CR/head 后写 Prepared/Published 并再次 load；Split status 同样会读 + attempt、head，某分支重复查 commit_status。这些只恢复已经取消的请求历史合同。 + 改为读取已存草稿，旧 Committing 在响应中解释为可重新发布的 Prepared，不写回、不查 + 发布账本、不确认 GET；完整资源准入仍由 CB/SR 和删除事实决定，后台旧资源 GC 解码不变。 + 不把它当作 P2/P3 的 edit/evaluation 状态机已经删除。Files: `composition/{edit,publication}.rs`、 + `split/{lifecycle,recovery,publication}.rs`；验证存在/缺失旧回执均单次 bounded GET、零写。 - [ ] **R2 审查条件发布的回复丢失窗口**：用 H0→H1→H2 测试区分条件失败、相同结果与历史未知； 列出确实依赖历史精确结果的调用者，没有这种需求就移除相应 ledger/response replay。 @@ -479,9 +486,6 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 无条件覆盖与 expected-base 重试测试。断言总 GET+写，不仅是零写。Files: Dataset `tests/*cost*`、`read_query*`、upload/Composition/Split/inspection；S3/Iceberg 对应 repository 测试。 - [ ] **V2 HTTP/native/SDK 集成**：各受影响 route 的鉴权、错误、协议/SDK 一致性；保留删除准入、 迟到写者、非空 namespace、共享 owner、held buffer、cancel/drop 和损坏内容测试。Files: Access/native tests、`sdk/tests/`、native Python consumer。 -- [~] **V2.4 Iceberg 配额清理后容器复验**：构建本批源码镜像，运行 - `access_iceberg::native_files::native_file_tree_publication_and_ranges_survive_catalog_storage_restart`， - 覆盖普通上传的后台过期、part 保留和文件重启读取；记录来源与结果，不能复用清理前镜像。 - [ ] **V3 工程门禁与文档**：每批执行适用 fmt/clippy/测试，全部经 `pixi run`；同步 owning design/spec/OpenAPI/SDK。只修改文档时检查 diff、引用和任务状态，不为此跑整套服务测试。 - [ ] **V4 同源真实负载验收**：重新构建当前源码的镜像，运行现有 HTTP/native Rust/native Python @@ -509,6 +513,16 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，V2.4**：从配额清理源码构建并导入 + `crowdb-node:access-cleanup-iceberg-credits-20261024`；14-node native files 用例通过， + 覆盖普通 MPU 的后台过期、已完成 part 保留、文件发布和存储重启读取。 + artifact `.crowdb-runtime/artifacts/container-e2e/run-ee87dde2591b484299f3c20be19991ff`； + image/client revision `4cb7accc4114357aa3aa6b45423610b3935e7394`，source tree + `c3589d7ccaee35d13260542f6966772c8db6b50b`，server_sources_match=true； + OCI `sha256:17621fe039b24c56fe8c2463fb45689622d93f680268a88700b27f1356ab4c56`， + Docker `sha256:d22daa4a26a5cd0aaa81b92b2086cdab91986c7d162c1421e68409c2dc0e6027`。 + 本机 TCP 功能验证，不作为性能测量或其他 Iceberg 用例全部通过的证明。 + - **2026-10-24，P6.2**：删除 `MultipartAdmission`、reserve/release/settle driver、journal proposal 和相关 test-util 写入口。后台每页不再读全局 policy，终态 upload 不再写 snapshot、 pending 或 released receipt；新 begin 拒绝 legacy credit 输入。旧记录仍可解码；已存在的 From e046236609ea114d9ec17938a01c5135ad8d17c2 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 20:54:19 +0800 Subject: [PATCH 130/258] Read draft status without publication recovery or helper writes --- .../dataset/design-dataset-control.md | 4 + .../dataset/spec-crowdb-dataset.md | 5 + doc/working/plan-access-simplification.md | 19 ++- .../src/authority/publication/receipt.rs | 18 --- .../src/composition/edit.rs | 9 +- .../src/composition/publication.rs | 47 +------ .../src/composition/record.rs | 2 +- .../src/split/lifecycle.rs | 9 +- .../src/split/publication.rs | 34 +---- .../src/split/recovery.rs | 37 ----- .../tests/draft_status_cost_test.rs | 132 ++++++++++++++++++ sdk/README.md | 8 +- 12 files changed, 175 insertions(+), 149 deletions(-) create mode 100644 lib/crowdb-access-dataset/tests/draft_status_cost_test.rs diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index 8889bb39d..7d1415708 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -74,6 +74,10 @@ remain readable. An empty set is a revision, distinct from an absent pointer. Split revision, edits, evaluation progress, candidate and ownership state. Edits use expected-revision CAS and immutable edit receipts. Historical retries return their original draft result; current status is a separate read. +Status reads load only the stored draft and never help edit receipts, query commit +ledgers or advance publication state. A legacy Committing phase is returned as +Prepared for an explicit caller retry; the stored bytes remain unchanged. A status +response does not establish whether that historical request selected the head. The default source is all parent Samples. Selection filters and a frozen named Split can restrict the source, with their intersection when both are present. diff --git a/doc/design/access-server/dataset/spec-crowdb-dataset.md b/doc/design/access-server/dataset/spec-crowdb-dataset.md index d2df8505e..83cfe521d 100644 --- a/doc/design/access-server/dataset/spec-crowdb-dataset.md +++ b/doc/design/access-server/dataset/spec-crowdb-dataset.md @@ -204,6 +204,11 @@ latest. A head conflict MUST return the observed latest/stable/generation withou a confirmation query. A reviewed retry uses that observation with caller-chosen content; the server MUST NOT rebase. Unknown storage outcomes MUST propagate. Existing legacy receipts remain readable without backfill. +Composition and Split status reads MUST NOT reconstruct a publication request, +help edit receipts or persist recovery state. Legacy Committing drafts are +presented as Prepared without rewriting the stored draft or querying attempt, +receipt or head records. This permits an explicit publication retry; it does not +prove that the historical request changed a head. Direct Snapshot head updates MUST accept the complete Snapshot ID and explicit nullable expected generation without an operation or draft identity. They MUST preserve stable and require the `snapshot-head-update` capability in portable SDKs. diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 7accfdfe4..eb4f4af6f 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -156,13 +156,6 @@ CAS 不再满足只证明“本次条件不成立”，不能谎称“第一次 历史回复查询；Composition/Split 的 Committing 恢复与 P2/P3 一起删除，客户端保留完整输入。 旧记录只作为兼容解码/GC 输入，不让旧账本控制新请求。head 保留 expected base；同一 candidate 的当前结果可判断相同，已被后续 head 遮住时不保证历史请求结果重放。 -- [ ] **R2.2e 旧草稿 status 不恢复发布过程**：Composition status 目前仍 help edit receipt， - Committing 还读 CA/CR/head 后写 Prepared/Published 并再次 load；Split status 同样会读 - attempt、head，某分支重复查 commit_status。这些只恢复已经取消的请求历史合同。 - 改为读取已存草稿，旧 Committing 在响应中解释为可重新发布的 Prepared，不写回、不查 - 发布账本、不确认 GET；完整资源准入仍由 CB/SR 和删除事实决定，后台旧资源 GC 解码不变。 - 不把它当作 P2/P3 的 edit/evaluation 状态机已经删除。Files: `composition/{edit,publication}.rs`、 - `split/{lifecycle,recovery,publication}.rs`；验证存在/缺失旧回执均单次 bounded GET、零写。 - [ ] **R2 审查条件发布的回复丢失窗口**：用 H0→H1→H2 测试区分条件失败、相同结果与历史未知； 列出确实依赖历史精确结果的调用者，没有这种需求就移除相应 ledger/response replay。 @@ -513,6 +506,18 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，R2.2e**：Composition/Split status 只读已存草稿；删除 Committing 的 + attempt/receipt/head 查询、二次确认、Prepared/Published 帮助写入，以及专用 + `stored_commit_attempt` helper。Composition status 也不补 edit receipt。旧 Committing + 只在响应中呈现为可重新发布的 Prepared，存储字节不变，不声称原请求曾赢 head。 + 新 cost 回归分别注入旧 Committing 和缺失 edit receipt，在 CA/CR 存在与缺失两种 + 情况下均断言每次 status 为 1 bounded GET、零 unbounded GET、零 mutation；重启后 + 也相同。18 项 native 草稿/发布/legacy/cost、4 项真实 HTTP recovery/SDK 测试通过； + Dataset/Access/native SDK all-target/all-feature clippy、fmt、diff 检查通过，均经 pixi。 + Native status 成本不包含 HTTP 的 Dataset 准入和 upload summary 查询；未冒充总路由 + RPC 数。显式旧 commit-status 仍为只读兼容，edit/evaluation/cleanup 工作状态仍由 + P2/P3/P13 删除；此项没有新增资源标记或放宽完整结果/删除准入。 + - **2026-10-24，V2.4**：从配额清理源码构建并导入 `crowdb-node:access-cleanup-iceberg-credits-20261024`；14-node native files 用例通过， 覆盖普通 MPU 的后台过期、已完成 part 保留、文件发布和存储重启读取。 diff --git a/lib/crowdb-access-dataset/src/authority/publication/receipt.rs b/lib/crowdb-access-dataset/src/authority/publication/receipt.rs index edcff0624..213cf3a90 100644 --- a/lib/crowdb-access-dataset/src/authority/publication/receipt.rs +++ b/lib/crowdb-access-dataset/src/authority/publication/receipt.rs @@ -84,24 +84,6 @@ impl DatasetAuthority { } Ok(None) } - - pub(crate) async fn stored_commit_attempt( - &self, - identity: &DatasetIdentity, - id: OperationId, - ) -> Result, AuthorityError> { - self.store - .get(&attempt_key(identity, id)?) - .await? - .map(|value| { - let attempt = CommitAttempt::decode(&value.bytes)?; - if attempt.id != id { - return Err(AuthorityError::Corrupt); - } - Ok(attempt) - }) - .transpose() - } } fn scoped(prefix: &[u8], identity: &DatasetIdentity, id: &[u8]) -> Result, DatasetError> { let mut key = prefix.to_vec(); diff --git a/lib/crowdb-access-dataset/src/composition/edit.rs b/lib/crowdb-access-dataset/src/composition/edit.rs index 5d38f54c6..2d7bfd4bd 100644 --- a/lib/crowdb-access-dataset/src/composition/edit.rs +++ b/lib/crowdb-access-dataset/src/composition/edit.rs @@ -12,7 +12,7 @@ use crate::{ use sha2::{Digest, Sha256}; impl DatasetAuthority { - /// Returns durable composition state, including edits whose response was lost. + /// Reads the stored composition without replaying edits or publication requests. /// # Errors /// Returns unknown operations, corrupt metadata and storage failures. pub async fn composition_status( @@ -20,7 +20,12 @@ impl DatasetAuthority { identity: &DatasetIdentity, operation: OperationId, ) -> Result { - self.reconcile_composition(identity, operation).await + let (_, state) = self.load_composition(identity, operation).await?; + let mut draft = state.draft; + if let DraftProgress::Committing { target, .. } = draft.record.progress { + draft.record.progress = DraftProgress::Prepared { target }; + } + Ok(draft) } /// Appends a bounded Sample patch page and changes schema/Splits with a revision CAS. /// # Errors diff --git a/lib/crowdb-access-dataset/src/composition/publication.rs b/lib/crowdb-access-dataset/src/composition/publication.rs index c3b50f8e0..8c9d43979 100644 --- a/lib/crowdb-access-dataset/src/composition/publication.rs +++ b/lib/crowdb-access-dataset/src/composition/publication.rs @@ -1,8 +1,8 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{state::CompositionState, CompositionDraft, CompositionError, DraftProgress}; -use crate::{CommitAttempt, DatasetAuthority, DatasetIdentity, OperationId, StoredValue, SuccessfulCommit}; +use super::{CompositionError, DraftProgress}; +use crate::{CommitAttempt, DatasetAuthority, DatasetIdentity, OperationId, SuccessfulCommit}; impl DatasetAuthority { /// Compatibility adapter for publishing a prepared immutable result without process writes. @@ -30,47 +30,4 @@ impl DatasetAuthority { } Ok(self.publish_snapshot_candidate(identity, attempt).await?) } - pub(super) async fn reconcile_composition( - &self, - identity: &DatasetIdentity, - operation: OperationId, - ) -> Result { - let (old, state) = self.load_composition(identity, operation).await?; - self.help_composition_edit(identity, &state).await?; - if let DraftProgress::Committing { target, attempt } = state.draft.record.progress { - let pending = self - .stored_commit_attempt(identity, attempt) - .await? - .ok_or(CompositionError::Corrupt)?; - if pending.target != target { - return Err(CompositionError::Corrupt); - } - if let Some(receipt) = self.commit_status(identity, attempt).await? { - self.finish_legacy_composition_publication(identity, &old, &state, &receipt) - .await?; - } else { - let mut next = state.clone(); - next.draft.record.progress = DraftProgress::Prepared { target }; - self.replace_composition(identity, &old, &state, &next).await?; - } - return Ok(self.load_composition(identity, operation).await?.1.draft); - } - Ok(state.draft) - } - pub(super) async fn finish_legacy_composition_publication( - &self, - identity: &DatasetIdentity, - old: &StoredValue, - state: &CompositionState, - receipt: &SuccessfulCommit, - ) -> Result<(), CompositionError> { - let mut next = state.clone(); - next.edit = None; - next.draft.record.progress = DraftProgress::Published { - target: receipt.attempt.target, - attempt: receipt.attempt.id, - }; - self.replace_composition(identity, old, state, &next).await?; - Ok(()) - } } diff --git a/lib/crowdb-access-dataset/src/composition/record.rs b/lib/crowdb-access-dataset/src/composition/record.rs index 1be40e2de..19f7c5f89 100644 --- a/lib/crowdb-access-dataset/src/composition/record.rs +++ b/lib/crowdb-access-dataset/src/composition/record.rs @@ -20,7 +20,7 @@ pub enum CompositionSource { } /// Evaluation output is valid only at the editing revision that produced it. -/// Prepared content is frozen; commit attempts are separate durable records. +/// Prepared content is frozen; older publication phases remain decodable. #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub enum DraftProgress { diff --git a/lib/crowdb-access-dataset/src/split/lifecycle.rs b/lib/crowdb-access-dataset/src/split/lifecycle.rs index 6e97161ae..037d8c6b9 100644 --- a/lib/crowdb-access-dataset/src/split/lifecycle.rs +++ b/lib/crowdb-access-dataset/src/split/lifecycle.rs @@ -106,7 +106,7 @@ impl DatasetAuthority { Ok(current.record) } - /// Reads the durable editing/evaluation/preparation state after restart. + /// Reads the stored draft without recovering a historical publication request. /// # Errors /// Returns unknown operations, corrupt records or storage failures. pub async fn split_draft_status( @@ -114,7 +114,12 @@ impl DatasetAuthority { identity: &DatasetIdentity, operation: OperationId, ) -> Result { - self.reconcile_split_draft(identity, operation).await + let (_, state) = self.load_split_draft(identity, operation).await?; + let mut record = state.record; + if let DraftProgress::Committing { target, .. } = record.progress { + record.progress = DraftProgress::Prepared { target }; + } + Ok(record) } /// Replaces the whole definition set under a draft revision CAS. diff --git a/lib/crowdb-access-dataset/src/split/publication.rs b/lib/crowdb-access-dataset/src/split/publication.rs index ebf975f9c..000dcedcc 100644 --- a/lib/crowdb-access-dataset/src/split/publication.rs +++ b/lib/crowdb-access-dataset/src/split/publication.rs @@ -1,10 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{ - key, state::DraftState, SplitHead, SplitHeadUpdate, SplitLifecycleError, SplitReference, - SplitRevisionRecord, -}; +use super::{key, SplitHead, SplitHeadUpdate, SplitLifecycleError, SplitReference, SplitRevisionRecord}; use crate::{ composition::DraftProgress, CasOutcome, CommitAttempt, CommitGeneration, CommitTarget, DatasetAuthority, DatasetIdentity, OperationId, SnapshotId, SplitRevisionId, StoredValue, SuccessfulCommit, @@ -144,35 +141,6 @@ impl DatasetAuthority { } } } - - pub(super) async fn reset_legacy_split_publication( - &self, - identity: &DatasetIdentity, - old: &StoredValue, - state: &DraftState, - ) -> Result<(), SplitLifecycleError> { - let DraftProgress::Committing { target, .. } = state.record.progress else { - return Err(SplitLifecycleError::Corrupt); - }; - let mut next = state.clone(); - next.record.progress = DraftProgress::Prepared { target }; - self.replace_split_draft(identity, old, state, &next).await - } - pub(super) async fn finish_legacy_split_publication( - &self, - identity: &DatasetIdentity, - receipt: &SuccessfulCommit, - old: &StoredValue, - state: &DraftState, - ) -> Result<(), SplitLifecycleError> { - let mut next = state.clone(); - next.edit = None; - next.record.progress = DraftProgress::Published { - target: receipt.attempt.target, - attempt: receipt.attempt.id, - }; - self.replace_split_draft(identity, old, state, &next).await - } } fn current_reference( diff --git a/lib/crowdb-access-dataset/src/split/recovery.rs b/lib/crowdb-access-dataset/src/split/recovery.rs index 438b12842..2ecf8f46a 100644 --- a/lib/crowdb-access-dataset/src/split/recovery.rs +++ b/lib/crowdb-access-dataset/src/split/recovery.rs @@ -8,43 +8,6 @@ use crate::{ }; impl DatasetAuthority { - pub(super) async fn reconcile_split_draft( - &self, - identity: &DatasetIdentity, - operation: OperationId, - ) -> Result { - let (old, state) = self.load_split_draft(identity, operation).await?; - if let crate::composition::DraftProgress::Committing { target, attempt: id } = state.record.progress { - let attempt = self - .stored_commit_attempt(identity, id) - .await? - .ok_or(SplitLifecycleError::Corrupt)?; - if attempt.target != target { - return Err(SplitLifecycleError::Corrupt); - } - if let Some(receipt) = self.commit_status(identity, id).await? { - self.finish_legacy_split_publication(identity, &receipt, &old, &state) - .await?; - return Ok(self.load_split_draft(identity, operation).await?.1.record); - } - let CommitTarget::SplitRevision { snapshot, .. } = target else { - return Err(SplitLifecycleError::Corrupt); - }; - let observed = self.inspect_split_head(identity, snapshot).await?; - if observed.generation != attempt.expected_head { - if let Some(receipt) = self.commit_status(identity, id).await? { - self.finish_legacy_split_publication(identity, &receipt, &old, &state) - .await?; - } else { - self.reset_legacy_split_publication(identity, &old, &state) - .await?; - } - return Ok(self.load_split_draft(identity, operation).await?.1.record); - } - } - Ok(state.record) - } - pub(crate) async fn recover_split_commit( &self, identity: &DatasetIdentity, diff --git a/lib/crowdb-access-dataset/tests/draft_status_cost_test.rs b/lib/crowdb-access-dataset/tests/draft_status_cost_test.rs new file mode 100644 index 000000000..3e1c6c9a7 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/draft_status_cost_test.rs @@ -0,0 +1,132 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/composition.rs"] +#[allow(dead_code)] +mod common; +#[path = "common/publication.rs"] +mod publication; +#[path = "common/split_store.rs"] +mod split_store; +use crowdb_access_dataset::{composition::DraftProgress, *}; +use serde_json::json; +use std::sync::atomic::Ordering; + +fn key(prefix: &[u8], identity: &DatasetIdentity, id: OperationId) -> Vec { + [prefix, identity.key().unwrap().as_slice(), id.as_bytes()].concat() +} +async fn legacy_state( + store: &dyn DatasetStore, + key: &[u8], + composition: bool, + attempt: &CommitAttempt, +) -> Vec { + let stored = store.get(key).await.unwrap().unwrap(); + let mut value: serde_json::Value = serde_json::from_slice(&stored.bytes).unwrap(); + let record = if composition { + &mut value["draft"]["record"] + } else { + &mut value["record"] + }; + record["progress"] = serde_json::to_value(DraftProgress::Committing { + target: attempt.target, + attempt: attempt.id, + }) + .unwrap(); + let digest = [0_u8; 32]; + value["edit"] = json!({"id": OperationId::random(), "digest": digest}); + let bytes = serde_json::to_vec(&value).unwrap(); + assert!(matches!( + store + .compare_exchange(key, Some(&stored.bytes), &bytes) + .await + .unwrap(), + CasOutcome::Applied(_) + )); + bytes +} +async fn legacy_receipt(store: &dyn DatasetStore, identity: &DatasetIdentity, attempt: &CommitAttempt) { + let receipt = SuccessfulCommit { + version: 1, + attempt: attempt.clone(), + committed_head: CommitGeneration::from_head(b"legacy successful head"), + }; + for (prefix, bytes) in [ + (b"DS\0CA\0".as_slice(), attempt.encode().unwrap()), + (b"DS\0CR\0", receipt.encode().unwrap()), + ] { + store + .compare_exchange(&key(prefix, identity, attempt.id), None, &bytes) + .await + .unwrap(); + } +} +fn costs(store: &split_store::TestDraftStore) -> (usize, usize, usize) { + ( + store.bounded_reads.load(Ordering::Relaxed), + store.unbounded_reads.load(Ordering::Relaxed), + store.mutations.load(Ordering::Relaxed), + ) +} + +#[tokio::test] +async fn legacy_composition_status_never_helps_edits_or_reconstructs_publication() { + for has_receipt in [false, true] { + let (store, authority, identity, parent) = split_store::fixture().await; + let operation = OperationId::random(); + common::begin_evaluated(&authority, &identity, operation, &common::input(parent)).await; + let prepared = common::prepare(&authority, &identity, operation, 1).await; + let attempt = common::attempt(&authority, &identity, prepared.preparation.unwrap().candidate).await; + let key = key(b"DS\0CD\0", &identity, operation); + let stored = legacy_state(store.as_ref(), &key, true, &attempt).await; + if has_receipt { + legacy_receipt(store.as_ref(), &identity, &attempt).await; + } + store.inspect_only.store(true, Ordering::Relaxed); + store.reject_receipts.store(true, Ordering::Relaxed); + let before = costs(&store); + let restarted = DatasetAuthority::new(store.clone()); + let status = restarted.composition_status(&identity, operation).await.unwrap(); + assert_eq!( + status.record.progress, + DraftProgress::Prepared { + target: attempt.target + } + ); + assert_eq!(costs(&store), (before.0 + 1, before.1, before.2)); + store.inspect_only.store(false, Ordering::Relaxed); + assert_eq!(store.get(&key).await.unwrap().unwrap().bytes, stored); + } +} + +#[tokio::test] +async fn legacy_split_status_observes_one_draft_without_receipt_head_or_recovery_writes() { + for has_receipt in [false, true] { + let (store, authority, identity, snapshot) = split_store::fixture().await; + let prepared = split_store::prepared(&authority, &identity, snapshot, None).await; + let attempt = split_store::attempt(&authority, &identity, &prepared).await; + let key = key(b"DS\0SD\0", &identity, prepared.operation); + let stored = legacy_state(store.as_ref(), &key, false, &attempt).await; + if has_receipt { + legacy_receipt(store.as_ref(), &identity, &attempt).await; + } + store.inspect_only.store(true, Ordering::Relaxed); + store.reject_split_proof_reads.store(true, Ordering::Relaxed); + store.reject_receipts.store(true, Ordering::Relaxed); + let before = costs(&store); + let restarted = DatasetAuthority::new(store.clone()); + let status = restarted + .split_draft_status(&identity, prepared.operation) + .await + .unwrap(); + assert_eq!( + status.progress, + DraftProgress::Prepared { + target: attempt.target + } + ); + assert_eq!(costs(&store), (before.0 + 1, before.1, before.2)); + store.inspect_only.store(false, Ordering::Relaxed); + assert_eq!(store.get(&key).await.unwrap().unwrap().bytes, stored); + } +} diff --git a/sdk/README.md b/sdk/README.md index df8b4fc61..0745665cc 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -148,10 +148,10 @@ It contains no benchmark driver or timing threshold. - Only documented idempotent operations retry; bodies/tokens remain unchanged within one overall deadline. Authentication, integrity and semantic conflicts are not blanket-retried. Rust ambiguous errors and Python exceptions retain - recovery identity. Inspect status and retry the identical publication after - response loss; the operation's Published record can precede head advancement. - A successful publish call establishes publication, rather than part completion - or preparation. Later latest changes preserve historical publish retries. + resource identity. After response loss, retry the identical publication condition; + a status query is optional and does not prove that the head advanced. A successful + publish confirms the current effect. If a later publication replaced it, the old + condition returns a conflict and its observed head for caller review. - Manifests declare the complete current sample membership. Omitted samples disappear; surviving samples may inherit omitted optional fields through parents. Explicit tombstones hide inherited fields, including required fields From f915399bb50b857c31559485c63cc6d4a2b37436 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 21:02:14 +0800 Subject: [PATCH 131/258] Detail remaining composition and split state removal --- doc/working/plan-access-simplification.md | 45 +++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index eb4f4af6f..db266016e 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -330,6 +330,30 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 ### P2 — Dataset Composition（执行已确认 D1） +**记录与成本盘点（2026-10-24，源码审查口径）**： + +- `CBG` 保存 begin 的原输入与初态，`CD` 再保存可变草稿。新建各一次 create-CAS， + 尚未生成最终资源就已有两份过程记录。重复 begin 也读 CBG/CD/最终 CD；父、schema、 + inherited Split 验证另外计算，不能把三次草稿读取当成总 RPC。 +- `CE` 保存每个 edit 输入，`CX` 保存原 draft 响应；新 edit 先 CAS CD 登记 ID，再写 + CE，再 CAS CD 接受修改，再写 CX。即使只有一个 inline sample，首次新 edit 也有 + 四次条件写，不包含最终 Manifest/index。`registered_edits/journal/edit_pages` 只是为了 + 找回这些中间输入、重放原响应和清理失败写入，不属于 Snapshot 业务结果。 +- 每个 evaluation window 写一份 `CW` Step，再对 `CM` 中间行执行 GET/条件写，再 CAS + CD 推进 phase/page/binding。没有 parent 的空 Source 阶段和 edit 列表尾阶段仍各写 CW + 和 CD，零输出也有两次写。CM 最后再被读回拼成 Manifest、写最终 inspection rows， + 因而不能把这些工作行说成“最终索引”。Source/Edits/Validate/Done 续算全部删除。 +- `UploadOwnership.candidates.after/complete` 在每个 owner 页前后各 CAS CD，中间重新 + load CD；staging owner 与最终 Snapshot owner 分开。前者是计算过程；最终 upload + owner 对共享引用/回收有实际作用，按 P4/R4 收敛，不能把共享 payload 一并删除。 +- `CCG/CEG/CCP` 是 scratch/edit/candidate 清理游标；CW/CM/CE 的负标记防旧写者复活。 + 新 API 不再生成这些工作 key；旧格式清理由 P13 保留必要解码。最终 CB/Snapshot/Manifest/ + inspection rows 是资源，CB complete-vs-retired 的同 key 条件隔离不能未经替代证明删除。 +- API/SDK 替换必须真正停止 begin/edit/evaluate/status 的服务端续算写入；不能把旧流程包进 + 一个 HTTP 请求后宣称无状态。客户端保留修改、parent 与显式 head base；服务器每次处理 + 完整有界输入，直接生成最终不可变内容。大输入由客户端组织最终页，再校验绑定并完成根； + 不另建上传式 work session 或逐页进度根。源码计数需由新路径的成本回归复核，尚非性能结论。 + - [ ] **P2.1 分清输入、计算进度与候选内容**：逐项列出 draft/edit receipt/work page/row/candidate/ cleanup seal 的用途、写入次数和调用者；按 D1 更新 API/SDK 合同，移除服务端续算接口。Files: Dataset `composition/{begin,edit,state,evaluation,evaluate_step,prepare,cleanup}.rs`。 - [ ] **P2.2 删除服务端编辑/计算过程**：删除 durable 编辑/进度阶段，客户端失败重新请求； @@ -337,8 +361,29 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 ### P3 — Dataset Split（执行已确认 D1） +**记录与成本盘点（2026-10-24，源码审查口径）**：begin 的 `SB` 与 `SD` 分别保存原输入 +和当前 draft；`SX` 重放 edit 响应。每二十个 sample 的 evaluation window 写 `SW` page +intent、P 份最终 `SM` membership page，再 CAS SD 推进 after/next_page/bindings/counts， +即 P+2 次条件写；重新检查 owner/schema/编译规则的读取还不含在内。prepare 再写最终 +`SR`、`SRI` 和一次 SD 转换。SM/SR/SRI 描述冻结成员和可枚举完整结果，需要保留； +SB/SD/SX/SW 和 SCG 清理游标描述编辑与计算过程，按 D1 删除。`source_leases` 字段当前 +只是旧草稿存的 source ID 集合，生产不再写 reader 租约，不能把字段名误当成仍有读者登记。 +替换顺序先抽取可重做的有界计算，直接写最终成员/完整 revision，独立 `update_split_head` +继续使用已有观察值 CAS；随后迁移 REST/SDK 并停止旧续算写入口。源 revision 固定，重试 +不得用当前 Split 重算历史成员；partial SM 的发现和迟到写隔离与 R4/P13 一起验证。 + - [ ] **P3.1 分开临时评估和已发布成员**：盘点 edits/evaluation/overlap examples/work pages； 保留历史 revision 和冻结成员，不能重算时偷偷改变既有 train/test。Files: Dataset `split/{lifecycle,state,work,evaluation,prepare,initial}.rs`。 +- [~] **P3.2a 原生无状态 Split 准备**：新增 `SplitPreparation { revision, policy, splits }` + 和 `prepare_split_revision(identity, snapshot, request, cancellation)`，输入由客户端完整保存， + 每次从固定 Snapshot/继承 revision 重算，内存只保留一个有界窗口与最多 1 MiB 的最终绑定。 + 外层 30 秒 deadline；最多 64 个定义，继续使用既有条件与覆盖/重叠校验。只写最终 + SM/SR/SRI，禁止 SB/SD/SX/SW 和阶段/进度写入；SRI 在成员前记录可枚举资源身份， + SR 最后安装完整根。相同 ID 完整结果按输入身份复用，异内容冲突；unknown 不确认查询。 + partial SM 可从 SRI 或所属 Snapshot 的 SM prefix 发现，Snapshot 回收覆盖无草稿的 + 输出；独立未引用 revision 期限和迟到元数据重扫仍由 R4/P13 完成，不能掩盖该缺口。 + 先交付 native/cost/取消/丢回复/回收回归，再迁移 REST/SDK 和关闭旧入口；不使用旧 + durable evaluator 的外层循环冒充无状态。Files: `split/preparation.rs`、共享定义/源校验、tests。 - [ ] **P3.2 实现选定 Split 流程**：按 D1 删除辅助进度与检查点，客户端重试，保留 currentSplit 发布 CAS、覆盖/重叠校验和旧 revision 读取。Files: `split/{publication,recovery,cleanup}.rs`、v2/SDK Split 和恢复测试。 From 75c44f218375ed786cf1ce5c2093a86969059a37 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 21:21:06 +0800 Subject: [PATCH 132/258] Prepare immutable split results without draft progress --- .../dataset/design-dataset-control.md | 13 + doc/working/plan-access-simplification.md | 31 +- lib/crowdb-access-dataset/src/split.rs | 2 + lib/crowdb-access-dataset/src/split/draft.rs | 40 +-- .../src/split/inheritance.rs | 20 +- .../src/split/preparation.rs | 248 ++++++++++++++ .../tests/common/split_store.rs | 23 ++ .../tests/split_preparation_reclaim_test.rs | 109 +++++++ .../tests/split_preparation_test.rs | 305 ++++++++++++++++++ 9 files changed, 757 insertions(+), 34 deletions(-) create mode 100644 lib/crowdb-access-dataset/src/split/preparation.rs create mode 100644 lib/crowdb-access-dataset/tests/split_preparation_reclaim_test.rs create mode 100644 lib/crowdb-access-dataset/tests/split_preparation_test.rs diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index 7d1415708..34d61a2eb 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -124,6 +124,19 @@ not a promise of exact cardinality. Overlap/coverage validation remains independ Membership pages are immutable and checksum-bound. Preparation persists the whole revision before currentSplit CAS. A conflict preserves the candidate and caller inputs; retry never silently rebases definitions or overwrites another revision. +The native `prepare_split_revision` alternative accepts caller-owned revision ID, +policy and complete definitions. It recomputes incomplete work from those inputs, +writing only final member pages, the resource enumeration index and complete +revision root. The index precedes pages; the root is installed last. No draft, +edit receipt, window intent or phase cursor is written. Input and final bindings +are capped at 1 MiB; computation uses twenty-Sample windows within a 30-second +request deadline. It retains one window and the bounded bindings, encodes each +member page once, and tracks binding size incrementally. An identical complete +resource is reusable without recomputation; different input at that ID conflicts. +Unknown writes propagate without confirmation reads. Frozen inherited sources use +the supplied revision, regardless of later head changes. Snapshot retirement +finds both complete and partial output through resource prefixes without drafts; +independent unused-revision expiry and delayed-write resweeps remain separate work. The complete `SplitRevisionRecord` is the final result: an explicit revision ID can be read, listed or used as an inherited source without winning currentSplit. Read admission still checks the owning Snapshot. Reads load that immutable record diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index db266016e..4afc4d9eb 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -372,18 +372,11 @@ SB/SD/SX/SW 和 SCG 清理游标描述编辑与计算过程，按 D1 删除。`s 继续使用已有观察值 CAS；随后迁移 REST/SDK 并停止旧续算写入口。源 revision 固定，重试 不得用当前 Split 重算历史成员；partial SM 的发现和迟到写隔离与 R4/P13 一起验证。 -- [ ] **P3.1 分开临时评估和已发布成员**：盘点 edits/evaluation/overlap examples/work pages； - 保留历史 revision 和冻结成员，不能重算时偷偷改变既有 train/test。Files: Dataset `split/{lifecycle,state,work,evaluation,prepare,initial}.rs`。 -- [~] **P3.2a 原生无状态 Split 准备**：新增 `SplitPreparation { revision, policy, splits }` - 和 `prepare_split_revision(identity, snapshot, request, cancellation)`，输入由客户端完整保存， - 每次从固定 Snapshot/继承 revision 重算，内存只保留一个有界窗口与最多 1 MiB 的最终绑定。 - 外层 30 秒 deadline；最多 64 个定义，继续使用既有条件与覆盖/重叠校验。只写最终 - SM/SR/SRI，禁止 SB/SD/SX/SW 和阶段/进度写入；SRI 在成员前记录可枚举资源身份， - SR 最后安装完整根。相同 ID 完整结果按输入身份复用，异内容冲突；unknown 不确认查询。 - partial SM 可从 SRI 或所属 Snapshot 的 SM prefix 发现，Snapshot 回收覆盖无草稿的 - 输出；独立未引用 revision 期限和迟到元数据重扫仍由 R4/P13 完成，不能掩盖该缺口。 - 先交付 native/cost/取消/丢回复/回收回归，再迁移 REST/SDK 和关闭旧入口；不使用旧 - durable evaluator 的外层循环冒充无状态。Files: `split/preparation.rs`、共享定义/源校验、tests。 +- [ ] **P3.2b 无状态 Split REST/SDK**：以完整资源 `PUT snapshots/{snapshot}/split-revisions/{revision}` + 提交 policy/definitions，不使用 operation/draft/expected page。补能力、鉴权、1 MiB body + 限制、错误、Rust/Python SDK 和真实 consumer；相同请求重试不新建 revision，不降级旧 + draft 流程。迁移 standalone Split 示例/容器流程后关闭旧续算写入口；initial Split 的 + 内部工作状态与 Composition 一起清理，不能仅新增 API 后保留全部旧生产写路径。 - [ ] **P3.2 实现选定 Split 流程**：按 D1 删除辅助进度与检查点，客户端重试，保留 currentSplit 发布 CAS、覆盖/重叠校验和旧 revision 读取。Files: `split/{publication,recovery,cleanup}.rs`、v2/SDK Split 和恢复测试。 @@ -551,6 +544,20 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，P3.1/P3.2a**：完成 Composition/Split 过程记录与最终资源边界盘点，见 + P2/P3；新增原生 `SplitPreparation` / `prepare_split_revision`。每次接收完整客户端输入， + 从固定 Snapshot 和继承 revision 重算；只写 SM/SRI/SR，不生成 SB/SD/SX/SW 或进度。 + SRI 在成员页前写，完整根最后安装；相同 ID 完整结果复用，异内容冲突；成员页或最终根 + 丢回复直接传播，跨实例重试不续算、不确认查询。输入与最终绑定各限 1 MiB，单窗口 + 二十个 sample、外层 30 秒；每个成员页只编码一次，绑定长度增量计算，避免每页重编码 + 越来越大的完整根。45 sample / 两个 Split 的 native 全流程实测从 32G + 6S + 17C + 降为 14G + 6S + 8C，总逻辑调用 55→28，其中 6C 为相同的最终成员页；不是 routed + RPC 或 TPS 测量。新路径写集合回归禁止任何工作状态 key。26 项准备、取消、非法输入、 + 头发布、冻结继承、丢回复、重算、完整/部分输出无草稿回收和 Composition 兼容测试通过； + Dataset/Access all-target/all-feature clippy、fmt、diff 通过，均经 pixi。 + 所属 Snapshot 回收能发现并封口已有成员页；独立未引用 revision 期限、迟到未见 key 的 + 重扫仍归 R4/P13。REST/SDK 和旧入口关闭由 P3.2b 跟进，不能视为 P3 全部完成。 + - **2026-10-24，R2.2e**：Composition/Split status 只读已存草稿；删除 Committing 的 attempt/receipt/head 查询、二次确认、Prepared/Published 帮助写入，以及专用 `stored_commit_attempt` helper。Composition status 也不补 edit receipt。旧 Committing diff --git a/lib/crowdb-access-dataset/src/split.rs b/lib/crowdb-access-dataset/src/split.rs index 08bebd1f1..a45d59ccf 100644 --- a/lib/crowdb-access-dataset/src/split.rs +++ b/lib/crowdb-access-dataset/src/split.rs @@ -49,7 +49,9 @@ pub use cleanup::SplitScratchCleanup; pub(crate) use cleanup::SEALED as RETIRED_RECORD; pub use evaluation::SplitPreview; +mod preparation; mod prepare; +pub use preparation::SplitPreparation; mod head; pub use head::{SplitHeadUpdate, SplitReference}; diff --git a/lib/crowdb-access-dataset/src/split/draft.rs b/lib/crowdb-access-dataset/src/split/draft.rs index 69bfad17b..1c7211c57 100644 --- a/lib/crowdb-access-dataset/src/split/draft.rs +++ b/lib/crowdb-access-dataset/src/split/draft.rs @@ -40,24 +40,7 @@ impl SplitDraftRecord { SplitRevisionId::from_bytes(revision.as_bytes()).map_err(|_| SplitError::InvalidRecord)?; } self.progress.validate(self.revision)?; - if self.splits.len() > 64 { - return Err(SplitError::Limit); - } - let mut names = std::collections::BTreeSet::new(); - for split in &self.splits { - if !names.insert(&split.name) { - return Err(SplitError::InvalidRecord); - } - split.definition.validate()?; - if let MembershipOrigin::Inherited(source) = &split.origin { - SnapshotId::from_bytes(source.snapshot.as_bytes()).map_err(|_| SplitError::InvalidRecord)?; - SplitRevisionId::from_bytes(source.revision.as_bytes()) - .map_err(|_| SplitError::InvalidRecord)?; - if source.snapshot == self.snapshot { - return Err(SplitError::InvalidRecord); - } - } - } + validate_definitions(self.snapshot, &self.splits)?; if let DraftProgress::Prepared { target } | DraftProgress::Published { target, .. } | DraftProgress::Committing { target, .. } = self.progress @@ -85,6 +68,27 @@ impl SplitDraftRecord { } } +pub(super) fn validate_definitions(snapshot: SnapshotId, splits: &[DraftSplit]) -> Result<(), SplitError> { + if splits.len() > 64 { + return Err(SplitError::Limit); + } + let mut names = std::collections::BTreeSet::new(); + for split in splits { + if !names.insert(&split.name) { + return Err(SplitError::InvalidRecord); + } + split.definition.validate()?; + if let MembershipOrigin::Inherited(source) = &split.origin { + SnapshotId::from_bytes(source.snapshot.as_bytes()).map_err(|_| SplitError::InvalidRecord)?; + SplitRevisionId::from_bytes(source.revision.as_bytes()).map_err(|_| SplitError::InvalidRecord)?; + if source.snapshot == snapshot { + return Err(SplitError::InvalidRecord); + } + } + } + Ok(()) +} + fn required_revision<'de, D: serde::Deserializer<'de>>(d: D) -> Result, D::Error> { Option::deserialize(d) } diff --git a/lib/crowdb-access-dataset/src/split/inheritance.rs b/lib/crowdb-access-dataset/src/split/inheritance.rs index 90b7026fb..e708e6d99 100644 --- a/lib/crowdb-access-dataset/src/split/inheritance.rs +++ b/lib/crowdb-access-dataset/src/split/inheritance.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::{ - key, MembershipOrigin, MembershipPage, SplitDraftRecord, SplitLifecycleError, SplitName, + key, DraftSplit, MembershipOrigin, MembershipPage, SplitDraftRecord, SplitLifecycleError, SplitName, SplitRevisionRecord, }; use crate::{ @@ -30,7 +30,19 @@ impl DatasetAuthority { identity: &DatasetIdentity, record: &SplitDraftRecord, ) -> Result<(), SplitLifecycleError> { - if source_owners(record).is_empty() { + self.validate_split_definitions(identity, record.snapshot, &record.splits) + .await + } + pub(super) async fn validate_split_definitions( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + splits: &[DraftSplit], + ) -> Result<(), SplitLifecycleError> { + if !splits + .iter() + .any(|split| matches!(split.origin, MembershipOrigin::Inherited(_))) + { return Ok(()); } let budget = MetadataScanBudget { @@ -40,11 +52,11 @@ impl DatasetAuthority { }; let cancel = ReadCancellation::new(); let owner = self - .snapshot_inspection_summary(identity, record.snapshot, budget, &cancel) + .snapshot_inspection_summary(identity, snapshot, budget, &cancel) .await?; let mut work = InspectionWork::new(self.store.as_ref(), &cancel, budget)?; let mut revisions = BTreeMap::new(); - for split in &record.splits { + for split in splits { if let MembershipOrigin::Inherited(source) = &split.origin { if owner.parent != Some(source.snapshot) { return Err(SplitLifecycleError::Conflict); diff --git a/lib/crowdb-access-dataset/src/split/preparation.rs b/lib/crowdb-access-dataset/src/split/preparation.rs new file mode 100644 index 000000000..2a8ca7de9 --- /dev/null +++ b/lib/crowdb-access-dataset/src/split/preparation.rs @@ -0,0 +1,248 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + key, DraftSplit, MembershipPage, MembershipPageBinding, NamedSplitRecord, SplitLifecycleError, + SplitPolicy, SplitRevisionRecord, SplitRevisionSummary, SplitValidation, MAX_SPLIT_RECORD_BYTES, +}; +use crate::{ + AuthorityError, DatasetAuthority, DatasetIdentity, MetadataScanBudget, OperationId, ReadCancellation, + SnapshotId, SplitRevisionId, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::time::Duration; + +/// Complete caller-owned input for one immutable Split result, without editing or progress state. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SplitPreparation { + pub revision: SplitRevisionId, + pub policy: SplitPolicy, + pub splits: Vec, +} + +impl DatasetAuthority { + /// Computes final membership pages and installs their complete root without a draft. + /// A retry recomputes incomplete work or reuses an identical complete resource. + /// # Errors + /// Rejects unavailable sources, conflicting identities, policy violations, cancellation and limits. + pub async fn prepare_split_revision( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + input: &SplitPreparation, + cancel: &ReadCancellation, + ) -> Result { + SnapshotId::from_bytes(snapshot.as_bytes()).map_err(AuthorityError::from)?; + SplitRevisionId::from_bytes(input.revision.as_bytes()).map_err(AuthorityError::from)?; + super::draft::validate_definitions(snapshot, &input.splits)?; + super::codec::encode(input)?; + tokio::time::timeout( + Duration::from_secs(30), + self.prepare_split_result(identity, snapshot, input, cancel), + ) + .await + .map_err(|_| SplitLifecycleError::Limit)? + } + + async fn prepare_split_result( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + input: &SplitPreparation, + cancel: &ReadCancellation, + ) -> Result { + cancelled(cancel)?; + self.require_committed_snapshot(identity, snapshot).await?; + let root = key::revision(identity, snapshot, input.revision).map_err(AuthorityError::from)?; + if let Some(stored) = self.split_state_get(&root).await? { + if stored.bytes == super::RETIRED_RECORD { + return Err(SplitLifecycleError::Frozen); + } + let record = SplitRevisionRecord::decode(&stored.bytes)?; + return if matches_input(&record, snapshot, input) { + Ok(record) + } else { + Err(SplitLifecycleError::Conflict) + }; + } + self.validate_split_definitions(identity, snapshot, &input.splits) + .await?; + let mut record = result(snapshot, input)?; + let encoded_size = record.encode()?.len(); + let summary = SplitRevisionSummary { + version: 1, + snapshot, + revision: input.revision, + operation: record.operation, + split_count: u32::try_from(record.splits.len()).map_err(|_| SplitLifecycleError::Limit)?, + }; + // Resource enumeration precedes its member pages, including failed preparations. + self.put_split_state( + &key::revision_summary(identity, snapshot, input.revision).map_err(AuthorityError::from)?, + &super::codec::encode(&summary)?, + ) + .await?; + self.compute_split_members(identity, input, &mut record, encoded_size, cancel) + .await?; + cancelled(cancel)?; + self.require_committed_snapshot(identity, snapshot).await?; + self.put_split_state(&root, &record.encode()?).await?; + Ok(record) + } + + async fn compute_split_members( + &self, + identity: &DatasetIdentity, + input: &SplitPreparation, + record: &mut SplitRevisionRecord, + mut encoded_size: usize, + cancel: &ReadCancellation, + ) -> Result<(), SplitLifecycleError> { + let mut after = None; + let mut validation = SplitValidation::default(); + loop { + cancelled(cancel)?; + let window = self + .evaluate_split_window( + identity, + record.snapshot, + &input.splits, + after, + MetadataScanBudget { + items: 20, + bytes: 1024 * 1024, + timeout: Duration::from_secs(2), + }, + cancel, + false, + ) + .await?; + validation.merge(&window.validation)?; + if validation.scanned > window.universe_count + || (window.after.is_none() && validation.scanned != window.universe_count) + { + return Err(SplitLifecycleError::Corrupt); + } + self.write_split_member_window(identity, record, window.members, &mut encoded_size, cancel) + .await?; + after = window.after; + if after.is_none() { + if !validation.satisfies(input.policy) { + return Err(SplitLifecycleError::PolicyViolation); + } + return Ok(()); + } + } + } + + async fn write_split_member_window( + &self, + identity: &DatasetIdentity, + record: &mut SplitRevisionRecord, + members: Vec>>, + encoded_size: &mut usize, + cancel: &ReadCancellation, + ) -> Result<(), SplitLifecycleError> { + for (split, members) in record.splits.iter_mut().zip(members) { + if members.is_empty() { + continue; + } + cancelled(cancel)?; + let page = MembershipPage { + version: 1, + snapshot: record.snapshot, + revision: record.revision, + name: split.name.clone(), + page: u32::try_from(split.pages.len()).map_err(|_| SplitLifecycleError::Limit)?, + members, + }; + let bytes = page.encode()?; + let binding = MembershipPageBinding { + page: page.page, + count: u32::try_from(page.members.len()).map_err(|_| SplitLifecycleError::Limit)?, + sha256: Sha256::digest(&bytes).into(), + }; + let count = split + .count + .checked_add(u64::try_from(page.members.len()).map_err(|_| SplitLifecycleError::Limit)?) + .ok_or(SplitLifecycleError::Limit)?; + let added = super::codec::encode(&binding)?.len() + + usize::from(!split.pages.is_empty()) + + decimal_digits(count) + - decimal_digits(split.count); + *encoded_size = encoded_size + .checked_add(added) + .ok_or(SplitLifecycleError::Limit)?; + if u64::try_from(*encoded_size).map_err(|_| SplitLifecycleError::Limit)? > MAX_SPLIT_RECORD_BYTES + { + return Err(SplitLifecycleError::Limit); + } + split.count = count; + split.pages.push(binding); + self.put_split_state( + &key::membership_page(identity, record.snapshot, record.revision, &page.name, page.page) + .map_err(AuthorityError::from)?, + &bytes, + ) + .await?; + } + Ok(()) + } +} + +fn result( + snapshot: SnapshotId, + input: &SplitPreparation, +) -> Result { + Ok(SplitRevisionRecord { + version: 1, + snapshot, + revision: input.revision, + operation: OperationId::from_bytes(input.revision.as_bytes()).map_err(AuthorityError::from)?, + evaluator_version: 1, + policy: input.policy, + splits: input + .splits + .iter() + .map(|split| NamedSplitRecord { + name: split.name.clone(), + definition: split.definition.clone(), + origin: split.origin.clone(), + count: 0, + pages: Vec::new(), + }) + .collect(), + }) +} +const fn decimal_digits(mut value: u64) -> usize { + let mut count = 1; + while value >= 10 { + value /= 10; + count += 1; + } + count +} +fn matches_input(record: &SplitRevisionRecord, snapshot: SnapshotId, input: &SplitPreparation) -> bool { + record.snapshot == snapshot + && record.revision == input.revision + && record.operation.as_bytes() == input.revision.as_bytes() + && record.policy == input.policy + && record.splits.len() == input.splits.len() + && record + .splits + .iter() + .zip(&input.splits) + .all(|(stored, requested)| { + stored.name == requested.name + && stored.definition == requested.definition + && stored.origin == requested.origin + }) +} +fn cancelled(cancel: &ReadCancellation) -> Result<(), SplitLifecycleError> { + if cancel.is_cancelled() { + return Err(AuthorityError::Invalid(crate::DatasetError::ReadCancelled).into()); + } + Ok(()) +} diff --git a/lib/crowdb-access-dataset/tests/common/split_store.rs b/lib/crowdb-access-dataset/tests/common/split_store.rs index 3f313f3b6..d7d193054 100644 --- a/lib/crowdb-access-dataset/tests/common/split_store.rs +++ b/lib/crowdb-access-dataset/tests/common/split_store.rs @@ -20,6 +20,7 @@ pub(super) struct TestDraftStore { pub revision_reads: AtomicUsize, pub bounded_reads: AtomicUsize, pub unbounded_reads: AtomicUsize, + pub metadata_scans: AtomicUsize, pub mutations: AtomicUsize, pub split_head_reads: AtomicUsize, pub candidate_reads: AtomicUsize, @@ -33,7 +34,21 @@ pub(super) struct TestDraftStore { pub reject_publishing_ledgers: AtomicBool, pub hide_revision_once: AtomicBool, pub lose_revision_reply: AtomicBool, + pub lose_member_reply: AtomicBool, pub reject_split_proof_reads: AtomicBool, + pub reject_draft_writes: AtomicBool, +} +impl TestDraftStore { + fn check_result_write(&self, key: &[u8]) { + if self.reject_draft_writes.load(Ordering::Relaxed) { + assert!( + [b"DS\0SR\0".as_slice(), b"DS\0SRI\0", b"DS\0SM\0"] + .iter() + .any(|prefix| key.starts_with(prefix)), + "stateless preparation wrote workflow state" + ); + } + } } #[async_trait::async_trait] impl DatasetStore for TestDraftStore { @@ -98,6 +113,7 @@ impl DatasetStore for TestDraftStore { request: &MetadataScan, cancel: &ReadCancellation, ) -> Result { + self.metadata_scans.fetch_add(1, Ordering::Relaxed); if self.cancel_scan.swap(false, Ordering::Relaxed) { cancel.cancel(); } @@ -113,6 +129,7 @@ impl DatasetStore for TestDraftStore { value: &[u8], ) -> Result { self.mutations.fetch_add(1, Ordering::Relaxed); + self.check_result_write(key); if self.reject_publishing_ledgers.load(Ordering::Relaxed) { assert!( ![b"DS\0CA\0".as_slice(), b"DS\0CR\0", b"DS\0SC\0", b"DS\0SRC\0"] @@ -192,6 +209,12 @@ impl DatasetStore for TestDraftStore { { return Err(StoreError::Rejected); } + if key.starts_with(b"DS\0SM\0") + && matches!(result, CasOutcome::Applied(_)) + && self.lose_member_reply.swap(false, Ordering::Relaxed) + { + return Err(StoreError::Rejected); + } if (key.starts_with(b"DS\0SD\0") || key.starts_with(b"DS\0CD\0")) && expected.is_some() && matches!(result, CasOutcome::Applied(_)) diff --git a/lib/crowdb-access-dataset/tests/split_preparation_reclaim_test.rs b/lib/crowdb-access-dataset/tests/split_preparation_reclaim_test.rs new file mode 100644 index 000000000..ea06feace --- /dev/null +++ b/lib/crowdb-access-dataset/tests/split_preparation_reclaim_test.rs @@ -0,0 +1,109 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/publication.rs"] +mod publication; +#[path = "common/split_store.rs"] +mod split_store; +use crowdb_access_dataset::{split::*, *}; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, +}; + +#[tokio::test] +async fn owner_reclaim_finds_complete_and_partial_resources_without_any_draft() { + let (store, authority, identity, snapshot) = split_store::fixture().await; + let clock = Arc::new(AtomicU64::new(1000)); + let authority = authority.with_reclaim_clock_for_tests(clock.clone()); + let input = SplitPreparation { + revision: SplitRevisionId::random(), + policy: SplitPolicy::default(), + splits: split_store::partition_splits(), + }; + store.reject_draft_writes.store(true, Ordering::Relaxed); + let complete = authority + .prepare_split_revision(&identity, snapshot, &input, &ReadCancellation::new()) + .await + .unwrap(); + let mut partial = input.clone(); + partial.revision = SplitRevisionId::random(); + for split in &mut partial.splits { + split.definition = SplitDefinition::Selection(Selection::Explicit(vec![b"sample-1".to_vec()])); + } + assert!(matches!( + authority + .prepare_split_revision(&identity, snapshot, &partial, &ReadCancellation::new()) + .await, + Err(SplitLifecycleError::PolicyViolation) + )); + let mut keys = vec![key::revision(&identity, snapshot, complete.revision).unwrap()]; + for split in &complete.splits { + for page in &split.pages { + keys.push( + key::membership_page(&identity, snapshot, complete.revision, &split.name, page.page).unwrap(), + ); + } + } + for split in &partial.splits { + keys.push(key::membership_page(&identity, snapshot, partial.revision, &split.name, 0).unwrap()); + } + let mut before = Vec::new(); + for key in &keys { + before.push(store.get(key).await.unwrap().unwrap().bytes); + } + store.reject_draft_writes.store(false, Ordering::Relaxed); + let next = authority + .prepare_snapshot_candidate( + &identity, + None, + publication::publication_manifest(2), + OperationId::random(), + ) + .await + .unwrap(); + authority + .update_snapshot_head( + &identity, + &SnapshotHeadUpdate { + snapshot: next, + expected_head: authority.inspect_head(&identity).await.unwrap().generation, + }, + ) + .await + .unwrap(); + authority.begin_reclaim(&identity, snapshot).await.unwrap(); + clock.store(1300, Ordering::Release); + let restarted = DatasetAuthority::new(store.clone()).with_reclaim_clock_for_tests(clock); + let mut finished = false; + for _ in 0..100 { + if restarted + .reclaim_snapshot_metadata_step(&identity, snapshot, &ReadCancellation::new()) + .await + .unwrap() + .complete + { + finished = true; + break; + } + } + assert!(finished); + for (key, previous) in keys.iter().zip(before) { + assert_eq!(store.get(key).await.unwrap().unwrap().bytes, b"split-retired-v1"); + assert!(matches!( + store.compare_exchange(key, None, &previous).await.unwrap(), + CasOutcome::Conflict(_) + )); + } + let writes = store.mutations.load(Ordering::Relaxed); + assert!(restarted + .prepare_split_revision(&identity, snapshot, &input, &ReadCancellation::new()) + .await + .is_err()); + assert!(restarted + .prepare_split_revision(&identity, snapshot, &partial, &ReadCancellation::new()) + .await + .is_err()); + assert_eq!(store.mutations.load(Ordering::Relaxed), writes); + assert!(restarted.get_snapshot(&identity, next).await.is_ok()); +} diff --git a/lib/crowdb-access-dataset/tests/split_preparation_test.rs b/lib/crowdb-access-dataset/tests/split_preparation_test.rs new file mode 100644 index 000000000..7c27b8437 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/split_preparation_test.rs @@ -0,0 +1,305 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/split_inheritance.rs"] +mod inheritance; +#[path = "common/publication.rs"] +mod publication; +#[path = "common/split_store.rs"] +mod split_store; +use crowdb_access_dataset::{split::*, *}; +use std::sync::atomic::Ordering; + +fn request() -> SplitPreparation { + SplitPreparation { + revision: SplitRevisionId::random(), + policy: SplitPolicy::default(), + splits: split_store::partition_splits(), + } +} +fn pages(record: &SplitRevisionRecord) -> usize { + record.splits.iter().map(|split| split.pages.len()).sum() +} +fn reads(store: &split_store::TestDraftStore) -> (usize, usize) { + ( + store.bounded_reads.load(Ordering::Relaxed) + store.unbounded_reads.load(Ordering::Relaxed), + store.metadata_scans.load(Ordering::Relaxed), + ) +} + +#[tokio::test] +async fn preparation_writes_only_final_resources_without_draft_or_head_state() { + let (store, authority, identity, parent) = split_store::fixture().await; + let snapshot = split_store::publish_binary_samples(&authority, &identity, parent).await; + let before = store.mutations.load(Ordering::Relaxed); + let previous_reads = reads(&store); + let legacy = split_store::prepared(&authority, &identity, snapshot, None).await; + let legacy_writes = store.mutations.load(Ordering::Relaxed) - before; + let legacy_reads = ( + reads(&store).0 - previous_reads.0, + reads(&store).1 - previous_reads.1, + ); + store.reject_draft_writes.store(true, Ordering::Relaxed); + let before = store.mutations.load(Ordering::Relaxed); + let previous_reads = reads(&store); + let input = request(); + let result = authority + .prepare_split_revision(&identity, snapshot, &input, &ReadCancellation::new()) + .await + .unwrap(); + let writes = store.mutations.load(Ordering::Relaxed) - before; + let new_reads = ( + reads(&store).0 - previous_reads.0, + reads(&store).1 - previous_reads.1, + ); + assert_eq!(writes, pages(&result) + 2); + assert_eq!(pages(&result), pages(&legacy)); + assert!(legacy_writes > writes, "old={legacy_writes}, new={writes}"); + assert!(new_reads.0 + new_reads.1 + writes < legacy_reads.0 + legacy_reads.1 + legacy_writes); + println!( + "45 Samples: old={legacy_reads:?} (GET, scan) + {legacy_writes} CAS, new={new_reads:?} + {writes} CAS, final pages={}", + pages(&result) + ); + assert_eq!(result.splits.iter().map(|split| split.count).sum::(), 45); + let before = store.mutations.load(Ordering::Relaxed); + assert_eq!( + DatasetAuthority::new(store.clone()) + .prepare_split_revision(&identity, snapshot, &input, &ReadCancellation::new()) + .await + .unwrap(), + result + ); + assert_eq!(store.mutations.load(Ordering::Relaxed), before); + assert_eq!( + authority + .inspect_split_head(&identity, snapshot) + .await + .unwrap() + .revision, + None + ); + assert_eq!( + authority + .get_split_revision( + &identity, + snapshot, + input.revision, + budget(), + &ReadCancellation::new() + ) + .await + .unwrap(), + result + ); + store.reject_draft_writes.store(false, Ordering::Relaxed); + authority + .update_split_head( + &identity, + snapshot, + &SplitHeadUpdate { + revision: result.revision, + expected_revision: None, + expected_head: None, + }, + ) + .await + .unwrap(); + assert_eq!( + authority + .inspect_split_head(&identity, snapshot) + .await + .unwrap() + .revision, + Some(result.revision) + ); +} + +#[tokio::test] +async fn an_incomplete_result_recomputes_from_caller_input_after_a_member_reply_is_lost() { + let (store, authority, identity, parent) = split_store::fixture().await; + let snapshot = split_store::publish_binary_samples(&authority, &identity, parent).await; + store.reject_draft_writes.store(true, Ordering::Relaxed); + store.lose_member_reply.store(true, Ordering::Relaxed); + let input = request(); + assert!(authority + .prepare_split_revision(&identity, snapshot, &input, &ReadCancellation::new()) + .await + .is_err()); + assert!(store + .get(&key::revision(&identity, snapshot, input.revision).unwrap()) + .await + .unwrap() + .is_none()); + let result = DatasetAuthority::new(store.clone()) + .prepare_split_revision(&identity, snapshot, &input, &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(result.splits.iter().map(|split| split.count).sum::(), 45); + assert_eq!(result.revision, input.revision); +} + +#[tokio::test] +async fn a_lost_final_reply_has_no_confirmation_query_and_reuses_the_complete_result() { + let (store, authority, identity, snapshot) = split_store::fixture().await; + store.reject_draft_writes.store(true, Ordering::Relaxed); + store.lose_revision_reply.store(true, Ordering::Relaxed); + let reads = store.revision_reads.load(Ordering::Relaxed); + let input = request(); + assert!(authority + .prepare_split_revision(&identity, snapshot, &input, &ReadCancellation::new()) + .await + .is_err()); + assert_eq!(store.revision_reads.load(Ordering::Relaxed) - reads, 1); + let before = store.mutations.load(Ordering::Relaxed); + let restarted = DatasetAuthority::new(store.clone()); + let result = restarted + .prepare_split_revision(&identity, snapshot, &input, &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(store.mutations.load(Ordering::Relaxed), before); + assert_eq!(result.revision, input.revision); + let mut changed = input; + changed.splits.clear(); + assert!(matches!( + restarted + .prepare_split_revision(&identity, snapshot, &changed, &ReadCancellation::new()) + .await, + Err(SplitLifecycleError::Conflict) + )); + assert_eq!(store.mutations.load(Ordering::Relaxed), before); +} + +#[tokio::test] +async fn cancellation_and_invalid_definitions_do_not_write_or_expose_a_revision() { + let (store, authority, identity, snapshot) = split_store::fixture().await; + store.reject_draft_writes.store(true, Ordering::Relaxed); + let before = store.mutations.load(Ordering::Relaxed); + let cancel = ReadCancellation::new(); + cancel.cancel(); + let mut input = request(); + assert!(authority + .prepare_split_revision(&identity, snapshot, &input, &cancel) + .await + .is_err()); + input.splits.push(input.splits[0].clone()); + assert!(authority + .prepare_split_revision(&identity, snapshot, &input, &ReadCancellation::new()) + .await + .is_err()); + assert_eq!(store.mutations.load(Ordering::Relaxed), before); +} + +#[tokio::test] +async fn violated_policy_leaves_discoverable_result_pages_but_no_complete_root() { + let (store, authority, identity, snapshot) = split_store::fixture().await; + store.reject_draft_writes.store(true, Ordering::Relaxed); + let mut input = request(); + for split in &mut input.splits { + split.definition = SplitDefinition::Selection(Selection::Explicit(vec![b"sample-1".to_vec()])); + } + assert!(matches!( + authority + .prepare_split_revision(&identity, snapshot, &input, &ReadCancellation::new()) + .await, + Err(SplitLifecycleError::PolicyViolation) + )); + assert!(store + .get(&key::revision_summary(&identity, snapshot, input.revision).unwrap()) + .await + .unwrap() + .is_some()); + assert!(store + .get(&key::revision(&identity, snapshot, input.revision).unwrap()) + .await + .unwrap() + .is_none()); + assert!(authority + .get_split_revision( + &identity, + snapshot, + input.revision, + budget(), + &ReadCancellation::new() + ) + .await + .is_err()); +} + +fn budget() -> MetadataScanBudget { + MetadataScanBudget { + items: 20, + bytes: 1024 * 1024, + timeout: std::time::Duration::from_secs(2), + } +} + +#[tokio::test] +async fn inherited_members_use_the_requested_revision_after_the_parent_head_changes() { + let inheritance::TestInheritedFixture { + authority, + identity, + parent, + child, + source, + inherited, + } = inheritance::fixture().await; + let head = authority.inspect_split_head(&identity, parent).await.unwrap(); + let empty = authority + .prepare_split_revision( + &identity, + parent, + &SplitPreparation { + revision: SplitRevisionId::random(), + policy: SplitPolicy::default(), + splits: Vec::new(), + }, + &ReadCancellation::new(), + ) + .await + .unwrap(); + assert!(empty.splits.is_empty()); + authority + .update_split_head( + &identity, + parent, + &SplitHeadUpdate { + revision: empty.revision, + expected_revision: Some(source.revision), + expected_head: head.generation, + }, + ) + .await + .unwrap(); + let frozen = authority + .prepare_split_revision( + &identity, + child, + &SplitPreparation { + revision: SplitRevisionId::random(), + policy: SplitPolicy::default(), + splits: vec![inherited.clone()], + }, + &ReadCancellation::new(), + ) + .await + .unwrap(); + assert_eq!(frozen.splits[0].count, 1); + assert_eq!(frozen.splits[0].origin, inherited.origin); + let evaluated = authority + .prepare_split_revision( + &identity, + child, + &SplitPreparation { + revision: SplitRevisionId::random(), + policy: SplitPolicy::default(), + splits: vec![DraftSplit { + origin: MembershipOrigin::Evaluated, + ..inherited + }], + }, + &ReadCancellation::new(), + ) + .await + .unwrap(); + assert_eq!(evaluated.splits[0].count, 2); +} From 8b8f5eb29147b6e0f66b2d6a8351574c4a927a8e Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 21:38:59 +0800 Subject: [PATCH 133/258] Prepare split resources directly through REST and portable SDKs --- .../src/dataset/api/v2.rs | 2 + .../src/dataset/api/v2/split_preparation.rs | 48 +++++ .../src/dataset/api/v2/split_requests.rs | 30 +-- .../src/dataset/api/v2/splits.rs | 99 +-------- .../tests/common/dataset_v2.rs | 44 ++-- .../tests/dataset_read_query_contract_test.rs | 20 +- .../tests/dataset_retirement_contract_test.rs | 26 +-- .../tests/dataset_split_contract_test.rs | 101 ++++----- .../tests/dataset_split_head_contract_test.rs | 23 +- ...dataset_split_preparation_contract_test.rs | 107 ++++++++++ .../dataset_split_recovery_contract_test.rs | 156 ++++---------- .../tests/dataset_v2_contract_test.rs | 1 + .../dataset/design-dataset-control.md | 26 ++- .../dataset/openapi-crowdb-dataset-v2.yaml | 199 +++++------------- .../dataset/spec-crowdb-dataset.md | 9 + doc/working/plan-access-simplification.md | 7 +- e2e-test/tests/access_dataset_split.rs | 84 ++------ sdk/README.md | 13 +- .../examples/split_control.rs | 41 +--- .../crowdb-sdk-dataset/src/capabilities.rs | 3 +- sdk/crates/crowdb-sdk-dataset/src/v2.rs | 2 + .../crowdb-sdk-dataset/src/v2/lifecycle.rs | 124 +---------- sdk/crates/crowdb-sdk-dataset/src/v2/split.rs | 38 +--- .../src/v2/split_preparation.rs | 38 ++++ .../tests/split_preparation_test.rs | 92 ++++++++ sdk/crates/crowdb-sdk-python/src/control.rs | 13 +- sdk/crowdb_dataset/control.py | 15 +- sdk/tests/split_consumer.py | 21 +- sdk/tests/test_control.py | 16 ++ 29 files changed, 572 insertions(+), 826 deletions(-) create mode 100644 app/crowdb-access-server/src/dataset/api/v2/split_preparation.rs create mode 100644 app/crowdb-access-server/tests/dataset_split_preparation_contract_test.rs create mode 100644 sdk/crates/crowdb-sdk-dataset/src/v2/split_preparation.rs create mode 100644 sdk/crates/crowdb-sdk-dataset/tests/split_preparation_test.rs diff --git a/app/crowdb-access-server/src/dataset/api/v2.rs b/app/crowdb-access-server/src/dataset/api/v2.rs index f009c0e59..134d1f8b3 100644 --- a/app/crowdb-access-server/src/dataset/api/v2.rs +++ b/app/crowdb-access-server/src/dataset/api/v2.rs @@ -17,6 +17,7 @@ mod query; mod read_models; mod reads; mod split_models; +mod split_preparation; mod split_requests; mod split_responses; mod splits; @@ -48,6 +49,7 @@ impl DatasetHttpService { "inspection-preparation", "dataset-management", "split-revisions", + "split-preparation", "split-head-update", "stateless-reads", ]; diff --git a/app/crowdb-access-server/src/dataset/api/v2/split_preparation.rs b/app/crowdb-access-server/src/dataset/api/v2/split_preparation.rs new file mode 100644 index 000000000..809b1c2c2 --- /dev/null +++ b/app/crowdb-access-server/src/dataset/api/v2/split_preparation.rs @@ -0,0 +1,48 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + super::{read_body_limit, snapshot}, + json, + split_models::{revision, DraftSplit, Policy}, + split_responses, ApiError, ApiResult, DatasetHttpService, +}; +use crowdb_access_dataset::{split::SplitPreparation, DatasetIdentity, ReadCancellation}; +use hyper::{body::Incoming, Request}; +use serde::Deserialize; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Prepare { + policy: Policy, + splits: Vec, +} +impl DatasetHttpService { + pub(super) async fn v2_prepare_split_revision( + &self, + identity: &DatasetIdentity, + owner: &str, + revision_id: &str, + request: Request, + ) -> ApiResult { + let snapshot = snapshot(owner)?; + let revision = revision(revision_id)?; + let body = read_body_limit(request, true, 1024 * 1024).await?; + let input: Prepare = serde_json::from_slice(&body).map_err(|_| ApiError::invalid())?; + let request = SplitPreparation { + revision, + policy: input.policy.into(), + splits: input + .splits + .into_iter() + .map(DraftSplit::native) + .collect::>()?, + }; + json(&split_responses::revision( + self.reads + .authority() + .prepare_split_revision(identity, snapshot, &request, &ReadCancellation::new()) + .await?, + )) + } +} diff --git a/app/crowdb-access-server/src/dataset/api/v2/split_requests.rs b/app/crowdb-access-server/src/dataset/api/v2/split_requests.rs index edf2119cf..c92fae654 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/split_requests.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/split_requests.rs @@ -1,30 +1,10 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{ - super::ApiError, - split_models::{required_option, DraftSplit, Policy}, -}; +use super::{super::ApiError, split_models::required_option}; use crowdb_access_dataset::CommitGeneration; use serde::Deserialize; -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct Begin { - pub operation: String, - #[serde(deserialize_with = "required_option")] - pub expected_revision: Option, - pub policy: Policy, - pub splits: Vec, -} -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct SplitEdit { - pub edit: String, - pub expected_revision: String, - pub policy: Policy, - pub splits: Vec, -} #[derive(Deserialize)] #[serde(deny_unknown_fields)] pub(super) struct Evaluate { @@ -38,14 +18,6 @@ pub(super) struct Prepare { } #[derive(Deserialize)] #[serde(deny_unknown_fields)] -pub(super) struct Publish { - pub attempt: String, - pub revision: String, - #[serde(deserialize_with = "required_option")] - pub expected_head: Option, -} -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] pub(super) struct UpdateHead { pub revision: String, #[serde(deserialize_with = "required_option")] diff --git a/app/crowdb-access-server/src/dataset/api/v2/splits.rs b/app/crowdb-access-server/src/dataset/api/v2/splits.rs index fd4a38c64..cad744a5d 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/splits.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/splits.rs @@ -7,14 +7,11 @@ use super::super::{ use super::{ inspection_models::Page, query::Query, - split_models::{decimal, revision, DraftSplit}, + split_models::{decimal, revision}, split_requests as input, split_responses as output, }; use base64::{engine::general_purpose::STANDARD, Engine}; -use crowdb_access_dataset::{ - split::{SplitDraftEdit, SplitName}, - CommitAttempt, CommitTarget, DatasetIdentity, ReadCancellation, -}; +use crowdb_access_dataset::{split::SplitName, DatasetIdentity, ReadCancellation}; use hyper::{body::Incoming, Method, Request}; impl DatasetHttpService { @@ -39,9 +36,7 @@ impl DatasetHttpService { )?; let authority = self.reads.authority(); match (request.method().clone(), tail) { - (Method::POST, ["snapshots", owner, "split-drafts"]) => { - self.v2_begin_split(identity, owner, request).await - } + (Method::POST, ["snapshots", _, "split-drafts"]) => Err(ApiError::unsupported()), (_, ["split-drafts", ..]) => self.v2_split_draft(identity, tail, request).await, (Method::GET, ["snapshots", owner, "split-head"]) => { read_body(request, false).await?; @@ -52,6 +47,10 @@ impl DatasetHttpService { (Method::PUT, ["snapshots", owner, "split-head"]) => { self.v2_update_split_head(identity, owner, request).await } + (Method::PUT, ["snapshots", owner, "split-revisions", revision]) => { + self.v2_prepare_split_revision(identity, owner, revision, request) + .await + } (Method::GET, ["snapshots", owner, "split-revisions"]) => { read_body(request, false).await?; let page = authority @@ -101,33 +100,6 @@ impl DatasetHttpService { _ => Err(ApiError::not_found()), } } - async fn v2_begin_split( - &self, - identity: &DatasetIdentity, - owner: &str, - request: Request, - ) -> ApiResult { - let input: input::Begin = read_json(request).await?; - let splits = input - .splits - .into_iter() - .map(DraftSplit::native) - .collect::>()?; - let record = self - .reads - .authority() - .begin_split_draft( - identity, - operation(&input.operation)?, - snapshot(owner)?, - input.expected_revision.as_deref().map(revision).transpose()?, - input.policy.into(), - splits, - ) - .await?; - json(&output::draft(record)) - } - async fn v2_update_split_head( &self, identity: &DatasetIdentity, @@ -175,62 +147,7 @@ impl DatasetHttpService { .await?, )) } - (Method::PUT, []) => { - let input: input::SplitEdit = read_json(request).await?; - let edit = SplitDraftEdit { - id: super::super::operation(&input.edit)?, - expected_revision: decimal(&input.expected_revision)?, - policy: input.policy.into(), - splits: input - .splits - .into_iter() - .map(DraftSplit::native) - .collect::>()?, - }; - json(&output::draft( - authority.edit_split_draft(identity, operation, &edit).await?, - )) - } - (Method::POST, ["evaluate"]) => { - let input: input::Evaluate = read_json(request).await?; - json(&output::preview( - authority - .evaluate_split_draft( - identity, - operation, - decimal(&input.expected_revision)?, - decimal(&input.expected_page)?, - &ReadCancellation::new(), - ) - .await?, - )) - } - (Method::POST, ["prepare"]) => { - let input: input::Prepare = read_json(request).await?; - json(&output::revision( - authority - .prepare_split_draft(identity, operation, decimal(&input.expected_revision)?) - .await?, - )) - } - (Method::POST, ["publish"]) => { - let input: input::Publish = read_json(request).await?; - let draft = authority.split_draft_status(identity, operation).await?; - let attempt = CommitAttempt { - version: 1, - id: super::super::operation(&input.attempt)?, - target: CommitTarget::SplitRevision { - snapshot: draft.snapshot, - revision: revision(&input.revision)?, - }, - expected_head: input::generation(input.expected_head.as_deref())?, - }; - json(&output::receipt( - &authority - .publish_split_draft(identity, operation, &attempt) - .await?, - )) - } + (Method::PUT | Method::POST, _) => Err(ApiError::unsupported()), _ => Err(ApiError::not_found()), } } diff --git a/app/crowdb-access-server/tests/common/dataset_v2.rs b/app/crowdb-access-server/tests/common/dataset_v2.rs index af2bc2fdd..4e4c3680b 100644 --- a/app/crowdb-access-server/tests/common/dataset_v2.rs +++ b/app/crowdb-access-server/tests/common/dataset_v2.rs @@ -33,37 +33,29 @@ pub fn definitions() -> Value { {"name":"test", "definition":{"kind":"weighted_partition", "seed":"18446744073709551615", "weights":[{"name":"train/a b", "weight":8},{"name":"test", "weight":2}], "index":1}, "origin":{"kind":"evaluated"}} ]) } -pub fn begin(operation: &str, expected: &Value) -> Value { - json!({"operation":operation, "expected_revision":expected, "policy":{"overlap":"disjoint", "coverage":"require_complete"}, "splits":definitions()}) -} -pub async fn prepare(client: &Client, draft: &str, revision: &str) -> Value { - for page in 0..10 { - let p = response( - client - .post(format!("{draft}/evaluate")) - .json(&json!({"expected_revision":revision,"expected_page":page.to_string()})), - 200, - ) - .await; - if p["complete"] == true { - assert_eq!(p["validation"]["scanned"], "45"); - return response( - client - .post(format!("{draft}/prepare")) - .json(&json!({"expected_revision":revision})), - 200, - ) - .await; - } +pub fn preparation(empty: bool) -> Value { + if empty { + json!({"policy":{"overlap":"disjoint", "coverage":"allow_unassigned"}, "splits":[]}) + } else { + json!({"policy":{"overlap":"disjoint", "coverage":"require_complete"}, "splits":definitions()}) } - panic!("Split evaluation did not finish"); } -pub async fn publish(client: &Client, draft: &str, prepared: &Value, attempt: &str, head: Value) -> Value { +pub async fn prepare(client: &Client, root: &str, snapshot: &str, revision: &str, empty: bool) -> Value { response( client - .post(format!("{draft}/publish")) - .json(&json!({"attempt":attempt,"revision":prepared["revision"],"expected_head":head})), + .put(format!("{root}/snapshots/{snapshot}/split-revisions/{revision}")) + .json(&preparation(empty)), 200, ) .await } +pub async fn publish( + client: &Client, + root: &str, + snapshot: &str, + prepared: &Value, + expected: &Value, +) -> Value { + response(client.put(format!("{root}/snapshots/{snapshot}/split-head")) + .json(&json!({"revision":prepared["revision"], "expected_revision":expected["revision"], "expected_head":expected["generation"]})), 200).await +} diff --git a/app/crowdb-access-server/tests/dataset_read_query_contract_test.rs b/app/crowdb-access-server/tests/dataset_read_query_contract_test.rs index 9761732ee..21f75c019 100644 --- a/app/crowdb-access-server/tests/dataset_read_query_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_read_query_contract_test.rs @@ -68,24 +68,8 @@ fn plan() -> Value { "split":{"revision":null,"name":"train/a b"}}) } async fn published_split(client: &Client, root: &str, snapshot: &str) -> String { - let op = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; - response( - client - .post(format!("{root}/snapshots/{snapshot}/split-drafts")) - .json(&common::begin(op, &Value::Null)), - 200, - ) - .await; - let draft = format!("{root}/split-drafts/{op}"); - let prepared = common::prepare(client, &draft, "1").await; - common::publish( - client, - &draft, - &prepared, - "cccccccccccccccccccccccccccccccc", - Value::Null, - ) - .await; + let prepared = common::prepare(client, root, snapshot, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", false).await; + common::publish(client, root, snapshot, &prepared, &Value::Null).await; prepared["revision"].as_str().unwrap().into() } async fn batch(client: &Client, url: &str, cursor: &Value) -> Value { diff --git a/app/crowdb-access-server/tests/dataset_retirement_contract_test.rs b/app/crowdb-access-server/tests/dataset_retirement_contract_test.rs index c2ea57173..2b5bdd1b8 100644 --- a/app/crowdb-access-server/tests/dataset_retirement_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_retirement_contract_test.rs @@ -18,23 +18,7 @@ async fn retirement_respects_retention_and_finishes_split_and_composition_cleanu let test = TestDatasetHttp::start_payload(Arc::new(TestStore::default())).await; let client = Client::new(); let (root, old) = common::fixture(&client, &test.origin).await; - let split_op = "16161616161616161616161616161616"; - let split = format!("{root}/split-drafts/{split_op}"); - response( - client - .post(format!("{root}/snapshots/{old}/split-drafts")) - .json(&common::begin(split_op, &Value::Null)), - 200, - ) - .await; - common::prepare(&client, &split, "1").await; - response(client.delete(&split).json(&json!({"expected_revision":"1"})), 200).await; - for step in 0..100 { - if response(client.post(format!("{split}/cleanup")), 200).await["complete"] == true { - break; - } - assert!(step < 99, "Split cleanup did not terminate"); - } + let revision = common::prepare(&client, &root, &old, "16161616161616161616161616161616", false).await; let operation = "17171717171717171717171717171717"; let draft = format!("{root}/compositions/{operation}"); response( @@ -74,6 +58,14 @@ async fn retirement_respects_retention_and_finishes_split_and_composition_cleanu } assert!(step < 99, "Snapshot retirement did not terminate"); } + response( + client.get(format!( + "{snapshot}/split-revisions/{}", + revision["revision"].as_str().unwrap() + )), + 404, + ) + .await; response(client.get(format!("{snapshot}/summary")), 404).await; assert_eq!( response(client.get(format!("{root}/head")), 200).await["latest"], diff --git a/app/crowdb-access-server/tests/dataset_split_contract_test.rs b/app/crowdb-access-server/tests/dataset_split_contract_test.rs index 019d1c0ce..fd9552d91 100644 --- a/app/crowdb-access-server/tests/dataset_split_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_split_contract_test.rs @@ -8,7 +8,10 @@ use common::response; use dataset::{TestDatasetHttp, TestStore}; use reqwest::Client; use serde_json::{json, Value}; -use std::{collections::BTreeSet, sync::Arc}; +use std::{ + collections::BTreeSet, + sync::{atomic::Ordering, Arc}, +}; #[tokio::test] async fn split_revision_flow_preserves_data_head_and_historical_members() { @@ -17,11 +20,14 @@ async fn split_revision_flow_preserves_data_head_and_historical_members() { let (root, snapshot) = common::fixture(&client, &test.origin).await; let base = format!("{root}/snapshots/{snapshot}"); let head = response(client.get(format!("{root}/head")), 200).await; - let operation = "02020202020202020202020202020202"; - let draft = format!("{root}/split-drafts/{operation}"); - let begin = common::begin(operation, &Value::Null); - response(client.post(format!("{base}/split-drafts")).json(&begin), 200).await; - let prepared = common::prepare(&client, &draft, "1").await; + let prepared = common::prepare( + &client, + &root, + &snapshot, + "02020202020202020202020202020202", + false, + ) + .await; let revision = prepared["revision"].as_str().unwrap(); assert_eq!( response(client.get(format!("{base}/split-revisions/{revision}")), 200).await, @@ -36,42 +42,16 @@ async fn split_revision_flow_preserves_data_head_and_historical_members() { &format!("{base}/split-revisions/{revision}/splits/train%2Fa%20b/members"), ) .await; - let receipt = common::publish( - &client, - &draft, - &prepared, - "03030303030303030303030303030303", - Value::Null, - ) - .await; + let current = common::publish(&client, &root, &snapshot, &prepared, &Value::Null).await; assert_eq!( - common::publish( - &client, - &draft, - &prepared, - "03030303030303030303030303030303", - Value::Null - ) - .await, - receipt + common::publish(&client, &root, &snapshot, &prepared, &Value::Null).await, + current ); assert_eq!(response(client.get(format!("{root}/head")), 200).await, head); assert_eq!( response(client.get(format!("{base}/split-revisions/{revision}")), 200).await, prepared ); - assert_eq!( - response(client.get(&draft), 200).await["progress"]["state"], - "prepared" - ); - assert_eq!( - response( - client.get(format!("{root}/commits/03030303030303030303030303030303")), - 200 - ) - .await["receipt"], - Value::Null - ); let current = response(client.get(format!("{base}/split-head")), 200).await; assert_eq!(current["revision"], revision); let train = members( @@ -115,43 +95,36 @@ async fn members(client: &Client, url: &str) -> BTreeSet { } #[tokio::test] -async fn split_drafts_require_explicit_preconditions_and_reject_stale_edits() { - let test = TestDatasetHttp::start(Arc::new(TestStore::default())).await; +async fn retired_split_workflow_writes_never_access_storage() { + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start(store.clone()).await; let client = Client::new(); let (root, snapshot) = common::fixture(&client, &test.origin).await; - let url = format!("{root}/snapshots/{snapshot}/split-drafts"); - let operation = "04040404040404040404040404040404"; - let mut begin = common::begin(operation, &Value::Null); - begin.as_object_mut().unwrap().remove("expected_revision"); - response(client.post(&url).json(&begin), 400).await; - begin["expected_revision"] = Value::Null; - begin["unsupported"] = json!(true); - response(client.post(&url).json(&begin), 400).await; - begin.as_object_mut().unwrap().remove("unsupported"); - response(client.post(&url).json(&begin), 200).await; - let draft = format!("{root}/split-drafts/{operation}"); - let mut edit = json!({"edit":"05050505050505050505050505050505", "expected_revision":"1", "policy":begin["policy"], "splits":begin["splits"]}); - let edited = response(client.put(&draft).json(&edit), 200).await; - assert_eq!(edited["revision"], "2"); - assert_eq!(response(client.put(&draft).json(&edit), 200).await, edited); - edit["edit"] = json!("06060606060606060606060606060606"); - response(client.put(&draft).json(&edit), 409).await; + let before = ( + store.reads.load(Ordering::Relaxed), + store.bounded_reads.load(Ordering::Relaxed), + store.mutations.load(Ordering::Relaxed), + ); response( client - .post(format!("{draft}/prepare")) - .json(&json!({"expected_revision":"2"})), - 409, + .post(format!("{root}/snapshots/{snapshot}/split-drafts")) + .json(&json!({})), + 501, ) .await; - let prepared = common::prepare(&client, &draft, "2").await; - let publish = json!({"attempt":"07070707070707070707070707070707", "revision":prepared["revision"]}); - response(client.post(format!("{draft}/publish")).json(&publish), 400).await; + let draft = format!("{root}/split-drafts/04040404040404040404040404040404"); + response(client.put(&draft).json(&json!({})), 501).await; + for suffix in ["evaluate", "prepare", "publish"] { + response(client.post(format!("{draft}/{suffix}")).json(&json!({})), 501).await; + } assert_eq!( - response(client.delete(&draft).json(&json!({"expected_revision":"2"})), 200).await["progress"] - ["state"], - "abandoned" + ( + store.reads.load(Ordering::Relaxed), + store.bounded_reads.load(Ordering::Relaxed), + store.mutations.load(Ordering::Relaxed) + ), + before ); - response(client.put(&draft).json(&edit), 409).await; drop(client); test.stop().await; } diff --git a/app/crowdb-access-server/tests/dataset_split_head_contract_test.rs b/app/crowdb-access-server/tests/dataset_split_head_contract_test.rs index 23958ab76..490e6ed7d 100644 --- a/app/crowdb-access-server/tests/dataset_split_head_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_split_head_contract_test.rs @@ -10,7 +10,7 @@ mod dataset; use crowdb_access_dataset::{DatasetIdentity, DatasetStore, NamespacePath}; use dataset::{TestDatasetHttp, TestStore}; use reqwest::Client; -use serde_json::{json, Value}; +use serde_json::json; use std::sync::{atomic::Ordering, Arc}; fn counts(store: &TestStore) -> (u64, u64) { @@ -31,24 +31,17 @@ async fn update_uses_only_complete_results_and_returns_the_failed_head_condition let client = Client::new(); let (root, snapshot) = common::fixture(&client, &test.origin).await; let mut prepared = Vec::new(); - for operation in ["22".repeat(16), "33".repeat(16)] { - common::response( - client - .post(format!("{root}/snapshots/{snapshot}/split-drafts")) - .json(&common::begin(&operation, &Value::Null)), - 200, - ) - .await; - let draft = format!("{root}/split-drafts/{operation}"); - prepared.push(common::prepare(&client, &draft, "1").await); + for revision in ["22".repeat(16), "33".repeat(16)] { + prepared.push(common::prepare(&client, &root, &snapshot, &revision, false).await); } - // Remove the editing identity; the final result and its members are independent. + // Preparing a complete result creates no editing identity. let identity = DatasetIdentity::new(NamespacePath::root(), "splits").unwrap(); for prefix in [b"DS\0SD\0".as_slice(), b"DS\0SB\0"] { - store - .delete(&[prefix, &identity.key().unwrap(), &[0x22; 16]].concat()) + assert!(store + .get(&[prefix, &identity.key().unwrap(), &[0x22; 16]].concat()) .await - .unwrap(); + .unwrap() + .is_none()); } let url = format!("{root}/snapshots/{snapshot}/split-head"); let first = json!({"revision":prepared[0]["revision"], "expected_revision":null, "expected_head":null}); diff --git a/app/crowdb-access-server/tests/dataset_split_preparation_contract_test.rs b/app/crowdb-access-server/tests/dataset_split_preparation_contract_test.rs new file mode 100644 index 000000000..d7f66443c --- /dev/null +++ b/app/crowdb-access-server/tests/dataset_split_preparation_contract_test.rs @@ -0,0 +1,107 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/dataset_v2.rs"] +#[allow(dead_code)] +mod common; +#[path = "common/dataset.rs"] +mod dataset; +use common::response; +use dataset::{TestDatasetHttp, TestStore}; +use reqwest::Client; +use serde_json::{json, Value}; +use std::sync::{atomic::Ordering, Arc}; + +fn counts(store: &TestStore) -> (u64, u64) { + ( + store.reads.load(Ordering::Relaxed) + store.bounded_reads.load(Ordering::Relaxed), + store.mutations.load(Ordering::Relaxed), + ) +} +fn input() -> Value { + let weights = json!([{"name":"train/a b", "weight":1}, {"name":"test", "weight":1}]); + json!({"policy":{"overlap":"disjoint", "coverage":"require_complete"}, "splits":[ + {"name":"train/a b", "definition":{"kind":"weighted_partition", "seed":"42", "weights":weights,"index":0}, "origin":{"kind":"evaluated"}}, + {"name":"test", "definition":{"kind":"weighted_partition", "seed":"42", "weights":weights,"index":1}, "origin":{"kind":"evaluated"}} + ]}) +} + +#[tokio::test] +async fn preparation_uses_the_resource_id_and_reuses_complete_results_without_a_draft() { + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start(store.clone()).await; + let client = Client::new(); + let (root, snapshot) = common::fixture(&client, &test.origin).await; + let url = format!("{root}/snapshots/{snapshot}/split-revisions/{}", "ab".repeat(16)); + let before = counts(&store); + let prepared = response(client.put(&url).json(&input()), 200).await; + let after = counts(&store); + assert_eq!((after.0 - before.0, after.1 - before.1), (14, 8)); + assert_eq!(prepared["revision"], "ab".repeat(16)); + assert_eq!(prepared["snapshot"], snapshot); + assert_eq!( + prepared["splits"] + .as_array() + .unwrap() + .iter() + .map(|split| split["count"].as_str().unwrap().parse::().unwrap()) + .sum::(), + 45 + ); + assert_eq!(response(client.get(&url), 200).await, prepared); + assert_eq!( + response(client.get(format!("{root}/snapshots/{snapshot}/split-head")), 200).await["revision"], + Value::Null + ); + let before = counts(&store); + assert_eq!(response(client.put(&url).json(&input()), 200).await, prepared); + let after = counts(&store); + assert_eq!((after.0 - before.0, after.1 - before.1), (3, 0)); + let mut changed = input(); + changed["splits"] = json!([]); + response(client.put(&url).json(&changed), 409).await; + assert_eq!(counts(&store).1, after.1); + assert_eq!(response(client.get(&url), 200).await, prepared); + test.stop().await; +} + +#[tokio::test] +async fn invalid_input_and_missing_auth_reject_before_storage_and_never_use_workflow_routes() { + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start(store.clone()).await; + let client = Client::new(); + let url = format!( + "{}/api/dataset/v2/namespaces/dataset-ns/datasets/absent/snapshots/{}/split-revisions/{}", + test.origin, + "01".repeat(16), + "02".repeat(16) + ); + let denied = client.put(&url).json(&input()).send().await.unwrap(); + assert_eq!(denied.status(), 401); + denied.bytes().await.unwrap(); + for body in [ + json!({}), + json!({"policy":input()["policy"], "splits":[], "operation":"03".repeat(16)}), + json!({"policy":input()["policy"], "splits":[input()["splits"][0], input()["splits"][0]]}), + ] { + response(client.put(&url).json(&body), 400).await; + } + response(client.put(format!("{url}?expected_page=1")).json(&input()), 400).await; + response( + client.put(&url).header("content-type", "text/plain").body("{}"), + 400, + ) + .await; + response( + client + .put(&url) + .header("content-type", "application/json") + .body(vec![b' '; 1024 * 1024 + 1]), + 413, + ) + .await; + assert_eq!(counts(&store), (0, 0)); + response(client.put(&url).json(&input()), 404).await; + assert_eq!(counts(&store).1, 0); + test.stop().await; +} diff --git a/app/crowdb-access-server/tests/dataset_split_recovery_contract_test.rs b/app/crowdb-access-server/tests/dataset_split_recovery_contract_test.rs index bc8b1c0dd..11dfbd620 100644 --- a/app/crowdb-access-server/tests/dataset_split_recovery_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_split_recovery_contract_test.rs @@ -11,140 +11,72 @@ use serde_json::{json, Value}; use std::sync::Arc; #[tokio::test] -async fn evaluation_resumes_after_server_restart_and_stale_publish_keeps_candidate() { +async fn complete_result_retries_after_restart_and_stale_publish_preserves_history() { let client = Client::new(); - let (test, root, snapshot, draft, first_prepared) = restart_preparation(&client).await; - common::publish( - &client, - &draft, - &first_prepared, - "22222222222222222222222222222222", - Value::Null, - ) - .await; - let head_url = format!("{root}/snapshots/{snapshot}/split-head"); - let head = response(client.get(&head_url), 200).await; - let a = new_prepared( + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start(store.clone()).await; + let (root, snapshot) = common::fixture(&client, &test.origin).await; + let first_id = "11111111111111111111111111111111"; + let first = common::prepare(&client, &root, &snapshot, first_id, false).await; + let old_origin = test.origin.clone(); + test.stop().await; + let test = TestDatasetHttp::start(store).await; + let root = root.replace(&old_origin, &test.origin); + assert_eq!( + common::prepare(&client, &root, &snapshot, first_id, false).await, + first + ); + let head = common::publish(&client, &root, &snapshot, &first, &Value::Null).await; + let a = common::prepare( &client, &root, &snapshot, "33333333333333333333333333333333", - head["revision"].clone(), true, ) .await; - let b = new_prepared( + let b = common::prepare( &client, &root, &snapshot, "44444444444444444444444444444444", - head["revision"].clone(), false, ) .await; - common::publish( - &client, - &a.0, - &a.1, - "55555555555555555555555555555555", - head["generation"].clone(), + let current = common::publish(&client, &root, &snapshot, &a, &head).await; + let url = format!("{root}/snapshots/{snapshot}/split-head"); + let conflict = response(client.put(&url).json(&json!({"revision":b["revision"], "expected_revision":head["revision"], "expected_head":head["generation"]})), 409).await; + assert_eq!(conflict["observed_head"], current); + response( + client + .put(&url) + .json(&json!({"revision":first["revision"], "expected_revision":null, "expected_head":null})), + 409, ) .await; - let current = response(client.get(&head_url), 200).await; - response(client.post(format!("{}/publish", b.0)).json(&json!({"attempt":"66666666666666666666666666666666", "revision":b.1["revision"], "expected_head":head["generation"]})), 409).await; - assert_eq!( - response(client.get(&b.0), 200).await["progress"]["state"], - "prepared" - ); - response(client.post(format!("{draft}/publish")).json(&json!({"attempt":"22222222222222222222222222222222", "revision":first_prepared["revision"], "expected_head":Value::Null})), 409).await; - assert_eq!(response(client.get(&head_url), 200).await, current); - assert_eq!( - response( - client.get(format!( - "{root}/snapshots/{snapshot}/split-revisions/{}", - b.1["revision"].as_str().unwrap() - )), - 200 - ) - .await, - b.1 - ); - let historic = response( + assert_eq!(response(client.get(&url), 200).await, current); + for prepared in [&first, &a, &b] { + assert_eq!( + response( + client.get(format!( + "{root}/snapshots/{snapshot}/split-revisions/{}", + prepared["revision"].as_str().unwrap() + )), + 200 + ) + .await, + *prepared + ); + } + assert_eq!(a["splits"], json!([])); + let historical = response( client.get(format!( - "{root}/snapshots/{snapshot}/split-revisions/{}", - first_prepared["revision"].as_str().unwrap() + "{root}/snapshots/{snapshot}/split-revisions/{first_id}/splits/train%2Fa%20b/members" )), 200, ) .await; - assert_eq!(historic, first_prepared); - assert_eq!( - response( - client.get(format!( - "{root}/snapshots/{snapshot}/split-revisions/{}", - a.1["revision"].as_str().unwrap() - )), - 200 - ) - .await["splits"], - json!([]) - ); + assert!(!historical["entries"].as_array().unwrap().is_empty()); drop(client); test.stop().await; } -async fn new_prepared( - client: &Client, - root: &str, - snapshot: &str, - operation: &str, - expected: Value, - empty: bool, -) -> (String, Value) { - let mut begin = common::begin(operation, &expected); - if empty { - begin["splits"] = json!([]); - begin["policy"]["coverage"] = json!("allow_unassigned"); - } - response( - client - .post(format!("{root}/snapshots/{snapshot}/split-drafts")) - .json(&begin), - 200, - ) - .await; - let draft = format!("{root}/split-drafts/{operation}"); - let prepared = common::prepare(client, &draft, "1").await; - (draft, prepared) -} - -async fn restart_preparation(client: &Client) -> (TestDatasetHttp, String, String, String, Value) { - let store = Arc::new(TestStore::default()); - let test = TestDatasetHttp::start(store.clone()).await; - let (root, snapshot) = common::fixture(client, &test.origin).await; - let operation = "11111111111111111111111111111111"; - response( - client - .post(format!("{root}/snapshots/{snapshot}/split-drafts")) - .json(&common::begin(operation, &Value::Null)), - 200, - ) - .await; - let draft = format!("{root}/split-drafts/{operation}"); - let first = response( - client - .post(format!("{draft}/evaluate")) - .json(&json!({"expected_revision":"1", "expected_page":"0"})), - 200, - ) - .await; - assert!(!first["complete"].as_bool().unwrap()); - let old_origin = test.origin.clone(); - test.stop().await; - let test = TestDatasetHttp::start(store).await; - let root = root.replace(&old_origin, &test.origin); - let draft = draft.replace(&old_origin, &test.origin); - let status = response(client.get(&draft), 200).await; - assert_eq!(status["progress"]["next_page"], first["next_page"]); - let first_prepared = common::prepare(client, &draft, "1").await; - (test, root, snapshot, draft, first_prepared) -} diff --git a/app/crowdb-access-server/tests/dataset_v2_contract_test.rs b/app/crowdb-access-server/tests/dataset_v2_contract_test.rs index 709116b6f..50d28eb84 100644 --- a/app/crowdb-access-server/tests/dataset_v2_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_v2_contract_test.rs @@ -39,6 +39,7 @@ async fn capabilities_use_existing_auth_and_match_configured_operations() { "inspection-preparation", "dataset-management", "split-revisions", + "split-preparation", "split-head-update", "stateless-reads" ]) diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index 34d61a2eb..5c77d287e 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -104,11 +104,13 @@ independent data head transition selects that resource as latest. ## 3. Split evaluation and publication -Standalone Split drafts pin owner Snapshot and expected current revision at begin. -Preparation checks source availability; deletion may interrupt unfinished work. Whole-set edits invalidate -evaluation; prepare requires complete membership and valid overlap/coverage policy. -Defaults are disjoint and allow-unassigned. Frozen inherited membership is explicit; -reevaluation is a deliberate definition change. +Standalone Split preparation accepts an owner Snapshot, caller-owned revision ID, +policy and complete definitions in one resource PUT. Preparation checks source +availability; deletion may interrupt unfinished work. It requires valid overlap/ +coverage policy. Defaults are disjoint and allow-unassigned. Frozen inherited +membership is explicit; reevaluation is a deliberate definition change. The caller +captures the intended currentSplit revision/generation before computation and keeps +that base separate from the immutable result. Predicates support all, explicit IDs, prefix, range, keyword, inline field match and boolean composition over the selected universe. Effective optional inheritance and @@ -124,7 +126,7 @@ not a promise of exact cardinality. Overlap/coverage validation remains independ Membership pages are immutable and checksum-bound. Preparation persists the whole revision before currentSplit CAS. A conflict preserves the candidate and caller inputs; retry never silently rebases definitions or overwrites another revision. -The native `prepare_split_revision` alternative accepts caller-owned revision ID, +The native `prepare_split_revision` operation accepts caller-owned revision ID, policy and complete definitions. It recomputes incomplete work from those inputs, writing only final member pages, the resource enumeration index and complete revision root. The index precedes pages; the root is installed last. No draft, @@ -148,9 +150,15 @@ after a later revision replaces it, the stale request returns a precondition conflict without replaying historical success or rewinding the pointer. A definite CAS conflict uses its returned head; a transport error remains unknown. Legacy receipts and attempt-bearing heads remain readable without backfilling receipts. -The draft publication route is a compatibility adapter for its prepared target and -expected revision; `publish_split_revision` accepts these caller-owned inputs -directly. New draft status remains Prepared after publication. +`PUT snapshots/{snapshot}/split-revisions/{revision}` exposes preparation with the +`split-preparation` capability. Rust `PrepareSplitRevision` and synchronous/async +Python `prepare_split_revision` retain the exact resource ID and complete input on +retry; an error carries the revision identity. Missing capability never invokes a +draft or v1 mutation. Standalone begin/edit/evaluate/prepare/publish workflow routes +return 501 before storage access; SDK workflow methods are removed. Only legacy +draft observation, abandonment and cleanup remain for existing records. Composition +initial Split computation still uses its internal work records until that flow is +replaced; the standalone route never invokes it. `PUT snapshots/{snapshot}/split-head` accepts the target revision and both explicit nullable expected revision/generation. It requires the `split-head-update` capability and has no draft or attempt identity. The failed head condition is diff --git a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml index 4d13e0251..e7740ac3c 100644 --- a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml +++ b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml @@ -13,11 +13,14 @@ info: bounded-inspection; explicit POST inspection preparation requires inspection-preparation. Dataset creation, stable/retention changes and receipts require dataset-management; direct Snapshot head updates require snapshot-head-update; - Split lifecycle requires split-revisions; direct Split head updates require split-head-update; versioned + Split reads require split-revisions; complete Split preparation requires split-preparation; + direct Split head updates require split-head-update; versioned stateless reads require stateless-reads. Upload, composition and Snapshot retirement require uploads, composition and snapshot-reclaim respectively, advertised only when the upload service is configured. Query parameters are strict and duplicate - keys fail. Composition JSON bodies are limited to 1 MiB. + keys fail. Composition and Split preparation JSON bodies are limited to 1 MiB. + Standalone Split begin/edit/evaluate/prepare/publish workflow routes return 501; + legacy draft reads, abandonment and cleanup remain available for old records. servers: - url: /api/dataset/v2 security: @@ -369,6 +372,44 @@ paths: default: $ref: '#/components/responses/InspectionError' /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/split-revisions/{revision}: + put: + operationId: prepareSplitRevision + summary: Prepare complete immutable membership from caller-owned inputs + description: >- + Requires split-preparation. The caller retains revision ID, policy and + complete definitions. No operation, edit or evaluation cursor is accepted. + Computation is bounded to 30 seconds and 1 MiB input/final bindings. + Retry the same ID and input after an unknown outcome; incomplete work is + recomputed, while an identical complete result is reused. Different input + at a complete ID conflicts. Frozen sources retain their supplied revision. + This operation does not update currentSplit; capture its intended base + before computation and update split-head separately. No draft fallback. + parameters: + - {$ref: '#/components/parameters/dataset'} + - {$ref: '#/components/parameters/snapshot'} + - in: path + name: revision + required: true + schema: {$ref: '#/components/schemas/OpaqueId'} + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/PrepareSplitRevision'} + responses: + '200': + description: Complete immutable resource, readable before head publication + content: + application/json: + schema: {$ref: '#/components/schemas/SplitRevision'} + '409': + description: Resource ID already has different input, or validation conflicts + $ref: '#/components/responses/InspectionError' + '413': + description: Request body exceeds 1 MiB + $ref: '#/components/responses/InspectionError' + default: + $ref: '#/components/responses/InspectionError' get: operationId: splitRevision description: Reads a complete immutable revision by ID without requiring currentSplit publication. Snapshot read admission applies. @@ -423,33 +464,10 @@ paths: schema: {$ref: '#/components/schemas/SamplePage'} default: $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/split-drafts: - post: - operationId: beginSplitDraft - parameters: - - in: path - name: dataset - required: true - schema: {type: string} - - in: path - name: snapshot - required: true - schema: {type: string} - requestBody: - required: true - content: - application/json: - schema: {$ref: '#/components/schemas/BeginSplit'} - responses: - '200': - description: Revision-bound operation result - content: - application/json: - schema: {$ref: '#/components/schemas/SplitDraft'} - default: - $ref: '#/components/responses/InspectionError' /namespaces/dataset-ns/datasets/{dataset}/split-drafts/{operation}: get: + deprecated: true + description: Read-only observation of an existing legacy draft; never resumes work. operationId: splitDraft parameters: - in: path @@ -468,31 +486,9 @@ paths: schema: {$ref: '#/components/schemas/SplitDraft'} default: $ref: '#/components/responses/InspectionError' - put: - operationId: editSplitDraft - parameters: - - in: path - name: dataset - required: true - schema: {type: string} - - in: path - name: operation - required: true - schema: {type: string} - requestBody: - required: true - content: - application/json: - schema: {$ref: '#/components/schemas/EditSplit'} - responses: - '200': - description: Revision-bound operation result - content: - application/json: - schema: {$ref: '#/components/schemas/SplitDraft'} - default: - $ref: '#/components/responses/InspectionError' delete: + deprecated: true + description: Abandon an existing legacy draft for cleanup. operationId: abandonSplitDraft parameters: - in: path @@ -516,82 +512,6 @@ paths: schema: {$ref: '#/components/schemas/SplitDraft'} default: $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/split-drafts/{operation}/evaluate: - post: - operationId: evaluateSplitDraft - parameters: - - in: path - name: dataset - required: true - schema: {type: string} - - in: path - name: operation - required: true - schema: {type: string} - requestBody: - required: true - content: - application/json: - schema: {$ref: '#/components/schemas/EvaluateDraft'} - responses: - '200': - description: Revision-bound operation result - content: - application/json: - schema: {$ref: '#/components/schemas/SplitPreview'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/split-drafts/{operation}/prepare: - post: - operationId: prepareSplitDraft - parameters: - - in: path - name: dataset - required: true - schema: {type: string} - - in: path - name: operation - required: true - schema: {type: string} - requestBody: - required: true - content: - application/json: - schema: {$ref: '#/components/schemas/DraftRevision'} - responses: - '200': - description: Revision-bound operation result - content: - application/json: - schema: {$ref: '#/components/schemas/SplitRevision'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/split-drafts/{operation}/publish: - post: - operationId: publishSplitDraft - description: Conditionally updates currentSplit without a request ledger or draft publication progress. The same current revision confirms the current effect; a later revision makes a stale retry conflict. Unknown outcomes propagate without a confirmation query. - parameters: - - in: path - name: dataset - required: true - schema: {type: string} - - in: path - name: operation - required: true - schema: {type: string} - requestBody: - required: true - content: - application/json: - schema: {$ref: '#/components/schemas/PublishSplit'} - responses: - '200': - description: Revision-bound operation result - content: - application/json: - schema: {$ref: '#/components/schemas/CommitReceipt'} - default: - $ref: '#/components/responses/InspectionError' /namespaces/dataset-ns/datasets/{dataset}/reads/bind: post: operationId: bindRead @@ -820,6 +740,7 @@ paths: $ref: '#/components/responses/InspectionError' /namespaces/dataset-ns/datasets/{dataset}/split-drafts/{operation}/cleanup: post: + deprecated: true operationId: cleanupSplitDraft summary: Retire obsolete Split work parameters: @@ -1449,22 +1370,11 @@ components: name: {type: string} definition: {$ref: '#/components/schemas/SplitDefinition'} origin: {$ref: '#/components/schemas/SplitOrigin'} - BeginSplit: - type: object - additionalProperties: false - required: [operation, expected_revision, policy, splits] - properties: - operation: {$ref: '#/components/schemas/OpaqueId'} - expected_revision: {$ref: '#/components/schemas/NullableRevision'} - policy: {$ref: '#/components/schemas/SplitPolicy'} - splits: {type: array, items: {$ref: '#/components/schemas/DraftSplit'}} - EditSplit: + PrepareSplitRevision: type: object additionalProperties: false - required: [edit, expected_revision, policy, splits] + required: [policy, splits] properties: - edit: {$ref: '#/components/schemas/OpaqueId'} - expected_revision: {$ref: '#/components/schemas/Decimal'} policy: {$ref: '#/components/schemas/SplitPolicy'} splits: {type: array, items: {$ref: '#/components/schemas/DraftSplit'}} DraftRevision: @@ -1488,15 +1398,6 @@ components: revision: {$ref: '#/components/schemas/OpaqueId'} expected_revision: {$ref: '#/components/schemas/NullableRevision'} expected_head: {$ref: '#/components/schemas/NullableGeneration'} - PublishSplit: - type: object - additionalProperties: false - required: [attempt, revision, expected_head] - properties: - attempt: {$ref: '#/components/schemas/OpaqueId'} - revision: {$ref: '#/components/schemas/OpaqueId'} - expected_head: {$ref: '#/components/schemas/NullableGeneration'} - description: Expected revision was pinned by BeginSplit. Null head is explicit; omission is invalid. A conflict retains the candidate. CommitTarget: oneOf: - type: object diff --git a/doc/design/access-server/dataset/spec-crowdb-dataset.md b/doc/design/access-server/dataset/spec-crowdb-dataset.md index 83cfe521d..9181969f2 100644 --- a/doc/design/access-server/dataset/spec-crowdb-dataset.md +++ b/doc/design/access-server/dataset/spec-crowdb-dataset.md @@ -230,6 +230,15 @@ base MUST fail without rewinding currentSplit. It does not promise historical response replay. Unknown storage outcomes MUST propagate without a confirmation query. Existing legacy receipts remain readable; new Split attempts are absent from commit-status history. +Independent Split preparation MUST accept caller-owned revision ID, policy and +complete definitions through `PUT snapshots/{snapshot}/split-revisions/{revision}` +with the `split-preparation` capability. It MUST NOT persist editing/evaluation +progress or invoke draft APIs. Input and final bindings are bounded to 1 MiB and +computation to 30 seconds. An identical complete result MUST be reusable without +recomputation; different input at that ID MUST conflict. Incomplete work is retried +from client inputs. SDKs MUST retain the same ID/input and never downgrade to a +workflow mutation. Standalone begin/edit/evaluate/prepare/publish routes return +501; legacy draft observation and cleanup are for existing records only. Direct Split head updates MUST accept the complete resource ID and explicit nullable expected revision and head generation without an operation/attempt ID or draft lookup. A failed head condition MUST return the observed revision/generation, diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 4afc4d9eb..b906aa5fa 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -372,11 +372,16 @@ SB/SD/SX/SW 和 SCG 清理游标描述编辑与计算过程，按 D1 删除。`s 继续使用已有观察值 CAS；随后迁移 REST/SDK 并停止旧续算写入口。源 revision 固定，重试 不得用当前 Split 重算历史成员；partial SM 的发现和迟到写隔离与 R4/P13 一起验证。 -- [ ] **P3.2b 无状态 Split REST/SDK**：以完整资源 `PUT snapshots/{snapshot}/split-revisions/{revision}` +- [~] **P3.2b 无状态 Split REST/SDK**：以完整资源 `PUT snapshots/{snapshot}/split-revisions/{revision}` 提交 policy/definitions，不使用 operation/draft/expected page。补能力、鉴权、1 MiB body 限制、错误、Rust/Python SDK 和真实 consumer；相同请求重试不新建 revision，不降级旧 draft 流程。迁移 standalone Split 示例/容器流程后关闭旧续算写入口；initial Split 的 内部工作状态与 Composition 一起清理，不能仅新增 API 后保留全部旧生产写路径。 + REST/SDK、旧写入口关闭和 HTTP/consumer 迁移已实现；14 项 HTTP/Rust consumer、35 项 + 独立 SDK 源码包测试、11 项 installed Python 单测及真实 sync/async Python consumer 通过。 + Dataset/Access/e2e-test 和独立 SDK all-target clippy、fmt、boundary、OpenAPI 322 个本地 + 引用通过；wheel 位于 `.crowdb-runtime/artifacts/sdk-split-preparation-20261024`。 + 新镜像与五项 Dataset 重启用例尚待验证，本项暂不关闭。 - [ ] **P3.2 实现选定 Split 流程**：按 D1 删除辅助进度与检查点，客户端重试，保留 currentSplit 发布 CAS、覆盖/重叠校验和旧 revision 读取。Files: `split/{publication,recovery,cleanup}.rs`、v2/SDK Split 和恢复测试。 diff --git a/e2e-test/tests/access_dataset_split.rs b/e2e-test/tests/access_dataset_split.rs index 1ee0316f0..c3531136a 100644 --- a/e2e-test/tests/access_dataset_split.rs +++ b/e2e-test/tests/access_dataset_split.rs @@ -186,61 +186,23 @@ async fn revise(client: &Client, snapshot: &str) { ) .await; let old = head["revision"].as_str().unwrap(); - let operation = "24242424242424242424242424242424"; - let draft = format!("/split-drafts/{operation}"); - let begin = json!({"operation":operation,"expected_revision":old,"policy":{"overlap":"disjoint","coverage":"allow_unassigned"},"splits":[]}); - call( - client, - Method::POST, - &format!("/snapshots/{snapshot}/split-drafts"), - Some(&begin), - 200, - ) - .await; - evaluate_split(client, &draft).await; - let prepared = call( - client, - Method::POST, - &format!("{draft}/prepare"), - Some(&json!({"expected_revision":"1"})), - 200, - ) - .await; + let revision = "24242424242424242424242424242424"; + let resource = format!("/snapshots/{snapshot}/split-revisions/{revision}"); + let input = json!({"policy":{"overlap":"disjoint","coverage":"allow_unassigned"},"splits":[]}); + let prepared = call(client, Method::PUT, &resource, Some(&input), 200).await; + assert_eq!(call(client, Method::GET, &resource, None, 200).await, prepared); + restart().await; assert_eq!( - call( - client, - Method::GET, - &format!( - "/snapshots/{snapshot}/split-revisions/{}", - prepared["revision"].as_str().unwrap() - ), - None, - 200 - ) - .await, + call(client, Method::PUT, &resource, Some(&input), 200).await, prepared ); - restart().await; - let publish = json!({"attempt":"25252525252525252525252525252525","revision":prepared["revision"],"expected_head":head["generation"]}); - let receipt = call( - client, - Method::POST, - &format!("{draft}/publish"), - Some(&publish), - 200, - ) - .await; + let update = json!({"revision":revision,"expected_revision":old,"expected_head":head["generation"]}); + let head_url = format!("/snapshots/{snapshot}/split-head"); + let current = call(client, Method::PUT, &head_url, Some(&update), 200).await; restart().await; assert_eq!( - call( - client, - Method::POST, - &format!("{draft}/publish"), - Some(&publish), - 200 - ) - .await, - receipt + call(client, Method::PUT, &head_url, Some(&update), 200).await, + current ); assert_eq!(call(client, Method::GET, "/head", None, 200).await, data_head); assert_eq!( @@ -266,25 +228,3 @@ async fn revise(client: &Client, snapshot: &str) { json!(["AAH/"]) ); } - -async fn evaluate_split(client: &Client, draft: &str) { - let mut page = json!("0"); - for _ in 0..100 { - let preview = call( - client, - Method::POST, - &format!("{draft}/evaluate"), - Some(&json!({"expected_revision":"1","expected_page":page})), - 200, - ) - .await; - if preview["complete"] == true { - assert_eq!(preview["validation"]["scanned"], "1"); - assert_eq!(preview["validation"]["unassigned"], "1"); - return; - } - assert_ne!(preview["next_page"], page, "evaluation must advance"); - page = preview["next_page"].clone(); - } - panic!("Split evaluation did not finish"); -} diff --git a/sdk/README.md b/sdk/README.md index 0745665cc..bdd6d85c0 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -206,7 +206,7 @@ permission or promise that a redeployed endpoint still supports the operation. `prepare_composition` are resumable bounded steps. Keep operation/edit IDs and input revisions. Preparation is ready to publish only when `uploads.publication_ready` is true. Initial Splits publish with the Snapshot. -- `publish_composition` advances latest; `publish_split` updates only currentSplit +- `publish_composition` advances latest; `update_split_head` updates only currentSplit on its owning Snapshot. Null expected heads/revisions are explicit. On an unknown outcome, retry the exact request. `commit_status` exposes only legacy receipts; new Snapshot/Split publication saves no request ledger. A retry @@ -223,6 +223,17 @@ permission or promise that a redeployed endpoint still supports the operation. - Complete Split revisions can be read or used as frozen sources before updating currentSplit, after a head conflict, or after abandoning their draft. Each new read still checks Snapshot availability; incomplete revisions remain unavailable. +- `prepare_split_revision(snapshot, revision, request)` submits the complete policy + and definitions to an immutable resource PUT. Keep the revision ID and exact + input for retries. It requires `split-preparation`; different input at a complete + ID conflicts. There is no draft or evaluation cursor, and incomplete work is + recomputed. Input and final bindings are bounded to 1 MiB; computation has a + 30-second deadline. The complete result is readable before head publication. + Capture the intended head base before preparation. Rust uses + `PrepareSplitRevision`; synchronous and async Python expose the same operation. + Standalone Split workflow SDK methods are removed; their old HTTP write routes + return 501, with no fallback. Composition initial Splits still use composition + preparation until its workflow replacement is complete. - `update_split_head(snapshot, request)` updates a complete revision directly with `revision`, `expected_revision` and `expected_head`; both expected values must be supplied, including null. It requires `split-head-update`, takes no attempt ID diff --git a/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs b/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs index c929f8905..8d65f7767 100644 --- a/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs +++ b/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs @@ -203,40 +203,21 @@ async fn revise_splits( ) -> Result<()> { let head = client.head().await?; let split_head = client.split_head(snapshot).await?; - let begin = v2::BeginSplit { - operation: v2::OpaqueId::random(), - expected_revision: Some(previous.clone()), + let revision_id = v2::OpaqueId::random(); + let prepare = v2::PrepareSplitRevision { policy: v2::SplitPolicy::default(), - splits: vec![], + splits: Vec::new(), }; - client.begin_split(snapshot, &begin).await?; - let mut page = 0.into(); - let mut done = false; - for _ in 0..100 { - let p = client - .evaluate_split( - &begin.operation, - &v2::EvaluateDraft { - expected_revision: 1.into(), - expected_page: page, - }, - ) - .await?; - if p.complete { - done = true; - break; - } - page = p.next_page; - } - assert!(done); let revision = client - .prepare_split( - &begin.operation, - &v2::DraftRevision { - expected_revision: 1.into(), - }, - ) + .prepare_split_revision(snapshot, &revision_id, &prepare) .await?; + assert_eq!( + client + .prepare_split_revision(snapshot, &revision_id, &prepare) + .await? + .revision, + revision.revision + ); assert_eq!( client .split_revision(snapshot, &revision.revision) diff --git a/sdk/crates/crowdb-sdk-dataset/src/capabilities.rs b/sdk/crates/crowdb-sdk-dataset/src/capabilities.rs index f1e83c717..71cda1b56 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/capabilities.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/capabilities.rs @@ -7,7 +7,7 @@ use crate::{DatasetClient, Error}; use serde::{Deserialize, Serialize}; use std::sync::atomic::Ordering; -const KNOWN_CAPABILITIES: [&str; 11] = [ +const KNOWN_CAPABILITIES: [&str; 12] = [ "head-inspection", "bounded-inspection", "inspection-preparation", @@ -19,6 +19,7 @@ const KNOWN_CAPABILITIES: [&str; 11] = [ "composition", "snapshot-reclaim", "snapshot-head-update", + "split-preparation", ]; fn capability_bit(capability: &str) -> u64 { diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2.rs b/sdk/crates/crowdb-sdk-dataset/src/v2.rs index 9a9321e18..194ce36fb 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/v2.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/v2.rs @@ -14,6 +14,7 @@ mod requests; mod snapshot_head; mod split; mod split_head; +mod split_preparation; mod state; mod transfer; mod upload; @@ -27,6 +28,7 @@ pub use read::*; pub use snapshot_head::UpdateSnapshotHead; pub use split::*; pub use split_head::UpdateSplitHead; +pub use split_preparation::PrepareSplitRevision; pub use state::*; pub use upload::*; diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2/lifecycle.rs b/sdk/crates/crowdb-sdk-dataset/src/v2/lifecycle.rs index cd4465abb..38fe487a7 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/v2/lifecycle.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/v2/lifecycle.rs @@ -1,9 +1,8 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. use super::{ - BeginComposition, BeginSplit, CommitReceipt, CompositionCleanup, CompositionDraft, CompositionPreview, - ControlClient, DraftRevision, EditComposition, EditSplit, EvaluateDraft, OpaqueId, PublishComposition, - PublishSplit, SplitCleanup, SplitDraft, SplitHead, SplitPreview, SplitRevision, + BeginComposition, CommitReceipt, CompositionCleanup, CompositionDraft, CompositionPreview, ControlClient, + DraftRevision, EditComposition, EvaluateDraft, OpaqueId, PublishComposition, SplitHead, SplitRevision, }; use crate::Error; @@ -124,125 +123,6 @@ impl ControlClient { /// Executes one bounded operation; retain request IDs for exact retries. /// # Errors /// Returns capability, conflict, invalid input, transport or service errors. - pub async fn begin_split(&self, snapshot: &OpaqueId, request: &BeginSplit) -> Result { - self.call( - "split-revisions", - "POST", - &["snapshots", snapshot.as_str(), "split-drafts"], - Some(request), - ) - .await - .map_err(|error| error.with_identity(request.operation.as_str())) - } - /// Executes one bounded operation; retain request IDs for exact retries. - /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. - pub async fn split_draft(&self, operation: &OpaqueId) -> Result { - self.call::<(), _>( - "split-revisions", - "GET", - &["split-drafts", operation.as_str()], - None, - ) - .await - } - /// Executes one bounded operation; retain request IDs for exact retries. - /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. - pub async fn edit_split(&self, operation: &OpaqueId, request: &EditSplit) -> Result { - self.call( - "split-revisions", - "PUT", - &["split-drafts", operation.as_str()], - Some(request), - ) - .await - .map_err(|error| error.with_identity(request.edit.as_str())) - } - /// Executes one bounded operation; retain request IDs for exact retries. - /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. - pub async fn abandon_split( - &self, - operation: &OpaqueId, - request: &DraftRevision, - ) -> Result { - self.call( - "split-revisions", - "DELETE", - &["split-drafts", operation.as_str()], - Some(request), - ) - .await - .map_err(|error| error.with_identity(operation.as_str())) - } - /// Executes one bounded operation; retain request IDs for exact retries. - /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. - pub async fn evaluate_split( - &self, - operation: &OpaqueId, - request: &EvaluateDraft, - ) -> Result { - self.call( - "split-revisions", - "POST", - &["split-drafts", operation.as_str(), "evaluate"], - Some(request), - ) - .await - .map_err(|error| error.with_identity(operation.as_str())) - } - /// Executes one bounded operation; retain request IDs for exact retries. - /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. - pub async fn prepare_split( - &self, - operation: &OpaqueId, - request: &DraftRevision, - ) -> Result { - self.call( - "split-revisions", - "POST", - &["split-drafts", operation.as_str(), "prepare"], - Some(request), - ) - .await - .map_err(|error| error.with_identity(operation.as_str())) - } - /// Executes one bounded operation; retain request IDs for exact retries. - /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. - pub async fn publish_split( - &self, - operation: &OpaqueId, - request: &PublishSplit, - ) -> Result { - self.call( - "split-revisions", - "POST", - &["split-drafts", operation.as_str(), "publish"], - Some(request), - ) - .await - .map_err(|error| error.with_identity(request.attempt.as_str())) - } - /// Executes one bounded operation; retain request IDs for exact retries. - /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. - pub async fn cleanup_split(&self, operation: &OpaqueId) -> Result { - self.call::<(), _>( - "split-revisions", - "POST", - &["split-drafts", operation.as_str(), "cleanup"], - None, - ) - .await - .map_err(|error| error.with_identity(operation.as_str())) - } - /// Executes one bounded operation; retain request IDs for exact retries. - /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. pub async fn split_head(&self, snapshot: &OpaqueId) -> Result { self.call::<(), _>( "split-revisions", diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2/split.rs b/sdk/crates/crowdb-sdk-dataset/src/v2/split.rs index 33ef104b8..a5ceb8f82 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/v2/split.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/v2/split.rs @@ -1,6 +1,6 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{required_option, Decimal, DraftProgress, Generation, OpaqueId, SampleId}; +use super::{required_option, Decimal, Generation, OpaqueId, SampleId}; use serde::{Deserialize, Serialize}; /// Binary operands use canonical base64; explicit IDs retain binary identity. @@ -76,34 +76,6 @@ pub struct DraftSplit { pub origin: SplitOrigin, } #[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct BeginSplit { - pub operation: OpaqueId, - #[serde(deserialize_with = "required_option")] - pub expected_revision: Option, - pub policy: SplitPolicy, - pub splits: Vec, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct EditSplit { - pub edit: OpaqueId, - pub expected_revision: Decimal, - pub policy: SplitPolicy, - pub splits: Vec, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SplitDraft { - pub operation: OpaqueId, - pub snapshot: OpaqueId, - #[serde(deserialize_with = "required_option")] - pub expected_revision: Option, - pub revision: Decimal, - pub policy: SplitPolicy, - pub splits: Vec, - pub progress: DraftProgress, -} -#[derive(Clone, Debug, Serialize, Deserialize)] pub struct PublishedSplit { pub name: String, pub definition: SplitDefinition, @@ -137,14 +109,6 @@ pub struct SplitPreview { pub validation: SplitValidation, } #[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PublishSplit { - pub attempt: OpaqueId, - pub revision: OpaqueId, - #[serde(deserialize_with = "required_option")] - pub expected_head: Option, -} -#[derive(Clone, Debug, Serialize, Deserialize)] pub struct SplitHead { #[serde(deserialize_with = "required_option")] pub revision: Option, diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2/split_preparation.rs b/sdk/crates/crowdb-sdk-dataset/src/v2/split_preparation.rs new file mode 100644 index 000000000..6221afce6 --- /dev/null +++ b/sdk/crates/crowdb-sdk-dataset/src/v2/split_preparation.rs @@ -0,0 +1,38 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ControlClient, DraftSplit, OpaqueId, SplitPolicy, SplitRevision}; +use crate::Error; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PrepareSplitRevision { + pub policy: SplitPolicy, + pub splits: Vec, +} +impl ControlClient { + /// Prepares immutable membership from complete caller input, without editing or progress state. + /// # Errors + /// Returns unsupported capabilities, identity conflicts, source/policy errors and transfer failures. + pub async fn prepare_split_revision( + &self, + snapshot: &OpaqueId, + revision: &OpaqueId, + request: &PrepareSplitRevision, + ) -> Result { + self.call( + "split-preparation", + "PUT", + &[ + "snapshots", + snapshot.as_str(), + "split-revisions", + revision.as_str(), + ], + Some(request), + ) + .await + .map_err(|error| error.with_identity(revision.as_str())) + } +} diff --git a/sdk/crates/crowdb-sdk-dataset/tests/split_preparation_test.rs b/sdk/crates/crowdb-sdk-dataset/tests/split_preparation_test.rs new file mode 100644 index 000000000..264e704b6 --- /dev/null +++ b/sdk/crates/crowdb-sdk-dataset/tests/split_preparation_test.rs @@ -0,0 +1,92 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[allow(dead_code)] +#[path = "../../crowdb-sdk-core/tests/common/peer.rs"] +mod peer; +use crowdb_sdk_dataset::{ + v2::{OpaqueId, PrepareSplitRevision, SplitPolicy}, + DatasetClient, +}; +use serde_json::json; + +fn input() -> PrepareSplitRevision { + PrepareSplitRevision { + policy: SplitPolicy::default(), + splits: Vec::new(), + } +} +#[test] +fn preparation_has_no_draft_identity_or_progress_fields() { + let mut value = serde_json::to_value(input()).unwrap(); + assert_eq!( + value, + json!({"policy":{"overlap":"disjoint", "coverage":"allow_unassigned"}, "splits":[]}) + ); + for field in ["operation", "expected_page", "expected_revision"] { + value[field] = json!("1"); + assert!(serde_json::from_value::(value.clone()).is_err()); + value.as_object_mut().unwrap().remove(field); + } +} + +#[tokio::test] +async fn unavailable_preparation_never_downgrades_to_the_old_draft_protocol() { + let mut peer = + peer::TestPeer::start(r#"{"api_version":2,"capabilities":["split-revisions"]}"#, false).await; + let client = DatasetClient::new(&peer.endpoint, "test", None) + .unwrap() + .control(); + let revision = OpaqueId::random(); + assert!(client + .prepare_split_revision(&OpaqueId::random(), &revision, &input()) + .await + .is_err()); + assert!(peer + .requests + .recv() + .await + .unwrap() + .starts_with("GET /api/dataset/v2/capabilities ")); + assert!(peer.requests.try_recv().is_err()); +} + +#[tokio::test] +async fn transport_retry_preserves_the_resource_id_and_full_input_without_discovery_or_status_queries() { + let mut peer = peer::TestPeer::start_responses(vec![ + (200, r#"{"api_version":2,"capabilities":["split-preparation"]}"#), + (503, r#"{"code":"unavailable","message":"unknown outcome","request_id":"test","retryable":true}"#), + (200, r#"{"snapshot":"01010101010101010101010101010101","revision":"02020202020202020202020202020202","operation":"02020202020202020202020202020202","evaluator_version":1,"policy":{"overlap":"disjoint","coverage":"allow_unassigned"},"splits":[]}"#), + ], false).await; + let client = DatasetClient::new(&peer.endpoint, "test", None) + .unwrap() + .control(); + let snapshot: OpaqueId = "01".repeat(16).parse().unwrap(); + let revision: OpaqueId = "02".repeat(16).parse().unwrap(); + let request = input(); + assert_eq!( + client + .prepare_split_revision(&snapshot, &revision, &request) + .await + .unwrap() + .revision, + revision + ); + assert!(peer + .requests + .recv() + .await + .unwrap() + .starts_with("GET /api/dataset/v2/capabilities ")); + for _ in 0..2 { + let call = peer.requests.recv().await.unwrap(); + assert!(call.starts_with(&format!( + "PUT /api/dataset/v2/namespaces/dataset-ns/datasets/test/snapshots/{}/split-revisions/{} ", + snapshot.as_str(), + revision.as_str() + ))); + let body: serde_json::Value = serde_json::from_str(call.split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!(body, serde_json::to_value(&request).unwrap()); + } + assert!(peer.requests.try_recv().is_err()); +} diff --git a/sdk/crates/crowdb-sdk-python/src/control.rs b/sdk/crates/crowdb-sdk-python/src/control.rs index 24ab11ed6..e32600983 100644 --- a/sdk/crates/crowdb-sdk-python/src/control.rs +++ b/sdk/crates/crowdb-sdk-python/src/control.rs @@ -70,15 +70,12 @@ async fn lifecycle(client: &v2::ControlClient, name: &str, value: &Value) -> Res "publish_composition" => encode(&client.publish_composition(&id()?, &request(value)?).await?), "abandon_composition" => encode(&client.abandon_composition(&id()?, &request(value)?).await?), "cleanup_composition" => encode(&client.cleanup_composition(&id()?).await?), - "begin_split" => encode(&client.begin_split(&id()?, &request(value)?).await?), - "split_draft" => encode(&client.split_draft(&id()?).await?), - "edit_split" => encode(&client.edit_split(&id()?, &request(value)?).await?), - "evaluate_split" => encode(&client.evaluate_split(&id()?, &request(value)?).await?), - "prepare_split" => encode(&client.prepare_split(&id()?, &request(value)?).await?), - "publish_split" => encode(&client.publish_split(&id()?, &request(value)?).await?), + "prepare_split_revision" => encode( + &client + .prepare_split_revision(&id()?, &self::id(value, "revision")?, &request(value)?) + .await?, + ), "update_split_head" => encode(&client.update_split_head(&id()?, &request(value)?).await?), - "abandon_split" => encode(&client.abandon_split(&id()?, &request(value)?).await?), - "cleanup_split" => encode(&client.cleanup_split(&id()?).await?), "split_head" => encode(&client.split_head(&id()?).await?), "split_revision" => encode( &client diff --git a/sdk/crowdb_dataset/control.py b/sdk/crowdb_dataset/control.py index e5e22fa80..d1cf28142 100644 --- a/sdk/crowdb_dataset/control.py +++ b/sdk/crowdb_dataset/control.py @@ -26,7 +26,7 @@ def record(self): def _identity(arguments): request = arguments.get("request") or {} - return request.get("attempt", request.get("edit", request.get("operation", request.get("id", request.get("snapshot", arguments.get("id")))))) + return request.get("attempt", request.get("edit", request.get("operation", request.get("id", request.get("snapshot", arguments.get("revision", arguments.get("id"))))))) class ControlClient: @@ -72,18 +72,9 @@ def abandon_composition(self, operation, expected_revision): return self._call("abandon_composition", {"id": operation, "request": {"expected_revision": expected_revision}}) def cleanup_composition(self, operation): return self._call("cleanup_composition", {"id": operation}) - def begin_split(self, snapshot, request): return self._call("begin_split", {"id": snapshot, "request": request}) - def split_draft(self, operation): return self._call("split_draft", {"id": operation}) - def edit_split(self, operation, request): return self._call("edit_split", {"id": operation, "request": request}) - def evaluate_split(self, operation, expected_revision, expected_page): - return self._call("evaluate_split", {"id": operation, "request": {"expected_revision": expected_revision, "expected_page": expected_page}}) - def prepare_split(self, operation, expected_revision): - return self._call("prepare_split", {"id": operation, "request": {"expected_revision": expected_revision}}) - def publish_split(self, operation, request): return self._call("publish_split", {"id": operation, "request": request}) + def prepare_split_revision(self, snapshot, revision, request): + return self._call("prepare_split_revision", {"id": snapshot, "revision": revision, "request": request}) def update_split_head(self, snapshot, request): return self._call("update_split_head", {"id": snapshot, "request": request}) - def abandon_split(self, operation, expected_revision): - return self._call("abandon_split", {"id": operation, "request": {"expected_revision": expected_revision}}) - def cleanup_split(self, operation): return self._call("cleanup_split", {"id": operation}) def split_head(self, snapshot): return self._call("split_head", {"id": snapshot}) def split_revision(self, snapshot, revision): return self._call("split_revision", {"id": snapshot, "revision": revision}) diff --git a/sdk/tests/split_consumer.py b/sdk/tests/split_consumer.py index 72c8f3612..fcc6603e2 100644 --- a/sdk/tests/split_consumer.py +++ b/sdk/tests/split_consumer.py @@ -61,10 +61,13 @@ def upload_source(client, source, upload_id, length, md5, part_size=1024*1024): return client.complete_upload(upload_id) -async def verify_async(endpoint, name, token, snapshot, revision): +async def verify_async(endpoint, name, token, snapshot, revision, prepared_revision): async with AsyncDatasetClient(endpoint, name, token) as owner: client = owner.control() assert (await client.lifecycle(snapshot))["snapshot"] == snapshot + prepared = await client.prepare_split_revision(snapshot, prepared_revision, { + "policy": {"overlap": "disjoint", "coverage": "allow_unassigned"}, "splits": []}) + assert prepared["revision"] == prepared_revision try: await client.update_snapshot_head({"snapshot": snapshot, "expected_head": None}) except ConflictError as error: @@ -120,16 +123,10 @@ def main(): revision = head["revision"] assert client.matrix(snapshot, [sample], ["value"])["entries"][0]["fields"][0]["length"] == str(len(data)) assert client.preview(snapshot, sample, "value")["data"] == sample_id(data) - split_op = new_id() - client.begin_split(snapshot, {"operation": split_op, "expected_revision": revision, - "policy": {"overlap": "disjoint", "coverage": "allow_unassigned"}, "splits": []}) - page = "0" - for _ in range(100): - preview = client.evaluate_split(split_op, "1", page) - if preview["complete"]: break - page = preview["next_page"] - else: raise AssertionError("Split evaluation budget") - candidate = client.prepare_split(split_op, "1") + revision_id = new_id() + split_input = {"policy": {"overlap": "disjoint", "coverage": "allow_unassigned"}, "splits": []} + candidate = client.prepare_split_revision(snapshot, revision_id, split_input) + assert client.prepare_split_revision(snapshot, revision_id, split_input) == candidate update = {"revision": candidate["revision"], "expected_revision": revision, "expected_head": head["generation"]} changed = client.update_split_head(snapshot, update) try: @@ -155,7 +152,7 @@ def main(): raise AssertionError("stale Snapshot update must conflict") cleanup(client, child) assert client.summary(child_snapshot)["parent"] == snapshot - asyncio.run(verify_async(endpoint, name, token, snapshot, revision)) + asyncio.run(verify_async(endpoint, name, token, snapshot, revision, candidate["revision"])) print("installed Python Split SDK round trip passed", flush=True) diff --git a/sdk/tests/test_control.py b/sdk/tests/test_control.py index 2c7f0396e..7d9f5e972 100644 --- a/sdk/tests/test_control.py +++ b/sdk/tests/test_control.py @@ -61,6 +61,22 @@ def __exit__(self, *_): class TestControl(unittest.TestCase): + def test_split_preparation_uses_the_resource_path_and_preserves_its_identity(self): + snapshot, revision = new_id(), new_id() + request = {"policy": {"overlap": "disjoint", "coverage": "allow_unassigned"}, "splits": []} + prepared = dict(request, snapshot=snapshot, revision=revision, operation=revision, evaluator_version=1) + error = {"code": "conflict", "message": "resource ID has different input", "request_id": "test", "retryable": False} + with TestControlPeer(["split-preparation"], head_replies=[(409, error), (200, prepared)]) as peer, DatasetClient(peer.endpoint, "test") as owner: + control = owner.control() + with self.assertRaises(ConflictError) as failure: control.prepare_split_revision(snapshot, revision, request) + self.assertEqual(failure.exception.identity, revision) + self.assertEqual(control.prepare_split_revision(snapshot, revision, request), prepared) + self.assertTrue(peer.requests.get(timeout=10).endswith("/capabilities")) + path = f"/api/dataset/v2/namespaces/dataset-ns/datasets/test/snapshots/{snapshot}/split-revisions/{revision}" + self.assertEqual(peer.requests.get(timeout=10), (path, request)) + self.assertEqual(peer.requests.get(timeout=10), (path, request)) + self.assertTrue(peer.requests.empty()) + def test_snapshot_head_conflict_uses_the_observation_without_a_head_get(self): snapshot = new_id() observed = {"latest": new_id(), "stable": None, "generation": "11" * 32} From f4a6e0846e0551eedc60af21ac8bc1a6977d23a6 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 21:46:57 +0800 Subject: [PATCH 134/258] Record stateless split container validation and next composition work --- doc/working/plan-access-simplification.md | 37 +++++++++++++++++------ 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index b906aa5fa..2a0ce5e3b 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -356,6 +356,15 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 - [ ] **P2.1 分清输入、计算进度与候选内容**：逐项列出 draft/edit receipt/work page/row/candidate/ cleanup seal 的用途、写入次数和调用者；按 D1 更新 API/SDK 合同，移除服务端续算接口。Files: Dataset `composition/{begin,edit,state,evaluation,evaluate_step,prepare,cleanup}.rs`。 +- [~] **P2.2a 原生无状态 Composition 准备**：完整 parent/source/schema/changes/Split 输入， + source 和继承 revision 均显式固定，不在重试时重新解析 currentSplit。每次有界重算， + 直接生成最终 Manifest/index/初始 SM/SRI/SR 和 payload owners；完整根最后安装。资源 + Snapshot ID 由客户端指定，不另写 operation→随机 Snapshot 的过程索引，不写 CBG/CD/ + CE/CX/CW/CM 或 staging owner/进度。已有最终资源键的条件隔离保留；中途失败输出由 + Snapshot/成员前缀发现，R4/P13 仍需补独立过期/迟到写扫描。输入/输出均以既有 8 MiB + Manifest 上限约束，窗口二十个 sample、外层 30 秒；大输入最终页合同仍在 P2.2。 + 比较总 GET/scan/mutation，验证新路径写集合、部分结果不可读、丢回复重算、冻结继承、 + owner 与回收竞争；REST/SDK 和所有旧调用者随后迁移，不能仅新增入口后关闭整个 P2。 - [ ] **P2.2 删除服务端编辑/计算过程**：删除 durable 编辑/进度阶段，客户端失败重新请求； 保持发布前不可见、expected head 及源 Snapshot 被删时的明确错误。孤儿清理通过 R4 验证，例外封口记录先 review。Files: `composition/`、Access v2 composition、SDK 和恢复测试。 @@ -372,16 +381,6 @@ SB/SD/SX/SW 和 SCG 清理游标描述编辑与计算过程，按 D1 删除。`s 继续使用已有观察值 CAS；随后迁移 REST/SDK 并停止旧续算写入口。源 revision 固定，重试 不得用当前 Split 重算历史成员；partial SM 的发现和迟到写隔离与 R4/P13 一起验证。 -- [~] **P3.2b 无状态 Split REST/SDK**：以完整资源 `PUT snapshots/{snapshot}/split-revisions/{revision}` - 提交 policy/definitions，不使用 operation/draft/expected page。补能力、鉴权、1 MiB body - 限制、错误、Rust/Python SDK 和真实 consumer；相同请求重试不新建 revision，不降级旧 - draft 流程。迁移 standalone Split 示例/容器流程后关闭旧续算写入口；initial Split 的 - 内部工作状态与 Composition 一起清理，不能仅新增 API 后保留全部旧生产写路径。 - REST/SDK、旧写入口关闭和 HTTP/consumer 迁移已实现；14 项 HTTP/Rust consumer、35 项 - 独立 SDK 源码包测试、11 项 installed Python 单测及真实 sync/async Python consumer 通过。 - Dataset/Access/e2e-test 和独立 SDK all-target clippy、fmt、boundary、OpenAPI 322 个本地 - 引用通过；wheel 位于 `.crowdb-runtime/artifacts/sdk-split-preparation-20261024`。 - 新镜像与五项 Dataset 重启用例尚待验证，本项暂不关闭。 - [ ] **P3.2 实现选定 Split 流程**：按 D1 删除辅助进度与检查点，客户端重试，保留 currentSplit 发布 CAS、覆盖/重叠校验和旧 revision 读取。Files: `split/{publication,recovery,cleanup}.rs`、v2/SDK Split 和恢复测试。 @@ -549,6 +548,24 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，P3.2b**：新增 `split-preparation` capability 与完整资源 PUT，严格 policy/ + definitions、鉴权、1 MiB body/30 秒计算边界；Rust/Python sync/async SDK 保存 revision + 身份并以同一输入重试，不查 draft/status，不降级。删除 standalone Split SDK 流程方法， + HTTP begin/edit/evaluate/prepare/publish 返回 501 且零存储访问；旧记录只保留观察、废弃和 + cleanup。14 项 HTTP/Rust consumer、35 项独立源码包、11 项 installed Python 单测与真实 + sync/async consumer 通过；Dataset/Access/e2e-test、SDK all-target clippy、fmt、boundary、 + OpenAPI 322 个本地引用通过，均经 pixi。45 sample 准备 14G/8C，同完整结果重试 3G/0C； + 不包含 scan 和 routed RPC。wheel `.crowdb-runtime/artifacts/sdk-split-preparation-20261024`。 + 从 `7f54415e780fd202745a4ca54e8b2de3a8444d48` 构建镜像 + `crowdb-node:access-cleanup-split-preparation-20261024`，五项 Dataset 容器用例全部通过； + 包含完整结果未发布读取、相同资源跨重启 PUT、currentSplit CAS 重试和历史成员保持。 + artifact `.crowdb-runtime/artifacts/container-e2e/run-cff52437c80847cfa1ef2ddd7b9535bd`， + source tree `43245f37763b5a45057078640917de46cd62986a`， + OCI `sha256:7f1af28e9cc85a2d96f8354e550858793449b37d5bc7eb385db5636296b32033`， + Docker `sha256:6f0a6f533a9d475b2d852806315b70b40554b614b2d2cb6ce23164ae910c76e9`。 + 此为本机 TCP 功能/重启证明，未声称 TPS/HCA 提升。Composition 初始 Split 仍用内部 + 工作状态，由 P2/P3.2 继续删除；独立孤儿/迟到新页重扫仍归 R4/P13。 + - **2026-10-24，P3.1/P3.2a**：完成 Composition/Split 过程记录与最终资源边界盘点，见 P2/P3；新增原生 `SplitPreparation` / `prepare_split_revision`。每次接收完整客户端输入， 从固定 Snapshot 和继承 revision 重算；只写 SM/SRI/SR，不生成 SB/SD/SX/SW 或进度。 From c26c654384b197aefb6c88b281d36f90412e4411 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 22:08:17 +0800 Subject: [PATCH 135/258] Build complete composition resources without editing or evaluation progress --- .../dataset/design-dataset-control.md | 34 +++ .../dataset/spec-crowdb-dataset.md | 11 + doc/working/plan-access-simplification.md | 36 ++- lib/crowdb-access-dataset/src/authority.rs | 16 +- .../src/authority/publication.rs | 74 +++-- .../src/authority/publication/candidate.rs | 20 ++ lib/crowdb-access-dataset/src/composition.rs | 3 + .../src/composition/resource.rs | 214 +++++++++++++ .../src/composition/resource/input.rs | 216 +++++++++++++ .../src/composition/resource/source.rs | 86 ++++++ lib/crowdb-access-dataset/src/manifest.rs | 2 +- .../src/split/inheritance.rs | 17 +- .../src/split/preparation.rs | 74 ++++- .../tests/common/composition_resource.rs | 56 ++++ .../tests/common/split_store.rs | 62 ++++ .../composition_resource_reclaim_test.rs | 85 ++++++ .../composition_resource_recovery_test.rs | 236 ++++++++++++++ .../tests/composition_resource_test.rs | 289 ++++++++++++++++++ 18 files changed, 1477 insertions(+), 54 deletions(-) create mode 100644 lib/crowdb-access-dataset/src/composition/resource.rs create mode 100644 lib/crowdb-access-dataset/src/composition/resource/input.rs create mode 100644 lib/crowdb-access-dataset/src/composition/resource/source.rs create mode 100644 lib/crowdb-access-dataset/tests/common/composition_resource.rs create mode 100644 lib/crowdb-access-dataset/tests/composition_resource_reclaim_test.rs create mode 100644 lib/crowdb-access-dataset/tests/composition_resource_recovery_test.rs create mode 100644 lib/crowdb-access-dataset/tests/composition_resource_test.rs diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index 5c77d287e..cc09b83ac 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -70,6 +70,40 @@ remain readable. An empty set is a revision, distinct from an absent pointer. ## 2. Composition and inheritance +The native `prepare_composition_resource` accepts a caller-owned Snapshot ID and +complete parent/source/schema/changes/Split inputs. It computes a bounded final +Manifest directly, then writes its immutable partitions and inspection index, +initial Split membership and final payload owners. It completes the existing +resource root last; latest/stable are updated separately. There is no operation +mapping, draft, edit receipt, working row, window intent, staging owner or durable +computation cursor. The native input and final Manifest are bounded to 8 MiB and +the request to 30 seconds, with twenty-Sample source windows. Partial field patches +keep other effective parent fields; source filtering does not prevent an explicit +upsert from reintroducing a parent Sample. Each patch ID appears once. Frozen +Split sources and inherited definitions name exact revisions; retries never resolve +currentSplit again. Patch application moves existing rows and tracks encoded size +incrementally rather than serializing the growing Manifest after every change. + +A retry recomputes incomplete work and reuses matching complete content. The +existing content binding follows all final Manifest/index rows, so retrying a +failed owner claim does not rewrite those rows; it still checks unfinished initial +membership and owners before completion. The Snapshot ID fixes its Manifest and initial Split binding; different final content +conflicts, including removing an existing initial Split. Identical complete retries +write no state. Unknown completion replies return directly without a confirmation +read. Resource preparation checks Dataset identity without loading the head. +Managed upload metadata is validated once per unique upload before output writes, +and the final Snapshot owner is acquired before completion. The final Manifest +finds every owner for cleanup; there is no parallel ownership-progress list. Opaque +legacy locators inherited from a parent retain their existing storage contract; +new chunk patches must be valid managed upload locators. + +Resource prefixes and the immutable Snapshot record expose interrupted output to +owner retirement. Existing member pages are sealed after the grace period, and +recomputation cannot complete a retired Snapshot. Independent unused-resource +expiry and discovery of arbitrarily late unseen keys remain separate work. The +REST/SDK Composition workflow below still uses the legacy path until migration; +the new native operation does not call it. + `CompositionRecord` persists the operation, input revision, source, pinned parent Split revision, edits, evaluation progress, candidate and ownership state. Edits use expected-revision CAS and immutable edit receipts. Historical retries return diff --git a/doc/design/access-server/dataset/spec-crowdb-dataset.md b/doc/design/access-server/dataset/spec-crowdb-dataset.md index 9181969f2..0db5daaf5 100644 --- a/doc/design/access-server/dataset/spec-crowdb-dataset.md +++ b/doc/design/access-server/dataset/spec-crowdb-dataset.md @@ -204,6 +204,17 @@ latest. A head conflict MUST return the observed latest/stable/generation withou a confirmation query. A reviewed retry uses that observation with caller-chosen content; the server MUST NOT rebase. Unknown storage outcomes MUST propagate. Existing legacy receipts remain readable without backfill. +Native Composition resource preparation MUST accept complete caller-owned inputs +and a fixed Snapshot ID without persisting editing or evaluation progress. Its +fixed source/Split revisions MUST survive retries unchanged. It MUST install the +complete resource root only after final Manifest/index, initial Split membership +and managed payload owners are ready; it MUST NOT move latest/stable. Input and +final Manifest are bounded to 8 MiB and computation to 30 seconds. Incomplete +computation is redone; identical complete content is reusable with zero writes. +Different resource content, including its initial Split binding, MUST conflict. +Unknown completion outcomes MUST NOT cause a confirmation query. This native +contract does not imply that the legacy Composition REST workflow is migrated. + Composition and Split status reads MUST NOT reconstruct a publication request, help edit receipts or persist recovery state. Legacy Committing drafts are presented as Prepared without rewriting the stored draft or querying attempt, diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 2a0ce5e3b..d77b94c1e 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -356,15 +356,13 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 - [ ] **P2.1 分清输入、计算进度与候选内容**：逐项列出 draft/edit receipt/work page/row/candidate/ cleanup seal 的用途、写入次数和调用者；按 D1 更新 API/SDK 合同，移除服务端续算接口。Files: Dataset `composition/{begin,edit,state,evaluation,evaluate_step,prepare,cleanup}.rs`。 -- [~] **P2.2a 原生无状态 Composition 准备**：完整 parent/source/schema/changes/Split 输入， - source 和继承 revision 均显式固定，不在重试时重新解析 currentSplit。每次有界重算， - 直接生成最终 Manifest/index/初始 SM/SRI/SR 和 payload owners；完整根最后安装。资源 - Snapshot ID 由客户端指定，不另写 operation→随机 Snapshot 的过程索引，不写 CBG/CD/ - CE/CX/CW/CM 或 staging owner/进度。已有最终资源键的条件隔离保留；中途失败输出由 - Snapshot/成员前缀发现，R4/P13 仍需补独立过期/迟到写扫描。输入/输出均以既有 8 MiB - Manifest 上限约束，窗口二十个 sample、外层 30 秒；大输入最终页合同仍在 P2.2。 - 比较总 GET/scan/mutation，验证新路径写集合、部分结果不可读、丢回复重算、冻结继承、 - owner 与回收竞争；REST/SDK 和所有旧调用者随后迁移，不能仅新增入口后关闭整个 P2。 +- [~] **P2.2b 无状态 Composition REST/SDK**：新增资源 `PUT snapshots/{snapshot}` 与独立 + `composition-preparation` capability，完整输入/明确 body 与计算边界；Rust/Python sync/async + SDK 以同一 Snapshot ID/完整请求重试，不查询草稿或降级。upload 字段直传已有完整结果 + 的 id/length/md5，转换不再为补 descriptor 额外查询；原生校验真实元数据后 claim 最终 owner。 + source/继承 revision 显式固定，移除隐式 currentSplit 续算。迁移 HTTP/SDK/容器的全部 + Composition 写调用者后关闭 begin/edit/evaluate/prepare/publish；保留必要 legacy decoder/GC + 到 P13，不能用包装旧方法代替删除工作状态。补成本、鉴权、错误、丢回复与重启验收。 - [ ] **P2.2 删除服务端编辑/计算过程**：删除 durable 编辑/进度阶段，客户端失败重新请求； 保持发布前不可见、expected head 及源 Snapshot 被删时的明确错误。孤儿清理通过 R4 验证，例外封口记录先 review。Files: `composition/`、Access v2 composition、SDK 和恢复测试。 @@ -548,6 +546,24 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，P2.2a**：新增原生 `CompositionRequest` / `prepare_composition_resource`， + 客户端指定 Snapshot ID 与完整源/修改/规则；直接构造最终 Manifest、索引、初始 Split 和 + 最终 payload owner，完整根最后安装。无 operation→随机 Snapshot、CBG/CD/CE/CX/CW/CM/ + SB/SD/SX/SW、staging owner 或计算进度写。source/继承 revision 固定，稀疏 patch 保留 + 其他有效父字段；同一 patch ID 只出现一次。输入/输出各 8 MiB、二十 sample 窗口、30 秒。 + Map 内移动旧行并增量计大小；已有最终内容绑定可复用，owner 回复丢失后重试仅需两次 + 最终根写，不重写全部 Manifest/index；unknown owner/完整根回复均直接返回，不确认读。 + 同完整资源重试零写；异内容或移除已有 initial Split 冲突。原 Dataset 读取 helper 会顺带 + 查 head，新 Resource 只验证 Dataset 记录，准备阶段不读 head。 + 45 sample、两 Split 的 native 成本从 368G + 21S + 237C（626 次）降为 + 187G + 14S + 153C（354 次）；这是逻辑 storage 调用，尚非 routed RPC/TPS。 + 35 项相关 native 测试（含九项新资源/cost/故障/回收用例）和六项真实 HTTP 兼容测试通过； + Dataset/Access/native SDK all-target/all-feature clippy、fmt、diff 通过，均经 pixi。 + 覆盖 frozen source/继承、部分成员不可读、跨实例重算、owner/完整根丢回复、错误 descriptor + 零写、共享 upload 在逐个 Snapshot grace/回收后才可释放、部分结果前缀发现与已见页封口。 + REST/SDK/旧入口关闭由 P2.2b 完成；独立过期、迟到未见 key 与大输入最终页合同仍在 + P2.2/R4/P13，不能视为整个 Composition/Split 清理完成。 + - **2026-10-24，P3.2b**：新增 `split-preparation` capability 与完整资源 PUT，严格 policy/ definitions、鉴权、1 MiB body/30 秒计算边界；Rust/Python sync/async SDK 保存 revision 身份并以同一输入重试，不查 draft/status，不降级。删除 standalone Split SDK 流程方法， @@ -560,7 +576,7 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 `crowdb-node:access-cleanup-split-preparation-20261024`，五项 Dataset 容器用例全部通过； 包含完整结果未发布读取、相同资源跨重启 PUT、currentSplit CAS 重试和历史成员保持。 artifact `.crowdb-runtime/artifacts/container-e2e/run-cff52437c80847cfa1ef2ddd7b9535bd`， - source tree `43245f37763b5a45057078640917de46cd62986a`， + source tree `43245f37763b5a45057078640917de46cd62986a`，server_sources_match=true； OCI `sha256:7f1af28e9cc85a2d96f8354e550858793449b37d5bc7eb385db5636296b32033`， Docker `sha256:6f0a6f533a9d475b2d852806315b70b40554b614b2d2cb6ce23164ae910c76e9`。 此为本机 TCP 功能/重启证明，未声称 TPS/HCA 提升。Composition 初始 Split 仍用内部 diff --git a/lib/crowdb-access-dataset/src/authority.rs b/lib/crowdb-access-dataset/src/authority.rs index 202e65470..2233584fb 100644 --- a/lib/crowdb-access-dataset/src/authority.rs +++ b/lib/crowdb-access-dataset/src/authority.rs @@ -132,12 +132,7 @@ impl DatasetAuthority { /// # Errors /// Returns malformed storage or a missing Dataset. pub async fn get_dataset(&self, identity: &DatasetIdentity) -> Result { - let value = self - .store - .get(&key::dataset(identity)?) - .await? - .ok_or(AuthorityError::NotFound)?; - let mut dataset: DatasetRecord = decode(&value.bytes)?; + let mut dataset = self.read_dataset_record(identity).await?; if let Some(head) = self.store.get(&key::head(identity)?).await? { let head = decode_head(&head.bytes)?; dataset.latest = head.latest; @@ -146,6 +141,15 @@ impl DatasetAuthority { Ok(dataset) } + async fn read_dataset_record(&self, identity: &DatasetIdentity) -> Result { + let value = self + .store + .get(&key::dataset(identity)?) + .await? + .ok_or(AuthorityError::NotFound)?; + decode(&value.bytes) + } + /// # Errors /// Returns a missing or malformed head. pub async fn latest(&self, identity: &DatasetIdentity) -> Result, AuthorityError> { diff --git a/lib/crowdb-access-dataset/src/authority/publication.rs b/lib/crowdb-access-dataset/src/authority/publication.rs index 395410fa0..3eaba47cb 100644 --- a/lib/crowdb-access-dataset/src/authority/publication.rs +++ b/lib/crowdb-access-dataset/src/authority/publication.rs @@ -17,6 +17,7 @@ enum PublicationMode { LegacyPrepare, LegacyPublish, Candidate(Option), + Resource(Option), } use super::{decode, decode_head, decode_manifest, encode, DatasetAuthority}; @@ -136,11 +137,15 @@ impl DatasetAuthority { operation: OperationId, mode: PublicationMode, ) -> Result { - self.get_dataset(identity).await?; + if matches!(mode, PublicationMode::Resource(_)) { + self.read_dataset_record(identity).await?; + } else { + self.get_dataset(identity).await?; + } let mut manifest = match decode_manifest(&manifest_bytes) { Ok(manifest) => manifest, Err(AuthorityError::Invalid(DatasetError::InvalidManifest)) - if !matches!(mode, PublicationMode::Candidate(_)) => + if !matches!(mode, PublicationMode::Candidate(_) | PublicationMode::Resource(_)) => { return self .prepare_opaque_snapshot(identity, parent, manifest_bytes, operation) @@ -166,7 +171,9 @@ impl DatasetAuthority { } } let operation_key = key::operation(identity, operation)?; - let snapshot = if let Some(value) = self.store.get(&operation_key).await? { + let snapshot = if matches!(mode, PublicationMode::Resource(_)) { + SnapshotId::from_bytes(operation.as_bytes())? + } else if let Some(value) = self.store.get(&operation_key).await? { if value.bytes == retirement::RETIRED { return Err(DatasetError::InvalidPublicationState.into()); } @@ -218,30 +225,47 @@ impl DatasetAuthority { { return Err(AuthorityError::AlreadyExists); } - if existing.publication.state == PublicationState::Published - && (self - .snapshot_candidate(identity, snapshot) - .await? - .is_some_and(|c| c.version == 2) - || self.snapshot_commit(identity, snapshot).await?.is_some()) - { - if matches!(mode, PublicationMode::LegacyPublish) { - return self.advance_head(identity, None, parent, snapshot).await; - } - return Ok(snapshot); - } - if existing.publication.state == PublicationState::Published - && self.snapshot_candidate(identity, snapshot).await?.is_none() - { - match self.require_committed_snapshot(identity, snapshot).await { - Ok(()) => return Ok(snapshot), - Err(AuthorityError::SnapshotNotFound) => {} - Err(error) => return Err(error), - } - } if existing.publication.state == PublicationState::Aborted { return Err(AuthorityError::Invalid(DatasetError::InvalidPublicationState)); } + if let PublicationMode::Resource(initial) = mode { + if let Some(mut content) = self.snapshot_candidate(identity, snapshot).await? { + let reusable = matches!(content.version, 2 | 3); + let expected = + candidate::binding(snapshot, parent, operation, &manifest_bytes, initial); + content.version = expected.version; + if content != expected { + return Err(AuthorityError::AlreadyExists); + } + // The content binding follows all immutable Manifest and inspection rows. + // Initial membership and payload owners are checked by final completion. + if reusable { + return Ok(snapshot); + } + } + } else { + if existing.publication.state == PublicationState::Published + && (self + .snapshot_candidate(identity, snapshot) + .await? + .is_some_and(|c| c.version == 2) + || self.snapshot_commit(identity, snapshot).await?.is_some()) + { + if matches!(mode, PublicationMode::LegacyPublish) { + return self.advance_head(identity, None, parent, snapshot).await; + } + return Ok(snapshot); + } + if existing.publication.state == PublicationState::Published + && self.snapshot_candidate(identity, snapshot).await?.is_none() + { + match self.require_committed_snapshot(identity, snapshot).await { + Ok(()) => return Ok(snapshot), + Err(AuthorityError::SnapshotNotFound) => {} + Err(error) => return Err(error), + } + } + } } None => match self .store @@ -263,7 +287,7 @@ impl DatasetAuthority { operation, &manifest_bytes, match mode { - PublicationMode::Candidate(initial) => initial, + PublicationMode::Candidate(initial) | PublicationMode::Resource(initial) => initial, _ => None, }, ), diff --git a/lib/crowdb-access-dataset/src/authority/publication/candidate.rs b/lib/crowdb-access-dataset/src/authority/publication/candidate.rs index 00b34d19b..b75ae03d5 100644 --- a/lib/crowdb-access-dataset/src/authority/publication/candidate.rs +++ b/lib/crowdb-access-dataset/src/authority/publication/candidate.rs @@ -78,6 +78,26 @@ impl DatasetAuthority { .await } + pub(crate) async fn prepare_composition_content( + &self, + identity: &DatasetIdentity, + parent: Option, + manifest: Vec, + snapshot: SnapshotId, + initial: Option, + ) -> Result<(), AuthorityError> { + let operation = OperationId::from_bytes(snapshot.as_bytes())?; + self.write_publication( + identity, + parent, + manifest, + operation, + PublicationMode::Resource(initial), + ) + .await?; + Ok(()) + } + pub(in crate::authority) async fn bind_snapshot_candidate( &self, identity: &DatasetIdentity, diff --git a/lib/crowdb-access-dataset/src/composition.rs b/lib/crowdb-access-dataset/src/composition.rs index 1b678c3db..b359e38ef 100644 --- a/lib/crowdb-access-dataset/src/composition.rs +++ b/lib/crowdb-access-dataset/src/composition.rs @@ -3,6 +3,9 @@ //! Server-owned child composition drafts and revision-bound evaluation state. +mod resource; +pub use resource::{CompositionRequest, CompositionResult}; + mod record; pub use record::{CompositionRecord, CompositionSource, DraftProgress}; diff --git a/lib/crowdb-access-dataset/src/composition/resource.rs b/lib/crowdb-access-dataset/src/composition/resource.rs new file mode 100644 index 000000000..1e917e271 --- /dev/null +++ b/lib/crowdb-access-dataset/src/composition/resource.rs @@ -0,0 +1,214 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Complete caller-owned composition inputs and final immutable resources. + +mod input; +mod source; + +use super::{CompositionError, CompositionSource, SampleChange}; +use crate::{ + split::{DraftSplit, SplitPolicy, SplitPreparation}, + AuthorityError, DatasetAuthority, DatasetIdentity, DatasetUploads, FieldLocator, ManifestRecord, + OperationId, ReadCancellation, SchemaRecord, SnapshotId, SplitRevisionId, UploadError, +}; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, time::Duration}; + +const MAX_MANIFEST_BYTES: u64 = 8 * 1024 * 1024; + +/// One complete computation, with frozen source identities and no editing revision. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CompositionRequest { + pub parent: Option, + pub source: CompositionSource, + pub schema: Option, + pub samples: Vec, + pub splits: Vec, + pub policy: SplitPolicy, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct CompositionResult { + pub snapshot: SnapshotId, + pub initial_split: Option, + pub sample_count: u64, +} + +impl DatasetAuthority { + /// Recomputes caller-owned inputs and completes the immutable Snapshot after its dependencies. + /// It writes no draft, edit receipt, evaluation progress or operation-to-Snapshot index. + /// # Errors + /// Rejects divergent resource content, unavailable sources, invalid rules, payloads and limits. + pub async fn prepare_composition_resource( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + request: &CompositionRequest, + uploads: Option<&DatasetUploads>, + cancel: &ReadCancellation, + ) -> Result { + request.validate(identity, snapshot)?; + tokio::time::timeout( + Duration::from_secs(30), + self.compute_composition_resource(identity, snapshot, request, uploads, cancel), + ) + .await + .map_err(|_| CompositionError::Limit)? + } + + async fn compute_composition_resource( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + request: &CompositionRequest, + uploads: Option<&DatasetUploads>, + cancel: &ReadCancellation, + ) -> Result { + cancelled(cancel)?; + let schema = self.composition_resource_schema(identity, request).await?; + self.validate_split_parent_sources(identity, request.parent, &request.splits) + .await?; + let manifest = self + .composition_resource_manifest(identity, snapshot, request, schema, cancel) + .await?; + let references = referenced_uploads(identity, &manifest)?; + validate_uploads(uploads, &references, cancel).await?; + let operation = OperationId::from_bytes(snapshot.as_bytes()).map_err(AuthorityError::from)?; + let initial_split = if request.splits.is_empty() { + None + } else { + Some(crate::split::candidate_revision(operation, 1)?) + }; + let bytes = bincode::serialize(&manifest).map_err(|_| CompositionError::Corrupt)?; + self.prepare_composition_content(identity, request.parent, bytes, snapshot, initial_split) + .await?; + if let Some(revision) = initial_split { + self.prepare_initial_split_revision( + identity, + snapshot, + &SplitPreparation { + revision, + policy: request.policy, + splits: request.splits.clone(), + }, + cancel, + ) + .await?; + } + claim_uploads(uploads, &references, operation, cancel).await?; + cancelled(cancel)?; + self.complete_snapshot_candidate(identity, snapshot).await?; + Ok(CompositionResult { + snapshot, + initial_split, + sample_count: u64::try_from(manifest.samples.len()).map_err(|_| CompositionError::Limit)?, + }) + } + + async fn composition_resource_schema( + &self, + identity: &DatasetIdentity, + request: &CompositionRequest, + ) -> Result { + let parent = match request.parent { + Some(parent) => Some(self.composition_parent_schema(identity, parent).await?), + None => None, + }; + let schema = request + .schema + .as_ref() + .or(parent.as_ref()) + .ok_or(CompositionError::Conflict)?; + super::state::validate_schema(schema)?; + if let Some(parent) = &parent { + super::edit::validate_schema_extension(parent, schema)?; + } + Ok(schema.clone()) + } +} + +struct ReferencedUpload<'a> { + location: &'a [u8], + length: u64, + md5: [u8; 16], +} + +fn referenced_uploads<'a>( + identity: &DatasetIdentity, + manifest: &'a ManifestRecord, +) -> Result>, CompositionError> { + let mut references: BTreeMap<_, ReferencedUpload<'_>> = BTreeMap::new(); + for sample in &manifest.samples { + for field in &sample.fields { + if let FieldLocator::Chunk { + location, + length, + md5, + } = &field.value + { + // Opaque legacy locators retain the parent's existing storage contract. + let Ok(id) = DatasetUploads::upload_id(identity, location) else { + continue; + }; + if let Some(existing) = references.get(&id) { + if existing.length != *length || existing.md5 != *md5 { + return Err(UploadError::Integrity.into()); + } + } else { + references.insert( + id, + ReferencedUpload { + location, + length: *length, + md5: *md5, + }, + ); + } + } + } + } + Ok(references) +} + +async fn validate_uploads( + uploads: Option<&DatasetUploads>, + references: &BTreeMap>, + cancel: &ReadCancellation, +) -> Result<(), CompositionError> { + for reference in references.values() { + cancelled(cancel)?; + let record = uploads + .ok_or(UploadError::Invalid)? + .metadata(reference.location) + .await?; + if record.length != reference.length || record.md5 != reference.md5 { + return Err(UploadError::Integrity.into()); + } + } + Ok(()) +} + +async fn claim_uploads( + uploads: Option<&DatasetUploads>, + references: &BTreeMap>, + owner: OperationId, + cancel: &ReadCancellation, +) -> Result<(), CompositionError> { + for reference in references.values() { + cancelled(cancel)?; + uploads + .ok_or(UploadError::Invalid)? + .claim(reference.location, owner, super::state::now()) + .await?; + } + Ok(()) +} + +fn cancelled(cancel: &ReadCancellation) -> Result<(), CompositionError> { + if cancel.is_cancelled() { + return Err(crate::inspection::InspectionError::Cancelled.into()); + } + Ok(()) +} diff --git a/lib/crowdb-access-dataset/src/composition/resource/input.rs b/lib/crowdb-access-dataset/src/composition/resource/input.rs new file mode 100644 index 000000000..b2ec68776 --- /dev/null +++ b/lib/crowdb-access-dataset/src/composition/resource/input.rs @@ -0,0 +1,216 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{cancelled, CompositionRequest, MAX_MANIFEST_BYTES}; +use crate::{ + composition::{CompositionError, CompositionSource, SampleChange}, + AuthorityError, DatasetAuthority, DatasetIdentity, DatasetUploads, FieldLocator, ManifestRecord, + ReadCancellation, SampleRecord, SchemaRecord, SnapshotId, +}; +use std::collections::{BTreeMap, BTreeSet}; + +impl CompositionRequest { + pub(super) fn validate( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + ) -> Result<(), CompositionError> { + SnapshotId::from_bytes(snapshot.as_bytes()).map_err(AuthorityError::from)?; + if self.parent == Some(snapshot) + || (self.parent.is_none() && self.source != CompositionSource::All) + || bincode::serialized_size(self).map_err(|_| CompositionError::Corrupt)? > MAX_MANIFEST_BYTES + { + return Err(CompositionError::Limit); + } + if let Some(parent) = self.parent { + SnapshotId::from_bytes(parent.as_bytes()).map_err(AuthorityError::from)?; + } + match &self.source { + CompositionSource::All => {} + CompositionSource::Selection(selection) => { + crate::split::definition::validate_selection(selection, 0)?; + } + CompositionSource::Split { source, selection } => { + if self.parent != Some(source.snapshot) { + return Err(CompositionError::Conflict); + } + crate::SplitRevisionId::from_bytes(source.revision.as_bytes()) + .map_err(AuthorityError::from)?; + if let Some(selection) = selection { + crate::split::definition::validate_selection(selection, 0)?; + } + } + } + super::super::state::validate_splits(&self.splits)?; + if let Some(schema) = &self.schema { + super::super::state::validate_schema(schema)?; + } + let mut ids = BTreeSet::new(); + for change in &self.samples { + let id = match change { + SampleChange::Upsert(sample) => { + let mut names = BTreeSet::new(); + if sample.fields.len() > 256 { + return Err(CompositionError::Limit); + } + for field in &sample.fields { + if field.name.is_empty() || field.name.len() > 1024 || !names.insert(&field.name) { + return Err(CompositionError::Conflict); + } + field.value.validate().map_err(AuthorityError::from)?; + if let FieldLocator::Chunk { location, .. } = &field.value { + DatasetUploads::upload_id(identity, location)?; + } + } + &sample.sample_id + } + SampleChange::Remove(id) => id, + }; + if id.is_empty() || id.len() > 16 * 1024 || !ids.insert(id) { + return Err(CompositionError::Conflict); + } + } + Ok(()) + } +} + +impl DatasetAuthority { + pub(super) async fn composition_resource_manifest( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + request: &CompositionRequest, + schema: SchemaRecord, + cancel: &ReadCancellation, + ) -> Result { + let mut builder = ManifestBuilder::new(snapshot, schema)?; + self.composition_resource_source(identity, request, &mut builder, cancel) + .await?; + if let Some(parent) = request.parent { + let missing: Vec<_> = request + .samples + .iter() + .filter_map(|change| match change { + SampleChange::Upsert(sample) if !builder.samples.contains_key(&sample.sample_id) => { + Some(sample.sample_id.clone()) + } + _ => None, + }) + .collect(); + for ids in missing.chunks(20) { + cancelled(cancel)?; + for sample in self + .composition_source_samples( + identity, + parent, + ids, + &builder.schema, + super::super::state::budget(), + cancel, + ) + .await? + { + builder.insert(sample)?; + } + } + } + for change in &request.samples { + cancelled(cancel)?; + builder.apply(change)?; + } + builder.finish(snapshot) + } +} + +pub(super) struct ManifestBuilder { + pub schema: SchemaRecord, + samples: BTreeMap, SampleRecord>, + bytes: u64, +} + +impl ManifestBuilder { + fn new(snapshot: SnapshotId, schema: SchemaRecord) -> Result { + let empty = ManifestRecord { + version: 1, + snapshot, + schema: schema.clone(), + samples: Vec::new(), + }; + Ok(Self { + schema, + samples: BTreeMap::new(), + bytes: bincode::serialized_size(&empty).map_err(|_| CompositionError::Corrupt)?, + }) + } + + pub(super) fn insert(&mut self, sample: SampleRecord) -> Result<(), CompositionError> { + let size = bincode::serialized_size(&sample).map_err(|_| CompositionError::Corrupt)?; + let old = self + .samples + .get(&sample.sample_id) + .map_or(Ok(0), bincode::serialized_size) + .map_err(|_| CompositionError::Corrupt)?; + let bytes = self + .bytes + .checked_sub(old) + .and_then(|n| n.checked_add(size)) + .ok_or(CompositionError::Limit)?; + if bytes > MAX_MANIFEST_BYTES { + return Err(CompositionError::Limit); + } + self.bytes = bytes; + self.samples.insert(sample.sample_id.clone(), sample); + Ok(()) + } + + fn apply(&mut self, change: &SampleChange) -> Result<(), CompositionError> { + match change { + SampleChange::Remove(id) => { + if let Some(sample) = self.samples.remove(id) { + self.bytes -= bincode::serialized_size(&sample).map_err(|_| CompositionError::Corrupt)?; + } + } + SampleChange::Upsert(patch) => { + if patch.fields.iter().any(|field| { + !self + .schema + .fields + .iter() + .any(|defined| defined.name == field.name) + }) { + return Err(CompositionError::Conflict); + } + let mut sample = if let Some(existing) = self.samples.remove(&patch.sample_id) { + self.bytes -= + bincode::serialized_size(&existing).map_err(|_| CompositionError::Corrupt)?; + existing + } else { + SampleRecord { + sample_id: patch.sample_id.clone(), + fields: Vec::new(), + } + }; + for field in &patch.fields { + sample.fields.retain(|existing| existing.name != field.name); + sample.fields.push(field.clone()); + } + self.insert(sample)?; + } + } + Ok(()) + } + + fn finish(self, snapshot: SnapshotId) -> Result { + let manifest = ManifestRecord { + version: 1, + snapshot, + schema: self.schema, + samples: self.samples.into_values().collect(), + }; + if manifest.samples.is_empty() { + return Err(CompositionError::Conflict); + } + manifest.validate().map_err(AuthorityError::from)?; + Ok(manifest) + } +} diff --git a/lib/crowdb-access-dataset/src/composition/resource/source.rs b/lib/crowdb-access-dataset/src/composition/resource/source.rs new file mode 100644 index 000000000..7650c7db0 --- /dev/null +++ b/lib/crowdb-access-dataset/src/composition/resource/source.rs @@ -0,0 +1,86 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{cancelled, input::ManifestBuilder, CompositionRequest}; +use crate::{ + composition::{CompositionError, CompositionSource}, + inspection::work::InspectionWork, + DatasetAuthority, DatasetIdentity, ReadCancellation, +}; + +impl DatasetAuthority { + pub(super) async fn composition_resource_source( + &self, + identity: &DatasetIdentity, + request: &CompositionRequest, + builder: &mut ManifestBuilder, + cancel: &ReadCancellation, + ) -> Result<(), CompositionError> { + let Some(parent) = request.parent else { + return Ok(()); + }; + let selection = match &request.source { + CompositionSource::All => None, + CompositionSource::Selection(selection) => Some(selection), + CompositionSource::Split { selection, .. } => selection.as_ref(), + }; + let budget = super::super::state::budget(); + let frozen = if let CompositionSource::Split { source, .. } = &request.source { + let record = self + .get_split_revision(identity, parent, source.revision, budget, cancel) + .await?; + if !record.splits.iter().any(|split| split.name == source.name) { + return Err(CompositionError::NotFound); + } + Some((source, record)) + } else { + None + }; + let mut after = None; + loop { + cancelled(cancel)?; + let mut page = match selection { + Some(selection) => { + self.inspection_selection_page(identity, parent, selection, after, budget, cancel) + .await? + } + None => { + self.inspection_samples_page(identity, parent, after, budget, cancel) + .await? + } + }; + if let Some((source, record)) = &frozen { + let mut work = InspectionWork::new(self.store.as_ref(), cancel, budget)?; + let mut members = Vec::new(); + for id in page.entries { + if self + .frozen_split_contains(identity, record, &source.name, &id, &mut work) + .await? + { + members.push(id); + } + } + page.entries = members; + } + if !page.entries.is_empty() { + for sample in self + .composition_source_samples( + identity, + parent, + &page.entries, + &builder.schema, + budget, + cancel, + ) + .await? + { + builder.insert(sample)?; + } + } + after = page.after; + if after.is_none() { + return Ok(()); + } + } + } +} diff --git a/lib/crowdb-access-dataset/src/manifest.rs b/lib/crowdb-access-dataset/src/manifest.rs index b9b06a09e..0e5bcddd1 100644 --- a/lib/crowdb-access-dataset/src/manifest.rs +++ b/lib/crowdb-access-dataset/src/manifest.rs @@ -140,7 +140,7 @@ impl SchemaRecord { } impl FieldLocator { - fn validate(&self) -> Result<(), DatasetError> { + pub(crate) fn validate(&self) -> Result<(), DatasetError> { match self { Self::Inline { value, md5 } => { if value.len() > MAX_INLINE_BYTES { diff --git a/lib/crowdb-access-dataset/src/split/inheritance.rs b/lib/crowdb-access-dataset/src/split/inheritance.rs index e708e6d99..4272b7ff5 100644 --- a/lib/crowdb-access-dataset/src/split/inheritance.rs +++ b/lib/crowdb-access-dataset/src/split/inheritance.rs @@ -54,11 +54,26 @@ impl DatasetAuthority { let owner = self .snapshot_inspection_summary(identity, snapshot, budget, &cancel) .await?; + self.validate_split_parent_sources(identity, owner.parent, splits) + .await + } + pub(crate) async fn validate_split_parent_sources( + &self, + identity: &DatasetIdentity, + parent: Option, + splits: &[DraftSplit], + ) -> Result<(), SplitLifecycleError> { + let budget = MetadataScanBudget { + items: 1, + bytes: 1024 * 1024, + timeout: Duration::from_secs(2), + }; + let cancel = ReadCancellation::new(); let mut work = InspectionWork::new(self.store.as_ref(), &cancel, budget)?; let mut revisions = BTreeMap::new(); for split in splits { if let MembershipOrigin::Inherited(source) = &split.origin { - if owner.parent != Some(source.snapshot) { + if parent != Some(source.snapshot) { return Err(SplitLifecycleError::Conflict); } let key = (source.snapshot, source.revision); diff --git a/lib/crowdb-access-dataset/src/split/preparation.rs b/lib/crowdb-access-dataset/src/split/preparation.rs index 2a8ca7de9..65db532bf 100644 --- a/lib/crowdb-access-dataset/src/split/preparation.rs +++ b/lib/crowdb-access-dataset/src/split/preparation.rs @@ -34,27 +34,39 @@ impl DatasetAuthority { input: &SplitPreparation, cancel: &ReadCancellation, ) -> Result { - SnapshotId::from_bytes(snapshot.as_bytes()).map_err(AuthorityError::from)?; - SplitRevisionId::from_bytes(input.revision.as_bytes()).map_err(AuthorityError::from)?; - super::draft::validate_definitions(snapshot, &input.splits)?; - super::codec::encode(input)?; + validate(snapshot, input)?; tokio::time::timeout( Duration::from_secs(30), - self.prepare_split_result(identity, snapshot, input, cancel), + self.prepare_split_result(identity, snapshot, input, cancel, false), ) .await .map_err(|_| SplitLifecycleError::Limit)? } + pub(crate) async fn prepare_initial_split_revision( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + input: &SplitPreparation, + cancel: &ReadCancellation, + ) -> Result { + validate(snapshot, input)?; + self.prepare_split_result(identity, snapshot, input, cancel, true) + .await + } + async fn prepare_split_result( &self, identity: &DatasetIdentity, snapshot: SnapshotId, input: &SplitPreparation, cancel: &ReadCancellation, + initial: bool, ) -> Result { cancelled(cancel)?; - self.require_committed_snapshot(identity, snapshot).await?; + let parent = self + .check_preparation_owner(identity, snapshot, input.revision, initial) + .await?; let root = key::revision(identity, snapshot, input.revision).map_err(AuthorityError::from)?; if let Some(stored) = self.split_state_get(&root).await? { if stored.bytes == super::RETIRED_RECORD { @@ -67,8 +79,13 @@ impl DatasetAuthority { Err(SplitLifecycleError::Conflict) }; } - self.validate_split_definitions(identity, snapshot, &input.splits) - .await?; + if initial { + self.validate_split_parent_sources(identity, parent, &input.splits) + .await?; + } else { + self.validate_split_definitions(identity, snapshot, &input.splits) + .await?; + } let mut record = result(snapshot, input)?; let encoded_size = record.encode()?.len(); let summary = SplitRevisionSummary { @@ -84,14 +101,40 @@ impl DatasetAuthority { &super::codec::encode(&summary)?, ) .await?; - self.compute_split_members(identity, input, &mut record, encoded_size, cancel) + self.compute_split_members(identity, input, &mut record, encoded_size, cancel, initial) .await?; cancelled(cancel)?; - self.require_committed_snapshot(identity, snapshot).await?; + self.check_preparation_owner(identity, snapshot, input.revision, initial) + .await?; self.put_split_state(&root, &record.encode()?).await?; Ok(record) } + async fn check_preparation_owner( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + revision: SplitRevisionId, + initial: bool, + ) -> Result, SplitLifecycleError> { + if !initial { + self.require_committed_snapshot(identity, snapshot).await?; + return Ok(None); + } + let candidate = self + .snapshot_candidate(identity, snapshot) + .await? + .ok_or(SplitLifecycleError::NotFound)?; + if candidate.initial_split != Some(revision) { + return Err(SplitLifecycleError::Conflict); + } + if let Some(parent) = candidate.parent { + self.require_committed_snapshot(identity, parent).await?; + } + self.check_snapshot_readable(identity, snapshot).await?; + Ok(candidate.parent) + } + async fn compute_split_members( &self, identity: &DatasetIdentity, @@ -99,6 +142,7 @@ impl DatasetAuthority { record: &mut SplitRevisionRecord, mut encoded_size: usize, cancel: &ReadCancellation, + initial: bool, ) -> Result<(), SplitLifecycleError> { let mut after = None; let mut validation = SplitValidation::default(); @@ -116,7 +160,7 @@ impl DatasetAuthority { timeout: Duration::from_secs(2), }, cancel, - false, + initial, ) .await?; validation.merge(&window.validation)?; @@ -192,6 +236,14 @@ impl DatasetAuthority { } } +fn validate(snapshot: SnapshotId, input: &SplitPreparation) -> Result<(), SplitLifecycleError> { + SnapshotId::from_bytes(snapshot.as_bytes()).map_err(AuthorityError::from)?; + SplitRevisionId::from_bytes(input.revision.as_bytes()).map_err(AuthorityError::from)?; + super::draft::validate_definitions(snapshot, &input.splits)?; + super::codec::encode(input)?; + Ok(()) +} + fn result( snapshot: SnapshotId, input: &SplitPreparation, diff --git a/lib/crowdb-access-dataset/tests/common/composition_resource.rs b/lib/crowdb-access-dataset/tests/common/composition_resource.rs new file mode 100644 index 000000000..02b350bdb --- /dev/null +++ b/lib/crowdb-access-dataset/tests/common/composition_resource.rs @@ -0,0 +1,56 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_access_dataset::{composition::*, split::SplitPolicy, *}; +use md5::{Digest, Md5}; +use std::sync::Arc; + +pub(super) struct TestUploadedResource { + pub uploads: DatasetUploads, + pub location: Vec, + pub request: CompositionRequest, +} +impl TestUploadedResource { + pub async fn new(store: Arc, identity: &DatasetIdentity) -> Self { + let uploads = DatasetUploads::new(store, Arc::new(crate::upload::TestPayload::default())); + let location = DatasetUploads::location(identity, OperationId::random()).unwrap(); + let md5 = Md5::digest(b"data").into(); + uploads.create(&location, 4, 4, md5, u64::MAX).await.unwrap(); + uploads + .put_part(&location, 0, Bytes::from_static(b"data"), 1) + .await + .unwrap(); + uploads.complete(&location, 2).await.unwrap(); + let request = CompositionRequest { + parent: None, + source: CompositionSource::All, + schema: Some(SchemaRecord { + version: 1, + fields: vec![FieldDefinition { + name: "marker".into(), + required: true, + }], + }), + samples: vec![SampleChange::Upsert(SampleRecord { + sample_id: b"uploaded".to_vec(), + fields: vec![FieldRecord { + name: "marker".into(), + value: FieldLocator::Chunk { + location: location.clone(), + length: 4, + md5, + }, + }], + })], + splits: Vec::new(), + policy: SplitPolicy::default(), + }; + + Self { + uploads, + location, + request, + } + } +} diff --git a/lib/crowdb-access-dataset/tests/common/split_store.rs b/lib/crowdb-access-dataset/tests/common/split_store.rs index d7d193054..09e7228f1 100644 --- a/lib/crowdb-access-dataset/tests/common/split_store.rs +++ b/lib/crowdb-access-dataset/tests/common/split_store.rs @@ -37,9 +37,62 @@ pub(super) struct TestDraftStore { pub lose_member_reply: AtomicBool, pub reject_split_proof_reads: AtomicBool, pub reject_draft_writes: AtomicBool, + pub reject_workflow_writes: AtomicBool, + pub lose_complete_reply: AtomicBool, + pub lose_claim_reply: AtomicBool, + pub reject_after_unknown: AtomicBool, } impl TestDraftStore { + fn check_unknown_reply(&self, key: &[u8], value: &[u8], result: &CasOutcome) -> Result<(), StoreError> { + if key.starts_with(b"DS\0CB\0") + && matches!(result, CasOutcome::Applied(_)) + && self.lose_complete_reply.load(Ordering::Relaxed) + && serde_json::from_slice::(value).is_ok_and(|v| v["version"] == 2) + { + self.lose_complete_reply.store(false, Ordering::Relaxed); + self.reject_after_unknown.store(true, Ordering::Relaxed); + return Err(StoreError::Rejected); + } + if key.starts_with(b"DS\0U\0") + && matches!(result, CasOutcome::Applied(_)) + && self.lose_claim_reply.load(Ordering::Relaxed) + && serde_json::from_slice::(value).is_ok_and(|v| v["state"] == "claimed") + { + self.lose_claim_reply.store(false, Ordering::Relaxed); + self.reject_after_unknown.store(true, Ordering::Relaxed); + return Err(StoreError::Rejected); + } + Ok(()) + } fn check_result_write(&self, key: &[u8]) { + assert!( + !self.reject_after_unknown.load(Ordering::Relaxed), + "mutation after unknown completion" + ); + if self.reject_workflow_writes.load(Ordering::Relaxed) { + assert!( + ![ + b"DS\0CBG\0".as_slice(), + b"DS\0CD\0", + b"DS\0CE\0", + b"DS\0CX\0", + b"DS\0CW\0", + b"DS\0CM\0", + b"DS\0SB\0", + b"DS\0SD\0", + b"DS\0SX\0", + b"DS\0SW\0", + b"DS\0O\0", + b"DS\0CA\0", + b"DS\0CR\0", + b"DS\0SC\0", + b"DS\0SRC\0" + ] + .iter() + .any(|prefix| key.starts_with(prefix)), + "stateless composition wrote process metadata" + ); + } if self.reject_draft_writes.load(Ordering::Relaxed) { assert!( [b"DS\0SR\0".as_slice(), b"DS\0SRI\0", b"DS\0SM\0"] @@ -53,6 +106,10 @@ impl TestDraftStore { #[async_trait::async_trait] impl DatasetStore for TestDraftStore { async fn get(&self, key: &[u8]) -> Result, StoreError> { + assert!( + !self.reject_after_unknown.load(Ordering::Relaxed), + "GET after unknown completion" + ); self.unbounded_reads.fetch_add(1, Ordering::Relaxed); if key.starts_with(b"DS\0H\0") { self.data_head_reads.fetch_add(1, Ordering::Relaxed); @@ -81,6 +138,10 @@ impl DatasetStore for TestDraftStore { budget: MetadataScanBudget, cancel: &ReadCancellation, ) -> Result, StoreError> { + assert!( + !self.reject_after_unknown.load(Ordering::Relaxed), + "bounded GET after unknown completion" + ); self.bounded_reads.fetch_add(1, Ordering::Relaxed); if key.starts_with(b"DS\0H\0") { self.data_head_reads.fetch_add(1, Ordering::Relaxed); @@ -191,6 +252,7 @@ impl DatasetStore for TestDraftStore { return Err(StoreError::Rejected); } let result = self.inner.compare_exchange(key, expected, value).await?; + self.check_unknown_reply(key, value, &result)?; if key.starts_with(b"DS\0SR\0") && matches!(result, CasOutcome::Applied(_)) && self.lose_revision_reply.swap(false, Ordering::Relaxed) diff --git a/lib/crowdb-access-dataset/tests/composition_resource_reclaim_test.rs b/lib/crowdb-access-dataset/tests/composition_resource_reclaim_test.rs new file mode 100644 index 000000000..10af0ea32 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/composition_resource_reclaim_test.rs @@ -0,0 +1,85 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/composition.rs"] +#[allow(dead_code)] +mod common; +#[path = "common/publication.rs"] +mod publication; +#[path = "common/split_store.rs"] +mod split_store; +use crowdb_access_dataset::{composition::*, split::*, *}; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, +}; + +#[tokio::test] +async fn interrupted_resource_is_discoverable_and_retirement_blocks_recomputation() { + let (store, authority, identity, parent) = split_store::fixture().await; + let clock = Arc::new(AtomicU64::new(1000)); + let authority = authority.with_reclaim_clock_for_tests(clock.clone()); + let snapshot = SnapshotId::random(); + let request = CompositionRequest { + parent: Some(parent), + source: CompositionSource::All, + schema: None, + samples: Vec::new(), + splits: split_store::partition_splits(), + policy: SplitPolicy::default(), + }; + store.reject_workflow_writes.store(true, Ordering::Relaxed); + store.lose_member_reply.store(true, Ordering::Relaxed); + assert!(authority + .prepare_composition_resource(&identity, snapshot, &request, None, &ReadCancellation::new()) + .await + .is_err()); + assert!(authority.get_snapshot(&identity, snapshot).await.is_err()); + let mut prefix = b"DS\0SM\0".to_vec(); + prefix.extend(identity.key().unwrap()); + prefix.extend(snapshot.as_bytes()); + let pages = store + .scan_page( + &MetadataScan { + prefix, + after: None, + budget: common::budget(), + }, + &ReadCancellation::new(), + ) + .await + .unwrap(); + assert_eq!(pages.entries.len(), 1); + assert!(pages.after.is_none()); + store.reject_workflow_writes.store(false, Ordering::Relaxed); + authority.begin_reclaim(&identity, snapshot).await.unwrap(); + clock.store(1300, Ordering::Release); + let restarted = DatasetAuthority::new(store.clone()).with_reclaim_clock_for_tests(clock); + let mut complete = false; + for _ in 0..100 { + if restarted + .reclaim_snapshot_metadata_step(&identity, snapshot, &ReadCancellation::new()) + .await + .unwrap() + .complete + { + complete = true; + break; + } + } + assert!(complete); + for (key, previous) in pages.entries { + assert_eq!(store.get(&key).await.unwrap().unwrap().bytes, b"split-retired-v1"); + assert!(matches!( + store.compare_exchange(&key, None, &previous.bytes).await.unwrap(), + CasOutcome::Conflict(_) + )); + } + let before = store.mutations.load(Ordering::Relaxed); + assert!(restarted + .prepare_composition_resource(&identity, snapshot, &request, None, &ReadCancellation::new()) + .await + .is_err()); + assert_eq!(store.mutations.load(Ordering::Relaxed), before); + assert!(restarted.get_snapshot(&identity, parent).await.is_ok()); +} diff --git a/lib/crowdb-access-dataset/tests/composition_resource_recovery_test.rs b/lib/crowdb-access-dataset/tests/composition_resource_recovery_test.rs new file mode 100644 index 000000000..67d037d9e --- /dev/null +++ b/lib/crowdb-access-dataset/tests/composition_resource_recovery_test.rs @@ -0,0 +1,236 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/composition.rs"] +#[allow(dead_code)] +mod common; +#[path = "common/publication.rs"] +mod publication; +#[path = "common/composition_resource.rs"] +mod resource; +#[path = "common/split_store.rs"] +mod split_store; +#[path = "common/upload.rs"] +#[allow(dead_code)] +mod upload; +use bytes::Bytes; +use crowdb_access_dataset::{composition::*, split::*, *}; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, +}; + +fn request(parent: SnapshotId) -> CompositionRequest { + CompositionRequest { + parent: Some(parent), + source: CompositionSource::All, + schema: None, + samples: vec![SampleChange::Upsert(common::sample(b"new", 9))], + splits: split_store::partition_splits(), + policy: SplitPolicy::default(), + } +} + +#[tokio::test] +async fn partial_members_are_invisible_and_retry_recomputes_without_draft_progress() { + let (store, authority, identity, parent) = split_store::fixture().await; + let parent = split_store::publish_binary_samples(&authority, &identity, parent).await; + let input = request(parent); + let snapshot = SnapshotId::random(); + store.reject_workflow_writes.store(true, Ordering::Relaxed); + store.lose_member_reply.store(true, Ordering::Relaxed); + assert!(authority + .prepare_composition_resource(&identity, snapshot, &input, None, &ReadCancellation::new()) + .await + .is_err()); + assert!(matches!( + authority.get_snapshot(&identity, snapshot).await, + Err(AuthorityError::SnapshotNotFound) + )); + let result = DatasetAuthority::new(store.clone()) + .prepare_composition_resource(&identity, snapshot, &input, None, &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(result.sample_count, 46); + let split = authority + .get_split_revision( + &identity, + snapshot, + result.initial_split.unwrap(), + common::budget(), + &ReadCancellation::new(), + ) + .await + .unwrap(); + assert_eq!(split.splits.iter().map(|split| split.count).sum::(), 46); + assert_eq!( + authority.inspect_head(&identity).await.unwrap().latest, + Some(parent) + ); +} + +#[tokio::test] +async fn unknown_complete_reply_returns_without_a_confirmation_read() { + let (store, authority, identity, parent) = split_store::fixture().await; + store.reject_workflow_writes.store(true, Ordering::Relaxed); + store.lose_complete_reply.store(true, Ordering::Relaxed); + let input = request(parent); + let snapshot = SnapshotId::random(); + assert!(authority + .prepare_composition_resource(&identity, snapshot, &input, None, &ReadCancellation::new()) + .await + .is_err()); + assert!(store.reject_after_unknown.load(Ordering::Relaxed)); + store.reject_after_unknown.store(false, Ordering::Relaxed); + let before = store.mutations.load(Ordering::Relaxed); + let result = DatasetAuthority::new(store.clone()) + .prepare_composition_resource(&identity, snapshot, &input, None, &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!(result.snapshot, snapshot); + assert_eq!(result.sample_count, 2); + assert_eq!(store.mutations.load(Ordering::Relaxed), before); +} + +#[tokio::test] +async fn final_snapshot_owners_keep_shared_uploads_without_staging_or_cleanup_journals() { + let (store, authority, identity, _) = split_store::fixture().await; + let clock = Arc::new(AtomicU64::new(1000)); + let authority = authority.with_reclaim_clock_for_tests(clock.clone()); + let resource::TestUploadedResource { + uploads, + location, + request: input, + } = resource::TestUploadedResource::new(store.clone(), &identity).await; + store.reject_workflow_writes.store(true, Ordering::Relaxed); + let mut invalid = input.clone(); + let SampleChange::Upsert(sample) = &mut invalid.samples[0] else { + panic!("upsert") + }; + let FieldLocator::Chunk { length, .. } = &mut sample.fields[0].value else { + panic!("chunk") + }; + *length = 5; + let before = store.mutations.load(Ordering::Relaxed); + assert!(matches!( + authority + .prepare_composition_resource( + &identity, + SnapshotId::random(), + &invalid, + Some(&uploads), + &ReadCancellation::new() + ) + .await, + Err(CompositionError::Upload(UploadError::Integrity)) + )); + assert_eq!(store.mutations.load(Ordering::Relaxed), before); + let a = SnapshotId::random(); + let b = SnapshotId::random(); + for snapshot in [a, b] { + authority + .prepare_composition_resource( + &identity, + snapshot, + &input, + Some(&uploads), + &ReadCancellation::new(), + ) + .await + .unwrap(); + } + assert!(matches!( + uploads.abort(&location).await, + Err(UploadError::Conflict) + )); + store.reject_workflow_writes.store(false, Ordering::Relaxed); + for (snapshot, now) in [(a, 1300), (b, 1600)] { + authority.begin_reclaim(&identity, snapshot).await.unwrap(); + clock.store(now, Ordering::Release); + let mut complete = false; + for _ in 0..10 { + if authority + .reclaim_snapshot_uploads_step(&identity, snapshot, &uploads) + .await + .unwrap() + .complete + { + complete = true; + break; + } + } + assert!(complete); + if snapshot == a { + assert!(matches!( + uploads.abort(&location).await, + Err(UploadError::Conflict) + )); + assert_eq!( + uploads.read_part_verified(&location, 0).await.unwrap().0, + Bytes::from_static(b"data") + ); + let before = store.mutations.load(Ordering::Relaxed); + assert!(authority + .prepare_composition_resource( + &identity, + snapshot, + &input, + Some(&uploads), + &ReadCancellation::new() + ) + .await + .is_err()); + assert_eq!(store.mutations.load(Ordering::Relaxed), before); + } + } + assert!(uploads.abort(&location).await.unwrap().cleanup_complete); +} + +#[tokio::test] +async fn unknown_owner_reply_keeps_the_resource_incomplete_until_client_retry() { + let (store, authority, identity, _) = split_store::fixture().await; + let resource = resource::TestUploadedResource::new(store.clone(), &identity).await; + let snapshot = SnapshotId::random(); + store.reject_workflow_writes.store(true, Ordering::Relaxed); + store.lose_claim_reply.store(true, Ordering::Relaxed); + assert!(authority + .prepare_composition_resource( + &identity, + snapshot, + &resource.request, + Some(&resource.uploads), + &ReadCancellation::new() + ) + .await + .is_err()); + assert!(store.reject_after_unknown.load(Ordering::Relaxed)); + store.reject_after_unknown.store(false, Ordering::Relaxed); + assert!(matches!( + authority.get_snapshot(&identity, snapshot).await, + Err(AuthorityError::SnapshotNotFound) + )); + let before = store.mutations.load(Ordering::Relaxed); + let restarted = DatasetAuthority::new(store.clone()); + let result = restarted + .prepare_composition_resource( + &identity, + snapshot, + &resource.request, + Some(&resource.uploads), + &ReadCancellation::new(), + ) + .await + .unwrap(); + assert_eq!(store.mutations.load(Ordering::Relaxed) - before, 2); + assert_eq!(result.snapshot, snapshot); + assert_eq!(result.sample_count, 1); + assert_eq!( + resource + .uploads + .read_part_verified(&resource.location, 0) + .await + .unwrap() + .0, + Bytes::from_static(b"data") + ); +} diff --git a/lib/crowdb-access-dataset/tests/composition_resource_test.rs b/lib/crowdb-access-dataset/tests/composition_resource_test.rs new file mode 100644 index 000000000..855a73ec1 --- /dev/null +++ b/lib/crowdb-access-dataset/tests/composition_resource_test.rs @@ -0,0 +1,289 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/composition.rs"] +#[allow(dead_code)] +mod common; +#[path = "common/publication.rs"] +mod publication; +#[path = "common/split_store.rs"] +mod split_store; +use crowdb_access_dataset::{composition::*, split::*, *}; +use std::sync::atomic::Ordering; + +fn request(parent: SnapshotId) -> CompositionRequest { + CompositionRequest { + parent: Some(parent), + source: CompositionSource::All, + schema: None, + samples: vec![ + SampleChange::Remove(vec![0, 0, 255]), + SampleChange::Upsert(common::sample(b"new", 9)), + ], + splits: split_store::partition_splits(), + policy: SplitPolicy::default(), + } +} +fn counts(store: &split_store::TestDraftStore) -> (usize, usize, usize) { + ( + store.bounded_reads.load(Ordering::Relaxed) + store.unbounded_reads.load(Ordering::Relaxed), + store.metadata_scans.load(Ordering::Relaxed), + store.mutations.load(Ordering::Relaxed), + ) +} +fn delta(store: &split_store::TestDraftStore, old: (usize, usize, usize)) -> (usize, usize, usize) { + let now = counts(store); + (now.0 - old.0, now.1 - old.1, now.2 - old.2) +} + +#[tokio::test] +async fn complete_input_replaces_workflow_writes_and_keeps_the_same_manifest_and_members() { + let (store, authority, identity, parent) = split_store::fixture().await; + let parent = split_store::publish_binary_samples(&authority, &identity, parent).await; + let input = request(parent); + let before = counts(&store); + let operation = OperationId::random(); + let mut old = common::input(parent); + old.splits = input.splits.clone(); + authority + .begin_composition(&identity, operation, &old) + .await + .unwrap(); + authority + .edit_composition(&identity, operation, &common::edit(1, input.samples.clone())) + .await + .unwrap(); + common::evaluate(&authority, &identity, operation, 2).await; + let old = common::prepare(&authority, &identity, operation, 2).await; + let old_cost = delta(&store, before); + let old_snapshot = old.preparation.unwrap().candidate; + store.reject_workflow_writes.store(true, Ordering::Relaxed); + let before = counts(&store); + let head_reads = store.data_head_reads.load(Ordering::Relaxed); + let snapshot = SnapshotId::random(); + let result = authority + .prepare_composition_resource(&identity, snapshot, &input, None, &ReadCancellation::new()) + .await + .unwrap(); + let new_cost = delta(&store, before); + assert_eq!(store.data_head_reads.load(Ordering::Relaxed), head_reads); + assert_eq!(result.snapshot, snapshot); + assert_eq!(result.sample_count, 45); + assert!(new_cost.0 + new_cost.1 + new_cost.2 < old_cost.0 + old_cost.1 + old_cost.2); + assert!(new_cost.2 < old_cost.2); + assert_eq!(new_cost, (187, 14, 153)); + println!("45 Samples, two Splits: workflow {old_cost:?}, resource {new_cost:?} (GET, scan, CAS)"); + assert_eq!( + authority + .get_manifest(&identity, old_snapshot) + .await + .unwrap() + .samples, + authority.get_manifest(&identity, snapshot).await.unwrap().samples + ); + let initial = result.initial_split.unwrap(); + let split = authority + .get_split_revision( + &identity, + snapshot, + initial, + common::budget(), + &ReadCancellation::new(), + ) + .await + .unwrap(); + assert_eq!(split.splits.iter().map(|split| split.count).sum::(), 45); + assert_eq!( + authority + .inspect_split_head(&identity, snapshot) + .await + .unwrap() + .revision, + Some(initial) + ); + assert_eq!( + authority.inspect_head(&identity).await.unwrap().latest, + Some(parent) + ); + let before = store.mutations.load(Ordering::Relaxed); + assert_eq!( + DatasetAuthority::new(store.clone()) + .prepare_composition_resource(&identity, snapshot, &input, None, &ReadCancellation::new()) + .await + .unwrap(), + result + ); + assert_eq!(store.mutations.load(Ordering::Relaxed), before); + let mut changed = input.clone(); + changed.splits.clear(); + assert!(authority + .prepare_composition_resource(&identity, snapshot, &changed, None, &ReadCancellation::new()) + .await + .is_err()); + changed = input; + changed.samples[1] = SampleChange::Upsert(common::sample(b"new", 10)); + assert!(authority + .prepare_composition_resource(&identity, snapshot, &changed, None, &ReadCancellation::new()) + .await + .is_err()); + assert_eq!(store.mutations.load(Ordering::Relaxed), before); +} + +#[tokio::test] +async fn source_filter_and_sparse_patch_preserve_unmodified_parent_fields() { + let (store, authority, identity, parent) = split_store::fixture().await; + store.reject_workflow_writes.store(true, Ordering::Relaxed); + let mut input = request(parent); + input.source = CompositionSource::Selection(Selection::Prefix(b"excluded".to_vec())); + input.splits.clear(); + input.samples = vec![SampleChange::Upsert(SampleRecord { + sample_id: b"sample-1".to_vec(), + fields: Vec::new(), + })]; + let snapshot = SnapshotId::random(); + authority + .prepare_composition_resource(&identity, snapshot, &input, None, &ReadCancellation::new()) + .await + .unwrap(); + assert_eq!( + authority.get_manifest(&identity, snapshot).await.unwrap().samples, + vec![common::sample(b"sample-1", 1)] + ); +} + +#[tokio::test] +async fn frozen_source_and_inherited_members_ignore_later_split_heads() { + let (store, authority, identity, parent) = split_store::fixture().await; + let parent = split_store::publish_binary_samples(&authority, &identity, parent).await; + let original = authority + .prepare_split_revision( + &identity, + parent, + &SplitPreparation { + revision: SplitRevisionId::random(), + policy: SplitPolicy::default(), + splits: vec![DraftSplit { + name: SplitName::new("fixed").unwrap(), + definition: SplitDefinition::Selection(Selection::Explicit(vec![vec![0, 0, 255]])), + origin: MembershipOrigin::Evaluated, + }], + }, + &ReadCancellation::new(), + ) + .await + .unwrap(); + let source = SplitSource { + snapshot: parent, + revision: original.revision, + name: original.splits[0].name.clone(), + }; + let later = split_store::prepared(&authority, &identity, parent, None).await; + authority + .update_split_head( + &identity, + parent, + &SplitHeadUpdate { + revision: later.revision, + expected_revision: None, + expected_head: None, + }, + ) + .await + .unwrap(); + store.reject_workflow_writes.store(true, Ordering::Relaxed); + let input = CompositionRequest { + parent: Some(parent), + source: CompositionSource::Split { + source: source.clone(), + selection: None, + }, + schema: None, + samples: vec![SampleChange::Upsert(common::sample(b"new", 10))], + splits: vec![DraftSplit { + name: source.name.clone(), + definition: original.splits[0].definition.clone(), + origin: MembershipOrigin::Inherited(source), + }], + policy: SplitPolicy::default(), + }; + let result = authority + .prepare_composition_resource( + &identity, + SnapshotId::random(), + &input, + None, + &ReadCancellation::new(), + ) + .await + .unwrap(); + assert_eq!(result.sample_count, 2); + let revision = authority + .get_split_revision( + &identity, + result.snapshot, + result.initial_split.unwrap(), + common::budget(), + &ReadCancellation::new(), + ) + .await + .unwrap(); + assert_eq!(revision.splits[0].count, 1); + assert_eq!( + authority + .split_members_page( + &identity, + result.snapshot, + revision.revision, + &revision.splits[0].name, + None, + common::budget(), + &ReadCancellation::new() + ) + .await + .unwrap() + .entries, + vec![vec![0, 0, 255]] + ); +} + +#[tokio::test] +async fn invalid_cancelled_and_oversized_inputs_cannot_write_a_resource() { + let (store, authority, identity, parent) = split_store::fixture().await; + store.reject_workflow_writes.store(true, Ordering::Relaxed); + let before = counts(&store); + let input = request(parent); + let cancel = ReadCancellation::new(); + cancel.cancel(); + assert!(authority + .prepare_composition_resource(&identity, SnapshotId::random(), &input, None, &cancel) + .await + .is_err()); + assert_eq!(counts(&store), before); + let mut invalid = input.clone(); + invalid.samples.push(invalid.samples[0].clone()); + assert!(authority + .prepare_composition_resource( + &identity, + SnapshotId::random(), + &invalid, + None, + &ReadCancellation::new() + ) + .await + .is_err()); + invalid = input; + invalid.samples = vec![SampleChange::Remove(vec![1; 9 * 1024 * 1024])]; + assert!(matches!( + authority + .prepare_composition_resource( + &identity, + SnapshotId::random(), + &invalid, + None, + &ReadCancellation::new() + ) + .await, + Err(CompositionError::Limit) + )); + assert_eq!(counts(&store), before); +} From fe58512d1e04ac1065e96d46ae31aa07d1f0562d Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 22:32:50 +0800 Subject: [PATCH 136/258] Replace composition workflow APIs with immutable snapshot preparation --- app/crowdb-access-server/src/dataset/api.rs | 2 + .../src/dataset/api/admission.rs | 27 +- .../src/dataset/api/v2.rs | 9 +- .../src/dataset/api/v2/composition_fields.rs | 94 ---- .../src/dataset/api/v2/composition_models.rs | 92 +--- .../dataset/api/v2/composition_preparation.rs | 72 +++ .../api/v2/composition_preparation/fields.rs | 80 +++ .../dataset/api/v2/composition_responses.rs | 6 +- .../src/dataset/api/v2/compositions.rs | 93 +--- .../src/dataset/api/v2/split_models.rs | 4 +- .../src/dataset/api/v2/split_requests.rs | 6 - .../src/dataset/api/v2/uploads.rs | 2 +- .../tests/common/dataset_composition.rs | 59 +-- .../tests/common/dataset_v2.rs | 8 +- .../dataset_composition_contract_test.rs | 135 ++--- .../dataset_composition_preparation_test.rs | 186 +++++++ .../dataset_composition_recovery_test.rs | 43 +- .../tests/dataset_retirement_contract_test.rs | 27 +- .../dataset/design-dataset-control.md | 84 +-- .../dataset/openapi-crowdb-dataset-v2.yaml | 479 ++++++------------ .../dataset/spec-crowdb-dataset.md | 22 +- doc/working/plan-access-simplification.md | 10 + e2e-test/tests/access_dataset_split.rs | 77 +-- .../src/composition/resource.rs | 4 +- .../composition_resource_recovery_test.rs | 2 +- sdk/README.md | 36 +- .../examples/split_control.rs | 154 ++---- .../crowdb-sdk-dataset/src/capabilities.rs | 2 +- sdk/crates/crowdb-sdk-dataset/src/v2.rs | 4 + .../crowdb-sdk-dataset/src/v2/composition.rs | 118 +---- .../src/v2/composition_preparation.rs | 79 +++ .../crowdb-sdk-dataset/src/v2/lifecycle.rs | 122 +---- sdk/crates/crowdb-sdk-dataset/src/v2/split.rs | 24 - sdk/crates/crowdb-sdk-dataset/src/v2/state.rs | 38 +- .../tests/composition_preparation_test.rs | 109 ++++ .../tests/v2_contract_test.rs | 56 +- sdk/crates/crowdb-sdk-python/src/control.rs | 16 +- sdk/crowdb_dataset/control.py | 13 +- sdk/tests/split_consumer.py | 50 +- sdk/tests/test_control.py | 25 +- 40 files changed, 1031 insertions(+), 1438 deletions(-) delete mode 100644 app/crowdb-access-server/src/dataset/api/v2/composition_fields.rs create mode 100644 app/crowdb-access-server/src/dataset/api/v2/composition_preparation.rs create mode 100644 app/crowdb-access-server/src/dataset/api/v2/composition_preparation/fields.rs create mode 100644 app/crowdb-access-server/tests/dataset_composition_preparation_test.rs create mode 100644 sdk/crates/crowdb-sdk-dataset/src/v2/composition_preparation.rs create mode 100644 sdk/crates/crowdb-sdk-dataset/tests/composition_preparation_test.rs diff --git a/app/crowdb-access-server/src/dataset/api.rs b/app/crowdb-access-server/src/dataset/api.rs index 8609ba8a5..84e8335e6 100644 --- a/app/crowdb-access-server/src/dataset/api.rs +++ b/app/crowdb-access-server/src/dataset/api.rs @@ -208,6 +208,8 @@ async fn read_body_limit( .and_then(|v| v.parse::().ok()) .is_some_and(|v| v > limit as u64) { + // Dropping an unread body can reset HTTP before its client receives the error. + admission::drain(request.into_body(), std::time::Duration::from_secs(5)).await; return Err(ApiError::too_large()); } Limited::new(request.into_body(), limit) diff --git a/app/crowdb-access-server/src/dataset/api/admission.rs b/app/crowdb-access-server/src/dataset/api/admission.rs index 08c5cd951..127fc4cc0 100644 --- a/app/crowdb-access-server/src/dataset/api/admission.rs +++ b/app/crowdb-access-server/src/dataset/api/admission.rs @@ -54,16 +54,7 @@ pub(super) async fn reject( mut response: hyper::Response, timeout: std::time::Duration, ) -> hyper::Response { - use http_body_util::BodyExt; - let mut body = http_body_util::Limited::new(request.into_body(), crate::dataset::MAX_REQUEST_BYTES); - let drained = tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async { - while let Some(frame) = body.frame().await { - frame.map_err(|_| ())?; - } - Ok::<_, ()>(()) - }) - .await; - if !matches!(drained, Ok(Ok(()))) { + if !drain(request.into_body(), timeout).await { response.headers_mut().insert( hyper::header::CONNECTION, hyper::header::HeaderValue::from_static("close"), @@ -71,3 +62,19 @@ pub(super) async fn reject( } response } + +/// Discards at most one admitted request window, without allocating a collected body. +pub(super) async fn drain(body: hyper::body::Incoming, timeout: std::time::Duration) -> bool { + use http_body_util::BodyExt; + let mut body = http_body_util::Limited::new(body, crate::dataset::MAX_REQUEST_BYTES); + matches!( + tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async { + while let Some(frame) = body.frame().await { + frame.map_err(|_| ())?; + } + Ok::<_, ()>(()) + }) + .await, + Ok(Ok(())) + ) +} diff --git a/app/crowdb-access-server/src/dataset/api/v2.rs b/app/crowdb-access-server/src/dataset/api/v2.rs index 134d1f8b3..bb4d1b77c 100644 --- a/app/crowdb-access-server/src/dataset/api/v2.rs +++ b/app/crowdb-access-server/src/dataset/api/v2.rs @@ -4,8 +4,8 @@ //! Capability-gated control API. Unsupported operations never fall back to v1. mod auxiliary; -mod composition_fields; mod composition_models; +mod composition_preparation; mod composition_responses; mod compositions; mod heads; @@ -54,7 +54,7 @@ impl DatasetHttpService { "stateless-reads", ]; if self.uploads.is_some() { - capabilities.extend(["uploads", "composition", "snapshot-reclaim"]); + capabilities.extend(["uploads", "composition-preparation", "snapshot-reclaim"]); } return json(&Capabilities { api_version: 2, @@ -91,6 +91,11 @@ impl DatasetHttpService { .await; } } + if let ["snapshots", id] = tail.as_slice() { + if request.method() == Method::PUT { + return self.v2_prepare_snapshot(&identity, id, request).await; + } + } if tail.first() == Some(&"compositions") { return self.v2_compositions(&identity, &tail, request).await; } diff --git a/app/crowdb-access-server/src/dataset/api/v2/composition_fields.rs b/app/crowdb-access-server/src/dataset/api/v2/composition_fields.rs deleted file mode 100644 index c1f6e0d18..000000000 --- a/app/crowdb-access-server/src/dataset/api/v2/composition_fields.rs +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::{ - super::{operation, ApiError}, - query::bytes, -}; -use crowdb_access_dataset::{ - composition, DatasetIdentity, DatasetUploads, FieldLocator, FieldRecord, SampleRecord, UploadState, -}; -use md5::{Digest, Md5}; -use serde::Deserialize; - -#[derive(Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub(super) enum SampleChange { - Remove { id: String }, - Upsert { id: String, fields: Vec }, -} -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct Field { - name: String, - value: Value, -} -#[derive(Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -enum Value { - Inline { data: String }, - Upload { id: String }, - Tombstone, -} -impl SampleChange { - pub(super) async fn native( - self, - identity: &DatasetIdentity, - uploads: &DatasetUploads, - ) -> Result { - match self { - Self::Remove { id } => Ok(composition::SampleChange::Remove(bytes(&id, 16384, false)?)), - Self::Upsert { id, fields } => { - if fields.len() > 256 { - return Err(ApiError::too_large()); - } - let sample_id = bytes(&id, 16384, false)?; - let mut values = Vec::new(); - for field in fields { - values.push(field.native(identity, uploads).await?); - } - Ok(composition::SampleChange::Upsert(SampleRecord { - sample_id, - fields: values, - })) - } - } - } -} -impl Field { - async fn native( - self, - identity: &DatasetIdentity, - uploads: &DatasetUploads, - ) -> Result { - if self.name.is_empty() || self.name.len() > 1024 { - return Err(ApiError::invalid()); - } - let value = match self.value { - Value::Inline { data } => { - let value = bytes(&data, 65536, true)?; - FieldLocator::Inline { - md5: Md5::digest(&value).into(), - value, - } - } - Value::Tombstone => FieldLocator::Tombstone, - Value::Upload { id } => { - let location = DatasetUploads::location(identity, operation(&id)?)?; - let record = uploads.metadata(&location).await?; - if !matches!(record.state, UploadState::Complete | UploadState::Claimed) { - return Err(ApiError::conflict()); - } - FieldLocator::Chunk { - location, - length: record.length, - md5: record.md5, - } - } - }; - Ok(FieldRecord { - name: self.name, - value, - }) - } -} diff --git a/app/crowdb-access-server/src/dataset/api/v2/composition_models.rs b/app/crowdb-access-server/src/dataset/api/v2/composition_models.rs index ef6ca141a..460459814 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/composition_models.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/composition_models.rs @@ -1,11 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{ - super::{operation, snapshot, ApiError}, - predicate::Predicate, - split_models::{decimal, required_option, revision, DraftSplit, Policy, Source}, -}; +use super::{super::ApiError, predicate::Predicate, split_models::Source}; use crowdb_access_dataset::{composition as native, FieldDefinition, SchemaRecord}; use serde::{Deserialize, Serialize}; @@ -67,7 +63,7 @@ pub(super) enum CompositionSource { }, } impl CompositionSource { - fn native(self) -> Result { + pub(super) fn native(self) -> Result { Ok(match self { Self::All => native::CompositionSource::All, Self::Selection { predicate } => native::CompositionSource::Selection(predicate.native()?), @@ -92,87 +88,3 @@ impl From for CompositionSource { } } } -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct Begin { - pub operation: String, - #[serde(deserialize_with = "required_option")] - pub parent: Option, - pub source: CompositionSource, - pub schema: Option, - pub inherit_parent_splits: bool, - pub parent_split_revision: Option, - pub splits: Vec, - pub policy: Option, -} -impl Begin { - pub(super) fn native( - self, - ) -> Result<(crowdb_access_dataset::OperationId, native::CompositionInput), ApiError> { - Ok(( - operation(&self.operation)?, - native::CompositionInput { - parent: self.parent.as_deref().map(snapshot).transpose()?, - source: self.source.native()?, - schema: self.schema.map(Schema::native).transpose()?, - inherit_parent_splits: self.inherit_parent_splits, - parent_split_revision: self.parent_split_revision.as_deref().map(revision).transpose()?, - splits: self - .splits - .into_iter() - .map(DraftSplit::native) - .collect::>()?, - policy: self.policy.map(Into::into), - }, - )) - } -} -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct CompositionEdit { - pub edit: String, - pub expected_revision: String, - pub schema: Option, - pub samples: Vec, - pub splits: Option>, - pub policy: Option, -} -impl CompositionEdit { - pub(super) async fn native( - self, - identity: &crowdb_access_dataset::DatasetIdentity, - uploads: &crowdb_access_dataset::DatasetUploads, - ) -> Result { - if self.samples.len() > 20 { - return Err(ApiError::too_large()); - } - let mut samples = Vec::new(); - for sample in self.samples { - samples.push(sample.native(identity, uploads).await?); - } - Ok(native::CompositionEdit { - id: operation(&self.edit)?, - expected_revision: decimal(&self.expected_revision)?, - schema: self.schema.map(Schema::native).transpose()?, - samples, - splits: self - .splits - .map(|splits| { - splits - .into_iter() - .map(DraftSplit::native) - .collect::>() - }) - .transpose()?, - policy: self.policy.map(Into::into), - }) - } -} -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct Publish { - pub attempt: String, - pub snapshot: String, - #[serde(deserialize_with = "required_option")] - pub expected_head: Option, -} diff --git a/app/crowdb-access-server/src/dataset/api/v2/composition_preparation.rs b/app/crowdb-access-server/src/dataset/api/v2/composition_preparation.rs new file mode 100644 index 000000000..27fc1a51f --- /dev/null +++ b/app/crowdb-access-server/src/dataset/api/v2/composition_preparation.rs @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +mod fields; + +use super::{ + super::{json, read_body_limit, snapshot, ApiError, ApiResult, DatasetHttpService}, + composition_models::{CompositionSource, Schema}, + query::Query, + split_models::{required_option, DraftSplit, Policy}, +}; +use crowdb_access_dataset::{composition::CompositionRequest, DatasetIdentity, ReadCancellation}; +use hyper::{body::Incoming, Request}; +use serde::Deserialize; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Prepare { + #[serde(deserialize_with = "required_option")] + parent: Option, + source: CompositionSource, + #[serde(deserialize_with = "required_option")] + schema: Option, + samples: Vec, + splits: Vec, + policy: Policy, +} + +impl DatasetHttpService { + pub(super) async fn v2_prepare_snapshot( + &self, + identity: &DatasetIdentity, + id: &str, + request: Request, + ) -> ApiResult { + Query::parse(request.uri().query(), &[])?; + let snapshot_id = snapshot(id)?; + let uploads = self.uploads.as_ref().ok_or_else(ApiError::unsupported)?; + let bytes = read_body_limit(request, true, 8 * 1024 * 1024).await?; + let input: Prepare = serde_json::from_slice(&bytes).map_err(|_| ApiError::invalid())?; + let input = CompositionRequest { + parent: input.parent.as_deref().map(snapshot).transpose()?, + source: input.source.native()?, + schema: input.schema.map(Schema::native).transpose()?, + samples: input + .samples + .into_iter() + .map(|sample| sample.native(identity)) + .collect::>()?, + splits: input + .splits + .into_iter() + .map(DraftSplit::native) + .collect::>()?, + policy: input.policy.into(), + }; + let result = self + .reads + .authority() + .prepare_composition_resource( + identity, + snapshot_id, + &input, + Some(uploads), + &ReadCancellation::new(), + ) + .await?; + json(&serde_json::json!({"snapshot":result.snapshot.to_string(), + "initial_split":result.initial_split.map(|revision| revision.to_string()), + "sample_count":result.sample_count.to_string()})) + } +} diff --git a/app/crowdb-access-server/src/dataset/api/v2/composition_preparation/fields.rs b/app/crowdb-access-server/src/dataset/api/v2/composition_preparation/fields.rs new file mode 100644 index 000000000..2a1b96687 --- /dev/null +++ b/app/crowdb-access-server/src/dataset/api/v2/composition_preparation/fields.rs @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::super::{ + super::{operation, ApiError}, + query::bytes, + split_models::decimal, + uploads::digest, +}; +use crowdb_access_dataset::{ + composition::SampleChange, DatasetIdentity, DatasetUploads, FieldLocator, FieldRecord, SampleRecord, +}; +use md5::{Digest, Md5}; +use serde::Deserialize; + +#[derive(Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub(super) enum Change { + Remove { id: String }, + Upsert { id: String, fields: Vec }, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Field { + name: String, + value: Value, +} +#[derive(Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum Value { + Inline { data: String }, + Upload { id: String, length: String, md5: String }, + Tombstone, +} + +impl Change { + pub(super) fn native(self, identity: &DatasetIdentity) -> Result { + match self { + Self::Remove { id } => Ok(SampleChange::Remove(bytes(&id, 16384, false)?)), + Self::Upsert { id, fields } => { + if fields.len() > 256 { + return Err(ApiError::too_large()); + } + Ok(SampleChange::Upsert(SampleRecord { + sample_id: bytes(&id, 16384, false)?, + fields: fields + .into_iter() + .map(|field| field.native(identity)) + .collect::>()?, + })) + } + } + } +} +impl Field { + fn native(self, identity: &DatasetIdentity) -> Result { + if self.name.is_empty() || self.name.len() > 1024 { + return Err(ApiError::invalid()); + } + let value = match self.value { + Value::Inline { data } => { + let value = bytes(&data, 65536, true)?; + FieldLocator::Inline { + md5: Md5::digest(&value).into(), + value, + } + } + Value::Tombstone => FieldLocator::Tombstone, + Value::Upload { id, length, md5 } => FieldLocator::Chunk { + location: DatasetUploads::location(identity, operation(&id)?)?, + length: decimal(&length)?, + md5: digest(&md5)?, + }, + }; + Ok(FieldRecord { + name: self.name, + value, + }) + } +} diff --git a/app/crowdb-access-server/src/dataset/api/v2/composition_responses.rs b/app/crowdb-access-server/src/dataset/api/v2/composition_responses.rs index e0c76fa76..484628a71 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/composition_responses.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/composition_responses.rs @@ -8,7 +8,7 @@ use super::{ split_responses, }; use crowdb_access_dataset::{ - composition::{CompositionDraft, CompositionError, CompositionPreview, CompositionUploadStatus}, + composition::{CompositionDraft, CompositionError, CompositionUploadStatus}, SnapshotReclaimError, UploadReclaimError, }; use hyper::StatusCode; @@ -21,9 +21,7 @@ pub(super) fn draft(d: CompositionDraft, uploads: &CompositionUploadStatus) -> V "preparation":d.preparation.map(|p| json!({"snapshot":p.candidate.to_string(), "split_preview":p.split_preview.map(split_responses::preview)})), "uploads":{"publication_ready":uploads.publication_ready, "staged_uploads":uploads.staged_uploads.to_string(), "candidate_uploads":uploads.candidate_uploads.to_string()}}) } -pub(super) fn preview(p: &CompositionPreview) -> Value { - json!({"input_revision":p.input_revision.to_string(), "next_page":p.next_page.to_string(), "members":p.members.to_string(), "complete":p.complete, "binding":hex::encode(p.binding)}) -} + impl From for ApiError { fn from(error: CompositionError) -> Self { match error { diff --git a/app/crowdb-access-server/src/dataset/api/v2/compositions.rs b/app/crowdb-access-server/src/dataset/api/v2/compositions.rs index cd964fad7..893954472 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/compositions.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/compositions.rs @@ -2,15 +2,13 @@ // Licensed under the Apache License, Version 2.0. use super::{ - super::{json, operation, read_body, snapshot, ApiError, ApiResult, DatasetHttpService}, - composition_models as input, composition_responses as output, + super::{json, operation, read_body, ApiError, ApiResult, DatasetHttpService}, + composition_responses as output, query::Query, split_models::decimal, - split_requests, split_responses, -}; -use crowdb_access_dataset::{ - composition::DraftProgress, CommitAttempt, CommitTarget, DatasetIdentity, OperationId, ReadCancellation, + split_requests, }; +use crowdb_access_dataset::{composition::DraftProgress, DatasetIdentity, OperationId, ReadCancellation}; use hyper::{body::Incoming, Method, Request}; impl DatasetHttpService { pub(super) async fn v2_compositions( @@ -20,11 +18,13 @@ impl DatasetHttpService { request: Request, ) -> ApiResult { Query::parse(request.uri().query(), &[])?; - self.reads.authority().get_dataset(identity).await?; - let uploads = self.uploads.as_ref().ok_or_else(ApiError::unsupported)?; - if tail == ["compositions"] && request.method() == Method::POST { - return self.v2_composition_begin(identity, request).await; + if request.method() == Method::PUT + || (request.method() == Method::POST && !matches!(tail, ["compositions", _, "cleanup"])) + { + return Err(ApiError::unsupported()); } + self.reads.authority().get_dataset(identity).await?; + self.uploads.as_ref().ok_or_else(ApiError::unsupported)?; let ["compositions", id, rest @ ..] = tail else { return Err(ApiError::not_found()); }; @@ -35,20 +35,6 @@ impl DatasetHttpService { read_body(request, false).await?; self.v2_composition_status(identity, operation).await } - (Method::PUT, []) => { - let input: input::CompositionEdit = read_json(request).await?; - let edit = - tokio::time::timeout(std::time::Duration::from_secs(2), input.native(identity, uploads)) - .await - .map_err(|_| ApiError::timeout())??; - let draft = authority - .edit_composition_with_uploads(identity, operation, &edit, uploads) - .await?; - json(&output::draft( - draft, - &authority.composition_upload_status(identity, operation).await?, - )) - } (Method::DELETE, []) => { let input: split_requests::Prepare = read_json(request).await?; authority @@ -56,53 +42,6 @@ impl DatasetHttpService { .await?; self.v2_composition_status(identity, operation).await } - (Method::POST, ["evaluate"]) => { - let input: split_requests::Evaluate = read_json(request).await?; - json(&output::preview( - &authority - .evaluate_composition( - identity, - operation, - decimal(&input.expected_revision)?, - decimal(&input.expected_page)?, - &ReadCancellation::new(), - ) - .await?, - )) - } - (Method::POST, ["prepare"]) => { - let input: split_requests::Prepare = read_json(request).await?; - let draft = authority - .prepare_composition( - identity, - operation, - decimal(&input.expected_revision)?, - &ReadCancellation::new(), - ) - .await?; - if matches!(draft.record.progress, DraftProgress::Prepared { .. }) { - authority - .prepare_composition_uploads(identity, operation, uploads, &ReadCancellation::new()) - .await?; - } - self.v2_composition_status(identity, operation).await - } - (Method::POST, ["publish"]) => { - let input: input::Publish = read_json(request).await?; - let attempt = CommitAttempt { - version: 1, - id: super::super::operation(&input.attempt)?, - target: CommitTarget::Snapshot { - snapshot: snapshot(&input.snapshot)?, - }, - expected_head: split_requests::generation(input.expected_head.as_deref())?, - }; - json(&split_responses::receipt( - &authority - .publish_composition(identity, operation, &attempt) - .await?, - )) - } (Method::POST, ["cleanup"]) => { read_body(request, false).await?; self.v2_composition_cleanup(identity, operation).await @@ -110,18 +49,6 @@ impl DatasetHttpService { _ => Err(ApiError::not_found()), } } - async fn v2_composition_begin( - &self, - identity: &DatasetIdentity, - request: Request, - ) -> ApiResult { - let input: input::Begin = read_json(request).await?; - let (operation, input) = input.native()?; - let authority = self.reads.authority(); - authority.begin_composition(identity, operation, &input).await?; - authority.manage_composition_uploads(identity, operation).await?; - self.v2_composition_status(identity, operation).await - } async fn v2_composition_status(&self, identity: &DatasetIdentity, operation: OperationId) -> ApiResult { let authority = self.reads.authority(); json(&output::draft( diff --git a/app/crowdb-access-server/src/dataset/api/v2/split_models.rs b/app/crowdb-access-server/src/dataset/api/v2/split_models.rs index c50b9580e..9667e852d 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/split_models.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/split_models.rs @@ -20,7 +20,9 @@ pub(super) fn decimal(value: &str) -> Result { } value.parse().map_err(|_| ApiError::invalid()) } -pub(super) fn required_option<'de, D: serde::Deserializer<'de>>(d: D) -> Result, D::Error> { +pub(super) fn required_option<'de, D: serde::Deserializer<'de>, T: Deserialize<'de>>( + d: D, +) -> Result, D::Error> { Option::deserialize(d) } diff --git a/app/crowdb-access-server/src/dataset/api/v2/split_requests.rs b/app/crowdb-access-server/src/dataset/api/v2/split_requests.rs index c92fae654..249b5cfc4 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/split_requests.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/split_requests.rs @@ -5,12 +5,6 @@ use super::{super::ApiError, split_models::required_option}; use crowdb_access_dataset::CommitGeneration; use serde::Deserialize; -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct Evaluate { - pub expected_revision: String, - pub expected_page: String, -} #[derive(Deserialize)] #[serde(deny_unknown_fields)] pub(super) struct Prepare { diff --git a/app/crowdb-access-server/src/dataset/api/v2/uploads.rs b/app/crowdb-access-server/src/dataset/api/v2/uploads.rs index 957244542..b75af7594 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/uploads.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/uploads.rs @@ -85,7 +85,7 @@ impl DatasetHttpService { } } } -fn digest(value: &str) -> Result<[u8; 16], ApiError> { +pub(super) fn digest(value: &str) -> Result<[u8; 16], ApiError> { if value.len() != 32 || !value .bytes() diff --git a/app/crowdb-access-server/tests/common/dataset_composition.rs b/app/crowdb-access-server/tests/common/dataset_composition.rs index 9b96300e2..df1f227ce 100644 --- a/app/crowdb-access-server/tests/common/dataset_composition.rs +++ b/app/crowdb-access-server/tests/common/dataset_composition.rs @@ -3,49 +3,22 @@ use crate::common::response; use reqwest::Client; use serde_json::{json, Value}; -pub fn begin(operation: &str, parent: &Value) -> Value { - json!({"operation":operation,"parent":parent,"source":{"kind":"all"},"schema":{"version":1,"fields":[{"name":"value","required":true}]},"inherit_parent_splits":false,"parent_split_revision":null,"splits":[],"policy":null}) +pub fn input(parent: &Value) -> Value { + json!({"parent":parent,"source":{"kind":"all"},"schema":{"version":1,"fields":[{"name":"value","required":true}]},"samples":[],"splits":[],"policy":{"overlap":"disjoint","coverage":"allow_unassigned"}}) } -pub async fn prepare(client: &Client, draft: &str, revision: &str, members: &str) -> Value { - let mut done = false; - for page in 0..100 { - let preview = response( - client - .post(format!("{draft}/evaluate")) - .json(&json!({"expected_revision":revision,"expected_page":page.to_string()})), - 200, - ) - .await; - if preview["complete"] == true { - assert_eq!(preview["members"], members); - done = true; - break; - } - } - assert!(done, "composition evaluation did not finish"); - for _ in 0..100 { - let prepared = response( - client - .post(format!("{draft}/prepare")) - .json(&json!({"expected_revision":revision})), - 200, - ) - .await; - if prepared["uploads"]["publication_ready"] == true { - return prepared; - } - } - panic!("composition preparation did not finish"); +pub async fn prepare(client: &Client, root: &str, snapshot: &str, input: &Value) -> Value { + response( + client.put(format!("{root}/snapshots/{snapshot}")).json(input), + 200, + ) + .await } -pub async fn publish(client: &Client, root: &str, draft: &str, prepared: &Value, attempt: &str) -> Value { - let head = response(client.get(format!("{root}/head")), 200).await; - response(client.post(format!("{draft}/publish")).json(&json!({"attempt":attempt,"snapshot":prepared["preparation"]["snapshot"],"expected_head":head["generation"]})), 200).await -} -pub async fn cleanup(client: &Client, draft: &str) { - for _ in 0..100 { - if response(client.post(format!("{draft}/cleanup")), 200).await["complete"] == true { - return; - } - } - panic!("composition cleanup did not finish"); +pub async fn publish(client: &Client, root: &str, prepared: &Value, expected_head: &Value) -> Value { + response( + client + .put(format!("{root}/head")) + .json(&json!({"snapshot":prepared["snapshot"],"expected_head":expected_head})), + 200, + ) + .await } diff --git a/app/crowdb-access-server/tests/common/dataset_v2.rs b/app/crowdb-access-server/tests/common/dataset_v2.rs index 4e4c3680b..5ef51f687 100644 --- a/app/crowdb-access-server/tests/common/dataset_v2.rs +++ b/app/crowdb-access-server/tests/common/dataset_v2.rs @@ -8,7 +8,13 @@ use serde_json::{json, Value}; pub async fn response(request: reqwest::RequestBuilder, expected: u16) -> Value { let response = request.bearer_auth("secret").send().await.unwrap(); let status = response.status().as_u16(); - let body: Value = response.json().await.unwrap(); + let url = response.url().to_string(); + let bytes = response + .bytes() + .await + .unwrap_or_else(|error| panic!("{status} {url}: {error}")); + let body: Value = serde_json::from_slice(&bytes) + .unwrap_or_else(|error| panic!("{status} {url}: {error}; {}", String::from_utf8_lossy(&bytes))); assert_eq!(status, expected, "{body}"); body } diff --git a/app/crowdb-access-server/tests/dataset_composition_contract_test.rs b/app/crowdb-access-server/tests/dataset_composition_contract_test.rs index 8df99605e..fda0415d4 100644 --- a/app/crowdb-access-server/tests/dataset_composition_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_composition_contract_test.rs @@ -11,10 +11,10 @@ use common::response; use dataset::{TestDatasetHttp, TestStore}; use reqwest::Client; use serde_json::{json, Value}; -use std::sync::Arc; +use std::sync::{atomic::Ordering, Arc}; #[tokio::test] -async fn uploaded_composition_publishes_initial_splits_and_branches_from_a_frozen_split() { +async fn uploaded_resources_publish_initial_splits_and_branch_from_a_frozen_split() { let test = TestDatasetHttp::start_payload(Arc::new(TestStore::default())).await; let client = Client::new(); let capabilities = response( @@ -22,12 +22,16 @@ async fn uploaded_composition_publishes_initial_splits_and_branches_from_a_froze 200, ) .await; - for name in ["composition", "uploads", "snapshot-reclaim"] { + for name in ["composition-preparation", "uploads", "snapshot-reclaim"] { assert!(capabilities["capabilities"] .as_array() .unwrap() .contains(&json!(name))); } + assert!(!capabilities["capabilities"] + .as_array() + .unwrap() + .contains(&json!("composition"))); let root = format!( "{}/api/dataset/v2/namespaces/dataset-ns/datasets/import", test.origin @@ -36,38 +40,20 @@ async fn uploaded_composition_publishes_initial_splits_and_branches_from_a_froze response(client.put(&root), 409).await; let upload = "11111111111111111111111111111111"; create_upload(&client, &root, upload).await; - let operation = "22222222222222222222222222222222"; - let draft = format!("{root}/compositions/{operation}"); - let mut begin = composition::begin(operation, &Value::Null); - begin["splits"] = common::definitions(); - begin["policy"] = json!({"overlap":"disjoint","coverage":"require_complete"}); - response(client.post(format!("{root}/compositions")).json(&begin), 200).await; - let edit = json!({"edit":"33333333333333333333333333333333","expected_revision":"1","samples":[{"kind":"upsert","id":"AAH/","fields":[{"name":"value","value":{"kind":"upload","id":upload}}]}]}); + let snapshot = "22222222222222222222222222222222"; + let mut input = composition::input(&Value::Null); + input["splits"] = common::definitions(); + input["policy"]["coverage"] = json!("require_complete"); + input["samples"] = json!([{"kind":"upsert","id":"AAH/","fields":[{"name":"value","value":{"kind":"upload","id":upload,"length":"4","md5":"8d777f385d3dfec8815d20f7496026dc"}}]}]); + let prepared = composition::prepare(&client, &root, snapshot, &input).await; + assert_eq!(prepared["sample_count"], "1"); assert_eq!( - response(client.put(&draft).json(&edit), 200).await["revision"], - "2" + composition::prepare(&client, &root, snapshot, &input).await, + prepared ); response(client.delete(format!("{root}/uploads/{upload}")), 409).await; - let prepared = composition::prepare(&client, &draft, "2", "1").await; - let snapshot = prepared["preparation"]["snapshot"].as_str().unwrap(); response(client.get(format!("{root}/snapshots/{snapshot}/summary")), 200).await; - composition::publish( - &client, - &root, - &draft, - &prepared, - "44444444444444444444444444444444", - ) - .await; - assert_eq!( - response( - client.get(format!("{root}/commits/44444444444444444444444444444444")), - 200 - ) - .await["receipt"], - Value::Null - ); - let head = response(client.get(format!("{root}/head")), 200).await; + let head = composition::publish(&client, &root, &prepared, &Value::Null).await; assert_eq!(head["latest"], snapshot); assert_eq!(head["stable"], Value::Null); let preview = response( @@ -79,25 +65,17 @@ async fn uploaded_composition_publishes_initial_splits_and_branches_from_a_froze .await; assert_eq!(preview["data"], "ZGF0YQ=="); let (revision, name) = populated_split(&client, &root, snapshot).await; - let child_op = "55555555555555555555555555555555"; - let child = format!("{root}/compositions/{child_op}"); - let mut begin = composition::begin(child_op, &json!(snapshot)); - begin["source"] = json!({"kind":"split","source":{"snapshot":snapshot,"revision":revision,"name":name},"selection":null}); - begin["inherit_parent_splits"] = json!(true); - begin["parent_split_revision"] = json!(revision); - response(client.post(format!("{root}/compositions")).json(&begin), 200).await; - let child_prepared = composition::prepare(&client, &child, "1", "1").await; - assert_eq!(child_prepared["uploads"]["candidate_uploads"], "1"); - composition::publish( - &client, - &root, - &child, - &child_prepared, - "66666666666666666666666666666666", - ) - .await; - composition::cleanup(&client, &draft).await; - composition::cleanup(&client, &child).await; + let mut child = composition::input(&json!(snapshot)); + child["schema"] = Value::Null; + child["source"] = json!({"kind":"split","source":{"snapshot":snapshot,"revision":revision,"name":name},"selection":null}); + child["splits"] = input["splits"].clone(); + child["policy"] = input["policy"].clone(); + for split in child["splits"].as_array_mut().unwrap() { + split["origin"] = json!({"kind":"inherited","source":{"snapshot":snapshot,"revision":revision,"name":split["name"]}}); + } + let prepared = composition::prepare(&client, &root, "55555555555555555555555555555555", &child).await; + assert_eq!(prepared["sample_count"], "1"); + composition::publish(&client, &root, &prepared, &head["generation"]).await; response( client .put(format!("{root}/names/stable")) @@ -107,50 +85,39 @@ async fn uploaded_composition_publishes_initial_splits_and_branches_from_a_froze .await; assert_eq!( response(client.get(format!("{root}/head")), 200).await["latest"], - child_prepared["preparation"]["snapshot"] + prepared["snapshot"] ); response(client.post(format!("{root}/snapshots/{snapshot}/reclaim")), 409).await; + drop(client); test.stop().await; } #[tokio::test] -async fn composition_conflicts_and_cancellation_preserve_exact_retry_identity() { - let test = TestDatasetHttp::start_payload(Arc::new(TestStore::default())).await; +async fn retired_composition_workflow_writes_never_access_storage() { + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start_payload(store.clone()).await; let client = Client::new(); - let (root, parent) = common::fixture(&client, &test.origin).await; - let operation = "77777777777777777777777777777777"; - let draft = format!("{root}/compositions/{operation}"); - let begin = composition::begin(operation, &json!(parent)); - response(client.post(format!("{root}/compositions")).json(&begin), 200).await; - let edit = json!({"edit":"88888888888888888888888888888888","expected_revision":"1","samples":[]}); - response(client.put(&draft).json(&edit), 200).await; - response( - client - .put(&draft) - .json(&json!({"edit":"99999999999999999999999999999999","expected_revision":"1","samples":[]})), - 409, - ) - .await; - response(client.put(&draft).json(&json!({"edit":"99999999999999999999999999999999","expected_revision":"2","samples":[{"kind":"upsert","id":"YQ==","fields":[{"name":"value","value":{"kind":"chunk","location":"arbitrary"}}]}]})),400).await; - response(client.delete(&draft).json(&json!({"expected_revision":"1"})), 409).await; - response(client.delete(&draft).json(&json!({"expected_revision":"2"})), 200).await; - composition::cleanup(&client, &draft).await; - assert_eq!( - response(client.put(&draft).json(&edit), 200).await["revision"], - "2" + let (root, _) = common::fixture(&client, &test.origin).await; + let before = ( + store.reads.load(Ordering::Relaxed), + store.bounded_reads.load(Ordering::Relaxed), + store.mutations.load(Ordering::Relaxed), ); + response(client.post(format!("{root}/compositions")).json(&json!({})), 501).await; + let draft = format!("{root}/compositions/77777777777777777777777777777777"); + response(client.put(&draft).json(&json!({})), 501).await; + for suffix in ["evaluate", "prepare", "publish"] { + response(client.post(format!("{draft}/{suffix}")).json(&json!({})), 501).await; + } assert_eq!( - response(client.get(format!("{root}/head")), 200).await["latest"], - parent + ( + store.reads.load(Ordering::Relaxed), + store.bounded_reads.load(Ordering::Relaxed), + store.mutations.load(Ordering::Relaxed) + ), + before ); - response(client.get(format!("{root}/head?unexpected=1")), 400).await; - response( - client - .post(format!("{draft}/publish")) - .json(&json!({"attempt":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","snapshot":parent})), - 400, - ) - .await; + drop(client); test.stop().await; } diff --git a/app/crowdb-access-server/tests/dataset_composition_preparation_test.rs b/app/crowdb-access-server/tests/dataset_composition_preparation_test.rs new file mode 100644 index 000000000..7f5311cd4 --- /dev/null +++ b/app/crowdb-access-server/tests/dataset_composition_preparation_test.rs @@ -0,0 +1,186 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/dataset_v2.rs"] +#[allow(dead_code)] +mod common; +#[path = "common/dataset.rs"] +mod dataset; +use common::response; +use dataset::{TestDatasetHttp, TestStore}; +use reqwest::Client; +use serde_json::{json, Value}; +use std::sync::{atomic::Ordering, Arc}; + +fn input() -> Value { + json!({"parent":null, "source":{"kind":"all"}, + "schema":{"version":1,"fields":[{"name":"value","required":true}]}, + "samples":[{"kind":"upsert","id":"AAH/","fields":[{"name":"value","value":{"kind":"inline","data":"ZGF0YQ=="}}]}], + "splits":[{"name":"train","definition":{"kind":"selection","predicate":{"kind":"all"}},"origin":{"kind":"evaluated"}}], + "policy":{"overlap":"disjoint","coverage":"require_complete"}}) +} +fn counts(store: &TestStore) -> (u64, u64) { + ( + store.reads.load(Ordering::Relaxed) + store.bounded_reads.load(Ordering::Relaxed), + store.mutations.load(Ordering::Relaxed), + ) +} +fn root(origin: &str) -> String { + format!("{origin}/api/dataset/v2/namespaces/dataset-ns/datasets/composition-resource") +} + +#[tokio::test] +async fn complete_resource_reuses_the_snapshot_id_across_restart_without_advancing_head() { + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start_payload(store.clone()).await; + let client = Client::new(); + let root = root(&test.origin); + response(client.put(&root), 200).await; + let id = "12".repeat(16); + let url = format!("{root}/snapshots/{id}"); + let before = counts(&store); + let prepared = response(client.put(&url).json(&input()), 200).await; + let after = counts(&store); + assert_eq!((after.0 - before.0, after.1 - before.1), (12, 16)); + assert_eq!(prepared["snapshot"], id); + assert_eq!(prepared["sample_count"], "1"); + assert!(prepared["initial_split"].is_string()); + assert_eq!( + response(client.get(format!("{root}/head")), 200).await["latest"], + Value::Null + ); + response(client.get(format!("{url}/summary")), 200).await; + drop(client); + test.stop().await; + let client = Client::new(); + let test = TestDatasetHttp::start_payload(store.clone()).await; + let root = self::root(&test.origin); + let url = format!("{root}/snapshots/{id}"); + let before = counts(&store); + assert_eq!(response(client.put(&url).json(&input()), 200).await, prepared); + assert_eq!(counts(&store).1, before.1); + let mut changed = input(); + changed["splits"] = json!([]); + response(client.put(&url).json(&changed), 409).await; + assert_eq!(counts(&store).1, before.1); + response( + client + .put(format!("{root}/head")) + .json(&json!({"snapshot":id,"expected_head":null})), + 200, + ) + .await; + assert_eq!( + response(client.get(format!("{root}/head")), 200).await["latest"], + id + ); + drop(client); + test.stop().await; +} + +#[tokio::test] +async fn uploaded_descriptors_are_validated_once_per_unique_upload_without_conversion_queries() { + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start_payload(store.clone()).await; + let client = Client::new(); + let root = root(&test.origin); + response(client.put(&root), 200).await; + let upload = "34".repeat(16); + let upload_url = format!("{root}/uploads/{upload}"); + let descriptor = json!({"id":upload,"length":"4","part_size":4,"md5":"8d777f385d3dfec8815d20f7496026dc"}); + response(client.post(format!("{root}/uploads")).json(&descriptor), 200).await; + response( + client + .put(format!("{upload_url}/parts/0")) + .header("content-type", "application/octet-stream") + .body("data"), + 200, + ) + .await; + response(client.post(format!("{upload_url}/complete")), 200).await; + let mut input = input(); + input["splits"] = json!([]); + input["schema"]["fields"] + .as_array_mut() + .unwrap() + .push(json!({"name":"other","required":true})); + let value = json!({"kind":"upload","id":upload,"length":descriptor["length"],"md5":descriptor["md5"]}); + input["samples"][0]["fields"] = json!([{"name":"value","value":value},{"name":"other","value":value}]); + let before = store.upload_reads.load(Ordering::Relaxed); + let snapshot = "56".repeat(16); + response( + client.put(format!("{root}/snapshots/{snapshot}")).json(&input), + 200, + ) + .await; + assert_eq!(store.upload_reads.load(Ordering::Relaxed) - before, 2); + let url = format!("{root}/snapshots/{snapshot}/preview?sample=AAH%2F&field=other"); + assert_eq!(response(client.get(url), 200).await["data"], "ZGF0YQ=="); + input["samples"][0]["fields"][0]["value"]["length"] = json!("5"); + let before = counts(&store).1; + response( + client + .put(format!("{root}/snapshots/{}", "78".repeat(16))) + .json(&input), + 409, + ) + .await; + assert_eq!(counts(&store).1, before); + drop(client); + test.stop().await; +} + +#[tokio::test] +async fn auth_strict_input_and_limits_reject_before_storage_and_no_service_has_no_capability() { + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start_payload(store.clone()).await; + let client = Client::new(); + let url = format!("{}/snapshots/{}", root(&test.origin), "12".repeat(16)); + let denied = client.put(&url).json(&input()).send().await.unwrap(); + assert_eq!(denied.status(), 401); + denied.bytes().await.unwrap(); + for field in ["parent", "schema", "samples", "policy"] { + let mut invalid = input(); + invalid.as_object_mut().unwrap().remove(field); + response(client.put(&url).json(&invalid), 400).await; + } + let mut invalid = input(); + invalid["operation"] = json!("ab".repeat(16)); + response(client.put(&url).json(&invalid), 400).await; + invalid = input(); + invalid["samples"][0]["fields"][0]["value"] = json!({"kind":"upload","id":"ab".repeat(16)}); + response(client.put(&url).json(&invalid), 400).await; + response( + client.put(format!("{url}?expected_revision=1")).json(&input()), + 400, + ) + .await; + response( + client.put(&url).header("content-type", "text/plain").body("{}"), + 400, + ) + .await; + response( + client + .put(&url) + .header("content-type", "application/json") + .body(vec![b' '; 8 * 1024 * 1024 + 1]), + 413, + ) + .await; + assert_eq!(counts(&store), (0, 0)); + drop(client); + test.stop().await; + let test = TestDatasetHttp::start(store.clone()).await; + let client = Client::new(); + response( + client + .put(format!("{}/snapshots/{}", root(&test.origin), "12".repeat(16))) + .json(&input()), + 501, + ) + .await; + assert_eq!(counts(&store), (0, 0)); + drop(client); + test.stop().await; +} diff --git a/app/crowdb-access-server/tests/dataset_composition_recovery_test.rs b/app/crowdb-access-server/tests/dataset_composition_recovery_test.rs index a3102eb8a..faf96fed8 100644 --- a/app/crowdb-access-server/tests/dataset_composition_recovery_test.rs +++ b/app/crowdb-access-server/tests/dataset_composition_recovery_test.rs @@ -11,7 +11,7 @@ mod dataset; use common::response; use dataset::{TestDatasetHttp, TestStore}; use reqwest::Client; -use serde_json::{json, Value}; +use serde_json::json; use std::sync::Arc; #[tokio::test] @@ -22,21 +22,18 @@ async fn restart_and_head_conflict_preserve_candidates_without_receipt_writes() let (root, parent) = common::fixture(&client, &first.origin).await; let origin = first.origin.clone(); let old_head = response(client.get(format!("{root}/head")), 200).await; - let operation = "12121212121212121212121212121212"; - response( - client - .post(format!("{root}/compositions")) - .json(&composition::begin(operation, &json!(parent))), - 200, - ) - .await; + let snapshot = "12121212121212121212121212121212"; + let input = composition::input(&json!(parent)); + let prepared = composition::prepare(&client, &root, snapshot, &input).await; drop(client); first.stop().await; let second = TestDatasetHttp::start_payload(store).await; let root = root.replacen(&origin, &second.origin, 1); let client = Client::new(); - let draft = format!("{root}/compositions/{operation}"); - let prepared = composition::prepare(&client, &draft, "1", "45").await; + assert_eq!( + composition::prepare(&client, &root, snapshot, &input).await, + prepared + ); response( client .put(format!("{root}/names/stable")) @@ -44,27 +41,15 @@ async fn restart_and_head_conflict_preserve_candidates_without_receipt_writes() 200, ) .await; - let attempt = "13131313131313131313131313131313"; - let mut publish = json!({"attempt":attempt,"snapshot":prepared["preparation"]["snapshot"],"expected_head":old_head["generation"]}); - let conflict = response(client.post(format!("{draft}/publish")).json(&publish), 409).await; + let mut publish = json!({"snapshot":snapshot,"expected_head":old_head["generation"]}); + let url = format!("{root}/head"); + let conflict = response(client.put(&url).json(&publish), 409).await; assert_eq!(conflict["observed_head"]["latest"], parent); assert_eq!(conflict["observed_head"]["stable"], parent); - assert_eq!( - response(client.get(&draft), 200).await["progress"]["state"], - "prepared" - ); - publish["attempt"] = json!("14141414141414141414141414141414"); + response(client.get(format!("{root}/snapshots/{snapshot}/summary")), 200).await; publish["expected_head"] = conflict["observed_head"]["generation"].clone(); - let receipt = response(client.post(format!("{draft}/publish")).json(&publish), 200).await; - composition::cleanup(&client, &draft).await; - assert_eq!( - response(client.post(format!("{draft}/publish")).json(&publish), 200).await, - receipt - ); - assert_eq!( - response(client.get(format!("{root}/commits/{attempt}")), 200).await["receipt"], - Value::Null - ); + let current = response(client.put(&url).json(&publish), 200).await; + assert_eq!(response(client.put(&url).json(&publish), 200).await, current); drop(client); second.stop().await; } diff --git a/app/crowdb-access-server/tests/dataset_retirement_contract_test.rs b/app/crowdb-access-server/tests/dataset_retirement_contract_test.rs index 2b5bdd1b8..cb6a082b6 100644 --- a/app/crowdb-access-server/tests/dataset_retirement_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_retirement_contract_test.rs @@ -19,26 +19,11 @@ async fn retirement_respects_retention_and_finishes_split_and_composition_cleanu let client = Client::new(); let (root, old) = common::fixture(&client, &test.origin).await; let revision = common::prepare(&client, &root, &old, "16161616161616161616161616161616", false).await; - let operation = "17171717171717171717171717171717"; - let draft = format!("{root}/compositions/{operation}"); - response( - client - .post(format!("{root}/compositions")) - .json(&composition::begin(operation, &Value::Null)), - 200, - ) - .await; - response(client.put(&draft).json(&json!({"edit":"18181818181818181818181818181818","expected_revision":"1","samples":[{"kind":"upsert","id":"YQ==","fields":[{"name":"value","value":{"kind":"inline","data":"Yg=="}}]}]})),200).await; - let prepared = composition::prepare(&client, &draft, "2", "1").await; - composition::publish( - &client, - &root, - &draft, - &prepared, - "19191919191919191919191919191919", - ) - .await; - composition::cleanup(&client, &draft).await; + let head = response(client.get(format!("{root}/head")), 200).await; + let mut input = composition::input(&Value::Null); + input["samples"] = json!([{"kind":"upsert","id":"YQ==","fields":[{"name":"value","value":{"kind":"inline","data":"Yg=="}}]}]); + let prepared = composition::prepare(&client, &root, "17171717171717171717171717171717", &input).await; + composition::publish(&client, &root, &prepared, &head["generation"]).await; let snapshot = format!("{root}/snapshots/{old}"); response(client.post(format!("{snapshot}/retain")), 200).await; response(client.post(format!("{snapshot}/reclaim")), 409).await; @@ -69,7 +54,7 @@ async fn retirement_respects_retention_and_finishes_split_and_composition_cleanu response(client.get(format!("{snapshot}/summary")), 404).await; assert_eq!( response(client.get(format!("{root}/head")), 200).await["latest"], - prepared["preparation"]["snapshot"] + prepared["snapshot"] ); drop(client); test.stop().await; diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md index cc09b83ac..b66e5cf43 100644 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ b/doc/design/access-server/dataset/design-dataset-control.md @@ -100,18 +100,26 @@ new chunk patches must be valid managed upload locators. Resource prefixes and the immutable Snapshot record expose interrupted output to owner retirement. Existing member pages are sealed after the grace period, and recomputation cannot complete a retired Snapshot. Independent unused-resource -expiry and discovery of arbitrarily late unseen keys remain separate work. The -REST/SDK Composition workflow below still uses the legacy path until migration; -the new native operation does not call it. - -`CompositionRecord` persists the operation, input revision, source, pinned parent -Split revision, edits, evaluation progress, candidate and ownership state. Edits -use expected-revision CAS and immutable edit receipts. Historical retries return -their original draft result; current status is a separate read. -Status reads load only the stored draft and never help edit receipts, query commit -ledgers or advance publication state. A legacy Committing phase is returned as -Prepared for an explicit caller retry; the stored bytes remain unchanged. A status -response does not establish whether that historical request selected the head. +expiry and discovery of arbitrarily late unseen keys remain separate work. +`PUT snapshots/{snapshot}` exposes this operation with `composition-preparation`, +when an upload service is configured. Rust `PrepareSnapshot` and sync/async Python +`prepare_snapshot` preserve the same ID/input and never query workflow status or +downgrade. Explicit nullable parent/schema and strict DTOs reject process IDs. +Upload values carry id/length/md5, converted to locators without a metadata query; +the authority checks each unique upload before claiming it. Descriptor mismatches +return conflict. HTTP body/input/Manifest are bounded to 8 MiB. Declared oversized +bodies use the existing bounded discard before response, avoiding an unread-body +reset that can truncate the 413 JSON. Composition and standalone Split workflow +writes return 501 before storage access, and the workflow SDK methods are removed. +Only legacy HTTP observation, abandonment and cleanup remain for existing records. + +Legacy `CompositionRecord` stores old input revisions, source IDs, edits, work +progress, candidate and ownership lists. These formats remain cleanup inputs; +current routes neither append edits nor resume them. Status only reads the stored +draft and never helps receipts, queries commit ledgers or advances publication. +Legacy Committing is presented as Prepared without changing stored bytes. A caller +may select an already complete resource with a direct head update; the status does +not prove that a historical request selected the head. The default source is all parent Samples. Selection filters and a frozen named Split can restrict the source, with their intersection when both are present. @@ -129,12 +137,11 @@ fields from an ancestor and report that ancestor as the source. An unchanged required field must remain represented in the final complete membership. Tombstone, missing and empty inline value remain distinct. The current Schema contains only version/name/requiredness. -Evaluation advances revision-bound windows of at most twenty Samples. Durable -redo windows precede row writes; CAS progress and content bindings prevent stale -work from becoming the accepted result. Preparation validates the server-assembled -Manifest, initial Split set and all candidate upload claims before publication. -The complete resource root makes data and initial Splits available together; the -independent data head transition selects that resource as latest. +Computation reads windows of at most twenty Samples into a bounded request-local +Manifest builder. It writes final content directly; no redo window or working row +precedes it. Final resource keys reject conflicting immutable content. The complete +root makes data and initial Splits available together, and the independent data +head transition selects that resource as latest. ## 3. Split evaluation and publication @@ -190,9 +197,10 @@ Python `prepare_split_revision` retain the exact resource ID and complete input retry; an error carries the revision identity. Missing capability never invokes a draft or v1 mutation. Standalone begin/edit/evaluate/prepare/publish workflow routes return 501 before storage access; SDK workflow methods are removed. Only legacy -draft observation, abandonment and cleanup remain for existing records. Composition -initial Split computation still uses its internal work records until that flow is -replaced; the standalone route never invokes it. +draft observation, abandonment and cleanup remain for existing records. Initial Split computation in `prepare_snapshot` shares the same pure result +preparation against its fixed incomplete owner; it writes no draft or progress. +Legacy internal draft drivers remain for migration/cleanup and are not called by +the resource preparation routes. `PUT snapshots/{snapshot}/split-head` accepts the target revision and both explicit nullable expected revision/generation. It requires the `split-head-update` capability and has no draft or attempt identity. The failed head condition is @@ -280,31 +288,27 @@ protects any indexed child without terminal retirement, including an incomplete child awaiting cleanup. Thus deletion either observes the child or the child's last parent check rejects completion. Readers never register in this protocol. -Uploads are Dataset-scoped and content-verified. Composition registers staging and -candidate upload IDs durably before claims. Publication is gated on complete ownership -preparation, including unchanged shared parent uploads. Retiring one owner never -releases a different owner; permanent owner revocation rejects delayed claims. - -Abandonment first fences the draft, then cleanup proceeds in bounded resumable steps. -Obsolete evaluation pages and candidates use durable cursors. Cleanup seals possible -late write keys, including currently absent keys, rather than deleting a value and -allowing a stale CAS(None) writer to recreate it. Edit IDs are registered before -payload writes so failed admissions can also be retired. Small receipt/fence records -remain after bulk cleanup to preserve replay and non-resurrection guarantees. -Split publication does not create those replay records; existing records and -editing/evaluation cleanup remain compatibility inputs. -Helping an already stored edit receipt only reads and validates it; retries do not -submit a doomed CAS for the same bytes. Evaluation journals and cleanup cursors -are scoped to one draft, not to unrelated readers. They retain exact partial-work -recovery and prevent a delayed writer from reviving cleaned candidate data. +Uploads are Dataset-scoped and content-verified. Resource preparation claims only +the final Snapshot owner for each unique managed upload in its final Manifest, +including unchanged parent uploads. Completion requires all those owners. The +Manifest supplies cleanup's owner references without a second process list. +Retiring one owner preserves other owners; permanent revocation rejects delayed +claims. Legacy staging/candidate lists remain cleanup inputs for old records. + +Legacy draft abandonment and cleanup use their existing bounded cursors and seals +for old work, including keys that could still receive delayed writes. Stored edit +receipts and progress can be decoded and reclaimed, but resource preparation never +creates or resumes them. Their independent discovery and replacement remain part +of migration work; removing the last discovery path would lose owned garbage. + Split cleanup competes with preparation on the final revision key. If the complete root already exists, including a root returned by a failed cleanup CAS, cleanup preserves its member pages and summary while retiring evaluation intents. If cleanup seals an absent root first, a delayed preparation cannot publish it. Abandoning the draft does not delete an already complete revision. Retiring the owning Snapshot still permits revision cleanup and closes new read admission. -Sources use read-only availability checks. The compatibility -`source_lease_deadline` field is zero for new drafts and edits. +Sources use read-only availability checks. Legacy `source_lease_deadline` is only +a compatibility field; resource preparation has no reader lease or draft. Snapshot reclaim checks latest/stable, retained Snapshots, live children, candidate ownership under a terminal owner fence. A durable five-minute grace deadline diff --git a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml index e7740ac3c..4a088f26e 100644 --- a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml +++ b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml @@ -15,17 +15,62 @@ info: require dataset-management; direct Snapshot head updates require snapshot-head-update; Split reads require split-revisions; complete Split preparation requires split-preparation; direct Split head updates require split-head-update; versioned - stateless reads require stateless-reads. Upload, composition and Snapshot retirement - require uploads, composition and snapshot-reclaim respectively, advertised only + stateless reads require stateless-reads. Upload, Snapshot composition preparation and retirement + require uploads, composition-preparation and snapshot-reclaim respectively, advertised only when the upload service is configured. Query parameters are strict and duplicate - keys fail. Composition and Split preparation JSON bodies are limited to 1 MiB. - Standalone Split begin/edit/evaluate/prepare/publish workflow routes return 501; + keys fail. Snapshot preparation JSON/input/Manifest are limited to 8 MiB; + standalone Split preparation JSON and final bindings are limited to 1 MiB. + Composition and standalone Split begin/edit/evaluate/prepare/publish workflow routes return 501; legacy draft reads, abandonment and cleanup remain available for old records. servers: - url: /api/dataset/v2 security: - bearerAuth: [] paths: + /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}: + put: + operationId: prepareSnapshot + summary: Prepare complete Snapshot content from retained caller inputs + description: >- + Requires composition-preparation and a configured upload service. Caller + chooses the Snapshot ID and retains the complete parent/source/schema/ + Sample patches/Split definitions. Parent and schema are required nullable + fields; null schema inherits the fixed parent's schema. Frozen sources and + inherited definitions name exact revisions; currentSplit is never resolved + during retries. Body, native input and final Manifest are bounded to 8 MiB, + with 30 seconds for computation. Each patch ID occurs once; upsert patches + specified fields and preserves other effective parent fields. Uploaded + fields carry id/length/md5 from a completed result; conversion performs no + metadata query and the authority validates the descriptor before claiming + its final Snapshot owner. The complete root is installed last, after final + index, initial Split membership and payload ownership. Complete results + are readable by ID before head publication. Retry the same ID and full + input after unknown outcomes; different content or initial Split binding + conflicts. No editing/evaluation progress, operation mapping or staging + owner is stored. Capture the intended head base before computation, then + update the Dataset head separately. There is no workflow or v1 fallback. + parameters: + - {$ref: '#/components/parameters/dataset'} + - {$ref: '#/components/parameters/snapshot'} + requestBody: + required: true + content: + application/json: + schema: {$ref: '#/components/schemas/PrepareSnapshot'} + responses: + '200': + description: Complete immutable result; latest/stable are unchanged + content: + application/json: + schema: {$ref: '#/components/schemas/PreparedSnapshot'} + '409': + description: Resource content, descriptor or Split policy conflicts + $ref: '#/components/responses/InspectionError' + '413': + description: Body, native input, final Manifest or computation exceeds bounds + $ref: '#/components/responses/InspectionError' + default: + $ref: '#/components/responses/InspectionError' /namespaces/dataset-ns/datasets/{dataset}/head: get: operationId: datasetHead @@ -867,49 +912,11 @@ paths: type: string format: binary maxLength: 4194304 - /namespaces/dataset-ns/datasets/{dataset}/compositions: - post: - operationId: beginComposition - summary: Pin source membership and begin a composition - parameters: - - $ref: '#/components/parameters/dataset' - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/CompositionDraft' - default: - $ref: '#/components/responses/InspectionError' - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/BeginComposition' /namespaces/dataset-ns/datasets/{dataset}/compositions/{operation}: get: + deprecated: true operationId: getComposition - summary: Read current composition status - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: operation - required: true - schema: *id001 - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/CompositionDraft' - default: - $ref: '#/components/responses/InspectionError' - put: - operationId: editComposition - summary: Apply an idempotent revision-bound edit + summary: Observe an existing legacy composition without resuming it parameters: - $ref: '#/components/parameters/dataset' - in: path @@ -925,16 +932,8 @@ paths: $ref: '#/components/schemas/CompositionDraft' default: $ref: '#/components/responses/InspectionError' - description: At most 20 Sample changes and a 1 MiB JSON body. Historical edit retries return the historical - draft result; use GET for current status. Upload references are resolved within this Dataset; raw storage - locators are rejected. - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/EditComposition' delete: + deprecated: true operationId: abandonComposition summary: Abandon an unpublished composition parameters: @@ -959,88 +958,9 @@ paths: application/json: schema: $ref: '#/components/schemas/DraftRevision' - /namespaces/dataset-ns/datasets/{dataset}/compositions/{operation}/evaluate: - post: - operationId: evaluateComposition - summary: Evaluate one bounded composition window - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: operation - required: true - schema: *id001 - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/CompositionPreview' - default: - $ref: '#/components/responses/InspectionError' - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/EvaluateDraft' - /namespaces/dataset-ns/datasets/{dataset}/compositions/{operation}/prepare: - post: - operationId: prepareComposition - summary: Prepare data, initial Splits and upload ownership - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: operation - required: true - schema: *id001 - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/CompositionDraft' - default: - $ref: '#/components/responses/InspectionError' - description: Repeat until uploads.publication_ready is true. Prepared candidates remain invisible until publication. - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/DraftRevision' - /namespaces/dataset-ns/datasets/{dataset}/compositions/{operation}/publish: - post: - operationId: publishComposition - summary: Publish data and initial Splits atomically - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: operation - required: true - schema: *id001 - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/CommitReceipt' - default: - $ref: '#/components/responses/InspectionError' - description: Successful publication advances latest only, leaving the draft Prepared and creating no attempt, - receipt or success-proof record. Null expected_head is explicit; omission fails. Retry the exact conditional - request on unknown outcome. A conflict preserves the complete candidate and returns observed_head for caller - review, without a confirmation query. A later publication prevents historical replay from rewinding latest. - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/PublishComposition' /namespaces/dataset-ns/datasets/{dataset}/compositions/{operation}/cleanup: post: + deprecated: true operationId: cleanupComposition summary: Advance bounded obsolete composition cleanup parameters: @@ -1383,13 +1303,6 @@ components: required: [expected_revision] properties: expected_revision: {$ref: '#/components/schemas/Decimal'} - EvaluateDraft: - type: object - additionalProperties: false - required: [expected_revision, expected_page] - properties: - expected_revision: {$ref: '#/components/schemas/Decimal'} - expected_page: {$ref: '#/components/schemas/Decimal'} UpdateSplitHead: type: object additionalProperties: false @@ -1556,6 +1469,71 @@ components: required: [cursor] properties: cursor: {type: string, maxLength: 349530} + PrepareSnapshot: + type: object + additionalProperties: false + required: [parent, source, schema, samples, splits, policy] + properties: + parent: {$ref: '#/components/schemas/NullableRevision'} + source: {$ref: '#/components/schemas/CompositionSource'} + schema: + anyOf: + - {$ref: '#/components/schemas/CompositionSchema'} + - {type: 'null'} + samples: {type: array, items: {$ref: '#/components/schemas/SnapshotChange'}} + splits: {type: array, maxItems: 64, items: {$ref: '#/components/schemas/DraftSplit'}} + policy: {$ref: '#/components/schemas/SplitPolicy'} + SnapshotValue: + oneOf: + - type: object + additionalProperties: false + required: [kind, data] + properties: + kind: {const: inline} + data: {type: string, description: Canonical padded base64 of at most 65536 decoded bytes} + - type: object + additionalProperties: false + required: [kind, id, length, md5] + properties: + kind: {const: upload} + id: {$ref: '#/components/schemas/OpaqueId'} + length: {$ref: '#/components/schemas/Decimal'} + md5: {type: string, pattern: '^[0-9a-f]{32}$'} + - type: object + additionalProperties: false + required: [kind] + properties: + kind: {const: tombstone} + SnapshotField: + type: object + additionalProperties: false + required: [name, value] + properties: + name: {type: string, minLength: 1, description: At most 1024 UTF-8 bytes} + value: {$ref: '#/components/schemas/SnapshotValue'} + SnapshotChange: + oneOf: + - type: object + additionalProperties: false + required: [kind, id] + properties: + kind: {const: remove} + id: {type: string} + - type: object + additionalProperties: false + required: [kind, id, fields] + properties: + kind: {const: upsert} + id: {type: string} + fields: {type: array, maxItems: 256, items: {$ref: '#/components/schemas/SnapshotField'}} + description: Canonical base64 Sample IDs decode to 1 through 16384 bytes. Each ID occurs once; an upsert preserves unspecified effective parent fields. + PreparedSnapshot: + type: object + required: [snapshot, initial_split, sample_count] + properties: + snapshot: {$ref: '#/components/schemas/OpaqueId'} + initial_split: {$ref: '#/components/schemas/NullableRevision'} + sample_count: {$ref: '#/components/schemas/Decimal'} CompositionSchema: type: object additionalProperties: false @@ -1565,7 +1543,8 @@ components: properties: version: type: integer - const: 1 + minimum: 0 + maximum: 4294967295 fields: type: array items: @@ -1604,152 +1583,6 @@ components: anyOf: - $ref: '#/components/schemas/Predicate' - type: 'null' - BeginComposition: - type: object - additionalProperties: false - required: - - operation - - parent - - source - - inherit_parent_splits - - splits - properties: - operation: &composition001 - $ref: '#/components/schemas/OpaqueId' - parent: - $ref: '#/components/schemas/NullableRevision' - source: - $ref: '#/components/schemas/CompositionSource' - schema: - anyOf: - - $ref: '#/components/schemas/CompositionSchema' - - type: 'null' - inherit_parent_splits: &composition005 - type: boolean - parent_split_revision: - $ref: '#/components/schemas/NullableRevision' - splits: &composition003 - type: array - items: - $ref: '#/components/schemas/DraftSplit' - maxItems: 64 - policy: - anyOf: - - $ref: '#/components/schemas/SplitPolicy' - - type: 'null' - CompositionValue: - oneOf: - - type: object - additionalProperties: false - required: - - kind - - data - properties: - kind: - const: inline - data: - type: string - description: Canonical padded base64; decoded length at most 65536 bytes, including empty. - - type: object - additionalProperties: false - required: - - kind - - id - properties: - kind: - const: upload - id: *composition001 - - type: object - additionalProperties: false - required: - - kind - properties: - kind: - const: tombstone - CompositionField: - type: object - additionalProperties: false - required: - - name - - value - properties: - name: &composition002 - type: string - value: - $ref: '#/components/schemas/CompositionValue' - SampleChange: - oneOf: - - type: object - additionalProperties: false - required: - - kind - - id - properties: - kind: - const: remove - id: *composition002 - - type: object - additionalProperties: false - required: - - kind - - id - - fields - properties: - kind: - const: upsert - id: *composition002 - fields: - type: array - items: - $ref: '#/components/schemas/CompositionField' - maxItems: 256 - description: Sample IDs are canonical padded base64 of 1 through 16384 bytes. Upsert replaces the complete field - set for that Sample. - EditComposition: - type: object - additionalProperties: false - required: - - edit - - expected_revision - - samples - properties: - edit: *composition001 - expected_revision: &composition004 - $ref: '#/components/schemas/Decimal' - schema: - anyOf: - - $ref: '#/components/schemas/CompositionSchema' - - type: 'null' - samples: - type: array - items: - $ref: '#/components/schemas/SampleChange' - maxItems: 20 - splits: - anyOf: - - *composition003 - - type: 'null' - policy: - anyOf: - - $ref: '#/components/schemas/SplitPolicy' - - type: 'null' - CompositionPreview: - type: object - additionalProperties: false - required: - - input_revision - - next_page - - members - - complete - - binding - properties: - input_revision: *composition004 - next_page: *composition004 - members: *composition004 - complete: *composition005 - binding: - type: string - pattern: ^[0-9a-f]{64}$ CompositionUploadStatus: type: object additionalProperties: false @@ -1758,9 +1591,9 @@ components: - staged_uploads - candidate_uploads properties: - publication_ready: *composition005 - staged_uploads: *composition004 - candidate_uploads: *composition004 + publication_ready: {type: boolean} + staged_uploads: {$ref: '#/components/schemas/Decimal'} + candidate_uploads: {$ref: '#/components/schemas/Decimal'} CompositionDraft: type: object additionalProperties: false @@ -1778,8 +1611,8 @@ components: - preparation - uploads properties: - operation: *composition001 - revision: *composition004 + operation: {$ref: '#/components/schemas/OpaqueId'} + revision: {$ref: '#/components/schemas/Decimal'} parent: $ref: '#/components/schemas/NullableRevision' source: @@ -1788,13 +1621,13 @@ components: $ref: '#/components/schemas/NullableRevision' schema: $ref: '#/components/schemas/CompositionSchema' - splits: *composition003 + splits: {type: array, maxItems: 64, items: {$ref: '#/components/schemas/DraftSplit'}} policy: $ref: '#/components/schemas/SplitPolicy' progress: $ref: '#/components/schemas/DraftProgress' source_lease_deadline: - <<: *composition004 + <<: {$ref: '#/components/schemas/Decimal'} description: Compatibility field; new drafts and edits use zero and hold no source lease. preparation: anyOf: @@ -1804,7 +1637,7 @@ components: - snapshot - split_preview properties: - snapshot: *composition001 + snapshot: {$ref: '#/components/schemas/OpaqueId'} split_preview: anyOf: - $ref: '#/components/schemas/SplitPreview' @@ -1812,18 +1645,6 @@ components: - type: 'null' uploads: $ref: '#/components/schemas/CompositionUploadStatus' - PublishComposition: - type: object - additionalProperties: false - required: - - attempt - - snapshot - - expected_head - properties: - attempt: *composition001 - snapshot: *composition001 - expected_head: - $ref: '#/components/schemas/NullableGeneration' CompositionCleanup: type: object additionalProperties: false @@ -1835,7 +1656,7 @@ components: - edits - physical_cleanup_deferred properties: - complete: *composition005 + complete: {type: boolean} candidates: type: object additionalProperties: false @@ -1843,8 +1664,8 @@ components: - input_revision - complete properties: - input_revision: *composition004 - complete: *composition005 + input_revision: {$ref: '#/components/schemas/Decimal'} + complete: {type: boolean} uploads: type: object additionalProperties: false @@ -1852,8 +1673,8 @@ components: - retired_owners - complete properties: - retired_owners: *composition004 - complete: *composition005 + retired_owners: {$ref: '#/components/schemas/Decimal'} + complete: {type: boolean} scratch: $ref: '#/components/schemas/SplitCleanup' edits: @@ -1864,8 +1685,8 @@ components: - retired_pages - complete properties: - retired_pages: *composition004 - complete: *composition005 + retired_pages: {$ref: '#/components/schemas/Decimal'} + complete: {type: boolean} - type: 'null' physical_cleanup_deferred: const: true @@ -1877,9 +1698,9 @@ components: - next_page - complete properties: - input_revision: *composition004 - next_page: *composition004 - complete: *composition005 + input_revision: {$ref: '#/components/schemas/Decimal'} + next_page: {$ref: '#/components/schemas/Decimal'} + complete: {type: boolean} CreateUpload: type: object additionalProperties: false @@ -1889,8 +1710,8 @@ components: - part_size - md5 properties: - id: *composition001 - length: *composition004 + id: {$ref: '#/components/schemas/OpaqueId'} + length: {$ref: '#/components/schemas/Decimal'} part_size: type: integer minimum: 1 @@ -1909,7 +1730,7 @@ components: index: type: integer minimum: 0 - length: *composition004 + length: {$ref: '#/components/schemas/Decimal'} md5: *composition006 UploadStatus: type: object @@ -1924,8 +1745,8 @@ components: - cleanup_complete - parts properties: - id: *composition001 - length: *composition004 + id: {$ref: '#/components/schemas/OpaqueId'} + length: {$ref: '#/components/schemas/Decimal'} part_size: type: integer md5: *composition006 @@ -1935,8 +1756,8 @@ components: - complete - claimed - aborted - expires_at: *composition004 - cleanup_complete: *composition005 + expires_at: {$ref: '#/components/schemas/Decimal'} + cleanup_complete: {type: boolean} parts: type: array items: @@ -1948,7 +1769,7 @@ components: required: - snapshot properties: - snapshot: *composition001 + snapshot: {$ref: '#/components/schemas/OpaqueId'} RetainedSnapshot: type: object additionalProperties: false @@ -1956,8 +1777,8 @@ components: - snapshot - retained properties: - snapshot: *composition001 - retained: *composition005 + snapshot: {$ref: '#/components/schemas/OpaqueId'} + retained: {type: boolean} SnapshotLifecycle: type: object required: [snapshot, operation, latest, stable, retained, active_read_lease, state, physical_cleanup_deferred, parent_protection] @@ -1996,7 +1817,7 @@ components: - complete - physical_cleanup_deferred properties: - snapshot: *composition001 + snapshot: {$ref: '#/components/schemas/OpaqueId'} phase: enum: - grace_period @@ -2005,7 +1826,7 @@ components: - index_fences - metadata - complete - complete: *composition005 + complete: {type: boolean} reclaim_after: type: integer minimum: 0 diff --git a/doc/design/access-server/dataset/spec-crowdb-dataset.md b/doc/design/access-server/dataset/spec-crowdb-dataset.md index 0db5daaf5..70dcab727 100644 --- a/doc/design/access-server/dataset/spec-crowdb-dataset.md +++ b/doc/design/access-server/dataset/spec-crowdb-dataset.md @@ -103,7 +103,7 @@ The Dataset authority provides these operations. Each operation is scoped by a Dataset identity; operations that take a Snapshot ID MUST use the immutable ID, not `latest` or `stable` as an implicit mutable reference. The table summarizes the legacy publication/read surface. Control v2 adds bounded -catalog/tree inspection, composition drafts and independent Split revisions as +catalog/tree inspection, immutable composition resources and independent Split revisions as specified below and in its versioned OpenAPI contract. | Operation | Input | Successful result | @@ -212,13 +212,21 @@ and managed payload owners are ready; it MUST NOT move latest/stable. Input and final Manifest are bounded to 8 MiB and computation to 30 seconds. Incomplete computation is redone; identical complete content is reusable with zero writes. Different resource content, including its initial Split binding, MUST conflict. -Unknown completion outcomes MUST NOT cause a confirmation query. This native -contract does not imply that the legacy Composition REST workflow is migrated. +Unknown completion outcomes MUST NOT cause a confirmation query. REST exposes +this operation through `PUT snapshots/{snapshot}` with the +`composition-preparation` capability when uploads are configured. The JSON body +is also bounded to 8 MiB. Parent/schema are explicit nullable fields. Upload input +MUST include completed id/length/md5; DTO conversion MUST NOT add metadata queries. +The authority validates unique upload descriptors and returns conflict on mismatch. +Rust/Python SDKs MUST retain the same ID/full input without status queries or a +workflow fallback. Legacy Composition begin/edit/evaluate/prepare/publish routes +return 501 before storage; only observation, abandonment and cleanup remain for +old records. Composition and Split status reads MUST NOT reconstruct a publication request, help edit receipts or persist recovery state. Legacy Committing drafts are presented as Prepared without rewriting the stored draft or querying attempt, -receipt or head records. This permits an explicit publication retry; it does not +receipt or head records. A caller may select a complete resource with the direct head API; this does not prove that the historical request changed a head. Direct Snapshot head updates MUST accept the complete Snapshot ID and explicit nullable expected generation without an operation or draft identity. They MUST @@ -339,11 +347,13 @@ Split-aware inputs to v1. Required nullable preconditions MUST be supplied expli Unsigned 64-bit control values use canonical decimal strings. Legacy routes and read-plan interpretations remain compatible; binary framing is unchanged. -Composition is durable and revision-CAS edited. Parent data defaults to all Samples; +Composition takes retained complete caller inputs and creates an immutable +resource without edit/evaluation progress. Parent data defaults to all Samples; Selection, frozen named Split and their intersection can restrict it. Source S1/r2/train creates a child with parent S1 and frozen source membership. Inherited Splits intersect surviving IDs; new IDs are unassigned until explicit assignment or -reevaluation. Defaults are disjoint/allow-unassigned/frozen inheritance. Weighted +reevaluation. Defaults are disjoint/allow-unassigned; inheritance is explicit and +frozen to the supplied revision. Weighted partition follows the versioned SHA-256 algorithm in the [control lifecycle](design-dataset-control.md#3-split-evaluation-and-publication). diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index d77b94c1e..1bbb2e3b6 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -363,6 +363,16 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 source/继承 revision 显式固定，移除隐式 currentSplit 续算。迁移 HTTP/SDK/容器的全部 Composition 写调用者后关闭 begin/edit/evaluate/prepare/publish；保留必要 legacy decoder/GC 到 P13，不能用包装旧方法代替删除工作状态。补成本、鉴权、错误、丢回复与重启验收。 + REST/SDK、调用者迁移和旧写入口关闭已实现；11 项 HTTP/Rust consumer、36 项独立源码 + 包测试、12 项 installed Python 单测及真实 sync/async consumer 通过；已移除 SDK + workflow DTO/方法；OpenAPI 294 个本地引用有效。单 sample+ + 初始 Split 的 HTTP 准备为 12G/16C；同一 upload 在两个字段出现仍只读两次 U 元数据 + （一次唯一 descriptor 校验，一次 claim），无 DTO 补查。错误 descriptor 返回 409、零写。 + 修复声明过大 body 提前拒绝时未读 body 导致 413 JSON 被连接 reset 截断，复用现有有界 + drain，不放宽 body/存储限制。Dataset/Access/e2e-test 与独立 SDK all-target clippy、fmt、 + boundary、diff 通过，均经 pixi；wheel 为 `.crowdb-runtime/artifacts/sdk-composition-preparation-20261024`。 + 当前镜像尚未更新到此批，待新镜像验收后关闭；Native 旧 forward driver 与 legacy GC/ + decoder 的最终删除仍在 P2.2/P3.2/P13。 - [ ] **P2.2 删除服务端编辑/计算过程**：删除 durable 编辑/进度阶段，客户端失败重新请求； 保持发布前不可见、expected head 及源 Snapshot 被删时的明确错误。孤儿清理通过 R4 验证，例外封口记录先 review。Files: `composition/`、Access v2 composition、SDK 和恢复测试。 diff --git a/e2e-test/tests/access_dataset_split.rs b/e2e-test/tests/access_dataset_split.rs index c3531136a..3c808d993 100644 --- a/e2e-test/tests/access_dataset_split.rs +++ b/e2e-test/tests/access_dataset_split.rs @@ -39,42 +39,25 @@ async fn split_commit_restart_preserves_candidates_receipts_and_historical_membe .build() .unwrap(); call(&client, Method::PUT, "", None, 200).await; - let operation = "21212121212121212121212121212121"; - let draft = format!("/compositions/{operation}"); - let definitions = json!([{"name":"train","definition":{"kind":"selection","predicate":{"kind":"all"}},"origin":{"kind":"evaluated"}}]); - let begin = json!({"operation":operation,"parent":null,"source":{"kind":"all"},"schema":{"version":1,"fields":[{"name":"value","required":true}]},"inherit_parent_splits":false,"parent_split_revision":null,"splits":definitions,"policy":null}); - call(&client, Method::POST, "/compositions", Some(&begin), 200).await; - let edit = json!({"edit":"22222222222222222222222222222222","expected_revision":"1","samples":[{"kind":"upsert","id":"AAH/","fields":[{"name":"value","value":{"kind":"inline","data":"ZGF0YQ=="}}]}]}); - call(&client, Method::PUT, &draft, Some(&edit), 200).await; + let snapshot = "21212121212121212121212121212121"; + let resource = format!("/snapshots/{snapshot}"); + let input = json!({"parent":null,"source":{"kind":"all"},"schema":{"version":1,"fields":[{"name":"value","required":true}]}, + "samples":[{"kind":"upsert","id":"AAH/","fields":[{"name":"value","value":{"kind":"inline","data":"ZGF0YQ=="}}]}], + "splits":[{"name":"train","definition":{"kind":"selection","predicate":{"kind":"all"}},"origin":{"kind":"evaluated"}}], + "policy":{"overlap":"disjoint","coverage":"allow_unassigned"}}); + let prepared = call(&client, Method::PUT, &resource, Some(&input), 200).await; restart().await; assert_eq!( - call(&client, Method::PUT, &draft, Some(&edit), 200).await["revision"], - "2" + call(&client, Method::PUT, &resource, Some(&input), 200).await, + prepared ); - let prepared = prepare(&client, &draft).await; - let snapshot = prepared["preparation"]["snapshot"].as_str().unwrap(); complete_result_survives_restart(&client, snapshot).await; - let publish = - json!({"attempt":"23232323232323232323232323232323","snapshot":snapshot,"expected_head":null}); - let receipt = call( - &client, - Method::POST, - &format!("{draft}/publish"), - Some(&publish), - 200, - ) - .await; + let publish = json!({"snapshot":snapshot,"expected_head":null}); + let current = call(&client, Method::PUT, "/head", Some(&publish), 200).await; restart().await; assert_eq!( - call( - &client, - Method::POST, - &format!("{draft}/publish"), - Some(&publish), - 200 - ) - .await, - receipt + call(&client, Method::PUT, "/head", Some(&publish), 200).await, + current ); assert_eq!( call( @@ -141,40 +124,6 @@ async fn complete_result_survives_restart(client: &Client, snapshot: &str) { ); } -async fn prepare(client: &Client, draft: &str) -> Value { - let mut page = json!("0"); - let mut done = false; - for _ in 0..100 { - let p = call( - client, - Method::POST, - &format!("{draft}/evaluate"), - Some(&json!({"expected_revision":"2","expected_page":page})), - 200, - ) - .await; - if p["complete"] == true { - done = true; - break; - } - page = p["next_page"].clone(); - } - assert!(done, "evaluation did not finish"); - for _ in 0..100 { - let p = call( - client, - Method::POST, - &format!("{draft}/prepare"), - Some(&json!({"expected_revision":"2"})), - 200, - ) - .await; - if p["uploads"]["publication_ready"] == true { - return p; - } - } - panic!("preparation did not finish") -} async fn revise(client: &Client, snapshot: &str) { let data_head = call(client, Method::GET, "/head", None, 200).await; let head = call( diff --git a/lib/crowdb-access-dataset/src/composition/resource.rs b/lib/crowdb-access-dataset/src/composition/resource.rs index 1e917e271..4b2384256 100644 --- a/lib/crowdb-access-dataset/src/composition/resource.rs +++ b/lib/crowdb-access-dataset/src/composition/resource.rs @@ -154,7 +154,7 @@ fn referenced_uploads<'a>( }; if let Some(existing) = references.get(&id) { if existing.length != *length || existing.md5 != *md5 { - return Err(UploadError::Integrity.into()); + return Err(UploadError::Conflict.into()); } } else { references.insert( @@ -184,7 +184,7 @@ async fn validate_uploads( .metadata(reference.location) .await?; if record.length != reference.length || record.md5 != reference.md5 { - return Err(UploadError::Integrity.into()); + return Err(UploadError::Conflict.into()); } } Ok(()) diff --git a/lib/crowdb-access-dataset/tests/composition_resource_recovery_test.rs b/lib/crowdb-access-dataset/tests/composition_resource_recovery_test.rs index 67d037d9e..ae1c9aa62 100644 --- a/lib/crowdb-access-dataset/tests/composition_resource_recovery_test.rs +++ b/lib/crowdb-access-dataset/tests/composition_resource_recovery_test.rs @@ -122,7 +122,7 @@ async fn final_snapshot_owners_keep_shared_uploads_without_staging_or_cleanup_jo &ReadCancellation::new() ) .await, - Err(CompositionError::Upload(UploadError::Integrity)) + Err(CompositionError::Upload(UploadError::Conflict)) )); assert_eq!(store.mutations.load(Ordering::Relaxed), before); let a = SnapshotId::random(); diff --git a/sdk/README.md b/sdk/README.md index bdd6d85c0..7c68f8ec0 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -202,20 +202,25 @@ permission or promise that a redeployed endpoint still supports the operation. - `OpaqueId` preserves operation/Snapshot/revision IDs; `Generation` is an opaque head precondition. `Decimal` encodes exact u64 values as JSON strings, and `SampleId` preserves binary IDs through canonical base64. -- `begin_composition`, `edit_composition`, `evaluate_composition` and - `prepare_composition` are resumable bounded steps. Keep operation/edit IDs and - input revisions. Preparation is ready to publish only when - `uploads.publication_ready` is true. Initial Splits publish with the Snapshot. -- `publish_composition` advances latest; `update_split_head` updates only currentSplit - on its owning Snapshot. Null expected heads/revisions are explicit. On an - unknown outcome, retry the exact request. `commit_status` exposes only legacy - receipts; new Snapshot/Split publication saves no request ledger. A retry - confirms the same current target or returns a condition conflict after a later - target replaces it. `observed_head` supplies the failed condition's observation, - without a confirmation GET. A deliberate retry uses reviewed inputs and that - generation; content is never silently rebased. New publication leaves the - compatibility draft Prepared. Completed resources are readable and retain - payload ownership even before publication or after draft abandonment. +- `prepare_snapshot(snapshot, request)` submits complete parent/source/schema/ + Sample patches/Split definitions to an immutable resource PUT. Retain the Snapshot + ID and full input for retries. It requires `composition-preparation`; body/input/ + final Manifest are bounded to 8 MiB and computation to 30 seconds. Parent and + schema are explicitly nullable. Upsert patches specified fields and keeps other + effective parent fields; each Sample ID appears once. Frozen Split sources and + inherited definitions name exact revisions, so retries never resolve currentSplit. + Rust uses `PrepareSnapshot`, `SnapshotChange`, `SnapshotField` and `SnapshotValue`. + Upload values include the completed result's `id`, `length` and `md5`; conversion + adds no descriptor query. Synchronous and async Python expose the same operation. + Complete Snapshots and their initial Splits are readable before head publication. + Incomplete work is recomputed; matching final content is reused, while conflicting + content or initial Split binding is rejected. No workflow progress is saved. + Capture the intended Dataset head base before computation and publish separately. +- Composition workflow methods and standalone Split workflow methods are removed. + Their old begin/edit/evaluate/prepare/publish HTTP routes return 501; SDKs never + fall back to them. Legacy draft observation/abandonment/cleanup remain HTTP + maintenance operations for existing records. `commit_status` reads only legacy + receipts; new pointer updates create no request ledger or historical replay. - `split_members`, `split_history`, `inventory`, `roots`, `children`, `schema` and `samples` return lazy `PageCursor`s. Each `next_page` fetches at most 100 items; it never eagerly collects the Dataset. Persist `continuation()` for scoped @@ -232,8 +237,7 @@ permission or promise that a redeployed endpoint still supports the operation. Capture the intended head base before preparation. Rust uses `PrepareSplitRevision`; synchronous and async Python expose the same operation. Standalone Split workflow SDK methods are removed; their old HTTP write routes - return 501, with no fallback. Composition initial Splits still use composition - preparation until its workflow replacement is complete. + return 501, with no fallback. Initial Splits are prepared with `prepare_snapshot`. - `update_split_head(snapshot, request)` updates a complete revision directly with `revision`, `expected_revision` and `expected_head`; both expected values must be supplied, including null. It requires `split-head-update`, takes no attempt ID diff --git a/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs b/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs index 8d65f7767..6bf563832 100644 --- a/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs +++ b/sdk/crates/crowdb-sdk-dataset/examples/split_control.rs @@ -58,18 +58,8 @@ async fn import(client: &v2::ControlClient) -> Result<(v2::OpaqueId, v2::OpaqueI client.upload_reader(&mut &b"data"[..], &upload).await?; assert!(client.upload_reader(&mut &b"date"[..], &upload).await.is_err()); let sample = v2::SampleId::new(vec![0, 255, 128])?; - let operation = v2::OpaqueId::random(); - let splits = vec![v2::DraftSplit { - name: "train/a b".into(), - definition: v2::SplitDefinition::Selection { - predicate: v2::Predicate::Explicit { - ids: vec![sample.clone()], - }, - }, - origin: v2::SplitOrigin::Evaluated, - }]; - let begin = v2::BeginComposition { - operation: operation.clone(), + let snapshot = v2::OpaqueId::random(); + let prepare = v2::PrepareSnapshot { parent: None, source: v2::CompositionSource::All, schema: Some(v2::Schema { @@ -79,34 +69,33 @@ async fn import(client: &v2::ControlClient) -> Result<(v2::OpaqueId, v2::OpaqueI required: true, }], }), - inherit_parent_splits: false, - parent_split_revision: None, - splits, - policy: None, - }; - client.begin_composition(&begin).await?; - let edit = v2::EditComposition { - edit: v2::OpaqueId::random(), - expected_revision: 1.into(), - schema: None, - samples: vec![v2::SampleChange::Upsert { + samples: vec![v2::SnapshotChange::Upsert { id: sample.clone(), - fields: vec![v2::Field { + fields: vec![v2::SnapshotField { name: "value".into(), - value: v2::FieldValue::Upload { id: upload.id }, + value: v2::SnapshotValue::Upload { + id: upload.id.clone(), + length: upload.length, + md5: upload.md5.clone(), + }, }], }], - splits: None, - policy: None, + splits: vec![v2::DraftSplit { + name: "train/a b".into(), + definition: v2::SplitDefinition::Selection { + predicate: v2::Predicate::Explicit { + ids: vec![sample.clone()], + }, + }, + origin: v2::SplitOrigin::Evaluated, + }], + policy: v2::SplitPolicy::default(), }; - let edited = client.edit_composition(&operation, &edit).await?; - assert_eq!(edited.revision, 2.into()); + let prepared = client.prepare_snapshot(&snapshot, &prepare).await?; assert_eq!( - client.edit_composition(&operation, &edit).await?.revision, - edited.revision + client.prepare_snapshot(&snapshot, &prepare).await?.snapshot, + snapshot ); - let prepared = prepare(client, &operation, edited.revision).await?; - let snapshot = prepared.preparation.unwrap().snapshot; let request = v2::UpdateSnapshotHead { snapshot: snapshot.clone(), expected_head: None, @@ -120,58 +109,10 @@ async fn import(client: &v2::ControlClient) -> Result<(v2::OpaqueId, v2::OpaqueI client.update_snapshot_head(&request).await?.generation, head.generation ); - cleanup(client, &operation).await?; - let revision = client.split_head(&snapshot).await?.revision.unwrap(); + let revision = prepared.initial_split.unwrap(); Ok((snapshot, revision, sample)) } -async fn prepare( - client: &v2::ControlClient, - operation: &v2::OpaqueId, - revision: v2::Decimal, -) -> Result { - let mut page = 0.into(); - let mut done = false; - for _ in 0..100 { - let preview = client - .evaluate_composition( - operation, - &v2::EvaluateDraft { - expected_revision: revision, - expected_page: page, - }, - ) - .await?; - if preview.complete { - done = true; - break; - } - page = preview.next_page; - } - assert!(done, "composition evaluation exceeded the example's work budget"); - for _ in 0..100 { - let draft = client - .prepare_composition( - operation, - &v2::DraftRevision { - expected_revision: revision, - }, - ) - .await?; - if draft.uploads.publication_ready { - return Ok(draft); - } - } - Err("composition preparation exceeded the example's work budget".into()) -} -async fn cleanup(client: &v2::ControlClient, operation: &v2::OpaqueId) -> Result<()> { - for _ in 0..100 { - if client.cleanup_composition(operation).await?.complete { - return Ok(()); - } - } - Err("cleanup exceeded the example's work budget".into()) -} async fn inspect(client: &v2::ControlClient, snapshot: &v2::OpaqueId, sample: &v2::SampleId) -> Result<()> { assert_eq!(client.summary(snapshot).await?.sample_count, 1.into()); assert!(client.contains(snapshot, sample).await?); @@ -264,27 +205,39 @@ async fn revise_splits( } async fn branch(client: &v2::ControlClient, source: v2::SplitSource) -> Result { let expected_head = client.head().await?.generation; - let operation = v2::OpaqueId::random(); - client - .begin_composition(&v2::BeginComposition { - operation: operation.clone(), - parent: Some(source.snapshot.clone()), - source: v2::CompositionSource::Split { - source: source.clone(), - selection: None, + let snapshot = v2::OpaqueId::random(); + let inherited = client.split_revision(&source.snapshot, &source.revision).await?; + let policy = inherited.policy; + let splits = inherited + .splits + .into_iter() + .map(|split| v2::DraftSplit { + origin: v2::SplitOrigin::Inherited { + source: v2::SplitSource { + name: split.name.clone(), + ..source.clone() + }, }, - schema: None, - inherit_parent_splits: true, - parent_split_revision: Some(source.revision), - splits: vec![], - policy: None, + name: split.name, + definition: split.definition, }) + .collect(); + client + .prepare_snapshot( + &snapshot, + &v2::PrepareSnapshot { + parent: Some(source.snapshot.clone()), + source: v2::CompositionSource::Split { + source: source.clone(), + selection: None, + }, + schema: None, + samples: Vec::new(), + splits, + policy, + }, + ) .await?; - let snapshot = prepare(client, &operation, 1.into()) - .await? - .preparation - .unwrap() - .snapshot; client .update_snapshot_head(&v2::UpdateSnapshotHead { snapshot: snapshot.clone(), @@ -302,7 +255,6 @@ async fn branch(client: &v2::ControlClient, source: v2::SplitSource) -> Result u64 { diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2.rs b/sdk/crates/crowdb-sdk-dataset/src/v2.rs index 194ce36fb..d22d65a62 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/v2.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/v2.rs @@ -5,6 +5,7 @@ mod client; mod composition; +mod composition_preparation; mod identity; mod inspection; mod lifecycle; @@ -21,6 +22,9 @@ mod upload; pub use client::ControlClient; pub use composition::*; +pub use composition_preparation::{ + PrepareSnapshot, PreparedSnapshot, SnapshotChange, SnapshotField, SnapshotValue, +}; pub use identity::{Decimal, Generation, OpaqueId, SampleId}; pub use inspection::*; pub use paging::{PageCursor, PageKind}; diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2/composition.rs b/sdk/crates/crowdb-sdk-dataset/src/v2/composition.rs index 12a4e0097..0c50dcbe9 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/v2/composition.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/v2/composition.rs @@ -1,9 +1,6 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{ - required_option, Decimal, DraftProgress, DraftSplit, Generation, OpaqueId, Predicate, SampleId, - SplitCleanup, SplitPolicy, SplitPreview, SplitSource, -}; +use super::{Predicate, SplitSource}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Serialize, Deserialize)] @@ -24,116 +21,3 @@ pub enum CompositionSource { selection: Option, }, } -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct BeginComposition { - pub operation: OpaqueId, - #[serde(deserialize_with = "required_option")] - pub parent: Option, - pub source: CompositionSource, - pub schema: Option, - pub inherit_parent_splits: bool, - pub parent_split_revision: Option, - pub splits: Vec, - pub policy: Option, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum FieldValue { - Inline { data: String }, - Upload { id: OpaqueId }, - Tombstone, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct Field { - pub name: String, - pub value: FieldValue, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum SampleChange { - Remove { id: SampleId }, - Upsert { id: SampleId, fields: Vec }, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct EditComposition { - pub edit: OpaqueId, - pub expected_revision: Decimal, - pub schema: Option, - pub samples: Vec, - pub splits: Option>, - pub policy: Option, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CompositionPreparation { - pub snapshot: OpaqueId, - #[serde(deserialize_with = "required_option")] - pub split_preview: Option, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CompositionUploadStatus { - pub publication_ready: bool, - pub staged_uploads: Decimal, - pub candidate_uploads: Decimal, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CompositionDraft { - pub operation: OpaqueId, - pub revision: Decimal, - #[serde(deserialize_with = "required_option")] - pub parent: Option, - pub source: CompositionSource, - #[serde(deserialize_with = "required_option")] - pub parent_split_revision: Option, - pub schema: Schema, - pub splits: Vec, - pub policy: SplitPolicy, - pub progress: DraftProgress, - pub source_lease_deadline: Decimal, - #[serde(deserialize_with = "required_option")] - pub preparation: Option, - pub uploads: CompositionUploadStatus, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CompositionPreview { - pub input_revision: Decimal, - pub next_page: Decimal, - pub members: Decimal, - pub complete: bool, - pub binding: String, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PublishComposition { - pub attempt: OpaqueId, - pub snapshot: OpaqueId, - #[serde(deserialize_with = "required_option")] - pub expected_head: Option, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CandidateCleanup { - pub input_revision: Decimal, - pub complete: bool, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct OwnerCleanup { - pub retired_owners: Decimal, - pub complete: bool, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct EditCleanup { - pub retired_pages: Decimal, - pub complete: bool, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CompositionCleanup { - pub complete: bool, - pub candidates: CandidateCleanup, - pub uploads: OwnerCleanup, - pub scratch: SplitCleanup, - #[serde(deserialize_with = "required_option")] - pub edits: Option, - pub physical_cleanup_deferred: bool, -} diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2/composition_preparation.rs b/sdk/crates/crowdb-sdk-dataset/src/v2/composition_preparation.rs new file mode 100644 index 000000000..ea3b4320c --- /dev/null +++ b/sdk/crates/crowdb-sdk-dataset/src/v2/composition_preparation.rs @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + required_option, CompositionSource, ControlClient, Decimal, DraftSplit, OpaqueId, SampleId, Schema, + SplitPolicy, +}; +use crate::Error; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PrepareSnapshot { + #[serde(deserialize_with = "required_option")] + pub parent: Option, + pub source: CompositionSource, + #[serde(deserialize_with = "required_option")] + pub schema: Option, + pub samples: Vec, + pub splits: Vec, + pub policy: SplitPolicy, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum SnapshotChange { + Remove { + id: SampleId, + }, + Upsert { + id: SampleId, + fields: Vec, + }, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SnapshotField { + pub name: String, + pub value: SnapshotValue, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum SnapshotValue { + Inline { + data: String, + }, + Upload { + id: OpaqueId, + length: Decimal, + md5: String, + }, + Tombstone, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct PreparedSnapshot { + pub snapshot: OpaqueId, + #[serde(deserialize_with = "required_option")] + pub initial_split: Option, + pub sample_count: Decimal, +} + +impl ControlClient { + /// Computes complete immutable content from retained caller inputs; head publication is separate. + /// # Errors + /// Returns missing capability, source/content conflicts, payload validation or transfer errors. + pub async fn prepare_snapshot( + &self, + snapshot: &OpaqueId, + request: &PrepareSnapshot, + ) -> Result { + self.call( + "composition-preparation", + "PUT", + &["snapshots", snapshot.as_str()], + Some(request), + ) + .await + .map_err(|error| error.with_identity(snapshot.as_str())) + } +} diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2/lifecycle.rs b/sdk/crates/crowdb-sdk-dataset/src/v2/lifecycle.rs index 38fe487a7..ed8196546 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/v2/lifecycle.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/v2/lifecycle.rs @@ -1,128 +1,12 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{ - BeginComposition, CommitReceipt, CompositionCleanup, CompositionDraft, CompositionPreview, ControlClient, - DraftRevision, EditComposition, EvaluateDraft, OpaqueId, PublishComposition, SplitHead, SplitRevision, -}; +use super::{ControlClient, OpaqueId, SplitHead, SplitRevision}; use crate::Error; impl ControlClient { - /// Executes one bounded operation; retain request IDs for exact retries. - /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. - pub async fn begin_composition(&self, request: &BeginComposition) -> Result { - self.call("composition", "POST", &["compositions"], Some(request)) - .await - .map_err(|error| error.with_identity(request.operation.as_str())) - } - /// Executes one bounded operation; retain request IDs for exact retries. - /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. - pub async fn composition(&self, operation: &OpaqueId) -> Result { - self.call::<(), _>("composition", "GET", &["compositions", operation.as_str()], None) - .await - } - /// Executes one bounded operation; retain request IDs for exact retries. + /// Reads the current Split reference for one available Snapshot. /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. - pub async fn edit_composition( - &self, - operation: &OpaqueId, - request: &EditComposition, - ) -> Result { - self.call( - "composition", - "PUT", - &["compositions", operation.as_str()], - Some(request), - ) - .await - .map_err(|error| error.with_identity(request.edit.as_str())) - } - /// Executes one bounded operation; retain request IDs for exact retries. - /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. - pub async fn abandon_composition( - &self, - operation: &OpaqueId, - request: &DraftRevision, - ) -> Result { - self.call( - "composition", - "DELETE", - &["compositions", operation.as_str()], - Some(request), - ) - .await - .map_err(|error| error.with_identity(operation.as_str())) - } - /// Executes one bounded operation; retain request IDs for exact retries. - /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. - pub async fn evaluate_composition( - &self, - operation: &OpaqueId, - request: &EvaluateDraft, - ) -> Result { - self.call( - "composition", - "POST", - &["compositions", operation.as_str(), "evaluate"], - Some(request), - ) - .await - .map_err(|error| error.with_identity(operation.as_str())) - } - /// Executes one bounded operation; retain request IDs for exact retries. - /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. - pub async fn prepare_composition( - &self, - operation: &OpaqueId, - request: &DraftRevision, - ) -> Result { - self.call( - "composition", - "POST", - &["compositions", operation.as_str(), "prepare"], - Some(request), - ) - .await - .map_err(|error| error.with_identity(operation.as_str())) - } - /// Executes one bounded operation; retain request IDs for exact retries. - /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. - pub async fn publish_composition( - &self, - operation: &OpaqueId, - request: &PublishComposition, - ) -> Result { - self.call( - "composition", - "POST", - &["compositions", operation.as_str(), "publish"], - Some(request), - ) - .await - .map_err(|error| error.with_identity(request.attempt.as_str())) - } - /// Executes one bounded operation; retain request IDs for exact retries. - /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. - pub async fn cleanup_composition(&self, operation: &OpaqueId) -> Result { - self.call::<(), _>( - "composition", - "POST", - &["compositions", operation.as_str(), "cleanup"], - None, - ) - .await - .map_err(|error| error.with_identity(operation.as_str())) - } - /// Executes one bounded operation; retain request IDs for exact retries. - /// # Errors - /// Returns capability, conflict, invalid input, transport or service errors. + /// Returns capability, missing owner or service failures. pub async fn split_head(&self, snapshot: &OpaqueId) -> Result { self.call::<(), _>( "split-revisions", diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2/split.rs b/sdk/crates/crowdb-sdk-dataset/src/v2/split.rs index a5ceb8f82..cb18f6120 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/v2/split.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/v2/split.rs @@ -92,36 +92,12 @@ pub struct SplitRevision { pub splits: Vec, } #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SplitValidation { - pub scanned: Decimal, - pub assigned: Decimal, - pub unassigned: Decimal, - pub overlapping: Decimal, - pub unassigned_examples: Vec, - pub overlap_examples: Vec, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SplitPreview { - pub input_revision: Decimal, - pub next_page: Decimal, - pub complete: bool, - pub member_counts: Vec, - pub validation: SplitValidation, -} -#[derive(Clone, Debug, Serialize, Deserialize)] pub struct SplitHead { #[serde(deserialize_with = "required_option")] pub revision: Option, #[serde(deserialize_with = "required_option")] pub generation: Option, } -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SplitCleanup { - pub input_revision: Decimal, - pub next_page: Decimal, - pub complete: bool, -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SplitRevisionSummary { pub snapshot: OpaqueId, diff --git a/sdk/crates/crowdb-sdk-dataset/src/v2/state.rs b/sdk/crates/crowdb-sdk-dataset/src/v2/state.rs index ca50f3206..6aa0025e4 100644 --- a/sdk/crates/crowdb-sdk-dataset/src/v2/state.rs +++ b/sdk/crates/crowdb-sdk-dataset/src/v2/state.rs @@ -1,6 +1,6 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{required_option, Decimal, Generation, OpaqueId}; +use super::{required_option, Generation, OpaqueId}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Serialize, Deserialize)] @@ -10,31 +10,6 @@ pub enum CommitTarget { SplitRevision { snapshot: OpaqueId, revision: OpaqueId }, } #[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(tag = "state", rename_all = "snake_case")] -pub enum DraftProgress { - Editing, - Evaluating { - input_revision: Decimal, - next_page: Decimal, - }, - Evaluated { - input_revision: Decimal, - binding: String, - }, - Prepared { - target: CommitTarget, - }, - Committing { - target: CommitTarget, - attempt: OpaqueId, - }, - Published { - target: CommitTarget, - attempt: OpaqueId, - }, - Abandoned, -} -#[derive(Clone, Debug, Serialize, Deserialize)] pub struct CommitReceipt { pub attempt: OpaqueId, pub target: CommitTarget, @@ -51,14 +26,3 @@ pub struct Head { #[serde(deserialize_with = "required_option")] pub generation: Option, } -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct DraftRevision { - pub expected_revision: Decimal, -} -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct EvaluateDraft { - pub expected_revision: Decimal, - pub expected_page: Decimal, -} diff --git a/sdk/crates/crowdb-sdk-dataset/tests/composition_preparation_test.rs b/sdk/crates/crowdb-sdk-dataset/tests/composition_preparation_test.rs new file mode 100644 index 000000000..9bb33d00a --- /dev/null +++ b/sdk/crates/crowdb-sdk-dataset/tests/composition_preparation_test.rs @@ -0,0 +1,109 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[allow(dead_code)] +#[path = "../../crowdb-sdk-core/tests/common/peer.rs"] +mod peer; +use crowdb_sdk_dataset::{ + v2::{CompositionSource, OpaqueId, PrepareSnapshot, SplitPolicy}, + DatasetClient, +}; +use serde_json::json; + +fn input() -> PrepareSnapshot { + PrepareSnapshot { + parent: None, + source: CompositionSource::All, + schema: None, + samples: Vec::new(), + splits: Vec::new(), + policy: SplitPolicy::default(), + } +} + +#[test] +fn input_requires_explicit_parent_and_schema_without_process_identifiers() { + let original = serde_json::to_value(input()).unwrap(); + for field in ["parent", "schema"] { + let mut invalid = original.clone(); + invalid.as_object_mut().unwrap().remove(field); + assert!(serde_json::from_value::(invalid).is_err()); + } + for field in [ + "operation", + "edit", + "expected_revision", + "expected_page", + "inherit_parent_splits", + ] { + let mut invalid = original.clone(); + invalid[field] = json!("1"); + assert!(serde_json::from_value::(invalid).is_err()); + } +} + +#[tokio::test] +async fn unavailable_preparation_cannot_fall_back_to_a_composition_workflow() { + let mut peer = peer::TestPeer::start(r#"{"api_version":2,"capabilities":["composition"]}"#, false).await; + let client = DatasetClient::new(&peer.endpoint, "test", None) + .unwrap() + .control(); + assert!(client + .prepare_snapshot(&OpaqueId::random(), &input()) + .await + .is_err()); + assert!(peer + .requests + .recv() + .await + .unwrap() + .starts_with("GET /api/dataset/v2/capabilities ")); + assert!(peer.requests.try_recv().is_err()); +} + +#[tokio::test] +async fn an_unknown_reply_retries_the_same_snapshot_and_full_input_without_status_queries() { + let mut peer = peer::TestPeer::start_responses( + vec![ + ( + 200, + r#"{"api_version":2,"capabilities":["composition-preparation"]}"#, + ), + ( + 503, + r#"{"code":"unavailable","message":"unknown outcome","request_id":"test","retryable":true}"#, + ), + ( + 200, + r#"{"snapshot":"01010101010101010101010101010101","initial_split":null,"sample_count":"1"}"#, + ), + ], + false, + ) + .await; + let snapshot: OpaqueId = "01".repeat(16).parse().unwrap(); + let client = DatasetClient::new(&peer.endpoint, "test", None) + .unwrap() + .control(); + let input = input(); + assert_eq!( + client.prepare_snapshot(&snapshot, &input).await.unwrap().snapshot, + snapshot + ); + assert!(peer + .requests + .recv() + .await + .unwrap() + .starts_with("GET /api/dataset/v2/capabilities ")); + for _ in 0..2 { + let request = peer.requests.recv().await.unwrap(); + assert!(request.starts_with(&format!( + "PUT /api/dataset/v2/namespaces/dataset-ns/datasets/test/snapshots/{snapshot} " + ))); + let body: serde_json::Value = + serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!(body, serde_json::to_value(&input).unwrap()); + } + assert!(peer.requests.try_recv().is_err()); +} diff --git a/sdk/crates/crowdb-sdk-dataset/tests/v2_contract_test.rs b/sdk/crates/crowdb-sdk-dataset/tests/v2_contract_test.rs index 4aecc7c51..61e3744bb 100644 --- a/sdk/crates/crowdb-sdk-dataset/tests/v2_contract_test.rs +++ b/sdk/crates/crowdb-sdk-dataset/tests/v2_contract_test.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. use crowdb_sdk_dataset::{ - v2::{Decimal, OpaqueId, PublishComposition, SampleId}, + v2::{Decimal, OpaqueId, SampleId}, DatasetClient, }; use serde_json::json; @@ -65,37 +65,6 @@ fn exact_id_and_integer_contracts_reject_lossy_or_ambiguous_input() { for value in ["", "YQ", "YR==", "YQ==\n"] { assert!(serde_json::from_value::(json!(value)).is_err()); } - let mut publish = json!({"attempt":"01".repeat(16),"snapshot":"02".repeat(16),"expected_head":null}); - serde_json::from_value::(publish.clone()).unwrap(); - publish.as_object_mut().unwrap().remove("expected_head"); - assert!(serde_json::from_value::(publish).is_err()); -} - -#[tokio::test] -async fn unsupported_composition_cannot_issue_a_write() { - let mut peer = peer::TestPeer::start(r#"{"api_version":2,"capabilities":[]}"#, false).await; - let client = DatasetClient::new(&peer.endpoint, "test", None) - .unwrap() - .control(); - let id = OpaqueId::random(); - assert!(client - .publish_composition( - &id, - &PublishComposition { - attempt: OpaqueId::random(), - snapshot: OpaqueId::random(), - expected_head: None - } - ) - .await - .is_err()); - assert!(peer - .requests - .recv() - .await - .unwrap() - .starts_with("GET /api/dataset/v2/capabilities ")); - assert!(peer.requests.try_recv().is_err()); } #[tokio::test] @@ -120,26 +89,3 @@ async fn pages_are_lazy_bounded_and_reject_a_repeated_continuation() { assert_eq!(pages.continuation(), Some("YQ==")); assert!(peer.requests.recv().await.unwrap().contains("after=YQ%3D%3D")); } - -#[tokio::test] -async fn publication_transport_retry_preserves_exact_attempt_and_body() { - let mut peer = peer::TestPeer::start_statuses(r#"{"api_version":2,"capabilities":["composition"],"code":"busy","message":"busy","request_id":"retry","retryable":true,"attempt":"01010101010101010101010101010101","target":{"kind":"snapshot","snapshot":"02020202020202020202020202020202"},"expected_head":null,"committed_head":"0303030303030303030303030303030303030303030303030303030303030303"}"#,false,vec![200,503,200]).await; - let client = DatasetClient::new(&peer.endpoint, "test", None) - .unwrap() - .control(); - let request = PublishComposition { - attempt: "01".repeat(16).parse().unwrap(), - snapshot: "02".repeat(16).parse().unwrap(), - expected_head: None, - }; - let receipt = client - .publish_composition(&OpaqueId::random(), &request) - .await - .unwrap(); - assert_eq!(receipt.attempt, request.attempt); - peer.requests.recv().await.unwrap(); - let first = peer.requests.recv().await.unwrap(); - let retry = peer.requests.recv().await.unwrap(); - assert_eq!(first, retry); - assert!(first.contains("\"expected_head\":null")); -} diff --git a/sdk/crates/crowdb-sdk-python/src/control.rs b/sdk/crates/crowdb-sdk-python/src/control.rs index e32600983..0fdeb9006 100644 --- a/sdk/crates/crowdb-sdk-python/src/control.rs +++ b/sdk/crates/crowdb-sdk-python/src/control.rs @@ -5,7 +5,12 @@ use serde::{de::DeserializeOwned, Serialize}; use serde_json::Value; pub(super) async fn dispatch(client: DatasetClient, name: &str, args: &str) -> Result { - if args.len() > 1024 * 1024 { + let max_bytes = if name == "prepare_snapshot" { + 8 * 1024 * 1024 + 64 * 1024 + } else { + 1024 * 1024 + }; + if args.len() > max_bytes { return Err(Error::Limit); } if name == "capabilities" { @@ -62,14 +67,7 @@ pub(super) async fn dispatch(client: DatasetClient, name: &str, args: &str) -> R async fn lifecycle(client: &v2::ControlClient, name: &str, value: &Value) -> Result, Error> { let id = || id(value, "id"); let result = match name { - "begin_composition" => encode(&client.begin_composition(&request(value)?).await?), - "composition" => encode(&client.composition(&id()?).await?), - "edit_composition" => encode(&client.edit_composition(&id()?, &request(value)?).await?), - "evaluate_composition" => encode(&client.evaluate_composition(&id()?, &request(value)?).await?), - "prepare_composition" => encode(&client.prepare_composition(&id()?, &request(value)?).await?), - "publish_composition" => encode(&client.publish_composition(&id()?, &request(value)?).await?), - "abandon_composition" => encode(&client.abandon_composition(&id()?, &request(value)?).await?), - "cleanup_composition" => encode(&client.cleanup_composition(&id()?).await?), + "prepare_snapshot" => encode(&client.prepare_snapshot(&id()?, &request(value)?).await?), "prepare_split_revision" => encode( &client .prepare_split_revision(&id()?, &self::id(value, "revision")?, &request(value)?) diff --git a/sdk/crowdb_dataset/control.py b/sdk/crowdb_dataset/control.py index d1cf28142..8b8c5f4a2 100644 --- a/sdk/crowdb_dataset/control.py +++ b/sdk/crowdb_dataset/control.py @@ -60,18 +60,7 @@ def matrix(self, snapshot, sample_ids, fields): def preview(self, snapshot, sample, field, max_bytes=65536): return self._call("preview", {"id": snapshot, "sample": sample, "field": field, "max_bytes": max_bytes}) - def begin_composition(self, request): return self._call("begin_composition", {"request": request}) - def composition(self, operation): return self._call("composition", {"id": operation}) - def edit_composition(self, operation, request): return self._call("edit_composition", {"id": operation, "request": request}) - def evaluate_composition(self, operation, expected_revision, expected_page): - return self._call("evaluate_composition", {"id": operation, "request": {"expected_revision": expected_revision, "expected_page": expected_page}}) - def prepare_composition(self, operation, expected_revision): - return self._call("prepare_composition", {"id": operation, "request": {"expected_revision": expected_revision}}) - def publish_composition(self, operation, request): return self._call("publish_composition", {"id": operation, "request": request}) - def abandon_composition(self, operation, expected_revision): - return self._call("abandon_composition", {"id": operation, "request": {"expected_revision": expected_revision}}) - def cleanup_composition(self, operation): return self._call("cleanup_composition", {"id": operation}) - + def prepare_snapshot(self, snapshot, request): return self._call("prepare_snapshot", {"id": snapshot, "request": request}) def prepare_split_revision(self, snapshot, revision, request): return self._call("prepare_split_revision", {"id": snapshot, "revision": revision, "request": request}) def update_split_head(self, snapshot, request): return self._call("update_split_head", {"id": snapshot, "request": request}) diff --git a/sdk/tests/split_consumer.py b/sdk/tests/split_consumer.py index fcc6603e2..a84b31174 100644 --- a/sdk/tests/split_consumer.py +++ b/sdk/tests/split_consumer.py @@ -11,25 +11,6 @@ from crowdb_dataset.client import ConflictError -def prepare(client, operation, revision): - page = "0" - for _ in range(100): - preview = client.evaluate_composition(operation, revision, page) - if preview["complete"]: break - page = preview["next_page"] - else: raise AssertionError("composition evaluation budget") - for _ in range(100): - draft = client.prepare_composition(operation, revision) - if draft["uploads"]["publication_ready"]: return draft["preparation"]["snapshot"] - raise AssertionError("composition preparation budget") - - -def cleanup(client, operation): - for _ in range(100): - if client.cleanup_composition(operation)["complete"]: return - raise AssertionError("cleanup budget") - - def publish(client, snapshot, expected_head): request = {"snapshot": snapshot, "expected_head": expected_head} head = client.update_snapshot_head(request) @@ -61,9 +42,10 @@ def upload_source(client, source, upload_id, length, md5, part_size=1024*1024): return client.complete_upload(upload_id) -async def verify_async(endpoint, name, token, snapshot, revision, prepared_revision): +async def verify_async(endpoint, name, token, snapshot, revision, prepared_revision, snapshot_input): async with AsyncDatasetClient(endpoint, name, token) as owner: client = owner.control() + assert (await client.prepare_snapshot(snapshot, snapshot_input))["snapshot"] == snapshot assert (await client.lifecycle(snapshot))["snapshot"] == snapshot prepared = await client.prepare_split_revision(snapshot, prepared_revision, { "policy": {"overlap": "disjoint", "coverage": "allow_unassigned"}, "splits": []}) @@ -107,18 +89,19 @@ def main(): upload, data = new_id(), b"external source" for _ in range(2): upload_source(client, io.BytesIO(data), upload, len(data), hashlib.md5(data).hexdigest(), 4) - operation, sample = new_id(), sample_id(b"\x00\xff\x80") + snapshot, sample = new_id(), sample_id(b"\x00\xff\x80") split = {"name": "train/a b", "definition": {"kind": "selection", "predicate": {"kind": "all"}}, "origin": {"kind": "evaluated"}} - client.begin_composition({"operation": operation, "parent": None, "source": {"kind": "all"}, + snapshot_input = {"parent": None, "source": {"kind": "all"}, "schema": {"version": 1, "fields": [{"name": "value", "required": True}]}, - "inherit_parent_splits": False, "parent_split_revision": None, "splits": [split], "policy": None}) - client.edit_composition(operation, {"edit": new_id(), "expected_revision": "1", "schema": None, - "samples": [{"kind": "upsert", "id": sample, "fields": [{"name": "value", "value": {"kind": "upload", "id": upload}}]}], "splits": None, "policy": None}) - snapshot = prepare(client, operation, "2") + "samples": [{"kind": "upsert", "id": sample, "fields": [{"name": "value", "value": { + "kind": "upload", "id": upload, "length": str(len(data)), "md5": hashlib.md5(data).hexdigest()}}]}], + "splits": [split], "policy": {"overlap": "disjoint", "coverage": "allow_unassigned"}} + prepared = client.prepare_snapshot(snapshot, snapshot_input) + assert prepared["snapshot"] == snapshot + assert client.prepare_snapshot(snapshot, snapshot_input) == prepared published_head = publish(client, snapshot, None) lifecycle = client.lifecycle(snapshot) assert lifecycle["snapshot"] == snapshot and lifecycle["latest"] and not lifecycle["retained"] - cleanup(client, operation) head = client.split_head(snapshot) revision = head["revision"] assert client.matrix(snapshot, [sample], ["value"])["entries"][0]["fields"][0]["length"] == str(len(data)) @@ -138,11 +121,13 @@ def main(): assert client.head()["latest"] == snapshot source = SplitSource(snapshot, revision, "train/a b") assert next(client.pages("split_members", source=source))["entries"] == [sample] - child = new_id() - client.begin_composition({"operation": child, "parent": snapshot, + child_snapshot = new_id() + frozen = client.split_revision(snapshot, revision) + inherited = [{"name": item["name"], "definition": item["definition"], "origin": { + "kind": "inherited", "source": SplitSource(snapshot, revision, item["name"]).record()}} for item in frozen["splits"]] + client.prepare_snapshot(child_snapshot, {"parent": snapshot, "source": {"kind": "split", "source": source.record(), "selection": None}, "schema": None, - "inherit_parent_splits": True, "parent_split_revision": revision, "splits": [], "policy": None}) - child_snapshot = prepare(client, child, "1") + "samples": [], "splits": inherited, "policy": frozen["policy"]}) child_head = publish(client, child_snapshot, published_head["generation"]) try: client.update_snapshot_head({"snapshot": snapshot, "expected_head": None}) @@ -150,9 +135,8 @@ def main(): assert error.observed_head == child_head else: raise AssertionError("stale Snapshot update must conflict") - cleanup(client, child) assert client.summary(child_snapshot)["parent"] == snapshot - asyncio.run(verify_async(endpoint, name, token, snapshot, revision, candidate["revision"])) + asyncio.run(verify_async(endpoint, name, token, snapshot, revision, candidate["revision"], snapshot_input)) print("installed Python Split SDK round trip passed", flush=True) diff --git a/sdk/tests/test_control.py b/sdk/tests/test_control.py index 7d9f5e972..164984537 100644 --- a/sdk/tests/test_control.py +++ b/sdk/tests/test_control.py @@ -61,6 +61,23 @@ def __exit__(self, *_): class TestControl(unittest.TestCase): + def test_snapshot_preparation_keeps_resource_identity_and_exact_input(self): + snapshot = new_id() + request = {"parent": None, "source": {"kind": "all"}, "schema": None, "samples": [], "splits": [], + "policy": {"overlap": "disjoint", "coverage": "allow_unassigned"}} + prepared = {"snapshot": snapshot, "initial_split": None, "sample_count": "1"} + error = {"code": "conflict", "message": "content conflicts", "request_id": "test", "retryable": False} + with TestControlPeer(["composition-preparation"], head_replies=[(409, error), (200, prepared)]) as peer, DatasetClient(peer.endpoint, "test") as owner: + control = owner.control() + with self.assertRaises(ConflictError) as failure: control.prepare_snapshot(snapshot, request) + self.assertEqual(failure.exception.identity, snapshot) + self.assertEqual(control.prepare_snapshot(snapshot, request), prepared) + self.assertTrue(peer.requests.get(timeout=10).endswith("/capabilities")) + path = f"/api/dataset/v2/namespaces/dataset-ns/datasets/test/snapshots/{snapshot}" + self.assertEqual(peer.requests.get(timeout=10), (path, request)) + self.assertEqual(peer.requests.get(timeout=10), (path, request)) + self.assertTrue(peer.requests.empty()) + def test_split_preparation_uses_the_resource_path_and_preserves_its_identity(self): snapshot, revision = new_id(), new_id() request = {"policy": {"overlap": "disjoint", "coverage": "allow_unassigned"}, "splits": []} @@ -119,10 +136,12 @@ def test_exact_integer_and_unavailable_mutation(self): for value in (True, 1.0, -1, 2**64): with self.assertRaises(ValueError): decimal(value) with TestControlPeer() as peer, DatasetClient(peer.endpoint, "test") as owner: - request = {"attempt": new_id(), "snapshot": new_id(), "expected_head": None} + snapshot = new_id() + request = {"parent": None, "source": {"kind": "all"}, "schema": None, "samples": [], "splits": [], + "policy": {"overlap": "disjoint", "coverage": "allow_unassigned"}} with self.assertRaises(DatasetError) as error: - owner.control().publish_composition(new_id(), request) - self.assertEqual(error.exception.identity, request["attempt"]) + owner.control().prepare_snapshot(snapshot, request) + self.assertEqual(error.exception.identity, snapshot) self.assertTrue(peer.requests.get(timeout=10).endswith("capabilities")) self.assertTrue(peer.requests.empty()) From e9de65bd5dd4e6afff00a6bcaeb05f814f649c38 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 23:03:36 +0800 Subject: [PATCH 137/258] Prepare native Dataset resources without composition workflows --- .../access-server/design-crowdb-native-sdk.md | 12 +- doc/working/plan-access-simplification.md | 63 +++-- lib/crowdb-sdk-native-python/README.md | 27 +- lib/crowdb-sdk-native-python/src/client.rs | 4 +- lib/crowdb-sdk-native-python/src/control.rs | 121 ++++----- lib/crowdb-sdk-native-python/src/error.rs | 62 ++++- .../tests/real_consumer.py | 86 ++++--- lib/crowdb-sdk-native/README.md | 23 +- lib/crowdb-sdk-native/src/dataset.rs | 3 +- .../src/dataset/composition.rs | 226 ----------------- lib/crowdb-sdk-native/src/dataset/control.rs | 6 +- lib/crowdb-sdk-native/src/dataset/error.rs | 4 + .../src/dataset/preparation.rs | 90 +++++++ lib/crowdb-sdk-native/src/error.rs | 9 + .../tests/common/dataset_composition.rs | 138 +++------- .../tests/common/metadata.rs | 98 +++++++- .../tests/dataset_preparation_test.rs | 237 ++++++++++++++++++ 17 files changed, 737 insertions(+), 472 deletions(-) delete mode 100644 lib/crowdb-sdk-native/src/dataset/composition.rs create mode 100644 lib/crowdb-sdk-native/src/dataset/preparation.rs create mode 100644 lib/crowdb-sdk-native/tests/dataset_preparation_test.rs diff --git a/doc/design/access-server/design-crowdb-native-sdk.md b/doc/design/access-server/design-crowdb-native-sdk.md index a9c6d590c..42ee230fb 100644 --- a/doc/design/access-server/design-crowdb-native-sdk.md +++ b/doc/design/access-server/design-crowdb-native-sdk.md @@ -39,10 +39,14 @@ Copy captures a source generation. Multipart completion binds an exact selected part set; a repeated completed request returns its recorded outcome. `DatasetClient` attaches `DatasetAuthority`, `DatasetUploads` and -`DatasetReads` to these same clients. Shared composition helpers enroll -upload ownership before edits and bind candidate ownership before publication. -Snapshot/Read Plan/Split selection, worker partition, CAS, durable intent -recovery and cleanup remain in the existing Dataset services. +`DatasetReads` to these same clients. Complete Snapshot/Split preparation takes +caller-owned immutable inputs and resource identities. Snapshot completion follows +final upload-owner claims; it stores no editing or evaluation progress. Mutable +head updates use explicit expected bases and preserve structured conflict +observations. Complete results remain readable by ID independently of head +selection. Unknown mutations retain their resource identity, full input and base; +the SDK never retries or rebases automatically. Read selection, worker partition, +resource CAS and cleanup remain in the Dataset authority. ## 2. Shared runtime diff --git a/doc/working/plan-access-simplification.md b/doc/working/plan-access-simplification.md index 1bbb2e3b6..4928d0a0f 100644 --- a/doc/working/plan-access-simplification.md +++ b/doc/working/plan-access-simplification.md @@ -356,23 +356,14 @@ P1.1 已完成，见验证记录；当前 TableLoader 直接解析 canonical 内 - [ ] **P2.1 分清输入、计算进度与候选内容**：逐项列出 draft/edit receipt/work page/row/candidate/ cleanup seal 的用途、写入次数和调用者；按 D1 更新 API/SDK 合同，移除服务端续算接口。Files: Dataset `composition/{begin,edit,state,evaluation,evaluate_step,prepare,cleanup}.rs`。 -- [~] **P2.2b 无状态 Composition REST/SDK**：新增资源 `PUT snapshots/{snapshot}` 与独立 - `composition-preparation` capability，完整输入/明确 body 与计算边界；Rust/Python sync/async - SDK 以同一 Snapshot ID/完整请求重试，不查询草稿或降级。upload 字段直传已有完整结果 - 的 id/length/md5，转换不再为补 descriptor 额外查询；原生校验真实元数据后 claim 最终 owner。 - source/继承 revision 显式固定，移除隐式 currentSplit 续算。迁移 HTTP/SDK/容器的全部 - Composition 写调用者后关闭 begin/edit/evaluate/prepare/publish；保留必要 legacy decoder/GC - 到 P13，不能用包装旧方法代替删除工作状态。补成本、鉴权、错误、丢回复与重启验收。 - REST/SDK、调用者迁移和旧写入口关闭已实现；11 项 HTTP/Rust consumer、36 项独立源码 - 包测试、12 项 installed Python 单测及真实 sync/async consumer 通过；已移除 SDK - workflow DTO/方法；OpenAPI 294 个本地引用有效。单 sample+ - 初始 Split 的 HTTP 准备为 12G/16C；同一 upload 在两个字段出现仍只读两次 U 元数据 - （一次唯一 descriptor 校验，一次 claim），无 DTO 补查。错误 descriptor 返回 409、零写。 - 修复声明过大 body 提前拒绝时未读 body 导致 413 JSON 被连接 reset 截断，复用现有有界 - drain，不放宽 body/存储限制。Dataset/Access/e2e-test 与独立 SDK all-target clippy、fmt、 - boundary、diff 通过，均经 pixi；wheel 为 `.crowdb-runtime/artifacts/sdk-composition-preparation-20261024`。 - 当前镜像尚未更新到此批，待新镜像验收后关闭；Native 旧 forward driver 与 legacy GC/ - decoder 的最终删除仍在 P2.2/P3.2/P13。 +- [~] **P2.2d 控制台迁移 Snapshot/Split 完整资源 API**：删除服务端草稿、编辑/计算进度、 + attempt/receipt reconciliation；完整输入、资源 ID、显式 expected base 和 pending 请求由 + 客户端保存。上传完成返回已有 descriptor，显式冻结 Source/继承 revision；丢回复重试原 + 请求，409 使用 observed head 审阅后更新。迁移 import/Split/recovery/capability 组件与 + 五项浏览器 case，保留 binary ID、MPU、继承/历史成员和冲突检查。当前精确 import 基线 + 在新镜像上失败（3.6s，入口依赖已删除 composition capability，按钮 disabled）；不是既有 + 通过结果。基线 artifact `run-d463ff78dac94b26b59a028f52b9d565`。Files: console Dataset + UI、browser Dataset fixtures/flows、proxy 测试。 - [ ] **P2.2 删除服务端编辑/计算过程**：删除 durable 编辑/进度阶段，客户端失败重新请求； 保持发布前不可见、expected head 及源 Snapshot 被删时的明确错误。孤儿清理通过 R4 验证，例外封口记录先 review。Files: `composition/`、Access v2 composition、SDK 和恢复测试。 @@ -556,6 +547,44 @@ HTTP RetryLedger 的新请求入口与执行代码已删除，见验证记录。 ## 已完成与验证记录 +- **2026-10-24，P2.2c**：native Rust/Python SDK 删除 Composition 的 begin/edit/evaluate/ + prepare/publish/abandon/cleanup 包装，提供完整 Snapshot/Split 准备和独立显式 head 更新。 + 两种 head 冲突保留结构化 observation；Python `ConflictError.observed_head` 为原生 JSON + 数组身份/generation，不转成字符串，不额外 GET、不自动 rebase。Python head 条件必须 + 显式提供（可 null），旧流程 command 拒绝；完整资源按同 ID/输入重做。 + 新 Native 回归拒绝所有工作状态/attempt/receipt 写，准备不查 head，完整重试零写； + Snapshot head 成功 3G/1C，连续四次 stale 各 3G/0C，复用观察值后仍 3G/1C；Split + stale 4G/0C，沿观察值更新 4G/1C。未知成功回复为 UncertainMutation、无确认 GET， + 同原条件重试 3G/0C。这是逻辑元数据操作，bounded GET 的单 key scan 计一次，非 TPS。 + 66 项 native 全目标测试、两 crate all-target/all-feature clippy、fmt/diff 通过，均经 pixi。 + 修复测试 transport 仅对 point 注入 stale route、漏掉 bounded scan 的缺口，保留原拓扑断言。 + installed wheel 真实 TCP consumer 验证同资源重试、完整未发布 ID 可读、四次 stale 返回 + 原生观察值直接重试、Split 条件、共享 upload owner、MPU、65 秒闲置无 reader TTL、取消/ + held buffer 与存储重启均通过；未执行性能 trials。artifact + `.crowdb-runtime/artifacts/dataset-workload/12ca47dd8df242349847ddef31baf124`。 + 新镜像 `crowdb-node:access-cleanup-native-preparation-20261024` 在提交前完整源码快照 + tree `79249259d7bcbbdfa187d447c1209ba72cf852ad` 构建，server source 校验通过； + OCI `sha256:807498a58c20c93e05c7b5126a50019fada38bf243e462c599acd091a5e9ba99`。 + 控制台和旧 authority forward driver/legacy GC 仍按 P2.2d/P2.2/P3.2/P13 继续。 + +- **2026-10-24，P2.2b**：REST/portable Rust/Python sync/async SDK 改为完整 Snapshot + `PUT` 与独立 `composition-preparation` capability，关闭旧 begin/edit/evaluate/prepare/publish + 写入口（501、零存储访问），删除 portable workflow DTO/方法。上传字段直传 id/length/md5， + 不为补 descriptor 查询；固定 source/继承 revision。完整输入/Manifest 各 8 MiB、30 秒。 + 单 sample+初始 Split HTTP 准备 12G/16C；共享 descriptor 两字段仍仅两次 U GET；重试 + 完整资源零写，错误 descriptor 409/零写。修复提前拒绝超限 body 时连接 reset 截断 413 + JSON，复用有界 drain。11 项 HTTP/Rust consumer、36 项独立源码包、12 项 installed + Python 单测、真实 sync/async consumer、相关 all-target clippy/fmt/boundary/diff 通过； + OpenAPI 294 个本地引用有效，wheel `sdk-composition-preparation-20261024`。 + 源码 `6546daa9d0e757ad1bcf3dfc768da691989f220c` 新镜像五项 Dataset 容器验收全通过， + 覆盖完整资源按 ID 使用、同 ID 重试、独立 head、冲突观察和存储重启。artifact + `.crowdb-runtime/artifacts/container-e2e/run-6b9c1857a5e14a01a6d2f98486605cab`， + tree `83ccbe4d0822351dcf6a9895f805bcd44fed02bd`，server_sources_match=true； + OCI `sha256:01ebd67706b3345da2d1342ea841e6ea5054f45dc4d39090669fca1e83fd0abe`， + Docker `sha256:dbb49ee288b4cec45ffd93370b3e2b3c40da7f3d91df3c049fb27c3f6f3bc470`。 + 此证据只覆盖 REST/portable SDK；native RPC SDK 和控制台仍调用旧流程，分别 P2.2c/d； + forward driver/legacy decoder/GC 删除由 P2.2/P3.2/P13 跟踪，未声称 TPS 提升。 + - **2026-10-24，P2.2a**：新增原生 `CompositionRequest` / `prepare_composition_resource`， 客户端指定 Snapshot ID 与完整源/修改/规则；直接构造最终 Manifest、索引、初始 Split 和 最终 payload owner，完整根最后安装。无 operation→随机 Snapshot、CBG/CD/CE/CX/CW/CM/ diff --git a/lib/crowdb-sdk-native-python/README.md b/lib/crowdb-sdk-native-python/README.md index f116e185c..a73821aa6 100644 --- a/lib/crowdb-sdk-native-python/README.md +++ b/lib/crowdb-sdk-native-python/README.md @@ -61,11 +61,17 @@ runtime.close() compatibility-copy bytes. `runtime.topology(refresh=True)` observes metadata and data transport state; these hints cannot override routing authority. - `dataset.call(json, timeout=30)` uses a tagged `op` command and the existing - Rust Dataset JSON schema. Commands cover create/open/snapshot/head/commit - status, composition begin/status/edit/evaluate/prepare/publish/abandon/cleanup, - stateless read binding, and upload create/complete/status/abort. - `upload_location` returns the authority's opaque location for a composition - field. No command is retried automatically. + Rust Dataset JSON schema. Commands cover create/open/snapshot/head, complete + Snapshot/Split preparation, explicit head updates, stateless read binding, + upload create/complete/status/abort and read-only legacy commit status. + `prepare_snapshot` takes `snapshot` and a complete `input: CompositionRequest`; + `prepare_split_revision` takes `snapshot` and `input: SplitPreparation`. + `update_snapshot_head` requires `snapshot` and `expected_head` (nullable). + `update_split_head` requires `snapshot`, `revision`, `expected_revision` and + `expected_head` (both conditions nullable). The JSON envelope is bounded to + 8 MiB + 64 KiB; native preparation independently bounds content and work. + `upload_location` returns the authority's opaque field location. + No command is retried automatically. - `bind_read` accepts `spec: ReadSpec` and returns a `cursor`. Call `next_batch(cursor)` to fetch; retrying the same cursor replays its batch. Save `Batch.header.next` after successful processing. There is no remote @@ -73,11 +79,14 @@ runtime.close() identical in Rust and HTTP. - `upload_part(upload_id, index, bytes)` copies Python input once into final transport frame slots across allocations of at most 1 MiB and freezes them - before submission. A logical upload part can still contain 4 MiB. Publication uses - upload-managed compositions and preserves durable attempt identity. + before submission. A logical upload part can still contain 4 MiB. + Snapshot preparation claims final upload owners before marking the resource complete; + pointer publication is a separate conditional update. - Typed exceptions include capacity, integrity, cancellation, closed runtime, - deadline, conflict and uncertain mutation. An uncertain mutation does not - authorize replay with a new operation identity. + deadline, conflict and uncertain mutation. Head `ConflictError` carries + `observed_head`, with native JSON ID/generation arrays and the current pointer. + Use it directly for a deliberate retry without another GET. Unknown outcomes + retain the original resource ID, full input and publication conditions. `tests/real_consumer.py` validates the installed binding against real services; `tests/workload.py` includes explicit Python byte-copy costs. The comparison is diff --git a/lib/crowdb-sdk-native-python/src/client.rs b/lib/crowdb-sdk-native-python/src/client.rs index ee1206a99..1e082ad66 100644 --- a/lib/crowdb-sdk-native-python/src/client.rs +++ b/lib/crowdb-sdk-native-python/src/client.rs @@ -61,8 +61,8 @@ impl Dataset { #[pyo3(signature=(command, timeout=30.0))] fn call(&self, py: Python<'_>, command: &str, timeout: f64) -> PyResult { let context = self.context(timeout)?; - if command.len() > 1024 * 1024 { - return Err(NativeError::new_err("control input exceeds 1 MiB")); + if command.len() > 8 * 1024 * 1024 + 64 * 1024 { + return Err(NativeError::new_err("control input exceeds 8 MiB + 64 KiB")); } let command: control::Command = serde_json::from_str(command).map_err(|e| NativeError::new_err(e.to_string()))?; diff --git a/lib/crowdb-sdk-native-python/src/control.rs b/lib/crowdb-sdk-native-python/src/control.rs index 357636886..5ed2cc47b 100644 --- a/lib/crowdb-sdk-native-python/src/control.rs +++ b/lib/crowdb-sdk-native-python/src/control.rs @@ -2,13 +2,11 @@ // Licensed under the Apache License, Version 2.0. use crowdb_access_dataset::{ - composition::{CompositionEdit, CompositionInput}, - CommitAttempt, DatasetUploads, OperationId, ReadSpec, SnapshotId, -}; -use crowdb_sdk_native::{ - dataset::{CompositionStatus, DatasetClient}, - Error, OperationContext, Result, + composition::CompositionRequest, + split::{SplitHeadUpdate, SplitPreparation}, + CommitGeneration, DatasetUploads, OperationId, ReadSpec, SnapshotHeadUpdate, SnapshotId, SplitRevisionId, }; +use crowdb_sdk_native::{dataset::DatasetClient, Error, OperationContext, Result}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; @@ -24,36 +22,26 @@ pub enum Command { CommitStatus { attempt: OperationId, }, - BeginComposition { - operation: OperationId, - input: CompositionInput, - }, - CompositionStatus { - operation: OperationId, - }, - EditComposition { - operation: OperationId, - edit: CompositionEdit, - }, - EvaluateComposition { - operation: OperationId, - revision: u64, - page: u64, - }, - PrepareComposition { - operation: OperationId, - revision: u64, + PrepareSnapshot { + snapshot: SnapshotId, + input: CompositionRequest, }, - PublishComposition { - operation: OperationId, - attempt: CommitAttempt, + UpdateSnapshotHead { + snapshot: SnapshotId, + #[serde(deserialize_with = "Deserialize::deserialize")] + expected_head: Option, }, - AbandonComposition { - operation: OperationId, - revision: u64, + PrepareSplitRevision { + snapshot: SnapshotId, + input: SplitPreparation, }, - CleanupComposition { - operation: OperationId, + UpdateSplitHead { + snapshot: SnapshotId, + revision: SplitRevisionId, + #[serde(deserialize_with = "Deserialize::deserialize")] + expected_revision: Option, + #[serde(deserialize_with = "Deserialize::deserialize")] + expected_head: Option, }, BindRead { spec: ReadSpec, @@ -82,10 +70,6 @@ pub enum Command { fn encoded(value: impl Serialize) -> Result { serde_json::to_value(value).map_err(|e| Error::Storage(e.to_string())) } -fn composition(status: &CompositionStatus) -> Value { - json!({"draft":status.draft,"uploads":status.uploads}) -} - pub async fn execute(client: &DatasetClient, context: OperationContext, command: Command) -> Result { match command { Command::UploadLocation { upload } => encoded(DatasetUploads::location(client.identity(), upload)?), @@ -97,7 +81,7 @@ pub async fn execute(client: &DatasetClient, context: OperationContext, command: .control(context, |s| async move { { let head = s.authority.inspect_head(&s.identity).await?; - Ok(json!({"latest":head.latest,"generation":head.generation})) + Ok(json!({"latest":head.latest,"stable":head.stable,"generation":head.generation})) } }) .await @@ -109,40 +93,45 @@ pub async fn execute(client: &DatasetClient, context: OperationContext, command: }) .await } - Command::BeginComposition { operation, input } => Ok(composition( - &client.begin_composition(context, operation, &input).await?, - )), - Command::CompositionStatus { operation } => { - Ok(composition(&client.composition_status(context, operation).await?)) + Command::PrepareSnapshot { snapshot, input } => { + encoded(client.prepare_snapshot(context, snapshot, &input).await?) + } + Command::UpdateSnapshotHead { + snapshot, + expected_head, + } => { + let head = client + .update_snapshot_head( + context, + &SnapshotHeadUpdate { + snapshot, + expected_head, + }, + ) + .await?; + Ok(json!({"latest":head.latest,"stable":head.stable,"generation":head.generation})) } - Command::EditComposition { operation, edit } => Ok(composition( - &client.edit_composition(context, operation, &edit).await?, - )), - Command::EvaluateComposition { - operation, + Command::PrepareSplitRevision { snapshot, input } => { + encoded(client.prepare_split_revision(context, snapshot, &input).await?) + } + Command::UpdateSplitHead { + snapshot, revision, - page, + expected_revision, + expected_head, } => encoded( client - .evaluate_composition(context, operation, revision, page) + .update_split_head( + context, + snapshot, + &SplitHeadUpdate { + revision, + expected_revision, + expected_head, + }, + ) .await?, ), - Command::PrepareComposition { operation, revision } => Ok(composition( - &client.prepare_composition(context, operation, revision).await?, - )), - Command::PublishComposition { operation, attempt } => { - encoded(client.publish_composition(context, operation, &attempt).await?) - } - Command::AbandonComposition { operation, revision } => Ok(composition( - &client.abandon_composition(context, operation, revision).await?, - )), - Command::CleanupComposition { operation } => { - let status = client.cleanup_composition(context, operation).await?; - Ok( - json!({"complete":status.complete,"retired_upload_owners":status.retired_upload_owners, - "physical_cleanup_deferred":status.physical_cleanup_deferred}), - ) - } Command::BindRead { spec } => encoded(client.bind_read(context, spec).await?), Command::CreateUpload { upload, diff --git a/lib/crowdb-sdk-native-python/src/error.rs b/lib/crowdb-sdk-native-python/src/error.rs index 7655d7613..f99de47b2 100644 --- a/lib/crowdb-sdk-native-python/src/error.rs +++ b/lib/crowdb-sdk-native-python/src/error.rs @@ -2,7 +2,12 @@ // Licensed under the Apache License, Version 2.0. use crowdb_sdk_native::{dataset::FailureKind, Error}; -use pyo3::{create_exception, exceptions::PyException, prelude::*}; +use pyo3::{ + create_exception, + exceptions::PyException, + prelude::*, + types::{PyDict, PyList}, +}; create_exception!(crowdb_dataset_native, NativeError, PyException); create_exception!(crowdb_dataset_native, CapacityError, NativeError); @@ -30,6 +35,46 @@ pub fn failure(error: &Error) -> PyErr { Error::Cancelled => CancelledError::new_err(detail), Error::Shutdown => ClosedError::new_err(detail), Error::Deadline => DeadlineError::new_err(detail), + Error::SnapshotHeadConflict { observed } => head_conflict(detail, |head| { + head.set_item( + "latest", + bytes_list( + head.py(), + observed.latest.as_ref().map(|id| id.as_bytes().as_slice()), + )?, + )?; + head.set_item( + "stable", + bytes_list( + head.py(), + observed.stable.as_ref().map(|id| id.as_bytes().as_slice()), + )?, + )?; + head.set_item( + "generation", + bytes_list( + head.py(), + observed.generation.as_ref().map(|id| id.as_bytes().as_slice()), + )?, + ) + }), + Error::SplitHeadConflict { snapshot, observed } => head_conflict(detail, |head| { + head.set_item("snapshot", PyList::new(head.py(), snapshot.as_bytes())?)?; + head.set_item( + "revision", + bytes_list( + head.py(), + observed.revision.as_ref().map(|id| id.as_bytes().as_slice()), + )?, + )?; + head.set_item( + "generation", + bytes_list( + head.py(), + observed.generation.as_ref().map(|id| id.as_bytes().as_slice()), + )?, + ) + }), Error::Dataset { kind: FailureKind::Conflict, .. @@ -39,6 +84,21 @@ pub fn failure(error: &Error) -> PyErr { } } +fn head_conflict(detail: String, fill: impl FnOnce(&Bound<'_, PyDict>) -> PyResult<()>) -> PyErr { + Python::attach(|py| { + let error = ConflictError::new_err(detail); + let head = PyDict::new(py); + match fill(&head).and_then(|()| error.value(py).setattr("observed_head", head)) { + Ok(()) => error, + Err(error) => error, + } + }) +} + +fn bytes_list<'py>(py: Python<'py>, value: Option<&[u8]>) -> PyResult>> { + value.map(|bytes| PyList::new(py, bytes)).transpose() +} + pub fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { let py = module.py(); module.add("NativeError", py.get_type::())?; diff --git a/lib/crowdb-sdk-native-python/tests/real_consumer.py b/lib/crowdb-sdk-native-python/tests/real_consumer.py index af47969cf..046d7d223 100644 --- a/lib/crowdb-sdk-native-python/tests/real_consumer.py +++ b/lib/crowdb-sdk-native-python/tests/real_consumer.py @@ -103,7 +103,16 @@ def pressure_and_idle_replay(client, snapshot): def publish_native(runtime, root): client = runtime.dataset("native-python-write") call(client, "create") - upload, operation = identity(), identity() + for command in [dict(op="begin_composition", operation=identity(), input={}), + dict(op="update_snapshot_head", snapshot=identity()), + dict(op="update_split_head", snapshot=identity(), revision=identity(), expected_head=None)]: + try: + client.call(json.dumps(command)) + except native.NativeError: + pass + else: + raise AssertionError("removed workflow or missing explicit condition was accepted") + upload, snapshot = identity(), identity() data = native_data() digest = list(hashlib.md5(data).digest()) call(client, "create_upload", upload=upload, length=len(data), part_size=4 << 20, @@ -112,39 +121,22 @@ def publish_native(runtime, root): client.upload_part(bytes(upload), index, data[offset:offset + (4 << 20)]) call(client, "complete_upload", upload=upload) location = call(client, "upload_location", upload=upload) - begin = {"parent": None, "source": "All", "schema": {"version": 1, - "fields": [{"name": "value", "required": True}]}, - "inherit_parent_splits": False, "parent_split_revision": None, - "splits": [{"name": "train", "definition": {"Selection": {"Explicit": [list(b"one")]}}, - "origin": "Evaluated"}], "policy": None} - call(client, "begin_composition", operation=operation, input=begin) - edit = {"id": identity(), "expected_revision": 1, "schema": None, "splits": None, "policy": None, - "samples": [{"Upsert": {"sample_id": list(b"one"), "fields": [{"name": "value", - "value": {"Chunk": {"location": location, "length": len(data), "md5": digest}}}]}}]} - call(client, "edit_composition", operation=operation, edit=edit) - for page in range(32): - if call(client, "evaluate_composition", operation=operation, revision=2, page=page)["complete"]: - break - else: - raise AssertionError("composition evaluation did not complete") - for _ in range(32): - state = call(client, "prepare_composition", operation=operation, revision=2) - if state["uploads"]["publication_ready"]: - break - else: - raise AssertionError("composition ownership did not complete") - snapshot = state["uploads"]["candidate"] - call(client, "publish_composition", operation=operation, attempt={"version": 1, "id": identity(), - "target": {"Snapshot": {"snapshot": snapshot}}, "expected_head": None}) + request = {"parent": None, "source": "All", "schema": {"version": 1, + "fields": [{"name": "value", "required": True}]}, + "samples": [{"Upsert": {"sample_id": list(b"one"), "fields": [{"name": "value", + "value": {"Chunk": {"location": location, "length": len(data), "md5": digest}}}]}}], + "splits": [{"name": "train", "definition": {"Selection": {"Explicit": [list(b"one")]}}, + "origin": "Evaluated"}], "policy": {"overlap": "AllowOverlap", "coverage": "AllowUnassigned"}} + prepared = call(client, "prepare_snapshot", snapshot=snapshot, input=request) + assert prepared["snapshot"] == snapshot and prepared["sample_count"] == 1 + assert call(client, "prepare_snapshot", snapshot=snapshot, input=request) == prepared + head = call(client, "update_snapshot_head", snapshot=snapshot, expected_head=None) + assert head["latest"] == snapshot and head["stable"] is None + check_head_conflicts(client, snapshot, request, head) split_binding = bind(client, bytes(snapshot).hex(), ids=[b"one"], split={"revision": None, "name": "train"}) split_batch = client.next_batch(split_binding["cursor"]) assert json.loads(split_batch.header)["sample_ids"] == [list(b"one")] assert drain(split_batch)[0] == data - for _ in range(32): - if call(client, "cleanup_composition", operation=operation)["complete"]: - break - else: - raise AssertionError("composition cleanup did not complete") try: call(client, "abort_upload", upload=upload) except native.ConflictError: @@ -155,6 +147,40 @@ def publish_native(runtime, root): verify_native(runtime, root) +def check_head_conflicts(client, snapshot, request, head): + # A losing complete result remains readable; only explicit new conditions retry publication. + next_snapshot = identity() + request = dict(request, parent=snapshot, source="All", schema=None, samples=[]) + call(client, "prepare_snapshot", snapshot=next_snapshot, input=request) + assert call(client, "snapshot", snapshot=next_snapshot) + assert drain(client.read_batch(bytes(next_snapshot), [list(b"one")], ["value"]))[0] == native_data() + for _ in range(4): + try: + call(client, "update_snapshot_head", snapshot=next_snapshot, expected_head=None) + except native.ConflictError as error: + assert error.observed_head == head + observed = error.observed_head + else: + raise AssertionError("stale explicit base unexpectedly published") + head = call(client, "update_snapshot_head", snapshot=next_snapshot, expected_head=observed["generation"]) + assert head["latest"] == next_snapshot + revision = identity() + preparation = {"revision": revision, "policy": request["policy"], "splits": request["splits"]} + prepared = call(client, "prepare_split_revision", snapshot=snapshot, input=preparation) + assert call(client, "prepare_split_revision", snapshot=snapshot, input=preparation) == prepared + try: + call(client, "update_split_head", snapshot=snapshot, revision=revision, + expected_revision=None, expected_head=None) + except native.ConflictError as error: + observed = error.observed_head + assert observed["snapshot"] == snapshot and observed["revision"] is not None + else: + raise AssertionError("stale Split base unexpectedly published") + updated = call(client, "update_split_head", snapshot=snapshot, revision=revision, + expected_revision=observed["revision"], expected_head=observed["generation"]) + assert updated["revision"] == revision + + def verify_native(runtime, root): client = runtime.dataset("native-python-write") snapshot = bytes(json.loads(root.joinpath("python-snapshot.json").read_text())) diff --git a/lib/crowdb-sdk-native/README.md b/lib/crowdb-sdk-native/README.md index b3c883f94..8e6f5d12e 100644 --- a/lib/crowdb-sdk-native/README.md +++ b/lib/crowdb-sdk-native/README.md @@ -112,15 +112,20 @@ async fn read_object(seeds: Vec) -> Result<(), Box -// Licensed under the Apache License, Version 2.0. - -use super::{DatasetClient, DatasetControl}; -use crate::{OperationContext, Result}; -use crowdb_access_dataset::{ - composition::{ - CompositionDraft, CompositionEdit, CompositionInput, CompositionPreview, CompositionUploadStatus, - DraftProgress, - }, - CommitAttempt, OperationId, ReadCancellation, SuccessfulCommit, -}; - -#[derive(Debug)] -pub struct CompositionStatus { - pub draft: CompositionDraft, - pub uploads: CompositionUploadStatus, -} - -#[derive(Debug)] -pub struct CompositionCleanup { - pub complete: bool, - pub retired_upload_owners: u64, - pub physical_cleanup_deferred: bool, -} - -impl DatasetControl { - async fn composition_status(&self, operation: OperationId) -> Result { - Ok(CompositionStatus { - draft: self - .authority - .composition_status(&self.identity, operation) - .await?, - uploads: self - .authority - .composition_upload_status(&self.identity, operation) - .await?, - }) - } -} - -impl DatasetClient { - /// Starts an upload-managed composition before any edits can acquire payloads. - /// # Errors - /// Returns divergent identity/input or uncertain storage outcomes. - pub async fn begin_composition( - &self, - context: OperationContext, - operation: OperationId, - input: &CompositionInput, - ) -> Result { - Box::pin(self.control(context, |scope| async move { - scope - .authority - .begin_composition(&scope.identity, operation, input) - .await?; - scope - .authority - .manage_composition_uploads(&scope.identity, operation) - .await?; - scope.composition_status(operation).await - })) - .await - } - - /// # Errors - /// Returns missing/corrupt composition or unavailable metadata. - pub async fn composition_status( - &self, - context: OperationContext, - operation: OperationId, - ) -> Result { - Box::pin(self.control(context, |scope| async move { - scope.composition_status(operation).await - })) - .await - } - - /// Applies one revision-bound edit with the same durable upload owners as HTTP. - /// # Errors - /// Returns revision conflict, invalid payloads or uncertain metadata writes. - pub async fn edit_composition( - &self, - context: OperationContext, - operation: OperationId, - edit: &CompositionEdit, - ) -> Result { - Box::pin(self.control(context, |scope| async move { - scope - .authority - .edit_composition_with_uploads(&scope.identity, operation, edit, &scope.uploads) - .await?; - scope.composition_status(operation).await - })) - .await - } - - /// Advances one bounded evaluation page; callers carry the returned cursor. - /// # Errors - /// Returns stale revision/page, invalid selection or uncertain metadata writes. - pub async fn evaluate_composition( - &self, - context: OperationContext, - operation: OperationId, - revision: u64, - page: u64, - ) -> Result { - Box::pin(self.control(context, |scope| async move { - Ok(scope - .authority - .evaluate_composition( - &scope.identity, - operation, - revision, - page, - &ReadCancellation::new(), - ) - .await?) - })) - .await - } - - /// Advances preparation and upload ownership in bounded, resumable windows. - /// # Errors - /// Returns stale revision, invalid preparation or uncertain metadata writes. - pub async fn prepare_composition( - &self, - context: OperationContext, - operation: OperationId, - revision: u64, - ) -> Result { - Box::pin(self.control(context, |scope| async move { - let cancel = ReadCancellation::new(); - let draft = scope - .authority - .prepare_composition(&scope.identity, operation, revision, &cancel) - .await?; - if matches!(draft.record.progress, DraftProgress::Prepared { .. }) { - scope - .authority - .prepare_composition_uploads(&scope.identity, operation, &scope.uploads, &cancel) - .await?; - } - scope.composition_status(operation).await - })) - .await - } - - /// Publishes using the caller's durable attempt identity, without replay. - /// # Errors - /// Returns incomplete ownership, conflicting head or uncertain publication. - pub async fn publish_composition( - &self, - context: OperationContext, - operation: OperationId, - attempt: &CommitAttempt, - ) -> Result { - Box::pin(self.control(context, |scope| async move { - Ok(scope - .authority - .publish_composition(&scope.identity, operation, attempt) - .await?) - })) - .await - } - - /// # Errors - /// Returns stale revision, publication conflict or uncertain metadata writes. - pub async fn abandon_composition( - &self, - context: OperationContext, - operation: OperationId, - revision: u64, - ) -> Result { - Box::pin(self.control(context, |scope| async move { - scope - .authority - .abandon_composition(&scope.identity, operation, revision) - .await?; - scope.composition_status(operation).await - })) - .await - } - - /// Advances logical cleanup only; shared authority retains physical fences. - /// # Errors - /// Returns conflicting live state or uncertain cleanup progress. - pub async fn cleanup_composition( - &self, - context: OperationContext, - operation: OperationId, - ) -> Result { - Box::pin(self.control(context, |scope| async move { - let cancel = ReadCancellation::new(); - let authority = &scope.authority; - let identity = &scope.identity; - let candidates = authority - .cleanup_composition_candidates_step(identity, operation, &cancel) - .await?; - let owners = authority - .cleanup_composition_uploads_step(identity, operation, &scope.uploads) - .await?; - let scratch = authority - .cleanup_composition_scratch_step(identity, operation, &cancel) - .await?; - let draft = authority.composition_status(identity, operation).await?; - let edits_complete = if matches!( - draft.record.progress, - DraftProgress::Published { .. } | DraftProgress::Abandoned - ) { - authority - .cleanup_composition_edits_step(identity, operation, &cancel) - .await? - .complete - } else { - true - }; - Ok(CompositionCleanup { - complete: candidates.complete && owners.complete && scratch.complete && edits_complete, - retired_upload_owners: owners.retired_owners, - physical_cleanup_deferred: true, - }) - })) - .await - } -} diff --git a/lib/crowdb-sdk-native/src/dataset/control.rs b/lib/crowdb-sdk-native/src/dataset/control.rs index 7ea0392f8..ae893d5e2 100644 --- a/lib/crowdb-sdk-native/src/dataset/control.rs +++ b/lib/crowdb-sdk-native/src/dataset/control.rs @@ -9,7 +9,7 @@ use crowdb_access_dataset::{ use std::{future::Future, sync::Arc}; /// Existing authority services scoped by an explicitly supplied trusted identity. -/// Composition, Split and cleanup use their original typed contracts and CAS rules. +/// Complete resources and mutable pointers retain the authority's typed CAS rules. pub struct DatasetControl { pub authority: Arc, pub uploads: Arc, @@ -18,8 +18,8 @@ pub struct DatasetControl { impl DatasetClient { /// Runs one caller-selected control operation under shared request admission. - /// Use original authority operation/attempt IDs to inspect uncertain outcomes; - /// this method never retries the callback or switches to HTTP. + /// Retain resource IDs, complete inputs and explicit bases for uncertain outcomes. + /// This method never retries the callback, rebases a condition or switches to HTTP. /// # Errors /// Returns admission, domain failures or interrupted mutation uncertainty. pub async fn control(&self, context: OperationContext, operation: F) -> Result diff --git a/lib/crowdb-sdk-native/src/dataset/error.rs b/lib/crowdb-sdk-native/src/dataset/error.rs index efbadefb5..37b4c4958 100644 --- a/lib/crowdb-sdk-native/src/dataset/error.rs +++ b/lib/crowdb-sdk-native/src/dataset/error.rs @@ -35,6 +35,9 @@ impl From for Error { fn from(value: AuthorityError) -> Self { let kind = match value { AuthorityError::Invalid(value) => return value.into(), + AuthorityError::SnapshotHeadConflict { observed } => { + return Self::SnapshotHeadConflict { observed } + } AuthorityError::NotFound | AuthorityError::SnapshotNotFound => FailureKind::NotFound, AuthorityError::AncestryCycle | AuthorityError::Corrupt => FailureKind::Integrity, AuthorityError::Store(_) => FailureKind::Storage, @@ -82,6 +85,7 @@ impl From for Error { let kind = match value { E::Authority(value) => return value.into(), E::Inspection(value) => return value.into(), + E::HeadConflict { snapshot, observed } => return Self::SplitHeadConflict { snapshot, observed }, E::NotFound => FailureKind::NotFound, E::Conflict | E::Frozen => FailureKind::Conflict, E::Limit => FailureKind::Capacity, diff --git a/lib/crowdb-sdk-native/src/dataset/preparation.rs b/lib/crowdb-sdk-native/src/dataset/preparation.rs new file mode 100644 index 000000000..3a0d9f42a --- /dev/null +++ b/lib/crowdb-sdk-native/src/dataset/preparation.rs @@ -0,0 +1,90 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::DatasetClient; +use crate::{OperationContext, Result}; +use crowdb_access_dataset::{ + composition::{CompositionRequest, CompositionResult}, + split::{SplitHeadUpdate, SplitPreparation, SplitReference, SplitRevisionRecord}, + DatasetHead, ReadCancellation, SnapshotHeadUpdate, SnapshotId, +}; + +impl DatasetClient { + /// Prepares a complete immutable Snapshot and claims its final payload owners. + /// Retry with the same resource ID and complete input after an uncertain outcome. + /// # Errors + /// Rejects unavailable sources, divergent content, invalid payloads and work limits. + pub async fn prepare_snapshot( + &self, + context: OperationContext, + snapshot: SnapshotId, + input: &CompositionRequest, + ) -> Result { + self.control(context, |scope| async move { + Ok(scope + .authority + .prepare_composition_resource( + &scope.identity, + snapshot, + input, + Some(&scope.uploads), + &ReadCancellation::new(), + ) + .await?) + }) + .await + } + + /// Updates latest using the caller's exact base; a conflict carries the observed head. + /// # Errors + /// Rejects unavailable resources or changed bases; interrupted writes remain uncertain. + pub async fn update_snapshot_head( + &self, + context: OperationContext, + update: &SnapshotHeadUpdate, + ) -> Result { + self.control(context, |scope| async move { + Ok(scope + .authority + .update_snapshot_head(&scope.identity, update) + .await?) + }) + .await + } + + /// Prepares one immutable Split revision from complete, frozen definitions. + /// # Errors + /// Rejects divergent content, unavailable sources, invalid policies and work limits. + pub async fn prepare_split_revision( + &self, + context: OperationContext, + snapshot: SnapshotId, + input: &SplitPreparation, + ) -> Result { + self.control(context, |scope| async move { + Ok(scope + .authority + .prepare_split_revision(&scope.identity, snapshot, input, &ReadCancellation::new()) + .await?) + }) + .await + } + + /// Updates the Split pointer using both supplied revision and generation conditions. + /// # Errors + /// Rejects changed bases with an observed head or propagates uncertain writes. + pub async fn update_split_head( + &self, + context: OperationContext, + snapshot: SnapshotId, + update: &SplitHeadUpdate, + ) -> Result { + self.control(context, |scope| async move { + Ok(scope + .authority + .update_split_head(&scope.identity, snapshot, update) + .await?) + }) + .await + } +} diff --git a/lib/crowdb-sdk-native/src/error.rs b/lib/crowdb-sdk-native/src/error.rs index e8ea98672..b11e5be1b 100644 --- a/lib/crowdb-sdk-native/src/error.rs +++ b/lib/crowdb-sdk-native/src/error.rs @@ -75,6 +75,15 @@ pub enum Error { kind: crate::dataset::FailureKind, detail: String, }, + #[error("Snapshot head precondition changed")] + SnapshotHeadConflict { + observed: Box, + }, + #[error("Split head precondition changed")] + SplitHeadConflict { + snapshot: crowdb_access_dataset::SnapshotId, + observed: Box, + }, #[error("object was not modified")] NotModified, #[error("object checksum does not match its published metadata")] diff --git a/lib/crowdb-sdk-native/tests/common/dataset_composition.rs b/lib/crowdb-sdk-native/tests/common/dataset_composition.rs index a1ede9f61..47dd40734 100644 --- a/lib/crowdb-sdk-native/tests/common/dataset_composition.rs +++ b/lib/crowdb-sdk-native/tests/common/dataset_composition.rs @@ -1,113 +1,57 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use crowdb_access_dataset::{composition::*, *}; +use crowdb_access_dataset::{composition::*, split::SplitPolicy, *}; use crowdb_sdk_native::dataset::DatasetClient; pub async fn publish(client: &DatasetClient, location: Vec, digest: [u8; 16]) -> SnapshotId { - let operation = OperationId::random(); - client - .begin_composition( - super::context(20), - operation, - &CompositionInput { - parent: None, - source: CompositionSource::All, - schema: Some(SchemaRecord { - version: 1, - fields: vec![FieldDefinition { - name: "value".into(), - required: true, - }], - }), - inherit_parent_splits: false, - parent_split_revision: None, - splits: vec![], - policy: None, - }, - ) - .await - .unwrap(); - let edited = client - .edit_composition( - super::context(21), - operation, - &CompositionEdit { - id: OperationId::random(), - expected_revision: 1, - schema: None, - splits: None, - policy: None, - samples: vec![SampleChange::Upsert(SampleRecord { - sample_id: b"one".to_vec(), - fields: vec![FieldRecord { - name: "value".into(), - value: FieldLocator::Chunk { - location, - length: 8, - md5: digest, - }, - }], - })], - }, - ) + let snapshot = SnapshotId::random(); + let input = CompositionRequest { + parent: None, + source: CompositionSource::All, + schema: Some(SchemaRecord { + version: 1, + fields: vec![FieldDefinition { + name: "value".into(), + required: true, + }], + }), + samples: vec![SampleChange::Upsert(SampleRecord { + sample_id: b"one".to_vec(), + fields: vec![FieldRecord { + name: "value".into(), + value: FieldLocator::Chunk { + location, + length: 8, + md5: digest, + }, + }], + })], + splits: vec![], + policy: SplitPolicy::default(), + }; + let prepared = client + .prepare_snapshot(super::context(20), snapshot, &input) .await .unwrap(); - assert_eq!(edited.uploads.staged_uploads, 1); - let snapshot = prepare(client, operation).await; + assert_eq!(prepared.snapshot, snapshot); + assert_eq!(prepared.sample_count, 1); + assert_eq!( + client + .prepare_snapshot(super::context(21), snapshot, &input) + .await + .unwrap(), + prepared + ); client - .publish_composition( - super::context(24), - operation, - &CommitAttempt { - version: 1, - id: OperationId::random(), - target: CommitTarget::Snapshot { snapshot }, + .update_snapshot_head( + super::context(22), + &SnapshotHeadUpdate { + snapshot, expected_head: None, }, ) .await .unwrap(); - let mut cleaned = false; - for _ in 0..32 { - if client - .cleanup_composition(super::context(25), operation) - .await - .unwrap() - .complete - { - cleaned = true; - break; - } - } - assert!(cleaned); snapshot } - -async fn prepare(client: &DatasetClient, operation: OperationId) -> SnapshotId { - let mut evaluated = false; - for page in 0..32 { - if client - .evaluate_composition(super::context(22), operation, 2, page) - .await - .unwrap() - .complete - { - evaluated = true; - break; - } - } - assert!(evaluated); - let mut snapshot = None; - for _ in 0..32 { - let status = client - .prepare_composition(super::context(23), operation, 2) - .await - .unwrap(); - if status.uploads.publication_ready { - snapshot = status.uploads.candidate; - break; - } - } - snapshot.expect("bounded preparation and ownership complete") -} diff --git a/lib/crowdb-sdk-native/tests/common/metadata.rs b/lib/crowdb-sdk-native/tests/common/metadata.rs index 1c745cb37..5c7e18972 100644 --- a/lib/crowdb-sdk-native/tests/common/metadata.rs +++ b/lib/crowdb-sdk-native/tests/common/metadata.rs @@ -40,6 +40,11 @@ pub struct TestMutationGate { pub entered: tokio::sync::Semaphore, pub permits: tokio::sync::Semaphore, pub calls: std::sync::atomic::AtomicUsize, + pub reads: std::sync::atomic::AtomicUsize, + pub bounded_reads: std::sync::atomic::AtomicUsize, + pub writes: std::sync::atomic::AtomicUsize, + pub head_reads: std::sync::atomic::AtomicUsize, + pub reject_workflow_writes: AtomicBool, } impl Default for TestMutationGate { @@ -49,6 +54,11 @@ impl Default for TestMutationGate { entered: tokio::sync::Semaphore::new(0), permits: tokio::sync::Semaphore::new(0), calls: std::sync::atomic::AtomicUsize::new(0), + reads: std::sync::atomic::AtomicUsize::new(0), + bounded_reads: std::sync::atomic::AtomicUsize::new(0), + writes: std::sync::atomic::AtomicUsize::new(0), + head_reads: std::sync::atomic::AtomicUsize::new(0), + reject_workflow_writes: AtomicBool::new(false), } } } @@ -66,14 +76,51 @@ pub struct TestMutationBlackout { pub blocked: AtomicBool, } -#[async_trait] -impl ChunkKvTransport for TestMetadataTransport { - async fn point(&self, endpoint: &str, request: &PointRequest) -> Result { +impl TestMutationGate { + fn record(&self, operation: &PointOperation) { + let key = operation.key(); + if matches!(operation, PointOperation::Get { .. }) { + self.reads.fetch_add(1, Ordering::Relaxed); + if key.starts_with(b"DS\0H\0") { + self.head_reads.fetch_add(1, Ordering::Relaxed); + } + } else { + self.writes.fetch_add(1, Ordering::Relaxed); + if self.reject_workflow_writes.load(Ordering::Relaxed) { + for prefix in [ + b"CBG".as_slice(), + b"CD", + b"CE", + b"CX", + b"CW", + b"CM", + b"O", + b"SB", + b"SD", + b"SX", + b"SW", + b"CA", + b"CR", + b"SC", + b"SRC", + ] { + let mut forbidden = b"DS\0".to_vec(); + forbidden.extend_from_slice(prefix); + forbidden.push(0); + assert!(!key.starts_with(&forbidden), "workflow write: {key:?}"); + } + } + } + } +} + +impl TestMetadataTransport { + fn reject_route(&self, endpoint: &str, routing: &RequestRouting) -> Option { if let Some(route) = &self.route { let generation = route.generation.load(Ordering::Acquire); - if endpoint != format!("test-owner-{generation}") || request.routing.owner_epoch != generation { + if endpoint != format!("test-owner-{generation}") || routing.owner_epoch != generation { route.rejected.fetch_add(1, Ordering::Relaxed); - return Ok(ChunkKvResponse { + return Some(ChunkKvResponse { map_revision: generation, journal_position: None, result: Err(RpcFailure { @@ -87,6 +134,19 @@ impl ChunkKvTransport for TestMetadataTransport { } route.accepted.store(generation, Ordering::Release); } + None + } +} + +#[async_trait] +impl ChunkKvTransport for TestMetadataTransport { + async fn point(&self, endpoint: &str, request: &PointRequest) -> Result { + if let Some(stats) = &self.gate { + stats.record(&request.operation); + } + if let Some(response) = self.reject_route(endpoint, &request.routing) { + return Ok(response); + } if self .blackout .as_ref() @@ -141,7 +201,10 @@ impl ChunkKvTransport for TestMetadataTransport { panic!("unexpected seek") } - async fn scan(&self, _: &str, request: &ScanRequest) -> Result { + async fn scan(&self, endpoint: &str, request: &ScanRequest) -> Result { + if let Some(response) = self.reject_route(endpoint, &request.routing) { + return Ok(response); + } let (sender, receiver) = oneshot::channel(); self.sender .send(TestCommand::Scan(request.clone(), sender)) @@ -156,6 +219,9 @@ impl ChunkKvTransport for TestMetadataTransport { request: &ScanRequest, max_bytes: usize, ) -> Result { + if let Some(stats) = &self.gate { + stats.bounded_reads.fetch_add(1, Ordering::Relaxed); + } let response = self.scan(endpoint, request).await?; if let Ok(OperationResult::Scan { items, .. }) = &response.result { if items @@ -374,6 +440,26 @@ pub async fn client_with_gate(gate: Arc) -> Arc .await } +#[allow(dead_code)] // Used by resource publication uncertainty acceptance. +pub async fn client_with_loss_and_gate( + loss: Arc, + gate: Arc, +) -> Arc { + let (head, pages) = catalog(1, 1); + client_with_options( + Arc::new(TestCatalog(head, pages)), + loss, + ClientConfig { + max_attempts: 1, + ..ClientConfig::default() + }, + None, + None, + Some(gate), + ) + .await +} + async fn client_with_options( source: Arc, loss: Arc, diff --git a/lib/crowdb-sdk-native/tests/dataset_preparation_test.rs b/lib/crowdb-sdk-native/tests/dataset_preparation_test.rs new file mode 100644 index 000000000..f4163408d --- /dev/null +++ b/lib/crowdb-sdk-native/tests/dataset_preparation_test.rs @@ -0,0 +1,237 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/dataset.rs"] +#[allow(dead_code)] // Reuses the manifest fixture without its read-only client setup. +mod dataset; +#[path = "common/metadata.rs"] +mod metadata; +#[path = "common/storage.rs"] +mod storage; + +use crowdb_access_dataset::{composition::*, split::*, *}; +use crowdb_sdk_native::{dataset::DatasetClient, Error}; +use dataset::context; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; + +fn input() -> CompositionRequest { + let manifest: ManifestRecord = bincode::deserialize(&dataset::manifest()).unwrap(); + CompositionRequest { + parent: None, + source: CompositionSource::All, + schema: Some(manifest.schema), + samples: manifest.samples.into_iter().map(SampleChange::Upsert).collect(), + splits: vec![], + policy: SplitPolicy::default(), + } +} +fn counts(stats: &metadata::TestMutationGate) -> (usize, usize) { + ( + stats.reads.load(Ordering::Relaxed) + stats.bounded_reads.load(Ordering::Relaxed), + stats.writes.load(Ordering::Relaxed), + ) +} +fn delta(stats: &metadata::TestMutationGate, old: (usize, usize)) -> (usize, usize) { + let now = counts(stats); + (now.0 - old.0, now.1 - old.1) +} +async fn fixture() -> (DatasetClient, Arc, Arc) { + let stats = Arc::new(metadata::TestMutationGate::default()); + stats.reject_workflow_writes.store(true, Ordering::Relaxed); + let loss = Arc::new(AtomicBool::new(false)); + let metadata = metadata::client_with_loss_and_gate(loss.clone(), stats.clone()).await; + let runtime = storage::runtime_with_metadata(metadata); + let client = DatasetClient::new( + &runtime, + DatasetIdentity::new(NamespacePath::root(), "resources").unwrap(), + ) + .unwrap(); + client.create(context(1)).await.unwrap(); + (client, stats, loss) +} +async fn prepare(client: &DatasetClient, stats: &metadata::TestMutationGate) -> SnapshotId { + let snapshot = SnapshotId::random(); + let head_reads = stats.head_reads.load(Ordering::Relaxed); + let result = client + .prepare_snapshot(context(2), snapshot, &input()) + .await + .unwrap(); + assert_eq!(result.snapshot, snapshot); + assert_eq!(result.sample_count, 3); + let before = counts(stats); + assert_eq!( + client + .prepare_snapshot(context(3), snapshot, &input()) + .await + .unwrap(), + result + ); + assert_eq!(delta(stats, before).1, 0); + assert_eq!(stats.head_reads.load(Ordering::Relaxed), head_reads); + client.snapshot(context(4), snapshot).await.unwrap(); + snapshot +} + +#[tokio::test] +async fn complete_resources_and_conflict_observations_need_no_workflow_or_confirmation_get() { + let (client, stats, _) = fixture().await; + let first = prepare(&client, &stats).await; + let second = prepare(&client, &stats).await; + let before = counts(&stats); + let head = client + .update_snapshot_head( + context(5), + &SnapshotHeadUpdate { + snapshot: first, + expected_head: None, + }, + ) + .await + .unwrap(); + assert_eq!(delta(&stats, before), (3, 1)); + for _ in 0..4 { + let before = counts(&stats); + let error = client + .update_snapshot_head( + context(6), + &SnapshotHeadUpdate { + snapshot: second, + expected_head: None, + }, + ) + .await + .unwrap_err(); + assert_eq!( + error, + Error::SnapshotHeadConflict { + observed: Box::new(head.clone()) + } + ); + assert_eq!(delta(&stats, before), (3, 0)); + } + let before = counts(&stats); + let next = client + .update_snapshot_head( + context(7), + &SnapshotHeadUpdate { + snapshot: second, + expected_head: head.generation, + }, + ) + .await + .unwrap(); + assert_eq!(next.latest, Some(second)); + assert_eq!(delta(&stats, before), (3, 1)); + + assert_split_revisions(&client, &stats, second).await; +} + +async fn assert_split_revisions( + client: &DatasetClient, + stats: &metadata::TestMutationGate, + snapshot: SnapshotId, +) { + let prepare = |revision| SplitPreparation { + revision, + policy: SplitPolicy::default(), + splits: vec![DraftSplit { + name: SplitName::new("train").unwrap(), + definition: SplitDefinition::Selection(Selection::Explicit(vec![vec![1], vec![2], vec![3]])), + origin: MembershipOrigin::Evaluated, + }], + }; + let a = SplitRevisionId::random(); + let b = SplitRevisionId::random(); + let prepared = client + .prepare_split_revision(context(8), snapshot, &prepare(a)) + .await + .unwrap(); + let before = counts(stats); + assert_eq!( + client + .prepare_split_revision(context(9), snapshot, &prepare(a)) + .await + .unwrap(), + prepared + ); + assert_eq!(delta(stats, before).1, 0); + client + .prepare_split_revision(context(10), snapshot, &prepare(b)) + .await + .unwrap(); + let head = client + .update_split_head( + context(11), + snapshot, + &SplitHeadUpdate { + revision: a, + expected_revision: None, + expected_head: None, + }, + ) + .await + .unwrap(); + let before = counts(stats); + let error = client + .update_split_head( + context(12), + snapshot, + &SplitHeadUpdate { + revision: b, + expected_revision: None, + expected_head: None, + }, + ) + .await + .unwrap_err(); + let Error::SplitHeadConflict { + snapshot: observed_snapshot, + observed, + } = error + else { + panic!("missing observation: {error:?}") + }; + assert_eq!(observed_snapshot, snapshot); + assert_eq!(*observed, head); + assert_eq!(delta(stats, before), (4, 0)); + let before = counts(stats); + let next = client + .update_split_head( + context(13), + snapshot, + &SplitHeadUpdate { + revision: b, + expected_revision: observed.revision, + expected_head: observed.generation, + }, + ) + .await + .unwrap(); + assert_eq!(next.revision, Some(b)); + assert_eq!(delta(stats, before), (4, 1)); +} + +#[tokio::test] +async fn an_applied_head_with_a_lost_reply_stays_uncertain_and_replays_the_original_condition() { + let (client, stats, loss) = fixture().await; + let snapshot = prepare(&client, &stats).await; + let update = SnapshotHeadUpdate { + snapshot, + expected_head: None, + }; + let before = counts(&stats); + loss.store(true, Ordering::Relaxed); + assert!(matches!( + client.update_snapshot_head(context(20), &update).await, + Err(Error::UncertainMutation { request_id: 20, .. }) + )); + assert_eq!(delta(&stats, before), (3, 1)); + loss.store(false, Ordering::Relaxed); + let before = counts(&stats); + let head = client.update_snapshot_head(context(21), &update).await.unwrap(); + assert_eq!(head.latest, Some(snapshot)); + assert_eq!(delta(&stats, before), (3, 0)); +} From 9da29b6d82ee45d5039436ddcb32d2eb7ed39aa5 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 24 Oct 2026 23:26:27 +0800 Subject: [PATCH 138/258] Use complete Dataset resources in console workflows --- app/crowdb-web/src/access/dataset.rs | 19 ++- app/crowdb-web/tests/dataset_proxy_test.rs | 46 +++++++ app/crowdb-web/ui/src/dataset/Actions.tsx | 22 ++-- .../ui/src/dataset/BeginRecovery.tsx | 26 ---- app/crowdb-web/ui/src/dataset/Publication.tsx | 85 ++++-------- app/crowdb-web/ui/src/dataset/api.ts | 8 +- app/crowdb-web/ui/src/dataset/editor.test.tsx | 41 ++++-- .../ui/src/dataset/import/Editor.tsx | 40 +++--- .../ui/src/dataset/import/Splits.tsx | 24 ---- .../ui/src/dataset/import/cleanup.ts | 27 ---- .../ui/src/dataset/import/integrity.test.ts | 10 +- .../ui/src/dataset/import/mapping.ts | 27 ++-- .../ui/src/dataset/import/transfer.ts | 101 ++++++++------ .../ui/src/dataset/import/upload.ts | 32 +++-- app/crowdb-web/ui/src/dataset/operation.ts | 123 +++++++++++------- .../ui/src/dataset/operation/storage.ts | 42 ++++++ .../ui/src/dataset/splits/Editor.tsx | 40 +++--- app/crowdb-web/ui/src/dataset/types.ts | 34 ++--- .../console/design-crowdb-console-ui.md | 32 +++-- doc/working/plan-access-simplification.md | 45 +++++-- .../console-ui/e2e/fixtures/consoleProxy.ts | 4 +- .../console-ui/e2e/fixtures/dataset.ts | 67 +++------- .../e2e/flows/80-dataset-browse.spec.ts | 3 +- .../e2e/flows/81-dataset-import.spec.ts | 50 +++---- .../e2e/flows/82-dataset-splits.spec.ts | 48 ++++--- 25 files changed, 505 insertions(+), 491 deletions(-) delete mode 100644 app/crowdb-web/ui/src/dataset/BeginRecovery.tsx delete mode 100644 app/crowdb-web/ui/src/dataset/import/Splits.tsx delete mode 100644 app/crowdb-web/ui/src/dataset/import/cleanup.ts create mode 100644 app/crowdb-web/ui/src/dataset/operation/storage.ts diff --git a/app/crowdb-web/src/access/dataset.rs b/app/crowdb-web/src/access/dataset.rs index 55bdf4013..4be78cfb5 100644 --- a/app/crowdb-web/src/access/dataset.rs +++ b/app/crowdb-web/src/access/dataset.rs @@ -37,7 +37,7 @@ pub(super) async fn proxy(state: AppState, request: Request) -> Result Result { @@ -72,9 +72,18 @@ async fn forward(state: AppState, request: Request) -> Result>()).as_deref(), + Some(["", "namespaces", "dataset-ns", "datasets", dataset, "snapshots", snapshot]) + if !dataset.is_empty() && snapshot.len() == 32 && snapshot.bytes().all(|b| b.is_ascii_hexdigit())) + { + 8 * METADATA_LIMIT + } else { + 4 * METADATA_LIMIT + }; + let body = axum::body::to_bytes(request.into_body(), limit) .await - .map_err(|_| err_400("Dataset request exceeds 4 MiB"))?; + .map_err(|_| err_400("Dataset request exceeds its body limit"))?; let client = reqwest::Client::builder() .no_proxy() .redirect(reqwest::redirect::Policy::none()) @@ -88,7 +97,7 @@ async fn forward(state: AppState, request: Request) -> Result Result METADATA_LIMIT { return Err(err_502("Dataset metadata response exceeds 1 MiB")); diff --git a/app/crowdb-web/tests/dataset_proxy_test.rs b/app/crowdb-web/tests/dataset_proxy_test.rs index a5f721e73..c159a0318 100644 --- a/app/crowdb-web/tests/dataset_proxy_test.rs +++ b/app/crowdb-web/tests/dataset_proxy_test.rs @@ -24,6 +24,15 @@ async fn fixture(readonly: bool) -> (axum::Router, tokio::task::JoinHandle<()>) async move { assert_eq!(request.headers()["authorization"], expected); assert!(request.uri().path().starts_with("/api/dataset/v2/")); + if request.uri().query() == Some("body_size=1") { + let bytes = axum::body::to_bytes(request.into_body(), 8 * 1024 * 1024) + .await + .unwrap(); + return ( + StatusCode::OK, + serde_json::json!({"length":bytes.len()}).to_string(), + ); + } if request.uri().query() == Some("large=1") { return (StatusCode::OK, "x".repeat(1024 * 1024 + 1)); } @@ -93,6 +102,12 @@ async fn readonly_allows_exact_matrix_but_rejects_all_mutations() { ("POST", format!("/snapshots/{snapshot}/matrix"), 200), ("GET", format!("/snapshots/{snapshot}/contains"), 200), ("POST", "/compositions".into(), 403), + ("PUT", format!("/snapshots/{snapshot}"), 403), + ( + "PUT", + format!("/snapshots/{snapshot}/split-revisions/{snapshot}"), + 403, + ), ("POST", format!("/snapshots/{snapshot}/reclaim"), 403), ("PUT", String::new(), 403), ("DELETE", "/uploads/123".into(), 403), @@ -113,3 +128,34 @@ async fn readonly_allows_exact_matrix_but_rejects_all_mutations() { } task.abort(); } + +#[tokio::test] +async fn complete_snapshot_inputs_get_eight_mib_without_expanding_upload_part_limits() { + let (app, task) = fixture(false).await; + let root = "/api/access/dataset/api/dataset/v2/namespaces/dataset-ns/datasets/example"; + let snapshot = "11111111111111111111111111111111"; + for (suffix, bytes, expected) in [ + (format!("/snapshots/{snapshot}"), 6 * 1024 * 1024, 200), + (format!("/snapshots/{snapshot}"), 8 * 1024 * 1024 + 1, 400), + (format!("/uploads/{snapshot}/parts/0"), 4 * 1024 * 1024 + 1, 400), + ] { + let response = app + .clone() + .oneshot( + Request::builder() + .method("PUT") + .uri(format!("{root}{suffix}?body_size=1")) + .body(Body::from(vec![b'x'; bytes])) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status().as_u16(), expected); + if expected == 200 { + let body = axum::body::to_bytes(response.into_body(), 1024).await.unwrap(); + let value: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(value["length"], bytes); + } + } + task.abort(); +} diff --git a/app/crowdb-web/ui/src/dataset/Actions.tsx b/app/crowdb-web/ui/src/dataset/Actions.tsx index 493549855..2cdc747f0 100644 --- a/app/crowdb-web/ui/src/dataset/Actions.tsx +++ b/app/crowdb-web/ui/src/dataset/Actions.tsx @@ -4,26 +4,26 @@ import { useState } from 'react'; import { buttonClass, JsonView } from '../access/Workbench'; import * as api from './api'; -import type { DatasetSelection, Receipt, SplitSource } from './types'; +import type { DatasetSelection, PublicationResult, SplitSource } from './types'; import { savedOperation, type Operation } from './operation'; import { ImportEditor } from './import/Editor'; import { SplitEditor } from './splits/Editor'; -export function Actions({ selected, readonly, capabilities, complete }: { selected: DatasetSelection; readonly: boolean; capabilities: string[]; complete: (receipt?: Receipt) => void }) { - const [editor, setEditor] = useState<{ kind: 'composition' | 'split'; parent?: string; revision?: string; source?: SplitSource; recovery?: Operation }>(); +export function Actions({ selected, readonly, capabilities, complete }: { selected: DatasetSelection; readonly: boolean; capabilities: string[]; complete: (receipt?: PublicationResult) => void }) { + const [editor, setEditor] = useState<{ kind: 'snapshot' | 'split'; parent?: string; revision?: string; source?: SplitSource; recovery?: Operation }>(); const [error, setError] = useState(''); const [busy, setBusy] = useState(false); const [observation, setObservation] = useState(); const [confirm, setConfirm] = useState(); const [unknown, setUnknown] = useState(); if (selected.kind === 'root') return null; const dataset = selected.dataset; const snapshot = 'snapshot' in selected ? selected.snapshot : undefined; const run = async (action: () => Promise) => { setBusy(true); setError(''); try { await action(); } catch (reason) { setError(String(reason)); } finally { setBusy(false); } }; - const open = (kind: 'composition' | 'split') => { + const open = async (kind: 'snapshot' | 'split') => { try { - const recovery = savedOperation(dataset); + const recovery = await savedOperation(dataset); if (recovery) { setEditor({ kind: recovery.kind, recovery, parent: recovery.snapshot }); return; } const source = 'split' in selected && selected.split !== undefined && selected.revision && snapshot ? { snapshot, revision: selected.revision, name: selected.split } : undefined; setEditor({ kind, parent: snapshot, source, revision: 'revision' in selected ? selected.revision : undefined }); } catch (reason) { setError(String(reason)); } }; - const done = (receipt: Receipt) => { setEditor(undefined); complete(receipt); }; + const done = (receipt: PublicationResult) => { setEditor(undefined); complete(receipt); }; const mutate = (action: 'retain' | 'release' | 'reclaim', target = snapshot!) => run(async () => { try { const result = await api.mutate(action === 'reclaim' ? 'snapshot-reclaim' : 'dataset-management', `${api.snapshotPath(dataset, target)}/${action}`, 'POST'); setObservation(result); setConfirm(undefined); complete(); @@ -31,23 +31,23 @@ export function Actions({ selected, readonly, capabilities, complete }: { select }); return
{!readonly &&
- - + + {snapshot && <> - + }
} - {!readonly && confirm &&

Reclaim {confirm}? Eligible data and metadata become unavailable. Latest, stable, retained, reachable or actively read Snapshots remain protected by the server.

} + {!readonly && confirm &&

Reclaim {confirm}? Eligible data and metadata become unavailable. Latest, stable, retained or reachable Snapshots remain protected. New reads stop after marking; cleanup follows the grace period.

} {snapshot && } {unknown &&
Reclamation outcome unknown for {unknown}. Inspect before continuing.
} {error &&

{error}

}{observation !== undefined && } - {!readonly && editor?.kind === 'composition' && setEditor(undefined)} complete={done} />} + {!readonly && editor?.kind === 'snapshot' && setEditor(undefined)} complete={done} />} {!readonly && editor?.kind === 'split' && editor.parent && setEditor(undefined)} complete={done} />}
; } diff --git a/app/crowdb-web/ui/src/dataset/BeginRecovery.tsx b/app/crowdb-web/ui/src/dataset/BeginRecovery.tsx deleted file mode 100644 index 7686e732a..000000000 --- a/app/crowdb-web/ui/src/dataset/BeginRecovery.tsx +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -import { useState } from 'react'; -import { buttonClass } from '../access/Workbench'; -import * as api from './api'; -import { forgetOperation, operationPath, retryIdentical, type Operation } from './operation'; -import type { CompositionDraft, SplitDraft } from './types'; - -/** Creation may commit before its response reaches the browser. Keep its identity. */ -export function BeginRecovery({ operation, busy, run, loaded, discarded }: { - operation: Operation; busy: boolean; run: (action: () => Promise) => void; - discarded: () => void; - loaded: (draft: CompositionDraft | SplitDraft) => void; -}) { - const [missing, setMissing] = useState(false); - return
-

Draft creation needs inspection. A missing draft can be created by replaying the saved request with the same operation ID.

- - {missing && !operation.pending && } - {operation.pending && } -
; -} diff --git a/app/crowdb-web/ui/src/dataset/Publication.tsx b/app/crowdb-web/ui/src/dataset/Publication.tsx index 4f6b0f7e6..549ea0d18 100644 --- a/app/crowdb-web/ui/src/dataset/Publication.tsx +++ b/app/crowdb-web/ui/src/dataset/Publication.tsx @@ -1,75 +1,34 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -import { useState } from 'react'; import { buttonClass, JsonView } from '../access/Workbench'; -import * as api from './api'; -import type { CompositionDraft, CompositionPreview, Receipt, SplitDraft, SplitPreview, SplitRevision } from './types'; -import { forgetOperation, newAttempt, operationPath, reconcile, request, retryIdentical, saveOperation, type Operation } from './operation'; -import { cleanupUploads } from './import/cleanup'; -import { newId } from './import/upload'; +import type { PublicationResult } from './types'; +import { forgetOperation, prepare, publish, retryIdentical, useObservedHead, type Operation } from './operation'; -export function Publication({ operation: op, draft, update, complete, busy, run, restart }: { - operation: Operation; draft: CompositionDraft | SplitDraft; update: (draft: CompositionDraft | SplitDraft) => void; - restart?: () => Promise; - complete: (receipt: Receipt) => void; busy: boolean; run: (action: () => Promise) => void; +export function Publication({ operation: op, update, complete, close, busy, run }: { + operation: Operation; update: (op: Operation) => void; complete: (result: PublicationResult) => void; + close: () => void; busy: boolean; run: (action: () => Promise) => void; }) { - const [cleaned, setCleaned] = useState(false); - const [preview, setPreview] = useState(); - const [cleanup, setCleanup] = useState(); const [conflict, setConflict] = useState<{ generation: string | null }>(); - const path = operationPath(op); - const refresh = async () => update(await api.control(path)); - const state = draft.progress.state; - const evaluated = state === 'evaluated' || state === 'prepared'; - const importReady = op.kind === 'split' || op.importsComplete === true; - const ready = importReady && state === 'prepared' && (op.kind === 'split' || ('uploads' in draft && draft.uploads.publication_ready)); - const evaluate = async () => { - const expected_page = draft.progress.state === 'evaluating' ? draft.progress.next_page : '0'; - const value = await request(op, `${path}/evaluate`, 'POST', { expected_revision: draft.revision, expected_page }); - setPreview(value); await refresh(); + const finish = async (result?: PublicationResult) => { + if (result) { await forgetOperation(op); complete(result); } + else update({ ...op }); }; - const prepare = async () => { - await request(op, `${path}/prepare`, 'POST', { expected_revision: draft.revision }); await refresh(); - }; - const publish = async () => { - if (draft.progress.state !== 'prepared') throw new Error('Prepare this draft first.'); - op.attempt ??= newId(); saveOperation(op); - const target = draft.progress.target; - const receipt = await request(op, `${path}/publish`, 'POST', { attempt: op.attempt, expected_head: op.expectedHead, - ...(target.kind === 'snapshot' ? { snapshot: target.snapshot } : { revision: target.revision }) }); - forgetOperation(op); complete(receipt); - }; - return
-

{state === 'prepared' ? 'Prepared candidate' : `Draft · ${state}`}

-

Operation {op.operation} · edit revision {draft.revision}

-

{op.kind === 'composition' ? 'Successful publication advances Dataset latest. Stable and the chosen parent stay unchanged.' : 'Publication replaces the complete current Split set. Dataset latest stays unchanged.'}

- {preview && } - {'uploads' in draft &&
Upload ownership and initial Split validation
} - {draft.progress.state === 'prepared' && } + return
+

{op.prepared ? 'Complete result' : 'Complete input ready'}

+

{op.kind === 'snapshot' ? 'Snapshot' : 'Split revision'} {op.resource}

+

{op.kind === 'snapshot' ? 'Publishing selects this Snapshot as latest. Stable and the parent keep their current values.' : 'Publishing replaces the current Split set. Historical revisions remain available.'}

+ {op.prepared && } {op.pending ?
-

Request unresolved. Inspect status before retrying; do not start another operation.

- - - {op.kind === 'composition' && op.attempt && op.conflict && } - {op.kind === 'split' && op.conflict && } - {conflict &&

Observed head: {conflict.generation ?? 'none'}. Keep this candidate and parent; publish with a new commit attempt?

} +

Reply unavailable. The complete request is saved in this browser.

+
:
- - - - - - {state === 'abandoned' && } + {!op.prepared && op.input && } + + {op.conflict &&

The head changed. Review the returned values before replacing it with this complete result.

+ +
} +
} - {cleaned && restart && } - {cleanup !== undefined && } +

Complete results can be read by ID before publication. Discarding local input leaves unreferenced results for GC.

; } diff --git a/app/crowdb-web/ui/src/dataset/api.ts b/app/crowdb-web/ui/src/dataset/api.ts index 69df7d017..66fe356e8 100644 --- a/app/crowdb-web/ui/src/dataset/api.ts +++ b/app/crowdb-web/ui/src/dataset/api.ts @@ -17,7 +17,7 @@ export function decimal(value: string): bigint { return BigInt(value); } export class DatasetError extends Error { - constructor(readonly status: number, message: string, readonly unknownOutcome = false) { super(message); } + constructor(readonly status: number, message: string, readonly unknownOutcome = false, readonly observedHead?: unknown) { super(message); } } async function readBounded(response: Response): Promise { const reader = response.body?.getReader(); @@ -35,10 +35,10 @@ async function readBounded(response: Response): Promise { let result: unknown; try { result = JSON.parse(text); } catch { throw new DatasetError(response.status || 502, 'Dataset returned an invalid response.'); } if (!response.ok) { - const error = result as { message?: string; code?: string; error?: string | { message?: string; code?: string } }; + const error = result as { message?: string; code?: string; observed_head?: unknown; error?: string | { message?: string; code?: string } }; const message = error.message ?? (typeof error.error === 'string' ? error.error : error.error?.message); const detail = error.code && message !== error.code ? `${error.code}: ${message}` : message; - throw new DatasetError(response.status, detail ?? `Dataset HTTP ${response.status}`, response.status >= 500); + throw new DatasetError(response.status, detail ?? `Dataset HTTP ${response.status}`, response.status >= 500, error.observed_head); } return result; } @@ -52,7 +52,7 @@ export async function control(path: string, method = 'GET', body?: unknown, s }); } catch (reason) { if (signal?.aborted) throw reason; - throw new DatasetError(0, 'Dataset service is unavailable. Reconcile any pending operation before retrying.', method !== 'GET'); + throw new DatasetError(0, 'Dataset service is unavailable. Retry the saved complete request after reconnecting.', method !== 'GET'); } return await readBounded(response) as T; } diff --git a/app/crowdb-web/ui/src/dataset/editor.test.tsx b/app/crowdb-web/ui/src/dataset/editor.test.tsx index 911f097d7..388b015e5 100644 --- a/app/crowdb-web/ui/src/dataset/editor.test.tsx +++ b/app/crowdb-web/ui/src/dataset/editor.test.tsx @@ -8,9 +8,10 @@ import { defaultPolicy, type DraftSplit } from './types'; import { DomainProvider } from '../contexts/DomainContext'; import { DatasetView } from '../views/DatasetView'; import * as api from './api'; +vi.mock('./operation/storage', () => ({ readRequest: vi.fn(async () => undefined), writeRequest: vi.fn(async () => {}), removeRequest: vi.fn(async () => {}) })); vi.mock('./api', async original => ({ ...await original(), listDatasets: vi.fn(async () => ({ entries: [], after: null })), - capabilities: vi.fn(async () => ['dataset-management', 'composition']), + capabilities: vi.fn(async () => ['dataset-management', 'composition-preparation', 'snapshot-head-update']), mutate: vi.fn(async () => ({})), page: vi.fn(async () => ({ entries: [], after: null })), splitHead: vi.fn(async () => ({ revision: null, generation: null })), @@ -66,17 +67,33 @@ it('readonly exposes inspection without creation, import or lifecycle mutations' expect(api.mutate).not.toHaveBeenCalled(); expect(screen.queryByText('Session activity')).toBeNull(); }); -it('a conflict discovered by identical retry retains candidate and enables explicit head review', async () => { - const { request, retryIdentical, newAttempt } = await import('./operation'); - const operation: import('./operation').Operation = { dataset: 'retry', kind: 'composition', operation: '1'.repeat(32), expectedHead: null, attempt: '2'.repeat(32) }; - const body = { attempt: operation.attempt, snapshot: '3'.repeat(32), expected_head: null }; - vi.mocked(api.mutate).mockRejectedValueOnce(new api.DatasetError(502, 'outcome unknown', true)).mockRejectedValueOnce(new api.DatasetError(409, 'head conflict')); - await expect(request(operation, '/compositions/publish', 'POST', body)).rejects.toThrow('outcome unknown'); +it('identical retry preserves the candidate and uses the returned head without status queries', async () => { + const { publish, retryIdentical, useObservedHead } = await import('./operation'); + const operation: import('./operation').Operation = { version: 2, dataset: 'retry', kind: 'snapshot', resource: '3'.repeat(32), expectedHead: null, + prepared: { snapshot: '3'.repeat(32), initial_split: null, sample_count: '1' } }; + const observed = { latest: '5'.repeat(32), stable: null, generation: '4'.repeat(64) }; + vi.mocked(api.mutate).mockRejectedValueOnce(new api.DatasetError(502, 'outcome unknown', true)) + .mockRejectedValueOnce(new api.DatasetError(409, 'head conflict', false, observed)) + .mockResolvedValueOnce({ ...observed, latest: operation.resource }); + await expect(publish(operation)).rejects.toThrow('outcome unknown'); + expect(operation.pending).toEqual({ kind: 'publish', expectedHead: null, expectedRevision: undefined }); await expect(retryIdentical(operation)).rejects.toThrow('head conflict'); - expect(operation.pending?.body).toEqual(body); expect(operation.conflict).toBe(true); - newAttempt(operation, '4'.repeat(32)); - expect(operation.expectedHead).toBe('4'.repeat(32)); expect(operation.attempt).not.toBe(body.attempt); - expect(operation.pending).toBeUndefined(); + expect(operation.conflict).toEqual(observed); expect(operation.pending).toBeUndefined(); + await useObservedHead(operation); + expect(operation.expectedHead).toBe(observed.generation); expect(operation.resource).toBe('3'.repeat(32)); + expect((await publish(operation)).target).toEqual({ kind: 'snapshot', snapshot: operation.resource }); + const first = vi.mocked(api.mutate).mock.calls; + expect(first[0].slice(0, 4)).toEqual(first[1].slice(0, 4)); + expect(first[2][3]).toEqual({ snapshot: operation.resource, expected_head: observed.generation }); + expect(api.control).not.toHaveBeenCalled(); expect(api.head).not.toHaveBeenCalled(); +}); +it('storage failure stops a preparation before its network mutation', async () => { + const { prepare } = await import('./operation'); const { writeRequest } = await import('./operation/storage'); + vi.mocked(writeRequest).mockRejectedValueOnce(new Error('browser storage full')); + const operation: import('./operation').Operation = { version: 2, dataset: 'bounded', kind: 'split', resource: '3'.repeat(32), snapshot: '2'.repeat(32), expectedHead: null, expectedRevision: null, + input: { splits: [], policy: defaultPolicy } }; + await expect(prepare(operation)).rejects.toThrow('browser storage full'); + expect(api.mutate).not.toHaveBeenCalled(); }); it('reclamation confirmation keeps its exact Snapshot target across navigation', async () => { const { Actions } = await import('./Actions'); @@ -96,7 +113,7 @@ it('waits for the selected Split base before enabling edits so loading cannot er vi.mocked(api.splitHead).mockReturnValueOnce(new Promise(resolve => { loaded = resolve; })); render(); expect(screen.getByRole('button', { name: 'Add selection Split' })).toBeDisabled(); - expect(screen.getByRole('button', { name: 'Save Split draft' })).toBeDisabled(); + expect(screen.getByRole('button', { name: 'Prepare Split revision' })).toBeDisabled(); expect(screen.queryByLabelText('Split 1 name')).toBeNull(); await act(async () => loaded({ revision: null, generation: null })); fireEvent.click(screen.getByRole('button', { name: 'Add selection Split' })); diff --git a/app/crowdb-web/ui/src/dataset/import/Editor.tsx b/app/crowdb-web/ui/src/dataset/import/Editor.tsx index 1858fa76d..c6af1aeef 100644 --- a/app/crowdb-web/ui/src/dataset/import/Editor.tsx +++ b/app/crowdb-web/ui/src/dataset/import/Editor.tsx @@ -5,22 +5,19 @@ import { useEffect, useRef, useState } from 'react'; import { buttonClass, inputClass, JsonView } from '../../access/Workbench'; import * as api from '../api'; import { encoded } from '../identity'; -import { defaultPolicy, type CompositionDraft, type DraftSplit, type Predicate, type Receipt, type SplitPolicy, type SplitSource } from '../types'; -import { operationPath, type Operation } from '../operation'; +import { defaultPolicy, type DraftSplit, type Predicate, type PublicationResult, type SplitPolicy, type SplitSource } from '../types'; +import { forgetOperation, type Operation } from '../operation'; import { Definitions, PredicateInput, validateEditor } from '../splits/Definitions'; -import { BeginRecovery } from '../BeginRecovery'; import { Publication } from '../Publication'; import { parseMapping, type Mapping } from './mapping'; -import { CompositionSplits } from './Splits'; import { saveImport } from './transfer'; const emptyMapping = JSON.stringify({ version: 1, schema: { version: 1, fields: [] }, samples: [] }, null, 2); export function ImportEditor({ dataset, parent: initialParent, source: initialSource, recovery, close, complete }: { - dataset: string; parent?: string; source?: SplitSource; recovery?: Operation; close: () => void; complete: (receipt: Receipt) => void; + dataset: string; parent?: string; source?: SplitSource; recovery?: Operation; close: () => void; complete: (result: PublicationResult) => void; }) { const transfer = useRef(); useEffect(() => () => transfer.current?.abort(), []); - const [editingSplits, setEditingSplits] = useState(false); const [parent, setParent] = useState(initialParent ?? ''); const [frozenSource, setFrozenSource] = useState(initialSource); const [mode, setMode] = useState(initialSource ? 'split' : 'all'); @@ -31,16 +28,17 @@ export function ImportEditor({ dataset, parent: initialParent, source: initialSo const [files, setFiles] = useState([]); const [mapping, setMapping] = useState(); const [splits, setSplits] = useState([]); const [policy, setPolicy] = useState(defaultPolicy); const element = useRef(null); - const [op, setOp] = useState(recovery); const [draft, setDraft] = useState(); + const [op, setOp] = useState(recovery); const [busy, setBusy] = useState(false); const [error, setError] = useState(''); const [progress, setProgress] = useState(''); const run = (action: () => Promise) => { setBusy(true); setError(''); void action().catch(reason => setError(String(reason))).finally(() => setBusy(false)); }; useEffect(() => { - if (!recovery) return; - const cancel = new AbortController(); - void api.control(operationPath(recovery), 'GET', undefined, cancel.signal).then(value => { - if (!cancel.signal.aborted) { setDraft(value); setParent(value.parent ?? ''); setSplits(value.splits); setPolicy(value.policy); setMode(value.source.kind); if (value.source.kind === 'split') { setFrozenSource(value.source.source); setFilterSplit(!!value.source.selection); if (value.source.selection) setPredicate(value.source.selection); } if (value.source.kind === 'selection') setPredicate(value.source.predicate); } - }).catch(reason => { if (!cancel.signal.aborted) setError(String(reason)); }); - return () => cancel.abort(); + const input = recovery?.input && 'parent' in recovery.input ? recovery.input : recovery?.source; + if (!input) return; + setParent(input.parent ?? ''); setSplits(input.splits); setPolicy(input.policy); + setMode(input.source.kind); + if (input.source.kind === 'split') { setFrozenSource(input.source.source); setFilterSplit(!!input.source.selection); if (input.source.selection) setPredicate(input.source.selection); } + if (input.source.kind === 'selection') setPredicate(input.source.predicate); + if (recovery?.mapping) setText(JSON.stringify(recovery.mapping, null, 2)); }, [recovery]); const review = () => { try { validateEditor(element.current); const value = parseMapping(text, files); if (parent) api.opaqueId(parent); setMapping(value); setError(''); } @@ -49,13 +47,13 @@ export function ImportEditor({ dataset, parent: initialParent, source: initialSo const save = async () => { validateEditor(element.current); transfer.current = new AbortController(); const signal = transfer.current.signal; - try { await saveImport({ dataset, parent, inherit, mode, predicate, frozenSource, filterSplit, splits, policy, op, mapping, files, signal, setOp, setDraft, setProgress }); } + try { await saveImport({ dataset, parent, inherit, mode, predicate, frozenSource, filterSplit, splits, policy, op, mapping, files, signal, setOp, setProgress }); } finally { transfer.current = undefined; } }; return

Import / New Snapshot

Local test/demo import · namespace dataset-ns · {dataset}

- {(!op || !op.importsComplete) && <> + {(!op || !op.input) && <>