From 646beb70ac36265a964bbad3995ef830a5e898e4 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Sun, 5 Jul 2026 07:58:51 +0000 Subject: [PATCH 01/52] reboot: support the `PUT` method for custom HTTP routes Before this change, an application's custom HTTP routes (`application.http`) could only be registered for `GET`, `POST`, and `OPTIONS`; the docs listed the `PUT`/`DELETE`/... gap as a known limitation. This blocked serving a plain-HTTP upload endpoint, where `PUT` is the natural verb. Add `application.http.put(...)`, a sibling of the existing `post(...)` that forwards `methods=["PUT"]` to the underlying FastAPI route (the route-capture machinery already supports arbitrary methods; only the public sugar was missing). Update the custom-HTTP-routes documentation to list `PUT` among the supported methods. Co-Authored-By: Claude Fable 5 --- documentation/docs/implement/application.mdx | 2 +- reboot/aio/http.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/documentation/docs/implement/application.mdx b/documentation/docs/implement/application.mdx index 1b3749c20..d8228eb63 100644 --- a/documentation/docs/implement/application.mdx +++ b/documentation/docs/implement/application.mdx @@ -165,7 +165,7 @@ implemented with [Express.js](https://expressjs.com/). **Limitations of custom HTTP routes** -* Currently only `GET` and `POST` methods are supported. +* Currently only `GET`, `POST`, and `PUT` methods are supported. * The `/` route is currently used by Reboot itself to show a helpful page explaining that this is a Reboot diff --git a/reboot/aio/http.py b/reboot/aio/http.py index e4d91d2ec..3c1ae418a 100644 --- a/reboot/aio/http.py +++ b/reboot/aio/http.py @@ -149,6 +149,14 @@ def post(self, path: str, **kwargs): assert "methods" not in kwargs return self._api_route(path, methods=["POST"], **kwargs) + def put(self, path: str, **kwargs): + # Rather than list out all of the possible keyword args + # that `FastAPI` expects we'll just pass along any that + # are passed to us, but we don't expect `methods` as we + # override that below. + assert "methods" not in kwargs + return self._api_route(path, methods=["PUT"], **kwargs) + def options(self, path: str, **kwargs): # Used for CORS preflight handlers. assert "methods" not in kwargs From 0a452a2a0fe3f14158d71ed347f558cfab3d4cdb Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Sun, 5 Jul 2026 08:00:13 +0000 Subject: [PATCH 02/52] reboot: run library `pre_run` hooks in the test harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before this change, `Application.run()` invoked each library's `pre_run(application)` hook, but the `Reboot` in-process test harness (`reboot.aio.tests`) did not. A library that performs application setup in `pre_run` — for example, registering custom HTTP routes — therefore behaved differently under test than in a real run, and its routes were simply absent when brought up via the harness. Call `library.pre_run(...)` for every library in `Reboot.up()`, before deciding whether a local Envoy is needed, mirroring what `Application.run()` does. Libraries must already tolerate being `pre_run` more than once (a test may bring the same application up again after a `down`). This harness behavior is covered by unit tests introduced in a later commit (`reboot/std: add a `Blob` state machine with a gRPC blob data plane`): `blobs_tests.py` brings an application up with the `BlobsLibrary`, whose `pre_run` hook connects to the blob data plane and registers the byte-proxying HTTP routes the tests then exercise — which succeeds only when the harness has invoked `pre_run`. Co-Authored-By: Claude Fable 5 --- reboot/aio/tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/reboot/aio/tests.py b/reboot/aio/tests.py index 11b203349..108534116 100644 --- a/reboot/aio/tests.py +++ b/reboot/aio/tests.py @@ -423,6 +423,14 @@ async def up( # Should only have `application`, `local_envoy`, # `local_envoy_port`, `servers`, `effect_validation`. + # Do any pre-run library set up, just like `Application.run()` + # does; e.g. a library may register HTTP routes. Libraries must + # tolerate being `pre_run` more than once, since a test may + # `up` the same `Application` after a `down`. + if not in_nodejs(): + for library in application.libraries: + await library.pre_run(application) + # Check if application.http has methods or mounts (note this # isn't relevant for TypeScript, which doesn't have that # property). If yes, we need a local_envoy to be present to From 5f0c57f31095fa3d56b8dfe861640286942bf684 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 7 Jul 2026 08:59:16 +0000 Subject: [PATCH 03/52] `reboot/std`: add a `Blob` state machine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reboot had no first-class way to store large binary objects: state machines hold protobuf state, which is unsuited to multi-megabyte payloads, so applications had nowhere to put user uploads like images or videos. Add `rbt.std.blobs.v1.Blob`, a state machine that is the *control plane* for one immutable-once-committed binary object. Its state holds only metadata — content type, expected/maximum size, upload progress, lifecycle status — while the bytes live in a *data plane* and travel directly between the client and that data plane via URLs minted per part. Uploads are resumable (parts are idempotent by number), sizes are enforced against the real bytes at commit time, and blobs that are never committed expire automatically. The data plane is a gRPC service, `BlobDataPlane` (`data_plane.proto`), deliberately free of Reboot options so that anything can implement it; the control plane discovers it via `REBOOT_BLOB_DATA_PLANE_URL` and calls it to provision uploads, mint URLs, finalize objects, and delete bytes. Keeping the implementation behind a bare gRPC URL means developers can write their own data planes to fit any environment. A data plane whose URLs are not directly reachable by clients (e.g. when run adjacent to an `rbt dev run` that's used over an `ngrok` tunnel) asks, via its `Configuration`, for certain requests to the app's HTTP server (under the reserved `/__/reboot/blob/`) to be forwarded to it; the `Blob` library then registers reverse-proxying routes on the application, so a single application origin serves both control plane and bytes and no second port needs exposing. A data plane whose URLs _are_ directly reachable (e.g. presigned S3) asks for nothing and is never in the application's path. This commit contains just one implementation of a data plane: a filesystem-based server. It binds loopback only and relies on the forwarded-path proxying above, so it works anywhere a Reboot app may be deployed. The `reboot.aio.tests.Reboot` test harness runs the filesystem data plane in-process for every test, so applications using `reboot.std.blobs` work in unit tests out of the box — which is also how this commit is tested. A later commit has `rbt dev run`/`rbt serve run` provide the same data plane for local runs. Authorization model: blob creation is application-mediated (the application enforces quota and size policy), after which the blob's framework-generated random ID acts as a capability (only callers who know the ID can call, and thus read or write, the blob). In addition the backend may limit the identities of callers by setting `uploader_id` and `downloader_ids` at create-time. Co-Authored-By: Claude Fable 5 --- rbt/std/BUILD.bazel | 3 + rbt/std/blob/v1/BUILD.bazel | 92 +++ rbt/std/blob/v1/blob.proto | 451 +++++++++++++ rbt/std/blob/v1/data_plane.proto | 195 ++++++ rbt/std/blob/v1/package.json | 3 + reboot/BUILD.bazel | 9 + reboot/aio/BUILD.bazel | 1 + reboot/aio/tests.py | 44 ++ reboot/std/BUILD.bazel | 1 + reboot/std/blob/v1/BUILD.bazel | 55 ++ reboot/std/blob/v1/_content_type.py | 66 ++ reboot/std/blob/v1/_data_plane.py | 81 +++ reboot/std/blob/v1/_filesystem_server.py | 334 ++++++++++ reboot/std/blob/v1/_http.py | 252 ++++++++ reboot/std/blob/v1/_proxy.py | 161 +++++ reboot/std/blob/v1/_store.py | 388 +++++++++++ reboot/std/blob/v1/blob.py | 699 ++++++++++++++++++++ reboot/std/blob/v1/index.ts | 30 + reboot/std/blob/v1/package.json | 3 + tests/reboot/std/blob/v1/BUILD.bazel | 16 + tests/reboot/std/blob/v1/blob_tests.py | 785 +++++++++++++++++++++++ 21 files changed, 3669 insertions(+) create mode 100644 rbt/std/blob/v1/BUILD.bazel create mode 100644 rbt/std/blob/v1/blob.proto create mode 100644 rbt/std/blob/v1/data_plane.proto create mode 100644 rbt/std/blob/v1/package.json create mode 100644 reboot/std/blob/v1/BUILD.bazel create mode 100644 reboot/std/blob/v1/_content_type.py create mode 100644 reboot/std/blob/v1/_data_plane.py create mode 100644 reboot/std/blob/v1/_filesystem_server.py create mode 100644 reboot/std/blob/v1/_http.py create mode 100644 reboot/std/blob/v1/_proxy.py create mode 100644 reboot/std/blob/v1/_store.py create mode 100644 reboot/std/blob/v1/blob.py create mode 100644 reboot/std/blob/v1/index.ts create mode 100644 reboot/std/blob/v1/package.json create mode 100644 tests/reboot/std/blob/v1/BUILD.bazel create mode 100644 tests/reboot/std/blob/v1/blob_tests.py diff --git a/rbt/std/BUILD.bazel b/rbt/std/BUILD.bazel index f2ba5d5eb..0b75b606a 100644 --- a/rbt/std/BUILD.bazel +++ b/rbt/std/BUILD.bazel @@ -47,6 +47,9 @@ ts_project( }, visibility = ["//visibility:public"], deps = [ + "//rbt/std/blob/v1:blob_js_proto", + "//rbt/std/blob/v1:blob_js_reboot", + "//rbt/std/blob/v1:blob_js_reboot_react", "//rbt/std/ciphertext/v1:ciphertext_js_proto", "//rbt/std/ciphertext/v1:ciphertext_js_reboot", "//rbt/std/collections/ordered_map/v1:ordered_map_js_proto", diff --git a/rbt/std/blob/v1/BUILD.bazel b/rbt/std/blob/v1/BUILD.bazel new file mode 100644 index 000000000..a71933179 --- /dev/null +++ b/rbt/std/blob/v1/BUILD.bazel @@ -0,0 +1,92 @@ +load( + "@com_github_reboot_dev_reboot//reboot:rules.bzl", + "js_proto_library", + "js_reboot_library", + "js_reboot_react_library", + "py_reboot_library", +) +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") + +proto_library( + name = "blob_proto", + srcs = [ + ":blob.proto", + ], + visibility = ["//visibility:public"], + deps = [ + "@com_github_reboot_dev_reboot//rbt/v1alpha1:options_proto", + "@com_google_protobuf//:descriptor_proto", + ], +) + +# The blob data-plane interface: a plain gRPC service (no Reboot state +# options). Built with `py_reboot_library` — which, besides the plain +# `_pb2`/`_pb2_grpc` modules the filesystem server uses directly, emits +# the `_rbt` module that lets the Cloud facilitator register it via +# `legacy_grpc_servicers`. Python-only: the JS SDK talks to the `Blob` +# control plane, never the data plane directly. +proto_library( + name = "data_plane_proto", + srcs = [ + ":data_plane.proto", + ], + visibility = ["//visibility:public"], +) + +py_reboot_library( + name = "data_plane_py_reboot", + proto = "data_plane.proto", + proto_library = ":data_plane_proto", + visibility = ["//visibility:public"], +) + +py_reboot_library( + name = "blob_py_reboot", + proto = "blob.proto", + proto_library = ":blob_proto", + visibility = ["//visibility:public"], +) + +js_proto_library( + name = "blob_js_proto", + package_json = ":package.json", + proto = "blob.proto", + proto_deps = [ + ":blob_proto", + # ISSUE(https://github.com/reboot-dev/mono/issues/3218): Until we can + # use `create_protoc_plugin_rule` we need to repeat the dependencies of + # the `proto_libraries` here. + "@com_github_reboot_dev_reboot//rbt/v1alpha1:options_proto", + "@com_google_protobuf//:descriptor_proto", + ], + visibility = ["//visibility:public"], +) + +js_reboot_library( + name = "blob_js_reboot", + srcs = [ + ":blob_proto", + ], + proto = "blob.proto", + visibility = ["//visibility:public"], + deps = [ + ":blob_js_proto", + ], +) + +js_reboot_react_library( + name = "blob_js_reboot_react", + srcs = [ + ":blob_js_proto", + ], + proto = "blob.proto", + proto_deps = [ + ":blob_proto", + # ISSUE(https://github.com/reboot-dev/mono/issues/3218): Until we can + # use `create_protoc_plugin_rule` we need to repeat the dependencies of + # the `proto_libraries` here. + "@com_github_reboot_dev_reboot//rbt/v1alpha1:options_proto", + "@com_google_protobuf//:descriptor_proto", + ], + visibility = ["//visibility:public"], +) diff --git a/rbt/std/blob/v1/blob.proto b/rbt/std/blob/v1/blob.proto new file mode 100644 index 000000000..cb03817f1 --- /dev/null +++ b/rbt/std/blob/v1/blob.proto @@ -0,0 +1,451 @@ +syntax = "proto3"; + +package rbt.std.blob.v1; + +import "rbt/v1alpha1/options.proto"; + +//////////////////////////////////////////////////////////////////////// +// Errors. + +// Raised when an upload-side call arrives after the blob has already +// been committed (or is in the process of being deleted). +message AlreadyCommitted {} + +// Raised when `GetDownloadUrl` is called before the blob has been +// committed. +message NotCommitted {} + +// Raised when the uploaded parts violate the blob's declared `size` +// or `max_size`. +message SizeMismatch { + // The total size of all parts reported uploaded so far. + uint64 bytes_uploaded = 1; +} + +// Raised when the reported parts cannot form a complete object: +// `Commit` was called before any part was reported, or the reported +// part numbers are not contiguous starting at 1. Also raised when +// `PartUploaded` reports an out-of-range part number or a malformed +// ETag. +message IncompleteParts {} + +//////////////////////////////////////////////////////////////////////// + +// The set of users allowed to download a blob. Wrapping the list in a +// message lets the download allow-list be *omitted* (anyone who knows +// the blob's ID may download) distinctly from *present but empty* (no +// one but app-internal callers may download). +message Downloaders { + repeated string user_ids = 1; +} + +//////////////////////////////////////////////////////////////////////// +// Blob: the control plane for one immutable-once-committed binary +// object. This state holds only *metadata*; the bytes live in a +// `BlobDataPlane` (see `data_plane.proto` — e.g. the local filesystem +// server, or an S3-backed service) and travel directly between the +// client and that data plane via URLs minted by `GetPartUploadInstructions` +// and `GetDownloadUrl`. The upload protocol follows S3 multipart +// semantics: numbered parts PUT to per-part URLs, each returning an +// ETag, then a completion step that validates those ETags. + +message BlobPart { + // 1-based part number, following S3 multipart numbering. + uint32 number = 1; + + // The ETag the data plane returned when this part was uploaded. + string etag = 2; + + // Size of this part in bytes. + uint64 size = 3; +} + +message Blob { + option (rbt.v1alpha1.state) = { + trusted_effects: true, + }; + + enum Status { + // Parts may be uploaded. The initial status. + UPLOADING = 0; + + // `Commit` was called; the `CompleteUpload` workflow is + // finalizing the object in the data plane. + COMMITTING = 1; + + // The blob's bytes are immutable and downloadable. + COMMITTED = 2; + + // `Remove` was called (or the upload expired); the + // `PerformRemove` workflow is removing the bytes from the data + // plane. + REMOVING = 3; + + // The bytes are gone; only this metadata tombstone remains. + REMOVED = 4; + } + + Status status = 1; + + // MIME type served on download, e.g. `image/png`. + string content_type = 2; + + // The ceiling on this blob's total size, in bytes. Whichever form + // is set is enforced as parts are reported and again at `Commit`. + oneof size_limit { + // The exact total the committed bytes must add up to, when it is + // known at `Create` time. Also enables definite progress + // tracking. + uint64 size = 3; + + // An upper bound, when the exact total is not known up front. + uint64 max_size = 4; + } + + // The ID of the user that may upload into this blob. Empty means + // anyone who knows this blob's ID may upload; see the authorizer + // note in `reboot.std.blob.v1.blob`. + string uploader_id = 5; + + // The IDs of the users who may download this blob. When unset + // (omitted at `Create`), anyone who knows the blob's ID may + // download. When set, only the listed users may (an empty list + // means no one but app-internal callers). The uploader is NOT + // implicitly a downloader. See the authorizer note in + // `reboot.std.blob.v1.blob`. + optional Downloaders downloader_ids = 10; + + // Data-plane upload session ID, set by the `BeginUpload` workflow. + // `GetPartUploadInstructions` reports `ready: false` until this is set. + optional string upload_id = 6; + + // The parts reported uploaded so far, ordered by part number. + repeated BlobPart parts = 7; + + // The committed object's ETag, as reported by the data plane: an + // opaque token identifying the stored bytes, suitable as a cache + // key for this blob. Do not compute an expected value and compare, + // and do not treat it as a content hash: a data plane may derive it + // from how the object was uploaded, so the same bytes uploaded with + // different part boundaries can carry different ETags. + optional string etag = 8; + + // Why the most recent `CompleteUpload` attempt failed. Cleared on + // success; while set, `status` has reverted to `UPLOADING` so the + // client can re-upload parts and `Commit` again. The message + // describes the failure; it does not identify which parts, if any, + // were at fault. + optional string commit_error = 9; +} + +//////////////////////////////////////////////////////////////////////// + +message CreateRequest { + // MIME type served on download, e.g. `image/png`. + string content_type = 1; + + // The ceiling on this blob's total size, in bytes. Set at most one + // form; leave both unset for no limit at all. Either way the + // running total is enforced as parts are reported (via + // `PartUploaded`) and again at `Commit`. + oneof size_limit { + // The exact total in bytes, when known up front. Enables definite + // progress bars, and the committed bytes must add up to exactly + // this — so it doubles as the ceiling during upload. + uint64 size = 2; + + // An upper bound in bytes, when the exact total is not known up + // front. + uint64 max_size = 3; + } + + // The ID of the user that may upload into this blob. Leave empty to + // allow anyone who knows this blob's ID to upload. + string uploader_id = 4; + + // The IDs of the users who may download this blob. Omit the field + // entirely to let anyone who knows the blob's ID download it; set it + // (even to an empty list) to restrict downloads to the listed users. + // The uploader is NOT implicitly a downloader — omit them from the + // list if they shouldn't be able to download (e.g. a drop-box). Can + // be changed later with `SetDownloaders`. + optional Downloaders downloader_ids = 5; +} + +message CreateResponse {} + +//////////////////////////////////////////////////////////////////////// + +message SetDownloadersRequest { + // The download allow-list to apply, replacing any previous one. + // Omit the field entirely to remove any restriction (anyone who + // knows the blob's ID may download); set it (even to an empty list) + // to restrict downloads to the listed users. + optional Downloaders downloader_ids = 1; +} + +message SetDownloadersResponse {} + +//////////////////////////////////////////////////////////////////////// + +message BeginUploadRequest {} + +message BeginUploadResponse {} + +//////////////////////////////////////////////////////////////////////// + +message GetPartUploadInstructionsRequest { + // The 1-based part numbers to mint upload URLs for. + repeated uint32 part_numbers = 1; +} + +message PartUploadInstruction { + uint32 part_number = 1; + + // URL to `PUT` this part's bytes to. May be relative (resolve it + // against the application's URL) or absolute (e.g. a presigned S3 + // URL). The response's `ETag` header must be reported back via + // `PartUploaded`. + string url = 2; +} + +message GetPartUploadInstructionsResponse { + // False until the `BeginUpload` workflow has provisioned the + // data-plane upload session; read this method reactively and the + // instructions arrive as soon as it is true. + bool ready = 1; + + // The part size uploads should use: every part except the last + // must be exactly this many bytes. + uint64 part_size = 2; + + repeated PartUploadInstruction instructions = 3; +} + +//////////////////////////////////////////////////////////////////////// + +message PartUploadedRequest { + uint32 part_number = 1; + + // The ETag returned by the data plane for this part. Validated by + // the data plane at `Commit` time. + string etag = 2; + + // Size of this part in bytes. + uint64 size = 3; +} + +message PartUploadedResponse {} + +//////////////////////////////////////////////////////////////////////// + +message CommitRequest {} + +message CommitResponse {} + +//////////////////////////////////////////////////////////////////////// + +message CompleteUploadRequest {} + +message CompleteUploadResponse {} + +//////////////////////////////////////////////////////////////////////// + +message InfoRequest {} + +message InfoResponse { + Blob.Status status = 1; + string content_type = 2; + + // Mirrors `Blob.size_limit`. + oneof size_limit { + uint64 size = 3; + uint64 max_size = 4; + } + + string uploader_id = 5; + + // Total bytes across the parts reported uploaded so far. Together + // with `size` this gives upload progress, observable reactively by + // any client authorized to call `Info`. + uint64 bytes_uploaded = 6; + + repeated BlobPart parts = 7; + optional string etag = 8; + optional string commit_error = 9; +} + +//////////////////////////////////////////////////////////////////////// + +message GetDownloadUrlRequest { + // How long the minted URL should remain valid. The storage + // backend's default (and maximum) applies when unset. + optional uint32 ttl_seconds = 1; +} + +message GetDownloadUrlResponse { + // URL to `GET` the blob's bytes from. May be relative (resolve it + // against the application's URL) or absolute (e.g. a presigned S3 + // URL). + string url = 1; + + // How long `url` is actually valid for, which is at most the + // requested `ttl_seconds`: every store caps how far ahead it will + // sign, so a caller that wants an unexpired URL must refresh on + // this, not on what it asked for. + uint32 ttl_seconds = 2; +} + +//////////////////////////////////////////////////////////////////////// + +message RemoveRequest {} + +message RemoveResponse {} + +//////////////////////////////////////////////////////////////////////// + +message PerformRemoveRequest {} + +message PerformRemoveResponse {} + +//////////////////////////////////////////////////////////////////////// + +message ExpireIfNotCommittedRequest {} + +message ExpireIfNotCommittedResponse {} + +//////////////////////////////////////////////////////////////////////// + +service BlobMethods { + // Creates the blob's metadata and schedules the `BeginUpload` + // workflow that provisions an upload session in the storage + // backend. Only application code may call this; it is the + // application's chance to enforce quota and size policy (directly, + // or by setting `max_size`), to record which user may upload + // (`uploader_id`), and to restrict who may download (`downloader_ids`). + rpc Create(CreateRequest) returns (CreateResponse) { + option (rbt.v1alpha1.method) = { + writer: { constructor: {} }, + }; + } + + // Replaces the blob's download allow-list. Application-mediated + // (app-internal only), like `Create`. Omit `downloader_ids` in the + // request to remove any restriction (anyone who knows the ID may + // download again); set it to restrict downloads to the listed + // users. + rpc SetDownloaders(SetDownloadersRequest) returns (SetDownloadersResponse) { + option (rbt.v1alpha1.method) = { + writer: {}, + }; + } + + // Provisions the data-plane upload session and records its + // `upload_id`. Scheduled by `Create`; not for direct use. + rpc BeginUpload(BeginUploadRequest) returns (BeginUploadResponse) { + option (rbt.v1alpha1.method) = { + workflow: {}, + }; + } + + // Mints `PUT` URLs for the requested part numbers. Reports + // `ready: false` until `BeginUpload` has completed. + rpc GetPartUploadInstructions(GetPartUploadInstructionsRequest) + returns (GetPartUploadInstructionsResponse) { + option (rbt.v1alpha1.method) = { + reader: {}, + errors: [ "AlreadyCommitted" ], + }; + } + + // Records that a part was uploaded to the data plane, updating the + // progress observable via `Info`. Safe to call multiple times for the + // same part number; a re-uploaded part overwrites its previous + // record. + rpc PartUploaded(PartUploadedRequest) returns (PartUploadedResponse) { + option (rbt.v1alpha1.method) = { + writer: {}, + errors: [ "AlreadyCommitted", "SizeMismatch", "IncompleteParts" ], + }; + } + + // Validates the reported parts and schedules the `CompleteUpload` + // workflow that finalizes the object in the data plane. + // Observe the outcome reactively via `Info`: `status` becomes + // `COMMITTED`, or reverts to `UPLOADING` with `commit_error` set. + // + // Finalization is a workflow rather than part of this writer + // because it is a side effect on the data plane, which must be + // retried until it converges. This method is then a writer that + // schedules that workflow, rather than being a workflow itself, + // because workflows can only be run as tasks and cannot be called + // from React — and the browser is who commits an upload. Splitting + // it this way also suits the outcome, which belongs to every + // participant: anyone who may read the blob observes the same + // `status` and `commit_error`, not just whoever called `Commit`. + // What this writer *can* decide up front — the parts are + // contiguous, the sizes agree with `size`/`max_size` — it decides + // here, and reports as a declared error. + rpc Commit(CommitRequest) returns (CommitResponse) { + option (rbt.v1alpha1.method) = { + writer: {}, + errors: [ "AlreadyCommitted", "SizeMismatch", "IncompleteParts" ], + }; + } + + // Finalizes the object in the data plane (which validates the + // reported part ETags). Scheduled by `Commit`; not for direct use. + rpc CompleteUpload(CompleteUploadRequest) returns (CompleteUploadResponse) { + option (rbt.v1alpha1.method) = { + workflow: {}, + }; + } + + // Metadata and upload progress. Reactive: watch it to render a + // progress bar. Visible to anyone who may upload or download the + // blob: the `uploader_id` and any listed `downloader_ids`, plus anyone + // who knows the ID whenever either side is left open. + rpc Info(InfoRequest) returns (InfoResponse) { + option (rbt.v1alpha1.method) = { + reader: {}, + }; + } + + // Mints a time-limited URL from which the committed blob's bytes can + // be downloaded. + rpc GetDownloadUrl(GetDownloadUrlRequest) returns (GetDownloadUrlResponse) { + option (rbt.v1alpha1.method) = { + reader: {}, + errors: [ "NotCommitted" ], + }; + } + + // Marks the blob for deletion and schedules the `PerformRemove` + // workflow that removes its bytes from the data plane. + // (Named `Remove` because `Delete` is a reserved Reboot method + // name.) + rpc Remove(RemoveRequest) returns (RemoveResponse) { + option (rbt.v1alpha1.method) = { + writer: {}, + }; + } + + // Removes the blob's bytes from the data plane. Scheduled by + // `Remove` and `ExpireIfNotCommitted`; not for direct use. + rpc PerformRemove(PerformRemoveRequest) returns (PerformRemoveResponse) { + option (rbt.v1alpha1.method) = { + workflow: {}, + }; + } + + // Expunges the blob if its upload was abandoned: scheduled by + // `Create` to run after the upload expiration period, it deletes + // the blob unless it has been committed by then. Not for direct + // use. + rpc ExpireIfNotCommitted(ExpireIfNotCommittedRequest) + returns (ExpireIfNotCommittedResponse) { + option (rbt.v1alpha1.method) = { + writer: {}, + }; + } +} diff --git a/rbt/std/blob/v1/data_plane.proto b/rbt/std/blob/v1/data_plane.proto new file mode 100644 index 000000000..25dc6223d --- /dev/null +++ b/rbt/std/blob/v1/data_plane.proto @@ -0,0 +1,195 @@ +// The blob data-plane interface. +// +// A `BlobDataPlane` holds blob *bytes*; all blob *metadata* lives in +// the `Blob` control-plane state machine (see `blob.proto`). The +// control plane never touches bytes: it calls this service to +// provision uploads, finalize them, mint download URLs, and delete +// objects, and the bytes themselves travel directly between the client +// and wherever the data plane keeps them (a presigned-URL store like +// S3, or — for a data plane whose `Configuration` asks for forwarded +// paths — the application's own HTTP routes proxying to it). +// +// This is a plain gRPC service, deliberately free of any Reboot +// framework options, so that it can be implemented by anything and +// addressed by a bare URL (`REBOOT_BLOB_DATA_PLANE_URL`). +// +// The control plane retries `BeginUpload`, `CompleteUpload`, and +// `Delete` inside workflows, so every method must be *safe* to call +// more than once for the same `blob_id`. `CompleteUpload` and `Delete` +// are naturally idempotent (a completed blob returns its ETag; +// deleting an absent blob succeeds). `BeginUpload` should reuse an +// existing uncommitted session where it can; a backend that can only +// mint fresh upload IDs is still acceptable — a lost response then +// orphans a session, which is reclaimed when the blob expires +// uncommitted or is deleted. + +syntax = "proto3"; + +package rbt.std.blob.v1; + +service BlobDataPlane { + // Reports what the control plane needs to know to talk to this data + // plane: the client part size, and which paths (if any) the + // application must forward to it. Read once at application startup. + rpc Configuration(ConfigurationRequest) returns (ConfigurationResponse); + + // Provisions an upload session for a blob and returns its data-plane + // upload ID (e.g. an S3 multipart upload ID). Should reuse an + // existing uncommitted session for the same blob where possible; a + // fresh session is acceptable (see the idempotency note above). + rpc BeginUpload(DataPlaneBeginUploadRequest) + returns (DataPlaneBeginUploadResponse); + + // Mints URLs to which the client `PUT`s each requested part's bytes + // directly. A URL is either absolute (e.g. a presigned S3 URL, + // directly reachable by the client) or application-relative, under a + // path the data plane's `Configuration` asked to have forwarded. + rpc GetPartUploadInstructions(DataPlaneGetPartUploadInstructionsRequest) + returns (DataPlaneGetPartUploadInstructionsResponse); + + // Finalizes the object from its uploaded parts, validating the + // reported ETags/sizes against the real bytes and enforcing + // `max_size` against the real total. Returns the composite ETag, or + // a permanent-failure `error` (ETag mismatch, over `max_size`, ...) + // that the control plane surfaces so the client can re-upload. A + // *transient* failure is a gRPC error instead, so the control-plane + // workflow retries. Idempotent: completing an already-completed blob + // returns its ETag. + rpc CompleteUpload(DataPlaneCompleteUploadRequest) + returns (DataPlaneCompleteUploadResponse); + + // Mints a URL from which the committed object's bytes can be + // downloaded: absolute (e.g. a presigned S3/CloudFront URL) or + // application-relative under a forwarded path, like + // `GetPartUploadInstructions` URLs. + rpc GetDownloadUrl(DataPlaneGetDownloadUrlRequest) + returns (DataPlaneGetDownloadUrlResponse); + + // Removes the object's bytes and any incomplete upload session. + // Idempotent: deleting an absent blob succeeds. + rpc Delete(DataPlaneDeleteRequest) returns (DataPlaneDeleteResponse); +} + +message ConfigurationRequest {} + +// The HTTP method of a forwarded path. +enum HttpMethod { + HTTP_METHOD_UNSPECIFIED = 0; + HTTP_METHOD_GET = 1; + HTTP_METHOD_PUT = 2; +} + +// One path namespace the application must forward to the data plane: +// requests with the given method whose path starts with `path_prefix` +// are reverse-proxied to the data plane's HTTP endpoint verbatim +// (path and query). `path_prefix` must itself start with +// `/__/reboot/blob/` — the application refuses anything else, so a +// data plane can never claim application routes. +message ForwardedPath { + HttpMethod method = 1; + string path_prefix = 2; +} + +message ConfigurationResponse { + // The part size clients must use; every part but the last must be + // exactly this size. + uint64 part_size = 1; + + // Paths the application must forward to this data plane. Empty when + // the data plane's URLs are directly reachable by clients (e.g. + // presigned S3 or CloudFront URLs) and no forwarding is needed. + repeated ForwardedPath forwarded_paths = 2; + + // The port of this data plane's HTTP byte endpoint, to which + // forwarded requests are proxied. The host is the one the + // application already reaches this gRPC service on (so the data + // plane need not know how it is addressed externally), and the + // scheme follows that connection's transport security. Only + // meaningful when `forwarded_paths` is non-empty. + uint32 http_port = 3; +} + +//////////////////////////////////////////////////////////////////////// + +message DataPlaneBeginUploadRequest { + string blob_id = 1; + string content_type = 2; +} + +message DataPlaneBeginUploadResponse { + string upload_id = 1; +} + +//////////////////////////////////////////////////////////////////////// + +message DataPlaneGetPartUploadInstructionsRequest { + string blob_id = 1; + string upload_id = 2; + repeated uint32 part_numbers = 3; +} + +message DataPlanePartUploadInstruction { + uint32 part_number = 1; + string url = 2; +} + +message DataPlaneGetPartUploadInstructionsResponse { + repeated DataPlanePartUploadInstruction instructions = 1; +} + +//////////////////////////////////////////////////////////////////////// + +message DataPlaneUploadedPart { + uint32 number = 1; + string etag = 2; + uint64 size = 3; +} + +message DataPlaneCompleteUploadRequest { + string blob_id = 1; + string upload_id = 2; + string content_type = 3; + repeated DataPlaneUploadedPart parts = 4; + optional uint64 max_size = 5; +} + +message DataPlaneCompleteUploadResponse { + // The committed object's composite ETag. Empty when `error` is set. + string etag = 1; + + // A permanent-failure reason, if completion failed in a way the + // client can fix by re-uploading. When set, `etag` is empty and the + // control plane reverts the blob to UPLOADING with this message. + optional string error = 2; +} + +//////////////////////////////////////////////////////////////////////// + +message DataPlaneGetDownloadUrlRequest { + string blob_id = 1; + optional uint64 ttl_seconds = 2; +} + +message DataPlaneGetDownloadUrlResponse { + string url = 1; + + // How long `url` is valid for: the requested `ttl_seconds` capped + // by whatever this data plane's signing scheme allows. + uint32 ttl_seconds = 2; +} + +//////////////////////////////////////////////////////////////////////// + +message DataPlaneDeleteRequest { + string blob_id = 1; + + // Upload sessions the control plane knows are outstanding for this + // blob. Deleting the object does not reach the bytes an unfinished + // upload has already parked, and the data plane cannot discover + // those sessions itself without permission to enumerate every + // application's uploads, so the IDs come from the side that + // recorded them. + repeated string upload_ids = 2; +} + +message DataPlaneDeleteResponse {} diff --git a/rbt/std/blob/v1/package.json b/rbt/std/blob/v1/package.json new file mode 100644 index 000000000..3dbc1ca59 --- /dev/null +++ b/rbt/std/blob/v1/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/reboot/BUILD.bazel b/reboot/BUILD.bazel index b0b112882..926db7c7c 100644 --- a/reboot/BUILD.bazel +++ b/reboot/BUILD.bazel @@ -511,6 +511,7 @@ py_library( py_library( name = "python_std", deps = [ + "//reboot/std/blob/v1:blob_py", "//reboot/std/ciphertext/v1:ciphertext_py", "//reboot/std/collections/ordered_map/v1:ordered_map_py", "//reboot/std/collections/queue/v1:queue_py", @@ -648,8 +649,12 @@ sh_library( ":reboot.dev", "//rbt/v1alpha1:reboot-dev-reboot-api", "//reboot/create-ui:reboot-dev-create-ui", + "//reboot/nodejs:reboot-dev-reboot-" + REBOOT_VERSION + ".tgz", "//reboot/react:reboot-dev-reboot-react", + "//reboot/std:reboot-dev-reboot-std", + "//reboot/std/react:reboot-dev-reboot-std-react", "//reboot/web:reboot-dev-reboot-web", + "@com_github_reboot_dev_reboot//rbt/std:reboot-dev-reboot-std-api", ], ) @@ -666,8 +671,12 @@ sh_binary( ":reboot.dev", "//rbt/v1alpha1:reboot-dev-reboot-api", "//reboot/create-ui:reboot-dev-create-ui", + "//reboot/nodejs:reboot-dev-reboot-" + REBOOT_VERSION + ".tgz", "//reboot/react:reboot-dev-reboot-react", + "//reboot/std:reboot-dev-reboot-std", + "//reboot/std/react:reboot-dev-reboot-std-react", "//reboot/web:reboot-dev-reboot-web", + "@com_github_reboot_dev_reboot//rbt/std:reboot-dev-reboot-std-api", ], deps = [":stage_and_publish_local_lib"], ) diff --git a/reboot/aio/BUILD.bazel b/reboot/aio/BUILD.bazel index cd2321c56..68503ea5a 100644 --- a/reboot/aio/BUILD.bazel +++ b/reboot/aio/BUILD.bazel @@ -525,6 +525,7 @@ py_library( "//reboot/aio/auth:oauth_providers_py", "//reboot/aio/auth:oauth_py", "//reboot/aio/auth:oauth_server_py", + "//reboot/std/blob/v1:blob_py", ], ) diff --git a/reboot/aio/tests.py b/reboot/aio/tests.py index 108534116..ba68e62d9 100644 --- a/reboot/aio/tests.py +++ b/reboot/aio/tests.py @@ -2,6 +2,7 @@ import os import reboot.aio.reboot import secrets +import tempfile import unittest from reboot.aio.applications import Application, NodeApplication from reboot.aio.auth.oauth import OAuth @@ -26,6 +27,8 @@ ENVVAR_REBOOT_ENABLE_EVENT_LOOP_BLOCKED_WATCHDOG, ENVVAR_REBOOT_IN_TEST, ) +from reboot.std.blob.v1._data_plane import ENVVAR_BLOB_DATA_PLANE_URL +from reboot.std.blob.v1._filesystem_server import FilesystemDataPlane from typing import ( Any, Awaitable, @@ -163,6 +166,47 @@ def __init__(self) -> None: os.environ[ENVVAR_REBOOT_IN_TEST] = 'true' # The application under test, or `None` before one is started. self._application: Optional[Application] = None + self._blob_data_plane: Optional[FilesystemDataPlane] = None + self._blob_data_plane_directory: Optional[tempfile.TemporaryDirectory + ] = None + + async def start(self): + result = await super().start() + # Run a filesystem blob data plane for the duration of the + # test, so that applications using `reboot.std.blob` work in + # unit tests exactly as they do under `rbt dev run` (which + # spawns the same data plane). An already-configured data plane + # is honored, mirroring the CLI — including one set up by + # another live `Reboot` instance in this process, which then + # must outlive this instance's use of it. + if not os.environ.get(ENVVAR_BLOB_DATA_PLANE_URL): + self._blob_data_plane_directory = tempfile.TemporaryDirectory( + prefix="reboot-test-blobs-" + ) + self._blob_data_plane = await FilesystemDataPlane.start( + directory=self._blob_data_plane_directory.name, + ) + os.environ[ENVVAR_BLOB_DATA_PLANE_URL] = ( + self._blob_data_plane.url + ) + return result + + async def stop(self) -> None: + try: + await super().stop() + finally: + if self._blob_data_plane is not None: + await self._blob_data_plane.stop() + # Only clear the env var if it still points at our data + # plane; another `Reboot` instance may have replaced it + # with its own in the meantime. + if os.environ.get(ENVVAR_BLOB_DATA_PLANE_URL + ) == (self._blob_data_plane.url): + os.environ.pop(ENVVAR_BLOB_DATA_PLANE_URL, None) + self._blob_data_plane = None + if self._blob_data_plane_directory is not None: + self._blob_data_plane_directory.cleanup() + self._blob_data_plane_directory = None async def make_valid_oauth_access_token( self, diff --git a/reboot/std/BUILD.bazel b/reboot/std/BUILD.bazel index 8c81b120c..a625fc6a2 100644 --- a/reboot/std/BUILD.bazel +++ b/reboot/std/BUILD.bazel @@ -47,6 +47,7 @@ ts_project( }, visibility = ["//visibility:public"], deps = [ + "//reboot/std/blob/v1:blob_ts", "//reboot/std/ciphertext/v1:ciphertext_ts", "//reboot/std/collections/ordered_map/v1:ordered_map_ts", "//reboot/std/collections/queue/v1:queue_ts", diff --git a/reboot/std/blob/v1/BUILD.bazel b/reboot/std/blob/v1/BUILD.bazel new file mode 100644 index 000000000..2d35f4a20 --- /dev/null +++ b/reboot/std/blob/v1/BUILD.bazel @@ -0,0 +1,55 @@ +load("@aspect_rules_ts//ts:defs.bzl", "ts_project") +load("@rbt_pypi//:requirements.bzl", "requirement") +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "blob_py", + srcs = [ + "_content_type.py", + "_data_plane.py", + "_filesystem_server.py", + "_http.py", + "_proxy.py", + "_store.py", + "blob.py", + ], + visibility = ["//visibility:public"], + deps = [ + "//reboot/aio:applications_py", + "//reboot/aio:contexts_py", + "//reboot/aio:http_py", + "//reboot/aio:workflows_py", + "//reboot/aio/auth:authorizers_py", + "//reboot/crypto:root_keys_py", + "@com_github_reboot_dev_reboot//log:log_py", + "@com_github_reboot_dev_reboot//rbt/std/blob/v1:blob_py_reboot", + "@com_github_reboot_dev_reboot//rbt/std/blob/v1:data_plane_py_reboot", + requirement("aiohttp"), + requirement("grpcio"), + requirement("starlette"), + requirement("uvicorn"), + ], +) + +ts_project( + name = "blob_ts", + srcs = [ + "index.ts", + "package.json", + ], + declaration = True, + tsconfig = { + "compilerOptions": { + "declaration": True, + "module": "nodenext", + "moduleResolution": "nodenext", + "target": "es2020", + }, + }, + visibility = ["//visibility:public"], + deps = [ + "//:node_modules/@reboot-dev/reboot", + "//:node_modules/@reboot-dev/reboot-std-api", + "//:node_modules/@types/node", + ], +) diff --git a/reboot/std/blob/v1/_content_type.py b/reboot/std/blob/v1/_content_type.py new file mode 100644 index 000000000..0ea47be5c --- /dev/null +++ b/reboot/std/blob/v1/_content_type.py @@ -0,0 +1,66 @@ +"""How a blob's declared content type is served back to a browser.""" + +# Content types a browser renders without the bytes being able to act +# as the application: images and media decode to pixels or samples, +# and plain text renders as text once sniffing is refused. A PDF is +# rendered by a viewer that runs any script it contains in its own +# context rather than the page's. +# +# `image/svg+xml` is deliberately absent. An SVG is a document, it may +# script, and a browser runs that script in the origin that served it. +_RENDERABLE_CONTENT_TYPES = frozenset( + { + "audio/aac", + "audio/mpeg", + "audio/ogg", + "audio/wav", + "audio/webm", + "application/pdf", + "image/avif", + "image/bmp", + "image/gif", + "image/jpeg", + "image/png", + "image/webp", + "text/plain", + "video/mp4", + "video/ogg", + "video/webm", + } +) + +# What anything else is served as. +_DOWNLOAD_CONTENT_TYPE = "application/octet-stream" + + +def download_headers(content_type: str) -> tuple[str, dict[str, str]]: + """The content type to serve a blob as, and the headers that go + with it. + + A blob's declared type is whatever its uploader claimed, and the + bytes come back on the application's own origin, so a claimed + `text/html` would render as a same-origin document with the + reader's session -- able to call the application as them. Only + types that cannot carry script are served as declared; everything + else is served as an opaque download, whatever it claims to be. + + `X-Content-Type-Options: nosniff` still matters for what remains: + it stops a browser reading, say, `text/plain` bytes as something + richer. + """ + declared = content_type.split(";")[0].strip().lower() + if declared in _RENDERABLE_CONTENT_TYPES: + # The normalized type, not what was declared: everything + # after the first `;` was never looked at, and this value + # goes into a response header verbatim -- parameters, and + # anything an uploader put after them, included. + return declared, {"X-Content-Type-Options": "nosniff"} + return ( + _DOWNLOAD_CONTENT_TYPE, + { + "X-Content-Type-Options": "nosniff", + # Downloaded rather than rendered, so nothing this blob + # contains is interpreted on the application's origin. + "Content-Disposition": "attachment", + }, + ) diff --git a/reboot/std/blob/v1/_data_plane.py b/reboot/std/blob/v1/_data_plane.py new file mode 100644 index 000000000..33a2c8953 --- /dev/null +++ b/reboot/std/blob/v1/_data_plane.py @@ -0,0 +1,81 @@ +"""Client-side glue for talking to a `BlobDataPlane` gRPC service.""" + +import grpc +import os +from rbt.std.blob.v1.data_plane_pb2_grpc import BlobDataPlaneStub +from urllib.parse import urlparse + +# The URL of the `BlobDataPlane` gRPC service. A bare `host:port`, or a +# URL whose scheme selects transport security (`https`/`grpcs` -> +# secure, anything else -> insecure). +ENVVAR_BLOB_DATA_PLANE_URL = "REBOOT_BLOB_DATA_PLANE_URL" + +# The path namespace a data plane's forwarded paths must live under; +# the application refuses to forward anything else, so a data plane +# can never claim application routes (see `ForwardedPath` in +# `data_plane.proto`). +FORWARDED_PATH_PREFIX = "/__/reboot/blob/" + +_SECURE_SCHEMES = ("https", "grpcs") + + +class DataPlaneNotConfigured(RuntimeError): + """Raised when no data-plane URL is configured. Under `rbt dev + run`/`rbt serve run` and in `reboot.aio.tests.Reboot` unit tests + this never happens (they provide the filesystem data plane and set + the URL); it indicates the application was started some other way + without a data plane.""" + + +def channel_for_url(url: str) -> grpc.aio.Channel: + """Builds a gRPC channel to the data-plane service at `url`. The + scheme selects transport security; the host:port is the gRPC + target. Any URL path is ignored (gRPC addresses by host:port, not + path).""" + # `urlparse` needs a scheme to populate `netloc`; treat a bare + # `host:port` as such. + parsed = urlparse(url if "://" in url else f"grpc://{url}") + target = parsed.netloc + if parsed.scheme in _SECURE_SCHEMES: + return grpc.aio.secure_channel(target, grpc.ssl_channel_credentials()) + return grpc.aio.insecure_channel(target) + + +def proxy_target_from_environment(http_port: int) -> str: + """The base URL of the data plane's HTTP byte endpoint, to which + the application proxies forwarded paths: the host the application + already reaches the data plane's gRPC service on (from + `REBOOT_BLOB_DATA_PLANE_URL`), with the `http_port` its + `Configuration` reported, over the same transport security.""" + url = os.environ.get(ENVVAR_BLOB_DATA_PLANE_URL) + if not url: + raise DataPlaneNotConfigured( + f"`{ENVVAR_BLOB_DATA_PLANE_URL}` is not set." + ) + parsed = urlparse(url if "://" in url else f"grpc://{url}") + scheme = "https" if parsed.scheme in _SECURE_SCHEMES else "http" + host = parsed.hostname or "" + # `urlparse` strips the brackets off an IPv6 literal; put them + # back, since they are required in a URL authority. + if ":" in host: + host = f"[{host}]" + return f"{scheme}://{host}:{http_port}" + + +def stub_from_environment() -> BlobDataPlaneStub: + """Builds a data-plane stub from `REBOOT_BLOB_DATA_PLANE_URL`. A + fresh channel is created on the current event loop (rather than + memoized) because `grpc.aio` channels are event-loop-affine, and + an application may be brought up on a new loop. Called once per + library `pre_run`; the channel then lives as long as the + application (the `Library` has no shutdown hook on which to close + it).""" + url = os.environ.get(ENVVAR_BLOB_DATA_PLANE_URL) + if not url: + raise DataPlaneNotConfigured( + f"`{ENVVAR_BLOB_DATA_PLANE_URL}` is not set. Run the " + "application via `rbt dev run` or `rbt serve run` (which " + "start the filesystem blob data plane automatically), or " + "set the variable to your own data-plane service's URL." + ) + return BlobDataPlaneStub(channel_for_url(url)) diff --git a/reboot/std/blob/v1/_filesystem_server.py b/reboot/std/blob/v1/_filesystem_server.py new file mode 100644 index 000000000..eccb12a71 --- /dev/null +++ b/reboot/std/blob/v1/_filesystem_server.py @@ -0,0 +1,334 @@ +"""The filesystem blob data-plane server. + +Implements the `BlobDataPlane` gRPC service backed by the local +filesystem, plus the HTTP byte endpoint its minted URLs point at. It +runs in two ways: `rbt dev run` and `rbt serve run` spawn it as a +standalone program (via `main`) whenever `REBOOT_BLOB_DATA_PLANE_URL` +is not already set, and the `reboot.aio.tests.Reboot` test harness +runs it in-process (via `FilesystemDataPlane.start`), so that blob +storage works out of the box in every local run mode. + +Its URLs are application-relative proxy paths: its `Configuration` +asks the application to forward the blob path namespace to it, so the +application reverse-proxies byte `PUT`/`GET` here rather than exposing +this server on its own port (see `_proxy.py`). +""" + +import argparse +import asyncio +import contextlib +import grpc +import os +import uvicorn # type: ignore[import] +from rbt.std.blob.v1.data_plane_pb2 import ( + HTTP_METHOD_GET, + HTTP_METHOD_PUT, + ConfigurationResponse, + DataPlaneBeginUploadResponse, + DataPlaneCompleteUploadResponse, + DataPlaneDeleteResponse, + DataPlaneGetDownloadUrlResponse, + DataPlaneGetPartUploadInstructionsResponse, + DataPlanePartUploadInstruction, + ForwardedPath, +) +from rbt.std.blob.v1.data_plane_pb2_grpc import ( + BlobDataPlaneServicer, + add_BlobDataPlaneServicer_to_server, +) +from reboot.std.blob.v1._http import build_http_app +from reboot.std.blob.v1._store import ( + DEFAULT_PART_SIZE_BYTES, + HTTP_PATH_PREFIX, + BlobStoreError, + FilesystemBlobStore, + UploadedPart, +) +from typing import Generator, Optional +from uuid import uuid4 + +# The filesystem server binds loopback only: it has no authentication +# (its URLs are HMAC-signed, but the gRPC control surface is not), so +# it must never be reachable off the host. The application reaches it +# over localhost and proxies client byte traffic to it. +LOOPBACK_HOST = "127.0.0.1" + + +class FilesystemDataPlaneServicer(BlobDataPlaneServicer): + """Implements `BlobDataPlane` over a `FilesystemBlobStore`.""" + + def __init__( + self, + store: FilesystemBlobStore, + http_port: int, + ): + self._store = store + self._http_port = http_port + + async def Configuration(self, request, context): + # This server's URLs are application-relative under + # `HTTP_PATH_PREFIX`; ask the application to forward that + # namespace to the HTTP byte endpoint. + return ConfigurationResponse( + part_size=self._store.part_size, + forwarded_paths=[ + ForwardedPath( + method=HTTP_METHOD_GET, + path_prefix=HTTP_PATH_PREFIX + "/", + ), + ForwardedPath( + method=HTTP_METHOD_PUT, + path_prefix=HTTP_PATH_PREFIX + "/", + ), + ], + http_port=self._http_port, + ) + + async def BeginUpload(self, request, context): + upload_id = await self._store.begin_upload( + request.blob_id, + request.content_type, + ) + return DataPlaneBeginUploadResponse(upload_id=upload_id) + + async def GetPartUploadInstructions(self, request, context): + instructions = [ + DataPlanePartUploadInstruction( + part_number=part_number, + url=self._store.part_put_url( + request.blob_id, + request.upload_id, + part_number, + ), + ) for part_number in request.part_numbers + ] + return DataPlaneGetPartUploadInstructionsResponse( + instructions=instructions + ) + + async def CompleteUpload(self, request, context): + try: + etag = await self._store.complete( + request.blob_id, + request.upload_id, + request.content_type, + [ + UploadedPart( + number=part.number, etag=part.etag, size=part.size + ) for part in request.parts + ], + max_size=( + request.max_size if request.HasField("max_size") else None + ), + ) + return DataPlaneCompleteUploadResponse(etag=etag) + except BlobStoreError as error: + # A permanent failure: report it so the control plane can + # surface it and let the client re-upload. Transient + # failures raise other exceptions, which become gRPC errors + # so the control-plane workflow retries. + return DataPlaneCompleteUploadResponse(error=str(error)) + + async def GetDownloadUrl(self, request, context): + url, ttl_seconds = self._store.download_url( + request.blob_id, + request.ttl_seconds if request.HasField("ttl_seconds") else None, + ) + return DataPlaneGetDownloadUrlResponse( + url=url, + ttl_seconds=ttl_seconds, + ) + + async def Delete(self, request, context): + await self._store.delete(request.blob_id) + return DataPlaneDeleteResponse() + + +class FilesystemDataPlane: + """A running filesystem blob data plane: the `BlobDataPlane` gRPC + service plus the HTTP byte endpoint its minted URLs point at, both + bound to loopback. Construct via `start()`.""" + + def __init__( + self, + *, + grpc_server: grpc.aio.Server, + grpc_port: int, + http_server, + http_task: asyncio.Task, + http_port: int, + ): + self._grpc_server = grpc_server + self._grpc_port = grpc_port + self._http_server = http_server + self._http_task = http_task + self._http_port = http_port + + @classmethod + async def start( + cls, + *, + directory: str, + part_size: int = DEFAULT_PART_SIZE_BYTES, + grpc_port: int = 0, + http_port: int = 0, + ) -> "FilesystemDataPlane": + """Starts serving, with bytes stored under `directory`. Ports + default to 0 (an ephemeral port chosen by the OS). The HTTP + byte endpoint is brought up before the gRPC surface, so that by + the time `Configuration` is reachable the `http_port` it + reports is already serving.""" + store = FilesystemBlobStore(directory, part_size=part_size) + + class Server(uvicorn.Server): + """We need to override the installation of signal handlers as + Reboot is already handling this itself. + """ + + @contextlib.contextmanager + def capture_signals(self) -> Generator[None, None, None]: + # Do nothing + yield + + http_server = Server( + uvicorn.Config( + build_http_app(store), + host=LOOPBACK_HOST, + port=http_port, + log_level="warning", + ) + ) + http_task = asyncio.create_task(http_server.serve()) + while not http_server.started: + if http_task.done(): + # Startup failed (e.g. the requested port is in use); + # surface the underlying error. + http_task.result() + raise RuntimeError( + "The blob data plane's HTTP server exited during " + "startup" + ) + await asyncio.sleep(0.01) + actual_http_port = http_server.servers[0].sockets[0].getsockname()[1] + + # From here on the HTTP server is live; shut it down if the + # gRPC surface fails to come up, so a failed `start` leaves + # nothing running. + try: + grpc_server = grpc.aio.server() + add_BlobDataPlaneServicer_to_server( + FilesystemDataPlaneServicer(store, actual_http_port), + grpc_server, + ) + actual_grpc_port = grpc_server.add_insecure_port( + f"{LOOPBACK_HOST}:{grpc_port}" + ) + if actual_grpc_port == 0: + raise RuntimeError( + "The blob data plane's gRPC server could not bind " + f"port {grpc_port}" + ) + await grpc_server.start() + except BaseException: + http_server.should_exit = True + await http_task + raise + + return cls( + grpc_server=grpc_server, + grpc_port=actual_grpc_port, + http_server=http_server, + http_task=http_task, + http_port=actual_http_port, + ) + + @property + def grpc_port(self) -> int: + return self._grpc_port + + @property + def http_port(self) -> int: + return self._http_port + + @property + def url(self) -> str: + """The gRPC address to put in `REBOOT_BLOB_DATA_PLANE_URL`.""" + return f"{LOOPBACK_HOST}:{self._grpc_port}" + + async def stop(self) -> None: + await self._grpc_server.stop(grace=None) + self._http_server.should_exit = True + await self._http_task + + async def wait(self) -> None: + """Blocks until the servers terminate (they don't, absent + `stop()`; this is how the standalone program serves forever).""" + await asyncio.gather( + self._grpc_server.wait_for_termination(), + self._http_task, + ) + + +def _write_ready_file(path: str, grpc_port: int, http_port: int) -> None: + """Atomically writes the two chosen ports so the spawning process + learns them only once both servers are listening.""" + temp_path = f"{path}.{uuid4().hex}.tmp" + with open(temp_path, "w") as f: + f.write(f"{grpc_port}\n{http_port}\n") + f.flush() + os.fsync(f.fileno()) + os.replace(temp_path, path) + + +async def serve( + directory: str, + grpc_port: int = 0, + http_port: int = 0, + part_size: int = DEFAULT_PART_SIZE_BYTES, + ready_file: Optional[str] = None, +) -> None: + """Runs the data plane until terminated. When `ready_file` is + given, the actually-bound ports are written to it once both + endpoints are listening, for the spawning process to read.""" + data_plane = await FilesystemDataPlane.start( + directory=directory, + part_size=part_size, + grpc_port=grpc_port, + http_port=http_port, + ) + if ready_file is not None: + _write_ready_file( + ready_file, + data_plane.grpc_port, + data_plane.http_port, + ) + await data_plane.wait() + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Filesystem blob data-plane server." + ) + parser.add_argument("--directory", required=True) + parser.add_argument("--grpc-port", type=int, default=0) + parser.add_argument("--http-port", type=int, default=0) + parser.add_argument( + "--part-size", + type=int, + default=DEFAULT_PART_SIZE_BYTES, + ) + parser.add_argument("--ready-file", default=None) + args = parser.parse_args() + asyncio.run( + serve( + args.directory, + args.grpc_port, + args.http_port, + part_size=args.part_size, + ready_file=args.ready_file, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/reboot/std/blob/v1/_http.py b/reboot/std/blob/v1/_http.py new file mode 100644 index 000000000..c9926673e --- /dev/null +++ b/reboot/std/blob/v1/_http.py @@ -0,0 +1,252 @@ +"""The HTTP byte endpoint of the filesystem blob data-plane server. + +Serves `PUT` (part upload) and `GET` (download) under +`/__/reboot/blob/`. The filesystem server (`_filesystem_server.py`) +runs this on localhost; the application's `Blob` library reverse- +proxies to it (see `_proxy.py`), so the bytes never leave a single +origin even though they live in a separate process. + +These handlers are self-authorizing: every URL carries an expiring +HMAC signature minted by the data plane, so the handlers never call +back into Reboot state. They touch only the store's directory, +mirroring how a presigned S3 URL is served by S3 without consulting +the application. +""" + +import asyncio +import hashlib +import hmac +import os +import re +import time +from reboot.std.blob.v1._content_type import download_headers +from reboot.std.blob.v1._store import ( + HTTP_PATH_PREFIX, + MAX_PARTS, + FilesystemBlobStore, +) +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import Response, StreamingResponse +from starlette.routing import Route +from typing import Optional +from uuid import uuid4 + +_STREAM_CHUNK_SIZE = 1024 * 1024 + +# Path parameters are also filesystem path components; restrict them +# to the alphabets the store actually produces (URL-safe base64 blob +# IDs, hex upload IDs) as defense in depth against traversal — even +# though a forged path could never carry a valid signature. +_ENCODED_BLOB_ID_PATTERN = re.compile(r"^[A-Za-z0-9_-]+={0,2}$") +_UPLOAD_ID_PATTERN = re.compile(r"^[0-9a-f]{32}$") + +# Enough digits for any epoch second this will ever mint, and +# far below the length `int()` refuses. +_MAX_EXPIRATION_DIGITS = 20 + + +class _PartTooLarge(Exception): + pass + + +def _signature_matches(expected: str, actual: str) -> bool: + # Compared as bytes: `compare_digest` refuses `str` arguments + # that are not ASCII, and `actual` is a query parameter, so a + # request can otherwise choose to raise here. + return hmac.compare_digest( + expected.encode("utf-8"), + actual.encode("utf-8"), + ) + + +def _unexpired_expiration(request: Request) -> Optional[int]: + """The `exp` a signed URL carries, or `None` if it has passed or + is not one this endpoint ever mints. + + The single place `exp` is parsed. It is attacker-chosen and is + read before anything has been verified, so every way `int()` + can refuse a string has to be excluded before calling it: + `isdigit()` alone admits characters like superscript two, and + both it and `isascii()` admit digit strings longer than + `sys.get_int_max_str_digits()`, which `int()` refuses in order + to bound its own quadratic parse.""" + expiration = request.query_params.get("exp", "0") + if ( + not expiration.isascii() or not expiration.isdigit() or + len(expiration) > _MAX_EXPIRATION_DIGITS + ): + return None + parsed = int(expiration) + if parsed < time.time(): + return None + return parsed + + +def _make_put_part(store: FilesystemBlobStore): + + async def put_part(request: Request) -> Response: + blob = request.path_params["blob"] + upload = request.path_params["upload"] + try: + part = int(request.path_params["part"]) + except ValueError: + return Response(status_code=400, content="Invalid part number") + + if part < 1 or part > MAX_PARTS: + return Response(status_code=400, content="Invalid part number") + if ( + not _ENCODED_BLOB_ID_PATTERN.match(blob) or + not _UPLOAD_ID_PATTERN.match(upload) + ): + return Response(status_code=400, content="Invalid blob ID") + expiration = _unexpired_expiration(request) + if expiration is None: + return Response(status_code=403, content="URL expired") + expected = store.signature_for_put(blob, upload, part, expiration) + if not _signature_matches( + expected, request.query_params.get("sig", "") + ): + return Response(status_code=403, content="Invalid signature") + + path = store.part_path(blob, upload, part) + # The upload directory is created by `begin_upload`; a missing + # directory means the blob was never created (or was deleted). + if not os.path.isdir(os.path.dirname(path)): + return Response(status_code=404, content="No such upload") + + # Refuse to mutate a committed blob's bytes: the part files + # *are* the committed object's on-disk representation, so a + # part-PUT URL minted just before commit must not still be + # usable to tamper with the bytes afterwards. + meta = store.read_meta(blob) + if meta is not None and meta.get("committed", False): + return Response(status_code=409, content="Blob already committed") + + # Write somewhere else and publish with a rename, rather than + # writing `path` in place: completion reads the part files to + # validate them, and a part being rewritten underneath it + # would leave a committed blob whose bytes no longer match the + # ETag it recorded. A rename is atomic, so completion sees + # either the whole old part or the whole new one. + temporary = f"{path}.{uuid4().hex}.partial" + digest = hashlib.md5() + size = 0 + try: + with open(temporary, "wb") as f: + async for chunk in request.stream(): + if size + len(chunk) > store.part_size: + raise _PartTooLarge() + digest.update(chunk) + size += len(chunk) + f.write(chunk) + f.flush() + os.fsync(f.fileno()) + except _PartTooLarge: + os.unlink(temporary) + return Response( + status_code=413, + content=( + "Part exceeds the maximum part size of " + f"{store.part_size} bytes" + ), + ) + except BaseException: + # Never leave a partial file behind to be mistaken for a + # part. + if os.path.exists(temporary): + os.unlink(temporary) + raise + + # Publish under the blob's lock, and re-read the metadata + # inside it: completion may have run while these bytes were + # being uploaded, and a part must not appear after the blob it + # belongs to has been committed. + async with store.lock_for(blob): + meta = store.read_meta(blob) + if meta is not None and meta.get("committed", False): + os.unlink(temporary) + return Response( + status_code=409, content="Blob already committed" + ) + os.replace(temporary, path) + + # Match S3: the ETag response header is the part's MD5, quoted. + return Response( + status_code=200, + headers={"ETag": f'"{digest.hexdigest()}"'}, + ) + + return put_part + + +def _make_get_blob(store: FilesystemBlobStore): + + async def get_blob(request: Request) -> Response: + blob = request.path_params["blob"] + if not _ENCODED_BLOB_ID_PATTERN.match(blob): + return Response(status_code=400, content="Invalid blob ID") + expiration = _unexpired_expiration(request) + if expiration is None: + return Response(status_code=403, content="URL expired") + expected = store.signature_for_get(blob, expiration) + if not _signature_matches( + expected, request.query_params.get("sig", "") + ): + return Response(status_code=403, content="Invalid signature") + + meta = store.read_meta(blob) + if meta is None or not meta.get("committed", False): + return Response(status_code=404, content="No such blob") + + upload_id = meta["upload_id"] + parts = meta["parts"] + total_size = sum(part["size"] for part in parts) + + async def stream(): + for part in sorted(parts, key=lambda part: part["number"]): + path = store.part_path(blob, upload_id, part["number"]) + # Read off the event loop: this generator is driven by + # it, and a part is megabytes, so reading inline would + # stall every other request this worker is serving. + file = await asyncio.to_thread(open, path, "rb") + try: + while chunk := await asyncio.to_thread( + file.read, + _STREAM_CHUNK_SIZE, + ): + yield chunk + finally: + await asyncio.to_thread(file.close) + + media_type, safety_headers = download_headers(meta["content_type"]) + return StreamingResponse( + stream(), + media_type=media_type, + headers={ + "Content-Length": str(total_size), + "ETag": f'"{meta["etag"]}"', + "Accept-Ranges": "none", + **safety_headers, + }, + ) + + return get_blob + + +def build_http_app(store: FilesystemBlobStore) -> Starlette: + """Builds the Starlette app serving `store`'s byte `PUT`/`GET`.""" + return Starlette( + routes=[ + Route( + HTTP_PATH_PREFIX + "/{blob}/{upload}/parts/{part}", + _make_put_part(store), + methods=["PUT"], + ), + Route( + HTTP_PATH_PREFIX + "/{blob}", + _make_get_blob(store), + methods=["GET"], + ), + ], + ) diff --git a/reboot/std/blob/v1/_proxy.py b/reboot/std/blob/v1/_proxy.py new file mode 100644 index 000000000..8ea44d113 --- /dev/null +++ b/reboot/std/blob/v1/_proxy.py @@ -0,0 +1,161 @@ +"""Application-side reverse proxy for a blob data plane's forwarded +paths. + +A data plane whose URLs are not directly reachable by clients (e.g. +the filesystem server, which binds localhost only) asks, via its +`Configuration`, for path namespaces to be forwarded to it. The `Blob` +library then registers these routes on the application's own HTTP +server, forwarding the requests to the data plane's HTTP endpoint. +That keeps the data plane off any externally-exposed port: a single +application origin/tunnel serves both the control plane and the +bytes. + +The proxy is deliberately dumb: it forwards the request path and query +verbatim and never inspects them. The data plane minted the URL and +validates its own signature, so the proxy adds no trust. The one thing +it enforces is the namespace: every forwarded path must live under +`FORWARDED_PATH_PREFIX`, so a data plane can never claim application +routes. +""" + +import aiohttp +from rbt.std.blob.v1.data_plane_pb2 import ( + HTTP_METHOD_GET, + HTTP_METHOD_PUT, + ForwardedPath, + HttpMethod, +) +from reboot.aio.http import PythonWebFramework +from reboot.std.blob.v1._content_type import download_headers +from reboot.std.blob.v1._data_plane import FORWARDED_PATH_PREFIX +from starlette.requests import Request +from starlette.responses import Response, StreamingResponse +from typing import Iterable + +_STREAM_CHUNK_SIZE = 1024 * 1024 + +# Response headers worth carrying back from the data plane on a +# download; others are hop-by-hop or recomputed by the framework. +_FORWARDED_GET_HEADERS = ( + "Content-Type", + "Content-Length", + "ETag", + "Accept-Ranges", + # Without this the data plane's own `nosniff` would be dropped on + # the way through, which is exactly where it matters: these bytes + # reach the browser on the application's origin. + "X-Content-Type-Options", +) + + +def mount_proxy_routes( + http: PythonWebFramework.HTTP, + proxy_target_url: str, + forwarded_paths: Iterable[ForwardedPath], +) -> None: + """Registers a reverse-proxy route for each of the data plane's + `forwarded_paths`, forwarding to `proxy_target_url`. Refuses paths + outside `FORWARDED_PATH_PREFIX` and unknown methods.""" + + target = proxy_target_url.rstrip("/") + + def _forward_url(request: Request) -> str: + url = target + request.url.path + if request.url.query: + url += "?" + request.url.query + return url + + async def put_forward(rest: str, request: Request) -> Response: + session = aiohttp.ClientSession() + try: + upstream = await session.put( + _forward_url(request), + data=request.stream(), + ) + body = await upstream.read() + headers = {} + if "ETag" in upstream.headers: + headers["ETag"] = upstream.headers["ETag"] + return Response( + content=body, + status_code=upstream.status, + headers=headers, + ) + finally: + await session.close() + + async def get_forward(rest: str, request: Request) -> Response: + session = aiohttp.ClientSession() + # Close the session on every non-streaming path (connection + # error, non-200); on the streaming path the generator's + # `finally` closes it once the body is fully read or the client + # disconnects. + try: + upstream = await session.get(_forward_url(request)) + except Exception: + await session.close() + raise + + if upstream.status != 200: + try: + body = await upstream.read() + finally: + await session.close() + return Response(content=body, status_code=upstream.status) + + headers = { + name: upstream.headers[name] + for name in _FORWARDED_GET_HEADERS + if name in upstream.headers + } + # Applied again on the way out, not just where the bytes + # are stored: a data plane that is not ours -- S3, say -- + # returns whatever content type it was given at upload, + # and this hop is what puts it on the application's + # origin. + media_type, safety_headers = download_headers( + upstream.headers.get("Content-Type", "") + ) + headers.pop("Content-Type", None) + headers.update(safety_headers) + + async def stream(): + try: + async for chunk in upstream.content.iter_chunked( + _STREAM_CHUNK_SIZE + ): + yield chunk + finally: + await session.close() + + return StreamingResponse( + stream(), + status_code=200, + media_type=media_type, + headers=headers, + ) + + mounted: set[tuple[int, str]] = set() + for forwarded in forwarded_paths: + prefix = forwarded.path_prefix + if not prefix.startswith(FORWARDED_PATH_PREFIX): + raise ValueError( + f"Blob data plane requested forwarding of '{prefix}', " + f"which is outside `{FORWARDED_PATH_PREFIX}`; refusing" + ) + if (forwarded.method, prefix) in mounted: + continue + mounted.add((forwarded.method, prefix)) + # `{rest:path}` matches anything, including `/`s and the empty + # string, so the route covers exactly "path starts with + # `prefix`". + route = prefix + "{rest:path}" + if forwarded.method == HTTP_METHOD_GET: + http.get(route)(get_forward) + elif forwarded.method == HTTP_METHOD_PUT: + http.put(route)(put_forward) + else: + raise ValueError( + "Blob data plane requested forwarding with unsupported " + f"method {HttpMethod.Name(forwarded.method)}" + ) diff --git a/reboot/std/blob/v1/_store.py b/reboot/std/blob/v1/_store.py new file mode 100644 index 000000000..3d56dfe64 --- /dev/null +++ b/reboot/std/blob/v1/_store.py @@ -0,0 +1,388 @@ +"""The filesystem blob store: bytes storage for the open-source blob +data plane. + +A blob's *bytes* live here; all its metadata lives in the `Blob` state +machine (the control plane), which talks to the data plane only over +the `BlobDataPlane` gRPC interface (see `data_plane.proto` — that +interface, not this module, is the contract a data plane implements). +The store mimics S3's multipart-upload semantics (numbered parts, +per-part MD5 ETags, ETag-validating completion) so that clients drive +one protocol regardless of which data plane serves them. +""" + +import asyncio +import base64 +import hashlib +import hmac +import json +import os +import shutil +import time +from dataclasses import dataclass +from reboot.crypto import root_keys +from typing import Optional, Sequence +from uuid import uuid4 + +# The part size clients should use. Every part except the last must be +# exactly this size. Must be at least 5 MiB (the S3 minimum part size, +# mirrored here so that filesystem- and S3-backed data planes are +# interchangeable). +DEFAULT_PART_SIZE_BYTES = 8 * 1024 * 1024 + +# The maximum number of parts in one blob, following S3. +MAX_PARTS = 10000 + +# Default validity of minted upload/download URLs. +DEFAULT_URL_TTL_SECONDS = 15 * 60 + +# The longest this store will sign a URL for. A signed URL is a bearer +# capability that cannot be revoked before it lapses, so the ceiling is +# this store's own policy; other stores set their own, bounded by +# whatever their signing scheme allows. +_MAX_URL_TTL_SECONDS = 7 * 24 * 60 * 60 + +# The URL path prefix under which blob bytes are `PUT` and `GET`: the +# filesystem data-plane server serves it, and the application's proxy +# routes (see `_proxy.py`) forward it. +HTTP_PATH_PREFIX = "/__/reboot/blob" + +# HKDF `info` (domain separator) for the filesystem store's URL-signing +# key. +_SIGNING_INFO = b"reboot.std.blob.url-signing" + + +class BlobStoreError(Exception): + """A permanent storage failure (e.g. a part ETag mismatch at + completion time), reported to the control plane as a + `CompleteUpload` `error` so the client can re-upload. Transient + failures (e.g. network errors) are raised as their original + exception types instead, becoming gRPC errors that the control + plane's workflow retries.""" + + +@dataclass(frozen=True) +class UploadedPart: + """One part of an upload, as reported by the client.""" + number: int + etag: str + size: int + + +def _encode_blob_id(blob_id: str) -> str: + """Encodes a blob ID into a string safe for use as both a directory + name and a URL path segment.""" + return base64.urlsafe_b64encode(blob_id.encode()).decode() + + +def _fsync_path(path: str) -> None: + fd = os.open(path, os.O_RDONLY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +class FilesystemBlobStore: + """Stores blob bytes as part files on the local filesystem, served + over HTTP by the filesystem data-plane server (see `_http.py`). + + Layout, under `directory`: + + {encoded_blob_id}/ + meta.json Content type; part manifest and + composite ETag once committed. + {upload_id}/ + part.{number:08d} One file per uploaded part. + + Parts are written once under a random `upload_id` directory (so no + temp-file-and-rename protocol is needed) and fsynced before the + data plane returns their ETag. The part files remain the committed + object's on-disk representation: downloads stream them in part + order, so completion never rewrites bytes. + """ + + def __init__( + self, + directory: str, + part_size: int = DEFAULT_PART_SIZE_BYTES, + ): + self._directory = directory + self._part_size = part_size + self._locks: dict[str, asyncio.Lock] = {} + os.makedirs(directory, exist_ok=True) + + @property + def directory(self) -> str: + return self._directory + + def lock_for(self, encoded_blob_id: str) -> asyncio.Lock: + """Serializes one blob's completion against the part `PUT`s + that publish its bytes. + + Both run in this one process -- the server hosts the gRPC + service and the byte endpoint together -- so an in-process + lock is enough to make completion see a fixed set of parts. + Held only across a `PUT`'s final rename, not across the upload + itself, so parts still upload concurrently.""" + return self._locks.setdefault(encoded_blob_id, asyncio.Lock()) + + @property + def part_size(self) -> int: + return self._part_size + + def _signing_key(self) -> bytes: + """The URL-signing key, derived from the active version of the + Reboot-managed cryptographic root keys (see + `reboot.crypto.root_keys`) with a blob-specific domain + separator. Rotating the root keys therefore invalidates + outstanding URLs; that is acceptable because URLs are + short-lived and clients can always mint fresh ones.""" + return root_keys.derive_key( + info=_SIGNING_INFO, + version=root_keys.active_version(), + ) + + def _sign(self, *parts: str) -> str: + message = "\n".join(parts).encode() + return hmac.new(self._signing_key(), message, + hashlib.sha256).hexdigest() + + def signature_for_put( + self, + encoded_blob_id: str, + upload_id: str, + part_number: int, + expiration: int, + ) -> str: + return self._sign( + "PUT", encoded_blob_id, upload_id, str(part_number), + str(expiration) + ) + + def signature_for_get( + self, + encoded_blob_id: str, + expiration: int, + ) -> str: + return self._sign("GET", encoded_blob_id, str(expiration)) + + def blob_directory(self, encoded_blob_id: str) -> str: + return os.path.join(self._directory, encoded_blob_id) + + def part_path( + self, + encoded_blob_id: str, + upload_id: str, + part_number: int, + ) -> str: + return os.path.join( + self.blob_directory(encoded_blob_id), + upload_id, + f"part.{part_number:08d}", + ) + + def _meta_path(self, encoded_blob_id: str) -> str: + return os.path.join(self.blob_directory(encoded_blob_id), "meta.json") + + def read_meta(self, encoded_blob_id: str) -> Optional[dict]: + try: + with open(self._meta_path(encoded_blob_id), "r") as f: + return json.load(f) + except FileNotFoundError: + return None + + def _write_meta(self, encoded_blob_id: str, meta: dict) -> None: + # Write to a temp file and atomically rename, so a crash + # mid-write can never leave a torn `meta.json` that a + # concurrent `read_meta` would fail to parse. + path = self._meta_path(encoded_blob_id) + temp_path = f"{path}.{uuid4().hex}.tmp" + with open(temp_path, "w") as f: + json.dump(meta, f) + f.flush() + os.fsync(f.fileno()) + os.replace(temp_path, path) + _fsync_path(os.path.dirname(path)) + + async def begin_upload(self, blob_id: str, content_type: str) -> str: + encoded = _encode_blob_id(blob_id) + + def sync(): + # Idempotent by blob ID: if an uncommitted session already + # exists (a retried `BeginUpload`), reuse it rather than + # orphaning it under a fresh upload ID. + existing = self.read_meta(encoded) + if ( + existing is not None and + not existing.get("committed", False) and + "upload_id" in existing + ): + upload_id = existing["upload_id"] + reuse = True + else: + upload_id = uuid4().hex + reuse = False + # Create the upload directory (and, with it, the blob + # directory) before writing `meta.json` into the latter. + os.makedirs( + os.path.join(self.blob_directory(encoded), upload_id), + exist_ok=True, + ) + if not reuse: + self._write_meta( + encoded, + { + "content_type": content_type, + "committed": False, + "upload_id": upload_id, + }, + ) + return upload_id + + return await asyncio.to_thread(sync) + + def part_put_url( + self, + blob_id: str, + upload_id: str, + part_number: int, + ) -> str: + encoded = _encode_blob_id(blob_id) + expiration = int(time.time()) + DEFAULT_URL_TTL_SECONDS + signature = self.signature_for_put( + encoded, upload_id, part_number, expiration + ) + return ( + f"{HTTP_PATH_PREFIX}/{encoded}/{upload_id}/parts/{part_number}" + f"?exp={expiration}&sig={signature}" + ) + + async def complete( + self, + blob_id: str, + upload_id: str, + content_type: str, + parts: list[UploadedPart], + max_size: Optional[int] = None, + ) -> str: + + def sync(): + encoded = _encode_blob_id(blob_id) + digests = [] + manifest = [] + total_size = 0 + last_part_number = max(part.number for part in parts) + for part in sorted(parts, key=lambda part: part.number): + path = self.part_path(encoded, upload_id, part.number) + digest = hashlib.md5() + size = 0 + try: + with open(path, "rb") as f: + while chunk := f.read(1024 * 1024): + digest.update(chunk) + size += len(chunk) + except FileNotFoundError: + raise BlobStoreError( + f"part {part.number} was never uploaded" + ) + if digest.hexdigest() != part.etag.strip('"'): + raise BlobStoreError( + f"part {part.number} ETag mismatch: the uploaded " + "bytes do not match what was reported via " + "`PartUploaded`" + ) + if size != part.size: + raise BlobStoreError( + f"part {part.number} size mismatch: uploaded " + f"{size} bytes but {part.size} were reported via " + "`PartUploaded`" + ) + if part.number != last_part_number and size != self.part_size: + # S3 rejects a short middle part with + # `EntityTooSmall`; reject it here too, so that an + # upload which cannot commit against the S3 store + # cannot commit against this one either. + raise BlobStoreError( + f"part {part.number} is {size} bytes, but every " + f"part except the last must be exactly " + f"{self.part_size} bytes" + ) + total_size += size + digests.append(digest.digest()) + manifest.append({"number": part.number, "size": size}) + + # Verify the *real* total against `max_size` (not the + # already-checked reported sizes) as defense in depth. + if max_size is not None and total_size > max_size: + raise BlobStoreError( + f"uploaded {total_size} bytes exceeds the maximum of " + f"{max_size}" + ) + + # Composite ETag, S3-style: the MD5 of the concatenated + # part MD5 digests, suffixed with the part count. + etag = ( + hashlib.md5(b"".join(digests)).hexdigest() + f"-{len(digests)}" + ) + self._write_meta( + encoded, + { + "content_type": content_type, + "committed": True, + "etag": etag, + "upload_id": upload_id, + "parts": manifest, + }, + ) + return etag + + # Completion validates the bytes on disk and then records + # the ETag it computed from them. A part `PUT` landing in + # between would leave a committed blob whose bytes no + # longer match its recorded ETag, so hold the blob's lock + # across the whole of it. (S3 gets this for free: a + # concurrent `UploadPart` changes the part's ETag and + # `CompleteMultipartUpload` then fails with + # `InvalidPart`.) + async with self.lock_for(_encode_blob_id(blob_id)): + return await asyncio.to_thread(sync) + + def download_url( + self, + blob_id: str, + ttl_seconds: Optional[int] = None, + ) -> tuple[str, int]: + """Returns a URL to `GET` the blob's bytes from, and how long + that URL is valid for.""" + encoded = _encode_blob_id(blob_id) + ttl = min( + DEFAULT_URL_TTL_SECONDS if ttl_seconds is None else ttl_seconds, + _MAX_URL_TTL_SECONDS, + ) + expiration = int(time.time()) + ttl + signature = self.signature_for_get(encoded, expiration) + url = ( + f"{HTTP_PATH_PREFIX}/{encoded}?exp={expiration}&sig={signature}" + ) + return url, ttl + + async def delete( + self, + blob_id: str, + upload_ids: Sequence[str] = (), + ) -> None: + # `upload_ids` is not needed here: a part lives inside the + # blob's own directory, so removing the directory removes any + # unfinished upload with it. + encoded = _encode_blob_id(blob_id) + + def sync(): + # Only a blob that is already gone is ignored: any + # other failure must reach the caller, or `PerformRemove` + # would report bytes deleted that are still on disk. + try: + shutil.rmtree(self.blob_directory(encoded)) + except FileNotFoundError: + pass + + await asyncio.to_thread(sync) diff --git a/reboot/std/blob/v1/blob.py b/reboot/std/blob/v1/blob.py new file mode 100644 index 000000000..6a7349b7e --- /dev/null +++ b/reboot/std/blob/v1/blob.py @@ -0,0 +1,699 @@ +"""Blob storage: large binary objects whose bytes live outside state. + +A `Blob` is the *control plane* for one immutable-once-committed +binary object: its state holds only metadata (content type, size, +upload progress, lifecycle status) while the bytes live in a *data +plane* — a `BlobDataPlane` gRPC service (see `data_plane.proto`), +discovered via `REBOOT_BLOB_DATA_PLANE_URL` — and travel directly +between the client and that data plane via URLs minted by +`GetPartUploadInstructions` and `GetDownloadUrl`. Uploads are resumable: parts +are idempotent by number, so a client that lost its connection +re-fetches instructions and re-uploads whatever `Info` does not yet +report. + +Authorization model: blob *creation* is application-mediated — only +application code may call `Create`, which is where size and quota +policy belongs (enforced directly or via `size`/`max_size`). The +blob's framework-generated random ID then acts as a capability. +Upload-side calls (`GetPartUploadInstructions`, `PartUploaded`, `Commit`) and +`Remove` are restricted to the `uploader_id` recorded at `Create` — +unless `uploader_id` is left empty, which deliberately allows anyone +who knows the blob's ID to upload (for applications without end-user +authentication). Downloads (`GetDownloadUrl`) are open to anyone who knows +the ID by default, but if `Create` (or a later `SetDownloaders`) +records a `downloader_ids` allow-list only the listed users may download; +an empty list restricts downloads to app-internal callers, and the +uploader is *not* implicitly a downloader. `Info` (metadata and upload +progress, watchable reactively) is visible to anyone who may upload or +download the blob: the `uploader_id` and listed `downloader_ids`, plus +anyone who knows the ID whenever either side is left open. +""" + +import log.log +import rbt.v1alpha1.errors_pb2 +import re +import time +from datetime import timedelta +from grpc.aio import AioRpcError +from rbt.std.blob.v1.blob_rbt import ( + AlreadyCommitted, + BeginUploadRequest, + BeginUploadResponse, + Blob, + BlobPart, + CommitRequest, + CommitResponse, + CompleteUploadRequest, + CompleteUploadResponse, + CreateRequest, + CreateResponse, + ExpireIfNotCommittedRequest, + ExpireIfNotCommittedResponse, + GetDownloadUrlRequest, + GetDownloadUrlResponse, + GetPartUploadInstructionsRequest, + GetPartUploadInstructionsResponse, + IncompleteParts, + InfoRequest, + InfoResponse, + NotCommitted, + PartUploadedRequest, + PartUploadedResponse, + PartUploadInstruction, + PerformRemoveRequest, + PerformRemoveResponse, + RemoveRequest, + RemoveResponse, + SetDownloadersRequest, + SetDownloadersResponse, + SizeMismatch, +) +from rbt.std.blob.v1.data_plane_pb2 import ( + ConfigurationRequest, + ConfigurationResponse, + DataPlaneBeginUploadRequest, + DataPlaneCompleteUploadRequest, + DataPlaneDeleteRequest, + DataPlaneGetDownloadUrlRequest, + DataPlaneGetPartUploadInstructionsRequest, + DataPlaneUploadedPart, +) +from rbt.std.blob.v1.data_plane_pb2_grpc import BlobDataPlaneStub +from reboot.aio.applications import Application, Library +from reboot.aio.auth.authorizers import Authorizer, allow_if, is_app_internal +from reboot.aio.backoff import Backoff +from reboot.aio.contexts import ReaderContext, WorkflowContext, WriterContext +from reboot.aio.http import PythonWebFramework +from reboot.aio.workflows import at_least_once_per_workflow +from reboot.std.blob.v1._data_plane import ( + ENVVAR_BLOB_DATA_PLANE_URL, + proxy_target_from_environment, + stub_from_environment, +) +from reboot.std.blob.v1._proxy import mount_proxy_routes +from reboot.std.blob.v1._store import MAX_PARTS +from typing import Optional + +logger = log.log.get_logger(__name__) + +# How long to keep retrying `Configuration` while the data plane +# comes up, before `pre_run` gives up. The data plane is normally +# already running (spawned by `rbt` or a ready facilitator), so this is +# only a startup-race cushion. +_CONFIGURATION_RETRY_SECONDS = 30 +_CONFIGURATION_MAX_BACKOFF_SECONDS = 2 + +# How long an upload may remain uncommitted before the blob is +# expunged by the `ExpireIfNotCommitted` task `Create` schedules. +DEFAULT_UPLOAD_EXPIRATION = timedelta(hours=24) + +# A part ETag is opaque: `data_plane.proto` defines it as whatever the +# data plane returned, and a store is free to return something that is +# not an MD5 digest. This checks only that it is safe to carry — no +# quotes, no control characters, bounded length — so that a client +# cannot smuggle arbitrary content into a value a data plane later +# relies on to finalize the object. Any narrower format belongs to the +# store that produces it. +_PART_ETAG_PATTERN = re.compile(r'^[!#-~]{1,128}$') + + +def _size_ceiling(state: Blob.State) -> Optional[int]: + """The most bytes this blob may hold, or `None` when unlimited. An + exact `size` is its own ceiling: bytes beyond it could never be + committed, so there is no reason to accept them.""" + if state.HasField("size"): + return state.size + if state.HasField("max_size"): + return state.max_size + return None + + +def _uploader_or_open( + *, + context, + state=None, + request=None, + **kwargs, +) -> Authorizer.Decision: + """Allow app-internal callers and the blob's recorded uploader to + make upload-side calls, or anyone when no uploader was recorded. An + empty `uploader_id` means the blob was created without end-user + authentication, so anyone who knows the blob's ID may upload into + it. This handles the app-internal case itself (rather than + composing `is_app_internal` via `any=[...]`) so that an + unauthenticated non-uploader still surfaces as `Unauthenticated` + rather than `PermissionDenied`.""" + if context.app_internal: + return rbt.v1alpha1.errors_pb2.Ok() + if state is None: + return rbt.v1alpha1.errors_pb2.PermissionDenied() + if state.uploader_id == "": + return rbt.v1alpha1.errors_pb2.Ok() + if context.auth is None or context.auth.user_id is None: + return rbt.v1alpha1.errors_pb2.Unauthenticated() + if context.auth.user_id == state.uploader_id: + return rbt.v1alpha1.errors_pb2.Ok() + return rbt.v1alpha1.errors_pb2.PermissionDenied() + + +def _downloader_or_open( + *, + context, + state=None, + request=None, + **kwargs, +) -> Authorizer.Decision: + """Allow app-internal callers, and restrict `GetDownloadUrl` to the + blob's download allow-list. When no `downloader_ids` list was + recorded (the + field is unset) anyone who knows the blob's ID may download; when + one was recorded only the listed users may (an empty list means no + one but app-internal callers). Like `_uploader_or_open`, this + handles app-internal itself so that an unauthenticated non-listed + caller surfaces as `Unauthenticated` rather than + `PermissionDenied`.""" + if context.app_internal: + return rbt.v1alpha1.errors_pb2.Ok() + if state is None: + return rbt.v1alpha1.errors_pb2.PermissionDenied() + if not state.HasField("downloader_ids"): + return rbt.v1alpha1.errors_pb2.Ok() + if context.auth is None or context.auth.user_id is None: + return rbt.v1alpha1.errors_pb2.Unauthenticated() + if context.auth.user_id in state.downloader_ids.user_ids: + return rbt.v1alpha1.errors_pb2.Ok() + return rbt.v1alpha1.errors_pb2.PermissionDenied() + + +class BlobServicer(Blob.Servicer): + + # The data-plane gRPC stub and the part size it reported, both set + # by `BlobLibrary` once it has connected to the data plane. + _data_plane: BlobDataPlaneStub + _part_size: int + + def authorizer(self) -> Blob.Authorizer: + # Every method is listed explicitly so that none can be + # accidentally left without a rule. + return Blob.Authorizer( + create=allow_if(any=[is_app_internal]), + set_downloaders=allow_if(any=[is_app_internal]), + begin_upload=allow_if(any=[is_app_internal]), + complete_upload=allow_if(any=[is_app_internal]), + perform_remove=allow_if(any=[is_app_internal]), + expire_if_not_committed=allow_if(any=[is_app_internal]), + # Either side may watch a blob: the uploader to follow + # its own progress, a downloader to see when the bytes + # are ready. + info=allow_if(any=[_uploader_or_open, _downloader_or_open]), + get_download_url=allow_if(any=[_downloader_or_open]), + get_part_upload_instructions=allow_if(any=[_uploader_or_open]), + part_uploaded=allow_if(any=[_uploader_or_open]), + commit=allow_if(any=[_uploader_or_open]), + remove=allow_if(any=[_uploader_or_open]), + ) + + async def create( + self, + context: WriterContext, + request: CreateRequest, + ) -> CreateResponse: + self.state.status = Blob.State.UPLOADING + self.state.content_type = request.content_type + self.state.uploader_id = request.uploader_id + if request.HasField("downloader_ids"): + self.state.downloader_ids.CopyFrom(request.downloader_ids) + if request.HasField("size"): + self.state.size = request.size + if request.HasField("max_size"): + self.state.max_size = request.max_size + + # The data-plane side effect (provisioning the upload + # session) happens in the `BeginUpload` workflow, not here. + await self.ref().schedule().begin_upload(context) + + # Expunge this blob if it is never committed. + await self.ref().schedule( + when=DEFAULT_UPLOAD_EXPIRATION, + ).expire_if_not_committed(context) + + return CreateResponse() + + async def set_downloaders( + self, + context: WriterContext, + request: SetDownloadersRequest, + ) -> SetDownloadersResponse: + # Replace semantics: a present `downloader_ids` (even empty) + # restricts downloads to the listed users; an omitted one + # removes any restriction so anyone who knows the ID may + # download again. + if request.HasField("downloader_ids"): + self.state.downloader_ids.CopyFrom(request.downloader_ids) + else: + self.state.ClearField("downloader_ids") + return SetDownloadersResponse() + + @classmethod + async def begin_upload( + cls, + context: WorkflowContext, + request: BeginUploadRequest, + ) -> BeginUploadResponse: + state = await Blob.ref().read(context) + + async def provision() -> str: + response = await cls._data_plane.BeginUpload( + DataPlaneBeginUploadRequest( + blob_id=context.state_id, + content_type=state.content_type, + ) + ) + return response.upload_id + + upload_id = await at_least_once_per_workflow( + "provision upload session", context, provision + ) + + # A `Remove` may have landed between the read above and this + # write -- a workflow spans several transactions, so Reboot + # serializes each of them but holds nothing across the whole + # method. Recording an upload session on a removed blob would + # strand the data plane's directory forever, so drop the + # session instead. Deletion always wins, as in `CompleteUpload`. + removed = False + + async def record(state: Blob.State) -> None: + nonlocal removed + if state.status in ( + Blob.State.REMOVING, + Blob.State.REMOVED, + ): + removed = True + return + state.upload_id = upload_id + + await Blob.ref().write(context, record) + + if removed: + await cls._data_plane.Delete( + DataPlaneDeleteRequest( + blob_id=context.state_id, + upload_ids=[upload_id], + ) + ) + + return BeginUploadResponse() + + async def get_part_upload_instructions( + self, + context: ReaderContext, + request: GetPartUploadInstructionsRequest, + ) -> GetPartUploadInstructionsResponse: + if self.state.status != Blob.State.UPLOADING: + raise Blob.GetPartUploadInstructionsAborted(AlreadyCommitted()) + + if not self.state.HasField("upload_id"): + return GetPartUploadInstructionsResponse( + ready=False, + part_size=self._part_size, + ) + + # A minted URL is self-authorizing: whoever holds it can `PUT` a + # full part into the data plane and never report it, so the + # numbers handed out here are the only place a declared + # `size`/`max_size` can restrict how many bytes an upload + # session may occupy: past that, `PartUploaded` and `Commit` + # only ever see the sizes a client chose to report. + ceiling = _size_ceiling(self.state) + if ceiling is None: + max_part_number = MAX_PARTS + else: + # Round up, since a ceiling that doesn't fill a whole part + # still needs a part to carry it. + max_part_number = min( + MAX_PARTS, + max(1, (ceiling + self._part_size - 1) // self._part_size), + ) + + part_numbers = [ + number for number in request.part_numbers + if 1 <= number <= max_part_number + ] + response = await self._data_plane.GetPartUploadInstructions( + DataPlaneGetPartUploadInstructionsRequest( + blob_id=context.state_id, + upload_id=self.state.upload_id, + part_numbers=part_numbers, + ) + ) + instructions = [ + PartUploadInstruction( + part_number=instruction.part_number, + url=instruction.url, + ) for instruction in response.instructions + ] + + return GetPartUploadInstructionsResponse( + ready=True, + part_size=self._part_size, + instructions=instructions, + ) + + async def part_uploaded( + self, + context: WriterContext, + request: PartUploadedRequest, + ) -> PartUploadedResponse: + if self.state.status != Blob.State.UPLOADING: + raise Blob.PartUploadedAborted(AlreadyCommitted()) + + # Reject out-of-range part numbers: a bogus record (e.g. the + # proto default `0` from an omitted field) can only be + # overwritten, never removed, so it would make `Commit`'s + # contiguity check fail forever. + if request.part_number < 1 or request.part_number > MAX_PARTS: + raise Blob.PartUploadedAborted(IncompleteParts()) + + # Validate the ETag as an MD5 hex digest, as the data-plane + # contract requires, so a client can't smuggle arbitrary + # content into the value a data plane later relies on to + # finalize the object. + if not _PART_ETAG_PATTERN.match(request.etag): + raise Blob.PartUploadedAborted(IncompleteParts()) + + part = BlobPart( + number=request.part_number, + etag=request.etag, + size=request.size, + ) + + # Safe to call multiple times for the same part number: a + # re-uploaded part overwrites its previous record. + parts = [p for p in self.state.parts if p.number != part.number] + parts.append(part) + parts.sort(key=lambda part: part.number) + + total = sum(part.size for part in parts) + ceiling = _size_ceiling(self.state) + if ceiling is not None and total > ceiling: + raise Blob.PartUploadedAborted(SizeMismatch(bytes_uploaded=total)) + + del self.state.parts[:] + self.state.parts.extend(parts) + return PartUploadedResponse() + + async def commit( + self, + context: WriterContext, + request: CommitRequest, + ) -> CommitResponse: + if self.state.status == Blob.State.COMMITTING: + # Idempotent: the `CompleteUpload` workflow is already + # scheduled. + return CommitResponse() + if self.state.status != Blob.State.UPLOADING: + raise Blob.CommitAborted(AlreadyCommitted()) + + numbers = [part.number for part in self.state.parts] + if not numbers or numbers != list(range(1, len(numbers) + 1)): + raise Blob.CommitAborted(IncompleteParts()) + + total = sum(part.size for part in self.state.parts) + if self.state.HasField("size") and total != self.state.size: + raise Blob.CommitAborted(SizeMismatch(bytes_uploaded=total)) + ceiling = _size_ceiling(self.state) + if ceiling is not None and total > ceiling: + raise Blob.CommitAborted(SizeMismatch(bytes_uploaded=total)) + + self.state.status = Blob.State.COMMITTING + # Clear any error from a previous failed commit attempt, so a + # client watching `Info` doesn't observe the stale error while + # this fresh attempt is in flight. + self.state.ClearField("commit_error") + await self.ref().schedule().complete_upload(context) + return CommitResponse() + + @classmethod + async def complete_upload( + cls, + context: WorkflowContext, + request: CompleteUploadRequest, + ) -> CompleteUploadResponse: + state = await Blob.ref().read(context) + + # A concurrent `Remove` may have moved the blob out of + # COMMITTING (deletion always wins); if so, don't finalize. + if state.status != Blob.State.COMMITTING: + return CompleteUploadResponse() + + complete_request = DataPlaneCompleteUploadRequest( + blob_id=context.state_id, + upload_id=state.upload_id, + content_type=state.content_type, + parts=[ + DataPlaneUploadedPart( + number=part.number, etag=part.etag, size=part.size + ) for part in state.parts + ], + ) + ceiling = _size_ceiling(state) + if ceiling is not None: + complete_request.max_size = ceiling + + async def attempt() -> tuple: + # A response `error` is a *permanent* failure (e.g. an ETag + # mismatch): report it back onto the blob so the client can + # re-upload and re-commit. A gRPC error is transient and + # propagates, so the workflow retries. + response = await cls._data_plane.CompleteUpload(complete_request) + if response.HasField("error"): + return ("failed", response.error) + return ("committed", response.etag) + + outcome, detail = await at_least_once_per_workflow( + "complete upload", context, attempt + ) + + # Only transition if the blob is still COMMITTING: a + # concurrent `Remove` may have moved it to REMOVING/REMOVED, + # which must win (otherwise we'd resurrect a deleted blob or + # mark a bytes-less blob COMMITTED). + superseded = [False] + + async def record(state: Blob.State) -> None: + if state.status != Blob.State.COMMITTING: + superseded[0] = True + return + if outcome == "committed": + state.status = Blob.State.COMMITTED + state.etag = detail + state.ClearField("commit_error") + else: + state.status = Blob.State.UPLOADING + state.commit_error = detail + + await Blob.ref().write(context, record) + + # If a delete raced ahead of a successful completion, the + # bytes we just finalized are now orphaned; clean them up. + if superseded[0] and outcome == "committed": + + async def cleanup() -> None: + await cls._data_plane.Delete( + DataPlaneDeleteRequest(blob_id=context.state_id) + ) + + await at_least_once_per_workflow( + "cleanup orphaned bytes", context, cleanup + ) + + return CompleteUploadResponse() + + async def info( + self, + context: ReaderContext, + request: InfoRequest, + ) -> InfoResponse: + response = InfoResponse( + status=self.state.status, + content_type=self.state.content_type, + uploader_id=self.state.uploader_id, + bytes_uploaded=sum(part.size for part in self.state.parts), + parts=self.state.parts, + ) + if self.state.HasField("size"): + response.size = self.state.size + if self.state.HasField("max_size"): + response.max_size = self.state.max_size + if self.state.HasField("etag"): + response.etag = self.state.etag + if self.state.HasField("commit_error"): + response.commit_error = self.state.commit_error + return response + + async def get_download_url( + self, + context: ReaderContext, + request: GetDownloadUrlRequest, + ) -> GetDownloadUrlResponse: + if self.state.status != Blob.State.COMMITTED: + raise Blob.GetDownloadUrlAborted(NotCommitted()) + download_request = DataPlaneGetDownloadUrlRequest( + blob_id=context.state_id, + ) + if request.HasField("ttl_seconds"): + download_request.ttl_seconds = request.ttl_seconds + response = await self._data_plane.GetDownloadUrl(download_request) + return GetDownloadUrlResponse( + url=response.url, + ttl_seconds=response.ttl_seconds, + ) + + async def remove( + self, + context: WriterContext, + request: RemoveRequest, + ) -> RemoveResponse: + if self.state.status in ( + Blob.State.REMOVING, + Blob.State.REMOVED, + ): + return RemoveResponse() + self.state.status = Blob.State.REMOVING + await self.ref().schedule().perform_remove(context) + return RemoveResponse() + + @classmethod + async def perform_remove( + cls, + context: WorkflowContext, + request: PerformRemoveRequest, + ) -> PerformRemoveResponse: + + # An upload that never completed has parked bytes that + # deleting the object does not reach, and the ID naming that + # session is only known here. + state = await Blob.ref().read(context) + upload_ids = [state.upload_id] if state.HasField("upload_id") else [] + + async def remove() -> None: + await cls._data_plane.Delete( + DataPlaneDeleteRequest( + blob_id=context.state_id, + upload_ids=upload_ids, + ) + ) + + await at_least_once_per_workflow("remove bytes", context, remove) + + async def record(state: Blob.State) -> None: + state.status = Blob.State.REMOVED + del state.parts[:] + + await Blob.ref().write(context, record) + return PerformRemoveResponse() + + async def expire_if_not_committed( + self, + context: WriterContext, + request: ExpireIfNotCommittedRequest, + ) -> ExpireIfNotCommittedResponse: + if self.state.status == Blob.State.UPLOADING: + self.state.status = Blob.State.REMOVING + await self.ref().schedule().perform_remove(context) + elif self.state.status == Blob.State.COMMITTING: + # A commit is in flight. If it fails it will revert to + # UPLOADING and could then be abandoned, so re-arm the + # expiration check rather than dropping it. + await self.ref().schedule( + when=DEFAULT_UPLOAD_EXPIRATION, + ).expire_if_not_committed(context) + return ExpireIfNotCommittedResponse() + + +BLOBS_LIBRARY_NAME = "reboot.std.blob.v1.blob" + + +class BlobLibrary(Library): + name = BLOBS_LIBRARY_NAME + + def __init__(self) -> None: + self._connected = False + + def servicers(self) -> list[type[Blob.Servicer]]: + return [BlobServicer] + + async def pre_run(self, application: Application) -> None: + # `pre_run` may be called more than once (e.g. a test that + # `up`s an application after a `down`); connect once. + if self._connected: + return + + stub = stub_from_environment() + BlobServicer._data_plane = stub + + configuration = await self._configuration(stub) + if configuration.part_size == 0: + # Fail here rather than let a zero reach the browser + # uploader, which divides the blob's size by it. + raise RuntimeError( + "The blob data plane reported a part size of zero; " + f"check the service at `{ENVVAR_BLOB_DATA_PLANE_URL}`" + ) + BlobServicer._part_size = configuration.part_size + + # A data plane whose URLs are not directly reachable (e.g. the + # localhost filesystem server) asks for paths to be forwarded + # to it, and the application proxies those to its HTTP + # endpoint; one that serves its own URLs (e.g. S3) asks for + # none and needs no application routes. + if configuration.forwarded_paths: + if not isinstance(application.web_framework, PythonWebFramework): + # Better to fail fast here than to hand out URLs that + # will 404: without the proxy routes, a forwarded-path + # data plane cannot serve any bytes. + raise RuntimeError( + "This blob data plane needs paths forwarded to it, " + "which only Python applications currently support; " + "configure a data plane whose URLs are directly " + "reachable by clients via " + f"`{ENVVAR_BLOB_DATA_PLANE_URL}`." + ) + mount_proxy_routes( + application.http, + proxy_target_from_environment(configuration.http_port), + configuration.forwarded_paths, + ) + + self._connected = True + + async def _configuration( + self, + stub: BlobDataPlaneStub, + ) -> ConfigurationResponse: + # The data plane is normally already running, but tolerate a + # startup race by retrying while it becomes reachable. + backoff = Backoff( + max_backoff_seconds=_CONFIGURATION_MAX_BACKOFF_SECONDS, + ) + deadline = time.monotonic() + _CONFIGURATION_RETRY_SECONDS + while True: + try: + return await stub.Configuration(ConfigurationRequest()) + except AioRpcError as error: + if time.monotonic() >= deadline: + raise RuntimeError( + "Timed out waiting for the blob data plane to " + "become reachable via " + f"`{ENVVAR_BLOB_DATA_PLANE_URL}`." + ) from error + await backoff() + + +def servicers() -> list[type[Blob.Servicer]]: + return [BlobServicer] + + +def blob_library() -> BlobLibrary: + return BlobLibrary() diff --git a/reboot/std/blob/v1/index.ts b/reboot/std/blob/v1/index.ts new file mode 100644 index 000000000..1e0e369a4 --- /dev/null +++ b/reboot/std/blob/v1/index.ts @@ -0,0 +1,30 @@ +import { NativeLibrary, NativeServicer } from "@reboot-dev/reboot"; + +export * from "@reboot-dev/reboot-std-api/blob/v1/blob_rbt.js"; + +// The servicers are implemented in Python (the data-plane client +// lives there); Node.js applications host them as "native" servicers. +// +// NOTE: the application-side routes that proxy bytes to a data plane +// requesting forwarded paths (such as the local filesystem one) are +// currently only registered by Python applications; Node.js +// applications need a data plane whose URLs are directly reachable by +// clients (no forwarded paths, e.g. Reboot Cloud's). +export default { + servicers: (): NativeServicer[] => { + return [ + { + nativeServicerModule: "reboot.std.blob.v1.blob", + }, + ]; + }, +}; + +export const BLOBS_LIBRARY_NAME = "reboot.std.blob.v1.blob"; + +export function blobLibrary(): NativeLibrary { + return { + nativeLibraryModule: "reboot.std.blob.v1.blob", + nativeLibraryFunction: "blob_library", + }; +} diff --git a/reboot/std/blob/v1/package.json b/reboot/std/blob/v1/package.json new file mode 100644 index 000000000..3dbc1ca59 --- /dev/null +++ b/reboot/std/blob/v1/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/tests/reboot/std/blob/v1/BUILD.bazel b/tests/reboot/std/blob/v1/BUILD.bazel new file mode 100644 index 000000000..06c6c4a10 --- /dev/null +++ b/tests/reboot/std/blob/v1/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rbt_pypi//:requirements.bzl", "requirement") +load("@rules_python//python:defs.bzl", "py_test") + +py_test( + name = "blob_tests_py", + size = "medium", + srcs = [":blob_tests.py"], + main = "blob_tests.py", + deps = [ + "//reboot/aio:applications_py", + "//reboot/aio:tests_py", + "//reboot/std/blob/v1:blob_py", + "@com_github_reboot_dev_reboot//rbt/std/blob/v1:blob_py_reboot", + requirement("aiohttp"), + ], +) diff --git a/tests/reboot/std/blob/v1/blob_tests.py b/tests/reboot/std/blob/v1/blob_tests.py new file mode 100644 index 000000000..bbc2ec119 --- /dev/null +++ b/tests/reboot/std/blob/v1/blob_tests.py @@ -0,0 +1,785 @@ +import aiohttp +import asyncio +import hashlib +import tempfile +import threading +import unittest +from rbt.std.blob.v1.blob_rbt import ( + Blob, + Downloaders, + IncompleteParts, + InfoResponse, + NotCommitted, + SizeMismatch, +) +from reboot.aio.applications import Application +from reboot.aio.tests import Reboot +from reboot.std.blob.v1._store import ( + DEFAULT_PART_SIZE_BYTES, + BlobStoreError, + FilesystemBlobStore, + UploadedPart, + _encode_blob_id, +) +from reboot.std.blob.v1.blob import blob_library +from unittest import mock + +# How long the completion/`PUT` handshake waits before giving up, +# generous because it only ever elapses when the test is already +# failing. +_RACE_TIMEOUT_SECONDS = 10 + + +class TestBlobs(unittest.IsolatedAsyncioTestCase): + """Exercises the `Blob` control plane against the filesystem data + plane that the `Reboot` test harness runs, through the same + reverse-proxied byte routes an application serves under + `rbt dev run`.""" + + async def asyncSetUp(self) -> None: + self.rbt = Reboot() + await self.rbt.start() + + await self.rbt.up( + Application(libraries=[blob_library()]), + local_envoy=True, + ) + + self.context = self.rbt.create_external_context( + name=f"test-{self.id()}", + app_internal=True, + ) + self.external_context = self.rbt.create_external_context( + name=f"test-external-{self.id()}", + ) + + async def asyncTearDown(self) -> None: + await self.rbt.stop() + + async def _instructions(self, blob, part_numbers: list[int]): + """Fetches upload instructions, waiting for the `BeginUpload` + workflow to have provisioned the upload session.""" + while True: + response = await blob.get_part_upload_instructions( + self.context, + part_numbers=part_numbers, + ) + if response.ready: + return response + await asyncio.sleep(0.05) + + async def _part_numbers(self, blob, part_numbers: list[int]) -> list[int]: + """The part numbers that upload instructions were minted for, + of those asked for.""" + response = await self._instructions(blob, part_numbers) + return [ + instruction.part_number for instruction in response.instructions + ] + + async def _put(self, url: str, data: bytes) -> str: + """`PUT`s bytes to a (possibly relative) data-plane URL and + returns the response's ETag.""" + async with aiohttp.ClientSession(self.rbt.url()) as session: + async with session.put(url, data=data) as response: + if response.status != 200: + self.fail( + f"PUT failed ({response.status}): " + f"{await response.text()}" + ) + return response.headers["ETag"].strip('"') + + async def _put_returning_status(self, url: str, data: bytes) -> int: + """`PUT`s bytes and returns the status, for the cases where a + refusal is the expected outcome.""" + async with aiohttp.ClientSession(self.rbt.url()) as session: + async with session.put(url, data=data) as response: + return response.status + + async def _upload(self, blob, data: bytes) -> None: + """Uploads `data` in data-plane-sized parts and reports each + part, exactly as the browser SDK does.""" + part_size = (await self._instructions(blob, [])).part_size + parts = [ + data[offset:offset + part_size] + for offset in range(0, len(data), part_size) + ] or [b""] + instructions = await self._instructions( + blob, list(range(1, + len(parts) + 1)) + ) + for instruction, part in zip(instructions.instructions, parts): + etag = await self._put(instruction.url, part) + await blob.part_uploaded( + self.context, + part_number=instruction.part_number, + etag=etag, + size=len(part), + ) + + async def _wait_until_status(self, blob, statuses) -> InfoResponse: + while True: + info = await blob.info(self.context) + if info.status in statuses: + return info + await asyncio.sleep(0.05) + + async def _download(self, blob) -> tuple[bytes, str]: + """Downloads the blob's bytes via its download URL, returning + the bytes and the response's content type.""" + url = (await blob.get_download_url(self.context)).url + async with aiohttp.ClientSession(self.rbt.url()) as session: + async with session.get(url) as response: + if response.status != 200: + self.fail( + f"GET failed ({response.status}): " + f"{await response.text()}" + ) + return await response.read(), response.content_type + + async def test_upload_and_download(self) -> None: + # 2.5 data-plane parts, so the upload is genuinely multipart. + data = bytes(range(256)) * (DEFAULT_PART_SIZE_BYTES * 5 // 2 // 256) + + blob, _ = await Blob.create( + self.context, + content_type="application/octet-stream", + size=len(data), + ) + + await self._upload(blob, data) + + info = await blob.info(self.context) + self.assertEqual(info.bytes_uploaded, len(data)) + self.assertEqual(info.status, Blob.State.UPLOADING) + + await blob.commit(self.context) + + info = await self._wait_until_status(blob, {Blob.State.COMMITTED}) + self.assertTrue(info.etag.endswith("-3")) + + downloaded, content_type = await self._download(blob) + self.assertEqual(downloaded, data) + self.assertEqual(content_type, "application/octet-stream") + + async def test_commit_validates_etags(self) -> None: + data = b"hello, blobs!" + + blob, _ = await Blob.create( + self.context, + content_type="text/plain", + ) + + instructions = await self._instructions(blob, [1]) + await self._put(instructions.instructions[0].url, data) + + # Report a bogus ETag; the commit must fail and revert the + # blob to `UPLOADING` with `commit_error` set. + await blob.part_uploaded( + self.context, + part_number=1, + etag=hashlib.md5(b"not the data").hexdigest(), + size=len(data), + ) + await blob.commit(self.context) + + info = await self._wait_until_status(blob, {Blob.State.UPLOADING}) + self.assertIn("ETag mismatch", info.commit_error) + + # Re-report the correct ETag and commit again; now it must + # succeed. + await blob.part_uploaded( + self.context, + part_number=1, + etag=hashlib.md5(data).hexdigest(), + size=len(data), + ) + await blob.commit(self.context) + info = await self._wait_until_status(blob, {Blob.State.COMMITTED}) + self.assertFalse(info.HasField("commit_error")) + + downloaded, _ = await self._download(blob) + self.assertEqual(downloaded, data) + + async def test_size_validation(self) -> None: + blob, _ = await Blob.create( + self.context, + content_type="text/plain", + max_size=10, + ) + + with self.assertRaises(Blob.PartUploadedAborted) as raised: + await blob.part_uploaded( + self.context, + part_number=1, + etag="0" * 32, + size=100, + ) + self.assertIsInstance(raised.exception.error, SizeMismatch) + + # Declared `size` must match the sum of the parts at commit + # time. + sized_blob, _ = await Blob.create( + self.context, + content_type="text/plain", + size=5, + ) + await sized_blob.part_uploaded( + self.context, + part_number=1, + etag="0" * 32, + size=3, + ) + + # A declared `size` is its own ceiling: a part that overshoots + # it is rejected as it is reported, rather than being stored + # only to be refused at commit time. + with self.assertRaises(Blob.PartUploadedAborted) as raised: + await sized_blob.part_uploaded( + self.context, + part_number=2, + etag="0" * 32, + size=100, + ) + self.assertIsInstance(raised.exception.error, SizeMismatch) + + with self.assertRaises(Blob.CommitAborted) as commit_raised: + await sized_blob.commit(self.context) + self.assertIsInstance(commit_raised.exception.error, SizeMismatch) + + async def test_part_urls_are_bounded_by_the_size_ceiling(self) -> None: + # A part URL is self-authorizing, so a blob that declares how + # big it may become must not be handed URLs for parts beyond + # that: whoever holds them could fill the data plane with + # bytes the blob could never commit. + bounded, _ = await Blob.create( + self.context, + content_type="text/plain", + max_size=10, + ) + part_size = (await self._instructions(bounded, [])).part_size + self.assertEqual(await self._part_numbers(bounded, [1, 2, 3]), [1]) + + # A ceiling spanning several parts hands out exactly the parts + # it spans, rounding up for the remainder. + spanning, _ = await Blob.create( + self.context, + content_type="text/plain", + max_size=2 * part_size + 1, + ) + self.assertEqual( + await self._part_numbers(spanning, [1, 2, 3, 4]), + [1, 2, 3], + ) + + # An exact `size` is its own ceiling. + sized, _ = await Blob.create( + self.context, + content_type="text/plain", + size=part_size + 1, + ) + self.assertEqual(await self._part_numbers(sized, [1, 2, 3]), [1, 2]) + + # Declaring no ceiling at all leaves the app's own policy in + # charge, so every part the data plane supports is available. + unbounded, _ = await Blob.create( + self.context, + content_type="text/plain", + ) + self.assertEqual( + await self._part_numbers(unbounded, [1, 2, 3]), + [1, 2, 3], + ) + + async def test_commit_requires_contiguous_parts(self) -> None: + blob, _ = await Blob.create( + self.context, + content_type="text/plain", + ) + await blob.part_uploaded( + self.context, + part_number=2, + etag="0" * 32, + size=1, + ) + with self.assertRaises(Blob.CommitAborted) as raised: + await blob.commit(self.context) + self.assertIsInstance(raised.exception.error, IncompleteParts) + + async def test_get_download_url_requires_committed(self) -> None: + blob, _ = await Blob.create( + self.context, + content_type="text/plain", + ) + with self.assertRaises(Blob.GetDownloadUrlAborted) as raised: + await blob.get_download_url(self.context) + self.assertIsInstance(raised.exception.error, NotCommitted) + + async def test_store_complete_enforces_max_size(self) -> None: + # `complete()` verifies the *real* total size against + # `max_size` independently of the control plane's own + # (reported-size) check, as defense in depth against untruthful + # reported sizes. A store-level test, on its own store. + with tempfile.TemporaryDirectory() as directory: + store = FilesystemBlobStore(directory) + blob_id = "max-size-blob" + upload_id = await store.begin_upload(blob_id, "text/plain") + encoded = _encode_blob_id(blob_id) + data = b"x" * 100 + with open(store.part_path(encoded, upload_id, 1), "wb") as f: + f.write(data) + part = UploadedPart( + number=1, + etag=hashlib.md5(data).hexdigest(), + size=len(data), + ) + with self.assertRaises(BlobStoreError): + await store.complete( + blob_id, + upload_id, + "text/plain", + [part], + max_size=50, + ) + # Within the bound, it succeeds. + etag = await store.complete( + blob_id, + upload_id, + "text/plain", + [part], + max_size=1000, + ) + self.assertTrue(etag.endswith("-1")) + + async def test_a_part_put_cannot_land_inside_completion( + self, + ) -> None: + # The race the blob's lock exists to close, driven to the + # exact interleaving rather than raced for: completion is + # paused after it has read the parts and computed their ETag + # but before it records either, and a part `PUT` on a + # still-valid signed URL is issued into that window. + # + # `_write_meta` runs on a worker thread (completion's body + # goes through `asyncio.to_thread`), so the handshake is + # `threading.Event`, not `asyncio.Event`: the thread cannot + # await one, and setting one from off the loop is not safe. + data = b"original bytes" + replacement = b"REPLACED bytes" + self.assertEqual(len(data), len(replacement)) + + blob, _ = await Blob.create( + self.context, + content_type="text/plain", + size=len(data), + ) + instructions = await self._instructions(blob, [1]) + url = instructions.instructions[0].url + etag = await self._put(url, data) + await blob.part_uploaded( + self.context, + part_number=1, + etag=etag, + size=len(data), + ) + + reached_recording = threading.Event() + may_record = threading.Event() + write_meta = FilesystemBlobStore._write_meta + + def paused_write_meta(self, encoded_blob_id: str, meta) -> None: + # Only completion records a committed blob; `begin_upload` + # writes metadata too, and must not be paused. + if meta.get("committed", False): + reached_recording.set() + may_record.wait(timeout=_RACE_TIMEOUT_SECONDS) + write_meta(self, encoded_blob_id, meta) + + with mock.patch.object( + FilesystemBlobStore, + "_write_meta", + paused_write_meta, + ): + await blob.commit(self.context) + self.assertTrue( + await asyncio.to_thread( + reached_recording.wait, + _RACE_TIMEOUT_SECONDS, + ), + "completion never reached the point where it records " + "what it read", + ) + + # Deliberately not awaited yet: while completion holds the + # blob's lock this `PUT` cannot finish, so awaiting it + # before releasing completion would deadlock the test + # rather than test anything. + overwrite = asyncio.ensure_future( + self._put_returning_status(url, replacement) + ) + # Long enough for the `PUT` to reach the lock and block on + # it -- or, unlocked, to publish its bytes and return. + await asyncio.sleep(0.5) + may_record.set() + + # Bounded: a `PUT` that did land inside completion leaves the + # bytes disagreeing with the ETag completion computed, the + # commit fails its own digest check, and the blob never + # commits. Without a bound that is a test that hangs instead + # of a test that reports what broke. + try: + await asyncio.wait_for( + self._wait_until_status(blob, {Blob.State.COMMITTED}), + timeout=_RACE_TIMEOUT_SECONDS, + ) + except asyncio.TimeoutError: + self.fail( + "the blob never committed: a part `PUT` published " + "bytes inside completion, so the ETag completion " + "computed no longer described them" + ) + status = await overwrite + + # Serialized behind completion, the `PUT` finds the blob + # committed and refuses rather than publishing. + self.assertEqual(409, status) + + # The bytes served are the bytes completion actually read, so + # they still match the ETag and length it recorded. + downloaded, _ = await self._download(blob) + self.assertEqual(data, downloaded) + + async def test_html_blob_is_not_served_as_html(self) -> None: + # A blob's content type is whatever its uploader claimed, + # and the bytes come back on the application's own origin. + # Served as `text/html` they would be a same-origin + # document with the reader's session, so they are served as + # an opaque download instead. `nosniff` does not cover this: + # nothing needs sniffing when the declared type is already + # the dangerous one. + data = b"" + + blob, _ = await Blob.create( + self.context, + content_type="text/html", + size=len(data), + ) + await self._upload(blob, data) + await blob.commit(self.context) + await self._wait_until_status(blob, {Blob.State.COMMITTED}) + + url = (await blob.get_download_url(self.context)).url + async with aiohttp.ClientSession(self.rbt.url()) as session: + async with session.get(url) as response: + self.assertEqual(200, response.status) + body = await response.read() + self.assertEqual( + "application/octet-stream", + response.content_type, + ) + self.assertEqual( + "attachment", + response.headers.get("Content-Disposition"), + ) + self.assertEqual( + "nosniff", + response.headers.get("X-Content-Type-Options"), + ) + + # The bytes themselves are untouched; only how they are + # labelled changes. + self.assertEqual(data, body) + + async def test_renderable_blob_keeps_its_content_type(self) -> None: + # The coercion is narrow: a type that cannot carry script + # still renders inline, or every image attachment would + # download instead of showing. + data = b"\x89PNG\r\n\x1a\n" + + blob, _ = await Blob.create( + self.context, + content_type="image/png", + size=len(data), + ) + await self._upload(blob, data) + await blob.commit(self.context) + await self._wait_until_status(blob, {Blob.State.COMMITTED}) + + url = (await blob.get_download_url(self.context)).url + async with aiohttp.ClientSession(self.rbt.url()) as session: + async with session.get(url) as response: + self.assertEqual("image/png", response.content_type) + self.assertIsNone(response.headers.get("Content-Disposition")) + + async def test_malformed_signed_url_params_are_refused( + self, + ) -> None: + # Both parameters of a signed URL are attacker-chosen, and + # both are read before anything has been verified, so neither + # may be able to raise: a non-ASCII `sig` is rejected by + # `hmac.compare_digest`, and an `exp` like superscript two + # satisfies `str.isdigit()` but not `int()`. Either one + # unhandled turns an unauthenticated request into a 500. + data = b"signed url bytes" + blob, _ = await Blob.create( + self.context, + content_type="text/plain", + size=len(data), + ) + await self._upload(blob, data) + await blob.commit(self.context) + await self._wait_until_status(blob, {Blob.State.COMMITTED}) + url = (await blob.get_download_url(self.context)).url + path = url.split("?")[0] + + for query in ( + # Non-ASCII signature. + "exp=99999999999&sig=%C3%A9", + # `isdigit()` accepts this; `int()` does not. + "exp=%C2%B2&sig=whatever", + # All ASCII digits, but longer than `int()` will parse: + # it refuses beyond `sys.get_int_max_str_digits()`. + f"exp={'9' * 4301}&sig=whatever", + # Nothing at all. + "", + ): + async with aiohttp.ClientSession(self.rbt.url()) as session: + async with session.get(f"{path}?{query}") as response: + self.assertLess( + response.status, + 500, + f"query {query!r} produced a server error", + ) + self.assertNotEqual(200, response.status) + + async def test_content_type_parameters_are_not_served_back( + self, + ) -> None: + # A declared content type is matched against the allow-list + # by its type alone, so whatever follows the first `;` is + # never inspected -- and it would otherwise go into a + # response header verbatim. Starlette does not validate + # header values, so that is a header the uploader writes. + data = b"\x89PNG\r\n\x1a\n" + + blob, _ = await Blob.create( + self.context, + content_type="image/png; charset=utf-7", + size=len(data), + ) + await self._upload(blob, data) + await blob.commit(self.context) + await self._wait_until_status(blob, {Blob.State.COMMITTED}) + + url = (await blob.get_download_url(self.context)).url + async with aiohttp.ClientSession(self.rbt.url()) as session: + async with session.get(url) as response: + header = response.headers["Content-Type"] + self.assertEqual("image/png", header) + self.assertNotIn("utf-7", header) + + async def test_commit_rejects_zero_parts(self) -> None: + blob, _ = await Blob.create( + self.context, + content_type="text/plain", + ) + with self.assertRaises(Blob.CommitAborted) as raised: + await blob.commit(self.context) + self.assertIsInstance(raised.exception.error, IncompleteParts) + + async def test_part_uploaded_rejects_bad_input(self) -> None: + blob, _ = await Blob.create( + self.context, + content_type="text/plain", + ) + # Part number 0 (e.g. an omitted proto field) is rejected. + with self.assertRaises(Blob.PartUploadedAborted): + await blob.part_uploaded( + self.context, + part_number=0, + etag="0" * 32, + size=1, + ) + # A non-MD5-hex ETag is rejected (it would corrupt the S3 + # completion XML). + with self.assertRaises(Blob.PartUploadedAborted): + await blob.part_uploaded( + self.context, + part_number=1, + etag='">', + size=1, + ) + + async def test_uploader_gating(self) -> None: + # A blob with an uploader: external callers that are not the + # uploader may read `Info` but not upload. (This application + # has no token verifier, so the external context has no user + # at all.) + blob, _ = await Blob.create( + self.context, + content_type="text/plain", + uploader_id="alice", + ) + + info = await Blob.ref(blob.state_id).info(self.external_context) + self.assertEqual(info.uploader_id, "alice") + + # A non-uploader (here: an unauthenticated external caller) is + # denied upload-side calls. Authorization denials surface as + # the method's `Aborted` type. + with self.assertRaises(Blob.PartUploadedAborted): + await Blob.ref(blob.state_id).part_uploaded( + self.external_context, + part_number=1, + etag="0" * 32, + size=1, + ) + + # A blob without an uploader: anyone who knows the id may + # upload. Use a fresh external context: after the denied + # mutation above, the previous context considers the outcome + # of its last mutation uncertain and refuses further + # non-idempotent mutations. + open_blob, _ = await Blob.create( + self.context, + content_type="text/plain", + ) + open_context = self.rbt.create_external_context( + name=f"test-open-{self.id()}", + ) + await Blob.ref(open_blob.state_id).part_uploaded( + open_context, + part_number=1, + etag="0" * 32, + size=1, + ) + + async def test_download_gating(self) -> None: + data = b"secret bytes" + + # A blob with a download allow-list: only listed users may get + # a download URL. This application has no token verifier, so + # the external context has no user and is not on any list. + blob, _ = await Blob.create( + self.context, + content_type="text/plain", + downloader_ids=Downloaders(user_ids=["bob"]), + ) + await self._upload(blob, data) + await blob.commit(self.context) + await self._wait_until_status(blob, {Blob.State.COMMITTED}) + + # The app-internal caller may always download. + downloaded, _ = await self._download(blob) + self.assertEqual(downloaded, data) + + # A caller not on the allow-list is denied. + with self.assertRaises(Blob.GetDownloadUrlAborted): + await Blob.ref(blob.state_id + ).get_download_url(self.external_context) + + # A blob with no allow-list: anyone who knows the id may get a + # download URL. + open_blob, _ = await Blob.create( + self.context, + content_type="text/plain", + ) + await self._upload(open_blob, data) + await open_blob.commit(self.context) + await self._wait_until_status(open_blob, {Blob.State.COMMITTED}) + + response = await Blob.ref(open_blob.state_id + ).get_download_url(self.external_context) + self.assertNotEqual(response.url, "") + + async def test_set_downloaders(self) -> None: + data = b"mutable acl" + + # Start open: anyone who knows the id may download. + blob, _ = await Blob.create( + self.context, + content_type="text/plain", + ) + await self._upload(blob, data) + await blob.commit(self.context) + await self._wait_until_status(blob, {Blob.State.COMMITTED}) + + response = await Blob.ref(blob.state_id + ).get_download_url(self.external_context) + self.assertNotEqual(response.url, "") + + # Restrict downloads to a user the external caller is not. + await blob.set_downloaders( + self.context, + downloader_ids=Downloaders(user_ids=["bob"]), + ) + with self.assertRaises(Blob.GetDownloadUrlAborted): + await Blob.ref(blob.state_id + ).get_download_url(self.external_context) + + # Remove the restriction again by omitting `downloader_ids`. + await blob.set_downloaders(self.context) + response = await Blob.ref(blob.state_id + ).get_download_url(self.external_context) + self.assertNotEqual(response.url, "") + + async def test_info_gating(self) -> None: + # `Info` is visible to anyone who may upload or download the + # blob. With both sides restricted, an unauthenticated external + # caller (this app has no token verifier) can do neither, so it + # cannot read `Info` either. + locked, _ = await Blob.create( + self.context, + content_type="text/plain", + uploader_id="alice", + downloader_ids=Downloaders(user_ids=["bob"]), + ) + with self.assertRaises(Blob.InfoAborted): + await Blob.ref(locked.state_id).info(self.external_context) + + # Open upload side (empty `uploader_id`): anyone who may upload + # may also watch progress via `Info`, even behind a download + # allow-list. + upload_open, _ = await Blob.create( + self.context, + content_type="text/plain", + downloader_ids=Downloaders(user_ids=["bob"]), + ) + info = await Blob.ref(upload_open.state_id).info(self.external_context) + self.assertEqual(info.status, Blob.State.UPLOADING) + + # Open download side (omitted `downloader_ids`): anyone who may + # download may read `Info`, even with a specific uploader. + download_open, _ = await Blob.create( + self.context, + content_type="text/plain", + uploader_id="alice", + ) + info = await Blob.ref(download_open.state_id + ).info(self.external_context) + self.assertEqual(info.status, Blob.State.UPLOADING) + + async def test_delete(self) -> None: + data = b"delete me" + + blob, _ = await Blob.create( + self.context, + content_type="text/plain", + ) + await self._upload(blob, data) + await blob.commit(self.context) + await self._wait_until_status(blob, {Blob.State.COMMITTED}) + + url = (await blob.get_download_url(self.context)).url + + await blob.remove(self.context) + info = await self._wait_until_status(blob, {Blob.State.REMOVED}) + self.assertEqual(info.status, Blob.State.REMOVED) + + # The bytes must be gone from the data plane. + async with aiohttp.ClientSession(self.rbt.url()) as session: + async with session.get(url) as response: + self.assertEqual(response.status, 404) + + +if __name__ == "__main__": + unittest.main() From 9a7408409c91b8f0e1914be2cb8507154a66a749 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:01:41 +0000 Subject: [PATCH 04/52] rbt: run the filesystem blob data plane under `dev`/`serve run` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the blob data plane behind a gRPC interface, an application that uses `reboot.std.blobs` needs a data-plane service to talk to — but local development must keep working out of the box, with no external service to configure. Have `rbt dev run` and `rbt serve run` start the open-source filesystem data-plane server as a background subprocess whenever `REBOOT_BLOB_DATA_PLANE_URL` is not already set (in Reboot Cloud, or via `--env`, it is — and then nothing is spawned), and point the application at it on localhost. The server picks its own ports and reports them through a ready file only once both its gRPC and HTTP endpoints are listening, so there is no port-allocation race and the application can never observe a half-started data plane. Blob bytes live under the application's state directory, so `rbt dev expunge` removes them along with the rest of the state. Co-Authored-By: Claude Fable 5 --- reboot/cli/commands/BUILD.bazel | 2 + reboot/cli/commands/dev.py | 57 +++++++++++ reboot/cli/commands/serve.py | 35 ++++++- reboot/cli/common/BUILD.bazel | 11 +++ reboot/cli/common/blob_data_plane.py | 141 +++++++++++++++++++++++++++ 5 files changed, 244 insertions(+), 2 deletions(-) create mode 100644 reboot/cli/common/blob_data_plane.py diff --git a/reboot/cli/commands/BUILD.bazel b/reboot/cli/commands/BUILD.bazel index babc364a4..f5b91e00e 100644 --- a/reboot/cli/commands/BUILD.bazel +++ b/reboot/cli/commands/BUILD.bazel @@ -38,6 +38,7 @@ py_library( "//rbt/std/presence/v1:presence_py_reboot", "//reboot/aio:aborted_py", "//reboot/aio:external_py", + "//reboot/cli/common:blob_data_plane_py", "//reboot/cli/common:dev_extra_py", "//reboot/cli/common:directories_py", "//reboot/cli/common:frontend_py", @@ -123,6 +124,7 @@ py_library( deps = [ requirement("aiofiles"), ":dev_py", + "//reboot/cli/common:blob_data_plane_py", "//reboot/cli/common:detect_cores_py", "//reboot/cli/common:directories_py", "//reboot/cli/common:frontend_py", diff --git a/reboot/cli/commands/dev.py b/reboot/cli/commands/dev.py index 2c6241a25..0bd978c03 100644 --- a/reboot/cli/commands/dev.py +++ b/reboot/cli/commands/dev.py @@ -34,6 +34,10 @@ # We import the whole `terminal` module (as opposed to the methods it contains) # to allow us to mock these methods out in tests. from reboot.cli.common import terminal +from reboot.cli.common.blob_data_plane import ( + BLOBS_SUBDIRECTORY, + start_filesystem_data_plane, +) from reboot.cli.common.dev_extra import dev_extra_installed, missing_dev_extra from reboot.cli.common.directories import ( add_working_directory_options, @@ -91,6 +95,7 @@ RBT_APPLICATION_EXIT_CODE_BACKWARDS_INCOMPATIBILITY, LocalEnvoyMode, ) +from reboot.std.blob.v1._data_plane import ENVVAR_BLOB_DATA_PLANE_URL from reboot.version import REBOOT_VERSION from typing import Any, Awaitable, Callable, Optional, TextIO, TypeVar @@ -1668,6 +1673,58 @@ def crypto_root_keys() -> str: ) ) + # Honor a blob data plane configured via `--env-file`/`--env` (they + # are otherwise only composed into the application's environment at + # launch, below) so that no local one is spawned; mirror + # `compose_env`'s precedence: `--env` wins over `--env-file`. + if args.env_file is not None and os.path.isfile(args.env_file): + data_plane_url = _load_env_file(args.env_file + ).get(ENVVAR_BLOB_DATA_PLANE_URL) + if data_plane_url is not None: + env[ENVVAR_BLOB_DATA_PLANE_URL] = data_plane_url + for (key, value) in args.env or []: + if key == ENVVAR_BLOB_DATA_PLANE_URL: + env[key] = value + + # Start the filesystem blob data plane (unless a data plane is + # already configured) and point the application at it. Bytes live + # beside the rest of the run's state, and are reclaimed the same + # way it is: `rbt dev expunge --application-name=...` removes + # both for a named application, while an anonymous run keeps + # both under `.rbt/dev` until that directory is removed by hand. + blobs_directory = os.path.join( + env.get( + ENVVAR_RBT_STATE_DIRECTORY, + str(dot_rbt_dev_directory(args, parser)), + ), + BLOBS_SUBDIRECTORY, + ) + # The data plane derives its URL-signing key from the + # cryptographic root keys, so it needs them in its environment. + # They go in a copy rather than in `env`: `compose_env()` mints + # fresh keys for the *application* on every restart, and seeding + # `env` would satisfy its `not in composed` guard and freeze the + # application's keys for the whole run. The data plane both signs + # and verifies its own URLs, so it only needs a key that is stable + # for its own lifetime. + data_plane_env = env.copy() + data_plane_env.setdefault( + ENVVAR_REBOOT_CRYPTO_ROOT_KEYS, + crypto_root_keys(), + ) + await start_filesystem_data_plane( + data_plane_env, + blobs_directory, + subprocesses, + background_command_tasks, + ) + # `start_filesystem_data_plane` reports the URL it bound by + # writing it into the environment it was given. + if ENVVAR_BLOB_DATA_PLANE_URL in data_plane_env: + env[ENVVAR_BLOB_DATA_PLANE_URL] = ( + data_plane_env[ENVVAR_BLOB_DATA_PLANE_URL] + ) + if tracing == Tracing.JAEGER: # TODO: dynamic port. See comment in `_run_jaeger()`. env[OTEL_EXPORTER_OTLP_TRACES_ENDPOINT] = "localhost:4317" diff --git a/reboot/cli/commands/serve.py b/reboot/cli/commands/serve.py index c1ac74f1e..58a58ad92 100644 --- a/reboot/cli/commands/serve.py +++ b/reboot/cli/commands/serve.py @@ -1,5 +1,6 @@ import aiofiles.os import argparse +import asyncio import math import os import sys @@ -13,6 +14,10 @@ try_and_become_child_subreaper_on_linux, ) from reboot.cli.common import terminal +from reboot.cli.common.blob_data_plane import ( + BLOBS_SUBDIRECTORY, + start_filesystem_data_plane, +) from reboot.cli.common.detect_cores import detect_cores from reboot.cli.common.directories import ( add_working_directory_options, @@ -325,6 +330,24 @@ async def serve_run( for (key, value) in args.env or []: env[key] = value + # Start the filesystem blob data plane (unless a data plane is + # already configured, e.g. by Reboot Cloud provisioning or via + # `--env`) and point the application at it. Bytes live under the + # state directory alongside the rest of the application's state. + # Done after `--env` is applied so an explicitly-configured data + # plane is honored rather than spawning a redundant local one. + blobs_directory = os.path.join( + env.get(ENVVAR_RBT_STATE_DIRECTORY, os.getcwd()), + BLOBS_SUBDIRECTORY, + ) + data_plane_tasks: list[asyncio.Task] = [] + await start_filesystem_data_plane( + env, + blobs_directory, + subprocesses, + data_plane_tasks, + ) + # If 'PYTHONPATH' is not explicitly set, we'll set it to the # specified generated code directory plus each proto directory. # We want to get the directories from 'rbt generate' flags, @@ -381,8 +404,16 @@ async def serve_run( application if not auto_transpilation else str(bundle), ] - async with subprocesses.exec(*args, env=env) as process: - return await process.wait() + try: + async with subprocesses.exec(*args, env=env) as process: + return await process.wait() + finally: + for data_plane_task in data_plane_tasks: + data_plane_task.cancel() + try: + await data_plane_task + except asyncio.CancelledError: + pass async def handle_serve_subcommand( diff --git a/reboot/cli/common/BUILD.bazel b/reboot/cli/common/BUILD.bazel index 940497373..1a1a42aaf 100644 --- a/reboot/cli/common/BUILD.bazel +++ b/reboot/cli/common/BUILD.bazel @@ -100,6 +100,17 @@ py_library( ], ) +py_library( + name = "blob_data_plane_py", + srcs = ["blob_data_plane.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + ":subprocesses_py", + "@com_github_reboot_dev_reboot//reboot/std/blob/v1:blob_py", + ], +) + py_library( name = "monkeys_py", srcs = ["monkeys.py"], diff --git a/reboot/cli/common/blob_data_plane.py b/reboot/cli/common/blob_data_plane.py new file mode 100644 index 000000000..9e0c21356 --- /dev/null +++ b/reboot/cli/common/blob_data_plane.py @@ -0,0 +1,141 @@ +"""Spawning the filesystem blob data plane for local runs. + +`rbt dev run` and `rbt serve run` start the open-source filesystem blob +data-plane server (`reboot.std.blob.v1._filesystem_server`) as a +background subprocess whenever `REBOOT_BLOB_DATA_PLANE_URL` is not +already set, and point the application at it on localhost. In Reboot +Cloud the variable is set by provisioning (to the app's facilitator), +so nothing is spawned. The server is started unconditionally when the +variable is unset — it is cheap and idle unless the application +actually uses blobs. + +The server chooses its own ports and writes them to a "ready file" +once both its gRPC and HTTP endpoints are listening; we wait for that +file before pointing the application at it. That both avoids the +port-allocation race of picking a port in the parent and guarantees +the data plane is fully serving before the application can reach it. +""" + +import asyncio +import os +import shutil +import sys +import tempfile +from reboot.cli.common.subprocesses import Subprocesses +from reboot.std.blob.v1._data_plane import ENVVAR_BLOB_DATA_PLANE_URL +from typing import Optional + +# The filesystem data plane binds loopback only (its gRPC control +# surface is unauthenticated); the application reaches it here. +_LOOPBACK_HOST = "127.0.0.1" + +# Where blob bytes live, under the application's state directory (so +# `rbt dev expunge`, which removes the state directory, removes blobs +# too). +BLOBS_SUBDIRECTORY = "blobs" + +# How long to wait for the spawned server to report its ports before +# giving up. +_READY_TIMEOUT_SECONDS = 30 +_READY_POLL_INTERVAL_SECONDS = 0.1 + + +async def _run_and_await_ready( + subprocesses: Subprocesses, + argv: list[str], + env: dict[str, str], + ready_file: str, + ready: asyncio.Future[tuple[int, int]], +) -> None: + """Runs the data-plane server until cancelled, resolving `ready` + with its `(grpc_port, http_port)` once the server writes them.""" + async with subprocesses.exec(*argv, env=env) as process: + waited = 0.0 + while not ready.done(): + if process.returncode is not None: + ready.set_exception( + RuntimeError( + "The filesystem blob data-plane server exited " + f"before becoming ready (code {process.returncode})." + ) + ) + break + ports = _read_ready_file(ready_file) + if ports is not None: + ready.set_result(ports) + break + if waited >= _READY_TIMEOUT_SECONDS: + ready.set_exception( + RuntimeError( + "Timed out waiting for the filesystem blob " + "data-plane server to become ready." + ) + ) + break + await asyncio.sleep(_READY_POLL_INTERVAL_SECONDS) + waited += _READY_POLL_INTERVAL_SECONDS + await process.wait() + + +def _read_ready_file(path: str) -> Optional[tuple[int, int]]: + try: + with open(path) as f: + content = f.read().split() + except FileNotFoundError: + return None + if len(content) != 2: + return None + return int(content[0]), int(content[1]) + + +async def start_filesystem_data_plane( + env: dict[str, str], + blobs_directory: str, + subprocesses: Subprocesses, + background_command_tasks: list[asyncio.Task], +) -> None: + """Unless a data plane is already configured in `env`, spawns the + filesystem server as a background task, waits until it is fully + listening, and points `env[REBOOT_BLOB_DATA_PLANE_URL]` at it. The + server runs with `env` — which must already carry + `REBOOT_CRYPTO_ROOT_KEYS`, from which the server derives its + URL-signing key — and is cleaned up when its task is cancelled.""" + if env.get(ENVVAR_BLOB_DATA_PLANE_URL): + return + + # The ready file lives in a fresh private directory (a bare + # `mktemp` name in a shared `/tmp` would be squattable). + ready_directory = tempfile.mkdtemp(prefix="reboot-blob-data-plane-") + ready_file = os.path.join(ready_directory, "ready") + argv = [ + sys.executable, + "-m", + "reboot.std.blob.v1._filesystem_server", + "--directory", + blobs_directory, + "--ready-file", + ready_file, + ] + + loop = asyncio.get_event_loop() + ready: asyncio.Future[tuple[int, int]] = loop.create_future() + task = asyncio.create_task( + _run_and_await_ready(subprocesses, argv, env, ready_file, ready), + name="run_filesystem_data_plane(...)", + ) + background_command_tasks.append(task) + try: + grpc_port, _ = await ready + except BaseException: + # Reap the never-ready server immediately: its task would + # otherwise run until the caller's cleanup — which under + # `rbt serve run` only happens once the application exits. + task.cancel() + try: + await task + except BaseException: + pass + raise + finally: + shutil.rmtree(ready_directory, ignore_errors=True) + env[ENVVAR_BLOB_DATA_PLANE_URL] = f"{_LOOPBACK_HOST}:{grpc_port}" From ef1297e74085ce176f4acff9b63da349b47285a1 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Sun, 5 Jul 2026 08:02:30 +0000 Subject: [PATCH 05/52] reboot/std/react: add browser helpers for blob upload and download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Uploading a file to a `Blob` from the browser means driving the whole multipart protocol — fetching upload instructions, `PUT`ting each part to its URL, reporting ETags, committing, and polling for the result — plus resuming after a dropped connection. Before this change an application author had to write all of that by hand against the generated client. Add `@reboot-dev/reboot-std-react/blobs`: - `useBlobUpload()` — the dead-simple case: given a blob id (from an application RPC, since creation is app-mediated) and a `File`, it uploads every part directly to the data plane, resumes already- confirmed parts, reports progress, and commits. - `BlobUploader` — the same machinery for bytes that don't come from a `File` (media recorders, transforms), with explicit `putPart`/`commit` and a `writable()` stream. - `useBlobDownloadUrl()` — resolves to a URL for a committed blob (e.g. for an ``), plus a re-exported reactive `useBlob` so any participant — not just the uploader — can render live progress. The part-`PUT` response carries the part's ETag, which the browser must read to report it back; expose the `etag` response header through Envoy's CORS configuration so cross-origin uploads (including direct-to-S3 uploads on the Cloud) can see it. Co-Authored-By: Claude Fable 5 --- rbt/std/BUILD.bazel | 1 + rbt/std/blob/v1/BUILD.bazel | 20 ++ reboot/routing/cors_settings.py | 6 +- reboot/std/package.json | 1 + reboot/std/react/BUILD.bazel | 1 + reboot/std/react/blob/BUILD.bazel | 27 +++ reboot/std/react/blob/index.tsx | 349 +++++++++++++++++++++++++++++ reboot/std/react/blob/package.json | 3 + reboot/std/react/package.json | 1 + 9 files changed, 407 insertions(+), 2 deletions(-) create mode 100644 reboot/std/react/blob/BUILD.bazel create mode 100644 reboot/std/react/blob/index.tsx create mode 100644 reboot/std/react/blob/package.json diff --git a/rbt/std/BUILD.bazel b/rbt/std/BUILD.bazel index 0b75b606a..93611eacc 100644 --- a/rbt/std/BUILD.bazel +++ b/rbt/std/BUILD.bazel @@ -50,6 +50,7 @@ ts_project( "//rbt/std/blob/v1:blob_js_proto", "//rbt/std/blob/v1:blob_js_reboot", "//rbt/std/blob/v1:blob_js_reboot_react", + "//rbt/std/blob/v1:blob_js_reboot_web", "//rbt/std/ciphertext/v1:ciphertext_js_proto", "//rbt/std/ciphertext/v1:ciphertext_js_reboot", "//rbt/std/collections/ordered_map/v1:ordered_map_js_proto", diff --git a/rbt/std/blob/v1/BUILD.bazel b/rbt/std/blob/v1/BUILD.bazel index a71933179..82dfa257c 100644 --- a/rbt/std/blob/v1/BUILD.bazel +++ b/rbt/std/blob/v1/BUILD.bazel @@ -3,6 +3,7 @@ load( "js_proto_library", "js_reboot_library", "js_reboot_react_library", + "js_reboot_web_library", "py_reboot_library", ) load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") @@ -74,6 +75,25 @@ js_reboot_library( ], ) +# The browser client: the same API as the React client, but callable +# from anywhere rather than only from inside a component. +js_reboot_web_library( + name = "blob_js_reboot_web", + proto = "blob.proto", + proto_deps = [ + ":blob_proto", + # ISSUE(https://github.com/reboot-dev/mono/issues/3218): Until we can + # use `create_protoc_plugin_rule` we need to repeat the dependencies of + # the `proto_libraries` here. + "@com_github_reboot_dev_reboot//rbt/v1alpha1:options_proto", + "@com_google_protobuf//:descriptor_proto", + ], + visibility = ["//visibility:public"], + deps = [ + ":blob_js_proto", + ], +) + js_reboot_react_library( name = "blob_js_reboot_react", srcs = [ diff --git a/reboot/routing/cors_settings.py b/reboot/routing/cors_settings.py index bde599338..89a5ff074 100644 --- a/reboot/routing/cors_settings.py +++ b/reboot/routing/cors_settings.py @@ -41,8 +41,10 @@ 'ngrok-skip-browser-warning', ) -# Response headers cross-origin JavaScript is allowed to read. -CORS_EXPOSE_HEADERS = ('grpc-status', 'grpc-message') +# Response headers cross-origin JavaScript is allowed to read. `etag` +# is exposed so that browser blob uploads can read each part's ETag +# from the data plane's response (see `reboot.std.blob`). +CORS_EXPOSE_HEADERS = ('grpc-status', 'grpc-message', 'etag') # How long a browser may cache a CORS preflight response, in seconds # (20 days). diff --git a/reboot/std/package.json b/reboot/std/package.json index 1a6d52720..f3e1c191a 100644 --- a/reboot/std/package.json +++ b/reboot/std/package.json @@ -21,6 +21,7 @@ "exports": { "./package.json": "./package.json", ".": "./index.js", + "./blob/v1": "./blob/v1/index.js", "./ciphertext/v1": "./ciphertext/v1/index.js", "./collections/queue/v1": "./collections/queue/v1/index.js", "./collections/ordered_map/v1": "./collections/ordered_map/v1/index.js", diff --git a/reboot/std/react/BUILD.bazel b/reboot/std/react/BUILD.bazel index 9e43a0a8e..9f5bb1656 100644 --- a/reboot/std/react/BUILD.bazel +++ b/reboot/std/react/BUILD.bazel @@ -48,6 +48,7 @@ ts_project( }, visibility = ["//visibility:public"], deps = [ + "//reboot/std/react/blob:blob_ts", "//reboot/std/react/presence:presence_ts", ], ) diff --git a/reboot/std/react/blob/BUILD.bazel b/reboot/std/react/blob/BUILD.bazel new file mode 100644 index 000000000..2858ee68f --- /dev/null +++ b/reboot/std/react/blob/BUILD.bazel @@ -0,0 +1,27 @@ +load("@aspect_rules_ts//ts:defs.bzl", "ts_project") + +ts_project( + name = "blob_ts", + srcs = [ + "index.tsx", + "package.json", + ], + declaration = True, + tsconfig = { + "compilerOptions": { + "declaration": True, + "jsx": "react-jsx", + "module": "nodenext", + "moduleResolution": "nodenext", + "target": "es2020", + }, + }, + visibility = ["//visibility:public"], + deps = [ + "//:node_modules/@reboot-dev/reboot-api", + "//:node_modules/@reboot-dev/reboot-react", + "//:node_modules/@reboot-dev/reboot-std-api", + "//:node_modules/@reboot-dev/reboot-web", + "//:node_modules/react", + ], +) diff --git a/reboot/std/react/blob/index.tsx b/reboot/std/react/blob/index.tsx new file mode 100644 index 000000000..8e8aafdd8 --- /dev/null +++ b/reboot/std/react/blob/index.tsx @@ -0,0 +1,349 @@ +// Browser-side helpers for `reboot.std.blob`: a dead-simple hook for +// uploading a `File` into a `Blob` the application backend has +// created, plus the lower-level `BlobUploader` for bytes that come +// from somewhere other than a file input. +// +// The control-plane calls go through the generated browser client, +// which brings retry, reconnection and authentication with it; only +// the bytes are handled here directly, `PUT` to the URLs the control +// plane mints (the application's own data plane for the `filesystem` +// store, presigned S3 URLs for the `s3` store — the uploader neither +// knows nor cares which). Those `PUT`s are not Reboot RPCs, so they +// stay a plain `fetch`. + +import { useRebootClient } from "@reboot-dev/reboot-react"; +import { Blob_Status } from "@reboot-dev/reboot-std-api/blob/v1/blob_pb.js"; +import { useBlob } from "@reboot-dev/reboot-std-api/blob/v1/blob_rbt_react.js"; +import { Blob } from "@reboot-dev/reboot-std-api/blob/v1/blob_rbt_web.js"; +import { WebContext } from "@reboot-dev/reboot-web"; +import { useMemo } from "react"; + +// Re-exported so applications can reactively render blob metadata +// (e.g. a progress bar for an attachment some *other* client is +// uploading) without a separate import of the generated client. +export { useBlob }; + +// How many parts `upload()` has in flight at once. Parts are +// independent, and uploading them one at a time leaves most of the +// available bandwidth unused on any connection with real latency. +const UPLOAD_CONCURRENCY = 4; + +// How long `useBlobDownloadUrl` asks its URL to stay valid for. The +// store caps what it grants; the granted value comes back on the +// response. +const DOWNLOAD_URL_TTL_SECONDS = 60 * 60; + +export interface UploadProgress { + uploadedBytes: number; + totalBytes: number; +} + +export interface UploadOptions { + onProgress?: (progress: UploadProgress) => void; + signal?: AbortSignal; +} + +export interface UploadResult { + // The committed object's ETag: an opaque token from the data + // plane, not a digest of the uploaded bytes. + etag?: string; + error?: string; +} + +/** + * Uploads bytes into a `Blob` that the application backend has + * created (blob creation is always application-mediated; ask your + * backend for a blob ID first). + * + * Use `upload(...)` for a `File`/`Blob`/`Uint8Array` you already + * have, or `putPart(...)`/`commit()` directly when producing bytes + * incrementally from some other source. Attaching an uploader to a + * partially-uploaded blob resumes it: already-confirmed parts are + * skipped. + */ +export class BlobUploader { + private options: { url: string; blobId: string; bearerToken?: string }; + private confirmed: Map = new Map(); + private blob: Blob.WeakReference; + private context: WebContext; + + constructor(options: { url: string; blobId: string; bearerToken?: string }) { + this.options = options; + this.blob = Blob.ref(options.blobId); + this.context = new WebContext({ + url: options.url, + bearerToken: options.bearerToken, + }); + } + + /** + * Fetches upload instructions for the given part numbers, waiting + * for the blob's upload session to be provisioned. + */ + async instructions( + partNumbers: number[], + options?: { signal?: AbortSignal } + ): Promise<{ partSize: number; urls: Map }> { + // `ready` is false until the `BeginUpload` workflow has + // provisioned the data-plane upload session, so watch until it + // flips rather than asking again on a timer. + options?.signal?.throwIfAborted(); + const controller = new AbortController(); + options?.signal?.addEventListener("abort", () => controller.abort(), { + once: true, + }); + try { + const [responses] = await this.blob + .reactively() + .getPartUploadInstructions( + this.context, + { partNumbers }, + { signal: controller.signal } + ); + for await (const response of responses) { + if (!response.ready) { + continue; + } + const urls = new Map(); + for (const instruction of response.instructions) { + urls.set( + instruction.partNumber, + new URL(instruction.url, this.options.url).toString() + ); + } + return { partSize: Number(response.partSize), urls }; + } + options?.signal?.throwIfAborted(); + throw new Error( + `Stopped watching blob ${this.options.blobId} before its upload ` + + "session was provisioned" + ); + } finally { + // Tear the stream down as soon as we have our answer. + controller.abort(); + } + } + + /** + * `PUT`s one part's bytes to the data plane and reports it to the + * control plane. Idempotent per part number. + */ + async putPart( + partNumber: number, + bytes: globalThis.Blob | Uint8Array, + options?: { signal?: AbortSignal } + ): Promise { + const { urls } = await this.instructions([partNumber], options); + const url = urls.get(partNumber); + if (url === undefined) { + throw new Error(`No upload URL for part ${partNumber}`); + } + await this.putPartToUrl(partNumber, url, bytes, options); + } + + /** + * `PUT`s one part's bytes to an already-minted URL and reports it to + * the control plane. + */ + private async putPartToUrl( + partNumber: number, + url: string, + bytes: globalThis.Blob | Uint8Array, + options?: { signal?: AbortSignal } + ): Promise { + const response = await fetch(url, { + method: "PUT", + body: bytes, + signal: options?.signal, + }); + if (!response.ok) { + throw new Error( + `Part ${partNumber} upload failed (${response.status}): ` + + `${await response.text()}` + ); + } + const etag = (response.headers.get("ETag") ?? "").replace(/"/g, ""); + if (etag === "") { + throw new Error( + `Part ${partNumber} upload returned no ETag; if this ` + + "application uses an S3-compatible store, its bucket CORS " + + "configuration must expose the `ETag` header" + ); + } + const size = bytes instanceof Uint8Array ? bytes.byteLength : bytes.size; + await this.blob.partUploaded(this.context, { + partNumber, + etag, + size: BigInt(size), + }); + this.confirmed.set(partNumber, size); + } + + /** + * Commits the upload and waits for the data plane to confirm, + * returning the blob's ETag or the reason the commit failed. The + * failure reason describes what went wrong but does not identify + * which parts, if any, were at fault; to retry, re-`putPart` (parts + * are safe to re-upload) and call `commit` again. + */ + async commit(options?: { signal?: AbortSignal }): Promise { + // `Commit` returns as soon as the blob is marked COMMITTING; the + // data plane finalizes the object in a workflow, and the outcome + // lands back on the blob's state. Subscribe rather than re-read on + // a timer: `Info` is a reader, so the update is pushed. Committing + // first is safe because a reactive read always yields current + // state before any update, and `Commit` clears the error from a + // previous attempt as it marks the blob COMMITTING. + await this.blob.commit(this.context); + + options?.signal?.throwIfAborted(); + const controller = new AbortController(); + options?.signal?.addEventListener("abort", () => controller.abort(), { + once: true, + }); + try { + const [infos] = await this.blob + .reactively() + .info(this.context, {}, { signal: controller.signal }); + for await (const info of infos) { + if (info.status === Blob_Status.COMMITTED) { + return { etag: info.etag }; + } + if (info.commitError !== undefined && info.commitError !== "") { + return { error: info.commitError }; + } + if ( + info.status === Blob_Status.REMOVING || + info.status === Blob_Status.REMOVED + ) { + return { error: "The blob was removed before it committed" }; + } + } + options?.signal?.throwIfAborted(); + throw new Error( + `Stopped watching blob ${this.options.blobId} before it committed` + ); + } finally { + controller.abort(); + } + } + + /** + * Uploads `data` in parts and commits: the whole story for bytes + * you already have. Resumes where a previous attempt left off. + */ + async upload( + data: globalThis.Blob | Uint8Array, + options?: UploadOptions + ): Promise { + // Refresh what the control plane already has, so interrupted + // uploads resume rather than restart. + const info = await this.blob.info(this.context); + this.confirmed = new Map( + info.parts.map((part) => [part.number, Number(part.size)]) + ); + + const { partSize } = await this.instructions([], options); + const totalBytes = data instanceof Uint8Array ? data.byteLength : data.size; + const partCount = Math.max(1, Math.ceil(totalBytes / partSize)); + + let uploadedBytes = 0; + for (const [, size] of this.confirmed) { + uploadedBytes += size; + } + + const pending: number[] = []; + for (let partNumber = 1; partNumber <= partCount; partNumber++) { + if (!this.confirmed.has(partNumber)) { + pending.push(partNumber); + } + } + + // One `instructions` call per window rather than one per part, and + // no more URLs minted ahead of use than a window's worth: the URLs + // are short-lived, so fetching them all up front would see the + // later ones expire before their turn. + for (let index = 0; index < pending.length; index += UPLOAD_CONCURRENCY) { + const window = pending.slice(index, index + UPLOAD_CONCURRENCY); + const { urls } = await this.instructions(window, options); + await Promise.all( + window.map(async (partNumber) => { + const url = urls.get(partNumber); + if (url === undefined) { + throw new Error(`No upload URL for part ${partNumber}`); + } + const offset = (partNumber - 1) * partSize; + const bytes = data.slice( + offset, + Math.min(offset + partSize, totalBytes) + ); + await this.putPartToUrl(partNumber, url, bytes, options); + uploadedBytes += + bytes instanceof Uint8Array ? bytes.byteLength : bytes.size; + options?.onProgress?.({ uploadedBytes, totalBytes }); + }) + ); + } + + return await this.commit(options); + } +} + +/** + * The dead-simple upload hook. The blob ID comes from an + * application-level RPC (blob creation is application-mediated), and + * then: + * + * const { upload } = useBlobUpload(); + * ... + * const { etag, error } = await upload(blobId, file); + */ +export function useBlobUpload(): { + upload: ( + blobId: string, + data: globalThis.Blob | Uint8Array, + options?: UploadOptions + ) => Promise; +} { + const client = useRebootClient(); + const upload = useMemo(() => { + return async ( + blobId: string, + data: globalThis.Blob | Uint8Array, + options?: UploadOptions + ) => { + const uploader = new BlobUploader({ + url: client.url, + blobId, + bearerToken: client.bearerToken, + }); + return await uploader.upload(data, options); + }; + }, [client.url, client.bearerToken]); + return { upload }; +} + +/** + * Resolves to a URL from which a committed blob's bytes can be + * downloaded (e.g. for an ``), or `undefined` while the blob + * is still uploading. Render upload progress meanwhile via + * `useBlob(...).useInfo()`. + */ +export function useBlobDownloadUrl(blobId: string): string | undefined { + const client = useRebootClient(); + // The generated reader hook rather than a hand-rolled request: it + // brings the retry, reconnection and authentication the reactive + // machinery already implements, and re-delivers when the blob + // commits, so there is nothing here to gate on `status`. + const { response } = useBlob({ id: blobId }).useGetDownloadUrl({ + ttlSeconds: DOWNLOAD_URL_TTL_SECONDS, + }); + + return useMemo( + () => + response === undefined + ? undefined + : new URL(response.url, client.url).toString(), + [response, client.url] + ); +} diff --git a/reboot/std/react/blob/package.json b/reboot/std/react/blob/package.json new file mode 100644 index 000000000..3dbc1ca59 --- /dev/null +++ b/reboot/std/react/blob/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/reboot/std/react/package.json b/reboot/std/react/package.json index f06f9e37a..614c788a9 100644 --- a/reboot/std/react/package.json +++ b/reboot/std/react/package.json @@ -23,6 +23,7 @@ "exports": { "./package.json": "./package.json", ".": "./index.js", + "./blob": "./blob/index.js", "./presence": "./presence/index.js" } } From 3b9aef054a62063556256bb627412ece9de55be2 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Sun, 5 Jul 2026 08:02:55 +0000 Subject: [PATCH 06/52] reboot/examples/chat-room: support message attachments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chat-room example only sent text, so it demonstrated nothing about storing binary data. Give it file attachments, wired the way a real app would: message-first, with attachment bytes uploaded after the message is already visible. `Send` now takes a list of attachment descriptors; the servicer checks each against a per-attachment size limit, creates a `Blob` per attachment, embeds the blob ids in the immediately-published message, and returns them. The web frontend then uploads each file into its blob via `useBlobUpload`, and every participant renders the attachment off its reactive blob status — a progress bar while it uploads (visible to everyone, since progress lives on the blob's state), the image once committed. No end-user auth here, so blobs are created with an empty `owner_id` (anyone in the room may upload). The documentation snippets that embed the chat-room proto are regenerated to match. Co-Authored-By: Claude Fable 5 --- documentation/docs/call/from_react.mdx | 50 ++- .../docs/call/from_within_your_app.mdx | 14 +- documentation/docs/implement/application.mdx | 12 +- documentation/docs/testing.md | 7 +- .../examples/chat-room/.tests/serve_test.sh | 4 +- .../api/chat_room/v1/chat_room.proto | 45 ++- .../backend/src/chat_room_servicer.py | 43 ++- reboot/examples/chat-room/backend/src/main.py | 5 + .../chat-room/frontend/.tests/type_check.sh | 4 +- .../chat-room/frontend/mobile/src/App.tsx | 4 +- .../frontend/reboot-non-react-web/src/main.ts | 2 +- .../chat-room/frontend/web/package.json | 2 + .../chat-room/frontend/web/src/App.module.css | 227 +++++++++++++- .../chat-room/frontend/web/src/App.tsx | 294 ++++++++++++++++-- .../chat-room/tests/chat_room.feature | 5 +- tests/reboot/examples/chat-room/BUILD.bazel | 4 + .../chat-room/serve_expected_output.txt | 5 +- 17 files changed, 635 insertions(+), 92 deletions(-) diff --git a/documentation/docs/call/from_react.mdx b/documentation/docs/call/from_react.mdx index 4a375e7b1..1f06085d1 100644 --- a/documentation/docs/call/from_react.mdx +++ b/documentation/docs/call/from_react.mdx @@ -303,7 +303,7 @@ export const DashboardApp: FC = ({ personalizedMessage }) => { You can call a `reader` _reactively_ very simply: +(CODE:src=../../../reboot/examples/chat-room/frontend/web/src/App.tsx&&lines=160-161) --> ```tsx @@ -332,7 +332,7 @@ and [`transaction`](/define/methods#kinds) methods are both callable from React. +(CODE:src=../../../reboot/examples/chat-room/frontend/web/src/App.tsx&&lines=160-160) --> ```tsx @@ -349,11 +349,17 @@ This line calls the definition, using its lower camel case name: +(CODE:src=../../../reboot/examples/chat-room/frontend/web/src/App.tsx&&lines=181-187) --> ```tsx -const { aborted } = await send({ message: message }); +const { response, aborted } = await send({ + message: message, + attachments: + file !== null + ? [{ contentType: file.type, sizeBytes: BigInt(file.size) }] + : [], +}); ``` @@ -370,12 +376,17 @@ Reboot attaches all in-flight mutations to a `.pending` property of every mutato facilitate this, for example: +(CODE:src=../../../reboot/examples/chat-room/frontend/web/src/App.tsx&&lines=306-313) --> ```tsx -{send.pending.map(({ request: { message }, isLoading }) => ( - +{send.pending.map(({ request, isLoading }, index) => ( + ))} ``` @@ -402,15 +413,30 @@ Every call to a mutator returns both a `response` and an `aborted`; successful c [Learn more about Reboot errors and error types.](/errors) +(CODE:src=../../../reboot/examples/chat-room/frontend/web/src/App.tsx&&lines=181-200) --> ```tsx -const { aborted } = await send({ message: message }); +const { response, aborted } = await send({ + message: message, + attachments: + file !== null + ? [{ contentType: file.type, sizeBytes: BigInt(file.size) }] + : [], +}); if (aborted !== undefined) { - console.warn(aborted.error.getType()); - console.warn(aborted.message); -} + // Surface the failure to the user. The backend's + // `AttachmentTooLarge` error carries the limit, so we can say + // exactly what it is. + if (aborted.error instanceof AttachmentTooLarge) { + const maxMebibytes = Number(aborted.error.maxSizeBytes) / (1024 * 1024); + setError( + `That attachment is too large. The maximum is ${maxMebibytes} MiB.` + ); + } else { + setError(`Couldn't send your message: ${aborted.message}`); + } + return; ``` diff --git a/documentation/docs/call/from_within_your_app.mdx b/documentation/docs/call/from_within_your_app.mdx index 89570443f..de5b0396c 100644 --- a/documentation/docs/call/from_within_your_app.mdx +++ b/documentation/docs/call/from_within_your_app.mdx @@ -84,18 +84,16 @@ via `write()` — see +(CODE:src=../../../reboot/examples/chat-room/backend/src/chat_room_servicer.py&lines=23-28) --> ```py -async def send( +async def messages( self, - context: WriterContext, - request: SendRequest, -) -> SendResponse: - message = request.message - self.state.messages.extend([message]) - return SendResponse() + context: ReaderContext, + request: MessagesRequest, +) -> MessagesResponse: + return MessagesResponse(messages=self.state.messages) ``` diff --git a/documentation/docs/implement/application.mdx b/documentation/docs/implement/application.mdx index d8228eb63..24e3d6162 100644 --- a/documentation/docs/implement/application.mdx +++ b/documentation/docs/implement/application.mdx @@ -13,19 +13,19 @@ Your entrypoint will construct an `Application` and then `run` it: +(CODE:src=../../../reboot/examples/chat-room/backend/src/main.py&lines=26-34) --> ```py async def main(): await Application( servicers=[ChatRoomServicer], + # Message attachments are stored as blobs; `rbt dev run` and + # `rbt serve run` provide a local filesystem data plane (see + # `REBOOT_BLOB_DATA_PLANE_URL` for using a custom one). + libraries=[blob_library()], initialize=initialize, ).run() - - -if __name__ == '__main__': - asyncio.run(main()) ``` @@ -59,7 +59,7 @@ instances used by your application are created. +(CODE:src=../../../reboot/examples/chat-room/backend/src/main.py&lines=14-21) --> ```py diff --git a/documentation/docs/testing.md b/documentation/docs/testing.md index fb85de27b..9f76d5252 100644 --- a/documentation/docs/testing.md +++ b/documentation/docs/testing.md @@ -37,13 +37,16 @@ async def test_chat_room(self) -> None: await chat_room.send(context, message="Hello, World") response: ChatRoom.MessagesResponse = await chat_room.messages(context) - self.assertEqual(response.messages, ["Hello, World"]) + self.assertEqual( + [message.text for message in response.messages], + ["Hello, World"], + ) await chat_room.send(context, message="Hello, Reboot!") await chat_room.send(context, message="Hello, Peace of Mind!") response = await chat_room.messages(context) self.assertEqual( - response.messages, + [message.text for message in response.messages], [ "Hello, World", "Hello, Reboot!", diff --git a/reboot/examples/chat-room/.tests/serve_test.sh b/reboot/examples/chat-room/.tests/serve_test.sh index 1c8ae024d..7039251d8 100755 --- a/reboot/examples/chat-room/.tests/serve_test.sh +++ b/reboot/examples/chat-room/.tests/serve_test.sh @@ -168,8 +168,8 @@ if command -v docker &> /dev/null; then fi # Verify the response contains the message we sent. # Collapse whitespace since the JSON response is pretty-printed. - if ! tr -d ' \n' < "$actual_output_file" | grep -q '"messages":\["test"\]'; then - echo "Expected '\"messages\":[\"test\"]' in response for state ID '$state_id', got:" + if ! tr -d ' \n' < "$actual_output_file" | grep -q '"messages":\[{"text":"test"'; then + echo "Expected '\"messages\":[{\"text\":\"test\"' in response for state ID '$state_id', got:" cat "$actual_output_file" exit 1 fi diff --git a/reboot/examples/chat-room/api/chat_room/v1/chat_room.proto b/reboot/examples/chat-room/api/chat_room/v1/chat_room.proto index f3c437803..2ed45d946 100644 --- a/reboot/examples/chat-room/api/chat_room/v1/chat_room.proto +++ b/reboot/examples/chat-room/api/chat_room/v1/chat_room.proto @@ -6,10 +6,28 @@ import "rbt/v1alpha1/options.proto"; //////////////////////////////////////////////////////////////////////// +// Raised when a requested attachment exceeds the chat room's +// per-attachment size limit. +message AttachmentTooLarge { + // The maximum allowed attachment size, in bytes, so the client can + // show a helpful message. + uint64 max_size_bytes = 1; +} + +message Message { + string text = 1; + + // IDs of `rbt.std.blob.v1.Blob`s holding this message's + // attachments. Attachments are visible to all participants from the + // moment the message is sent, while their bytes may still be + // uploading; render their progress reactively via `Blob.Info`. + repeated string attachment_blob_ids = 2; +} + message ChatRoom { option (rbt.v1alpha1.state) = { }; - repeated string messages = 1; + repeated Message messages = 1; } service ChatRoomMethods { @@ -19,9 +37,14 @@ service ChatRoomMethods { }; } - // Adds a new message to the list of recorded messages. + // Adds a new message to the list of recorded messages, creating a + // `Blob` for each requested attachment. The message is published + // immediately; the caller then uploads the attachment bytes into + // the returned blob IDs. rpc Send(SendRequest) returns (SendResponse) { - option (rbt.v1alpha1.method).writer = { + option (rbt.v1alpha1.method) = { + transaction: {}, + errors: [ "AttachmentTooLarge" ], }; } } @@ -29,11 +52,23 @@ service ChatRoomMethods { message MessagesRequest {} message MessagesResponse { - repeated string messages = 1; + repeated Message messages = 1; +} + +// Describes an attachment the sender intends to upload. +message Attachment { + string content_type = 1; // E.g. "image/png". + uint64 size_bytes = 2; } message SendRequest { string message = 1; // E.g. "Hello, World". + repeated Attachment attachments = 2; } -message SendResponse {} +message SendResponse { + // One `Blob` ID per requested attachment, in request order. Upload + // the attachment bytes into these (e.g. with `useBlobUpload` from + // `@reboot-dev/reboot-std-react/blob`). + repeated string attachment_blob_ids = 1; +} diff --git a/reboot/examples/chat-room/backend/src/chat_room_servicer.py b/reboot/examples/chat-room/backend/src/chat_room_servicer.py index eaa0adc94..25c3f7480 100644 --- a/reboot/examples/chat-room/backend/src/chat_room_servicer.py +++ b/reboot/examples/chat-room/backend/src/chat_room_servicer.py @@ -1,12 +1,18 @@ from chat_room.v1.chat_room_rbt import ( + AttachmentTooLarge, ChatRoom, + Message, MessagesRequest, MessagesResponse, SendRequest, SendResponse, ) +from rbt.std.blob.v1.blob_rbt import Blob from reboot.aio.auth.authorizers import allow -from reboot.aio.contexts import ReaderContext, WriterContext +from reboot.aio.contexts import ReaderContext, TransactionContext + +# The chat room refuses attachments larger than this. +MAX_ATTACHMENT_BYTES = 300 * 1024 * 1024 class ChatRoomServicer(ChatRoom.Servicer): @@ -23,9 +29,36 @@ async def messages( async def send( self, - context: WriterContext, + context: TransactionContext, request: SendRequest, ) -> SendResponse: - message = request.message - self.state.messages.extend([message]) - return SendResponse() + # Create a `Blob` for every requested attachment. This is the + # application-mediated step where attachment policy is + # enforced; the blobs' random IDs then act as upload/download + # capabilities. This example has no end-user authentication, + # so `uploader_id` is left empty: anyone who knows a blob's ID + # may upload into it. `downloader_ids` is likewise omitted, so + # anyone who knows a blob's ID may download it too. + attachment_blob_ids = [] + for attachment in request.attachments: + if attachment.size_bytes > MAX_ATTACHMENT_BYTES: + raise ChatRoom.SendAborted( + AttachmentTooLarge(max_size_bytes=MAX_ATTACHMENT_BYTES) + ) + blob, _ = await Blob.create( + context, + content_type=attachment.content_type, + size=attachment.size_bytes, + ) + attachment_blob_ids.append(blob.state_id) + + # The message is published immediately: participants see it + # (and can watch its attachments' upload progress) before the + # bytes have been uploaded. + self.state.messages.append( + Message( + text=request.message, + attachment_blob_ids=attachment_blob_ids, + ) + ) + return SendResponse(attachment_blob_ids=attachment_blob_ids) diff --git a/reboot/examples/chat-room/backend/src/main.py b/reboot/examples/chat-room/backend/src/main.py index 9683a2299..2f258ff6d 100644 --- a/reboot/examples/chat-room/backend/src/main.py +++ b/reboot/examples/chat-room/backend/src/main.py @@ -4,6 +4,7 @@ from chat_room_servicer import ChatRoomServicer from reboot.aio.applications import Application from reboot.aio.external import InitializeContext +from reboot.std.blob.v1.blob import blob_library logging.basicConfig(level=logging.INFO) @@ -25,6 +26,10 @@ async def initialize(context: InitializeContext): async def main(): await Application( servicers=[ChatRoomServicer], + # Message attachments are stored as blobs; `rbt dev run` and + # `rbt serve run` provide a local filesystem data plane (see + # `REBOOT_BLOB_DATA_PLANE_URL` for using a custom one). + libraries=[blob_library()], initialize=initialize, ).run() diff --git a/reboot/examples/chat-room/frontend/.tests/type_check.sh b/reboot/examples/chat-room/frontend/.tests/type_check.sh index ea751190a..a58789ed8 100755 --- a/reboot/examples/chat-room/frontend/.tests/type_check.sh +++ b/reboot/examples/chat-room/frontend/.tests/type_check.sh @@ -60,7 +60,9 @@ if [[ -n "${REBOOT_NPM_PACKAGE:-}" ]]; then "${SANDBOX_ROOT}${REBOOT_NPM_PACKAGE}" \ "${SANDBOX_ROOT}${REBOOT_API_NPM_PACKAGE}" \ "${SANDBOX_ROOT}${REBOOT_WEB_NPM_PACKAGE}" \ - "${SANDBOX_ROOT}${REBOOT_REACT_NPM_PACKAGE}" + "${SANDBOX_ROOT}${REBOOT_REACT_NPM_PACKAGE}" \ + "${SANDBOX_ROOT}${REBOOT_STD_API_PACKAGE}" \ + "${SANDBOX_ROOT}${REBOOT_STD_REACT_PACKAGE}" else npm install fi diff --git a/reboot/examples/chat-room/frontend/mobile/src/App.tsx b/reboot/examples/chat-room/frontend/mobile/src/App.tsx index a162526d9..7cb6797f0 100644 --- a/reboot/examples/chat-room/frontend/mobile/src/App.tsx +++ b/reboot/examples/chat-room/frontend/mobile/src/App.tsx @@ -113,8 +113,8 @@ const ChatRoom = () => { `${index}-${item}`} - renderItem={({ item }) => } + keyExtractor={(item, index) => `${index}-${item.text}`} + renderItem={({ item }) => } ListFooterComponent={ {pending.map(({ request: { message }, isLoading }, index) => ( diff --git a/reboot/examples/chat-room/frontend/reboot-non-react-web/src/main.ts b/reboot/examples/chat-room/frontend/reboot-non-react-web/src/main.ts index 97e7cffde..666df24ca 100644 --- a/reboot/examples/chat-room/frontend/reboot-non-react-web/src/main.ts +++ b/reboot/examples/chat-room/frontend/reboot-non-react-web/src/main.ts @@ -33,7 +33,7 @@ async function bindToElement( ) { for await (const response of generator) { element.innerHTML = `${response.messages - .map((msg: string) => `
${msg}
`) + .map((msg) => `
${msg.text}
`) .join("")}`; } } diff --git a/reboot/examples/chat-room/frontend/web/package.json b/reboot/examples/chat-room/frontend/web/package.json index 2512c706a..1c8154f54 100644 --- a/reboot/examples/chat-room/frontend/web/package.json +++ b/reboot/examples/chat-room/frontend/web/package.json @@ -7,6 +7,8 @@ "@bufbuild/protobuf": "1.10.1", "@eslint/js": "^9.34.0", "@reboot-dev/reboot-react": "1.5.0", + "@reboot-dev/reboot-std-api": "1.5.0", + "@reboot-dev/reboot-std-react": "1.5.0", "@types/eslint__js": "^8.42.3", "@types/react": "^19.1.0", "@types/react-dom": "^19.1.0", diff --git a/reboot/examples/chat-room/frontend/web/src/App.module.css b/reboot/examples/chat-room/frontend/web/src/App.module.css index 2289ef53e..24744a8a7 100644 --- a/reboot/examples/chat-room/frontend/web/src/App.module.css +++ b/reboot/examples/chat-room/frontend/web/src/App.module.css @@ -1,22 +1,127 @@ -/* Styles by @sowg: https://codepen.io/sowg/pen/qBXexaZ */ +/* Styles originally by @sowg: https://codepen.io/sowg/pen/qBXexaZ */ .messages { - width: 300px; + width: 340px; margin: 0 auto; } +/* --- Composer: a rounded text field with a `+` attach button inside, + and a Send button beside it. --- */ + +.composer { + display: flex; + align-items: center; + gap: 0.5em; + margin: 1em 0 0.4em; +} + +.inputWrap { + position: relative; + flex: 1; +} + .textInput { - margin: 1em; + width: 100%; + box-sizing: border-box; appearance: none; - border: none; + border: 0.15em solid #e91e63; outline: none; - border-bottom: 0.2em solid #e91e63; - background: rgba(#e91e63, 0.2); - border-radius: 0.2em 0.2em 0 0; - padding: 0.4em; + border-radius: 1.6em; + padding: 0.55em 2.5em 0.55em 0.9em; + color: #e91e63; + background: #fff0f5; + font-family: sans-serif; +} + +.textInput::placeholder { + color: #f48fb1; +} + +.plusButton { + position: absolute; + /* Equal inset on the top, bottom, and right; `aspect-ratio` then + makes it a circle sized to fit, so its right margin matches its + vertical margin. */ + top: 0.35em; + bottom: 0.35em; + right: 0.35em; + aspect-ratio: 1 / 1; + padding: 0; + border: none; + border-radius: 50%; + background: #e91e63; + color: #fff; + font-size: 1em; + line-height: 1; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; +} + +.plusButton:hover { + background: #c2185b; +} + +.hiddenFileInput { + display: none; +} + +/* A dismissable error notification (e.g. an attachment that's too + large). */ +.errorBanner { + display: flex; + align-items: center; + justify-content: space-between; + gap: 0.6em; + margin: 0 0 0.6em; + padding: 0.6em 0.9em; + border-radius: 0.8em; + background: #ffe3ec; + border: 0.12em solid #e91e63; + color: #c2185b; + font-family: sans-serif; + font-size: 0.9em; + cursor: pointer; +} + +.errorDismiss { + font-size: 1.2em; + line-height: 1; + flex: none; +} + +/* A chip showing the file that's staged to send, before the message + is sent. */ +.pendingAttachment { + display: flex; + align-items: center; + gap: 0.3em; + margin: 0 0 0.6em 0.4em; + font-family: sans-serif; + font-size: 0.85em; + color: #e91e63; +} + +.pendingAttachmentName { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + max-width: 220px; +} + +.pendingAttachmentRemove { + border: none; + background: transparent; color: #e91e63; + font-size: 1.2em; + line-height: 1; + cursor: pointer; + padding: 0 0.2em; } +/* --- Messages --- */ + .message { appearance: none; border: 0.2em solid #e91e63; @@ -29,6 +134,10 @@ font-family: sans-serif; } +.messageText { + word-break: break-word; +} + .pendingMessage { border: 0.2em solid #a9a9a9; color: #a9a9a9; @@ -47,18 +156,21 @@ .buttonDisabled, .buttonEnabled { - float: right; + flex: none; appearance: none; - border: 0.2em solid; + border: 0.15em solid; background: hsl(0 0 0/0); - padding: 0.85em 1.5em; + padding: 0.55em 1.1em; border-radius: 2em; - transition: 1s; + font-family: sans-serif; + transition: 0.3s; + cursor: pointer; } .buttonDisabled { color: gray; border-color: darkgray; + cursor: default; } .buttonEnabled { @@ -71,3 +183,94 @@ color: #fff; } } + +/* --- Attachments: fixed clickable squares --- */ + +.attachments { + display: flex; + flex-wrap: wrap; + gap: 0.5em; + margin-top: 0.6em; +} + +.attachment, +.attachmentPending { + width: 72px; + height: 72px; + flex: none; + box-sizing: border-box; + padding: 0; + border: 0.15em solid #e91e63; + border-radius: 0.6em; + background: #fff0f5; + overflow: hidden; + display: flex; + align-items: center; + justify-content: center; +} + +.attachment { + cursor: pointer; +} + +.attachment:disabled { + cursor: default; +} + +.attachmentPending { + border-color: #cfcfcf; + background: #f4f4f4; +} + +.attachmentThumb { + width: 100%; + height: 100%; + object-fit: cover; + display: block; +} + +.attachmentGlyph { + font-size: 1.9em; + color: #a9a9a9; +} + +/* Circular upload progress. Rotated so it fills from the top. */ +.ring { + width: 40px; + height: 40px; + transform: rotate(-90deg); +} + +.ringTrack { + fill: none; + stroke: #f8bbd0; + stroke-width: 3; +} + +.ringFill { + fill: none; + stroke: #e91e63; + stroke-width: 3; + stroke-linecap: round; + transition: stroke-dashoffset 0.3s ease; +} + +/* --- Lightbox for viewing a committed image attachment --- */ + +.lightbox { + position: fixed; + inset: 0; + z-index: 10; + background: rgba(0, 0, 0, 0.75); + display: flex; + align-items: center; + justify-content: center; + cursor: zoom-out; +} + +.lightboxImage { + max-width: 90vw; + max-height: 90vh; + border-radius: 0.4em; + box-shadow: 0 0.5em 2em rgba(0, 0, 0, 0.5); +} diff --git a/reboot/examples/chat-room/frontend/web/src/App.tsx b/reboot/examples/chat-room/frontend/web/src/App.tsx index 123ea8898..d0e421330 100644 --- a/reboot/examples/chat-room/frontend/web/src/App.tsx +++ b/reboot/examples/chat-room/frontend/web/src/App.tsx @@ -1,18 +1,133 @@ -import { FC, useState } from "react"; +import { Blob_Status } from "@reboot-dev/reboot-std-api/blob/v1/blob_pb.js"; +import { + useBlob, + useBlobDownloadUrl, + useBlobUpload, +} from "@reboot-dev/reboot-std-react/blob"; +import { FC, useRef, useState } from "react"; import css from "./App.module.css"; -import { useChatRoom } from "../../api/chat_room/v1/chat_room_rbt_react"; +import { + AttachmentTooLarge, + useChatRoom, +} from "../../api/chat_room/v1/chat_room_rbt_react"; // We can choose any id we want because the state will be constructed when we // make the first .writer call. const STATE_MACHINE_ID = "reboot-chat-room"; -const Message: FC<{ text: string }> = ({ text }) => { - return
{text}
; +// A circular upload-progress indicator that fills clockwise. `fraction` +// is 0..1; when the total size isn't known yet it stays near empty. +const ProgressRing: FC<{ fraction: number }> = ({ fraction }) => { + const radius = 13; + const circumference = 2 * Math.PI * radius; + return ( + + + + + ); }; -const PendingMessage: FC<{ text: string; isLoading: boolean }> = ({ +// A fixed-size square for one attachment. While the blob is uploading +// it shows a filling ring (progress lives on the blob's state, so this +// is visible to every participant); once committed it shows a +// thumbnail (or a file glyph) that can be clicked to view the content. +const Attachment: FC<{ blobId: string }> = ({ blobId }) => { + const { response: info } = useBlob({ id: blobId }).useInfo(); + const url = useBlobDownloadUrl(blobId); + const [viewing, setViewing] = useState(false); + + const isImage = info?.contentType.startsWith("image/") ?? false; + const committed = info?.status === Blob_Status.COMMITTED && url !== undefined; + const gone = + info?.status === Blob_Status.REMOVING || + info?.status === Blob_Status.REMOVED; + + // Only an exact `size` gives a definite total to measure against; a + // `max_size` upper bound leaves progress indeterminate. + const total = Number( + info?.sizeLimit.case === "size" ? info.sizeLimit.value : 0 + ); + const uploaded = Number(info?.bytesUploaded ?? 0); + const fraction = total > 0 ? uploaded / total : 0; + + const view = () => { + if (!committed) return; + if (isImage) { + setViewing(true); + } else { + window.open(url, "_blank", "noopener"); + } + }; + + return ( + <> + + {viewing && committed && isImage && ( +
setViewing(false)}> + attachment +
+ )} + + ); +}; + +const Attachments: FC<{ blobIds: string[] }> = ({ blobIds }) => + blobIds.length > 0 ? ( +
+ {blobIds.map((blobId) => ( + + ))} +
+ ) : null; + +const Message: FC<{ text: string; attachmentBlobIds: string[] }> = ({ text, - isLoading, + attachmentBlobIds, }) => { + return ( +
+ {text !== "" &&
{text}
} + +
+ ); +}; + +// An optimistically-rendered message that hasn't been confirmed by the +// backend yet. Its attachments have no blob id, so we show empty +// placeholder squares to keep the layout stable until the real message +// arrives (with squares that then track upload progress). +const PendingMessage: FC<{ + text: string; + attachmentCount: number; + isLoading: boolean; +}> = ({ text, attachmentCount, isLoading }) => { return (
= ({ : `${css.message} ${css.pendingMessage}` } > - {text} + {text !== "" &&
{text}
} + {attachmentCount > 0 && ( +
+ {Array.from({ length: attachmentCount }, (_, index) => ( +
+ +
+ ))} +
+ )}
); }; const App = () => { - // State of the input component. + // State of the input components. const [message, setMessage] = useState("Hello, Reboot!"); + const [file, setFile] = useState(null); + const [error, setError] = useState(null); + const fileInputRef = useRef(null); const { useMessages, send } = useChatRoom({ id: STATE_MACHINE_ID }); const { response } = useMessages(); + const { upload } = useBlobUpload(); - const handleClick = async () => { - const { aborted } = await send({ message: message }); + const clearFile = () => { + setFile(null); + if (fileInputRef.current !== null) { + fileInputRef.current.value = ""; + } + }; + + const canSend = message !== "" || file !== null; + + const handleSend = async () => { + if (!canSend) { + return; + } + setError(null); + // Publishes the message immediately; the backend creates a `Blob` + // per requested attachment and returns its id. Every participant + // sees the message right away, with attachments visibly uploading. + const { response, aborted } = await send({ + message: message, + attachments: + file !== null + ? [{ contentType: file.type, sizeBytes: BigInt(file.size) }] + : [], + }); if (aborted !== undefined) { - console.warn(aborted.error.getType()); - console.warn(aborted.message); + // Surface the failure to the user. The backend's + // `AttachmentTooLarge` error carries the limit, so we can say + // exactly what it is. + if (aborted.error instanceof AttachmentTooLarge) { + const maxMebibytes = Number(aborted.error.maxSizeBytes) / (1024 * 1024); + setError( + `That attachment is too large. The maximum is ${maxMebibytes} MiB.` + ); + } else { + setError(`Couldn't send your message: ${aborted.message}`); + } + return; } + const uploadFile = file; setMessage(""); + clearFile(); + + if (uploadFile !== null && response !== undefined) { + // Uploads directly to the data plane (the app's filesystem + // store locally, S3 on the Cloud -- this code neither knows + // nor cares), reporting progress onto the blob's state as it + // goes. + const { error } = await upload(response.attachmentBlobIds[0], uploadFile); + if (error !== undefined) { + console.warn(`Attachment upload failed: ${error}`); + } + } }; return (
+
+
+ + setMessage(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter" && canSend) { + handleSend(); + } + }} + value={message} + placeholder="Your message here..." + /> +
+ +
+ setMessage(e.target.value)} - onKeyDown={(e) => { - if (e.key === "Enter" && message !== "") { - handleClick(); - } + ref={fileInputRef} + type="file" + className={css.hiddenFileInput} + onChange={(e) => { + setError(null); + setFile(e.target.files?.[0] ?? null); }} - value={message} - placeholder="Your message here..." /> - + + {error !== null && ( +
setError(null)} + title="Dismiss" + > + {error} + × +
+ )} + + {file !== null && ( +
+ 📎 {file.name} + +
+ )} + {(response !== undefined && response.messages.length > 0 && - response.messages.map((message: string) => ( - + response.messages.map((message, index) => ( + ))) || (response !== undefined && response.messages.length === 0 && (

No messages yet!

@@ -78,8 +303,13 @@ const App = () => { been received that includes the mutation's updates so you don't have to worry about mutators racing with readers! */} - {send.pending.map(({ request: { message }, isLoading }) => ( - + {send.pending.map(({ request, isLoading }, index) => ( + ))} {/* If we're loading our first response, show the user a loading message, diff --git a/reboot/examples/chat-room/tests/chat_room.feature b/reboot/examples/chat-room/tests/chat_room.feature index 7cd8f605b..f38a18ad4 100644 --- a/reboot/examples/chat-room/tests/chat_room.feature +++ b/reboot/examples/chat-room/tests/chat_room.feature @@ -6,8 +6,7 @@ Feature: Chat room Scenario: Messages record in order When "anonymous" does a `send` with `message="Hello, World"` on `ChatRoom` of "testing-chat-room" - Then as "anonymous", `messages` on the `ChatRoom` for "testing-chat-room" has `messages=["Hello, World"]` + Then as "anonymous", `messages` on the `ChatRoom` for "testing-chat-room" has `messages` of length `1` and `messages[0].text="Hello, World"` When "anonymous" does a `send` with `message="Hello, Reboot!"` on `ChatRoom` of "testing-chat-room" And "anonymous" does a `send` with `message="Hello, Peace of Mind!"` on `ChatRoom` of "testing-chat-room" - Then as "anonymous", `messages` on the `ChatRoom` for "testing-chat-room" has `messages=["Hello, World", "Hello, Reboot!", "Hello, Peace of Mind!"]` - And as "anonymous", `messages` on the `ChatRoom` for "testing-chat-room" has `messages` of length `3` and `messages` containing `"Hello, Reboot!"` + Then as "anonymous", `messages` on the `ChatRoom` for "testing-chat-room" has `messages` of length `3`, `messages[0].text="Hello, World"`, `messages[1].text="Hello, Reboot!"` and `messages[2].text="Hello, Peace of Mind!"` diff --git a/tests/reboot/examples/chat-room/BUILD.bazel b/tests/reboot/examples/chat-room/BUILD.bazel index a01a110ad..b56412bb2 100644 --- a/tests/reboot/examples/chat-room/BUILD.bazel +++ b/tests/reboot/examples/chat-room/BUILD.bazel @@ -124,9 +124,11 @@ sh_test_in_working_directory( # The locally built Reboot npm packages that the frontends are # type-checked against, instead of the published releases. frontend_packages = [ + "//rbt/std:reboot-dev-reboot-std-api", "//rbt/v1alpha1:reboot-dev-reboot-api", "//reboot:reboot.dev", "//reboot/nodejs:reboot-dev-reboot-" + REBOOT_VERSION + ".tgz", + "//reboot/std/react:reboot-dev-reboot-std-react", "//reboot/web:reboot-dev-reboot-web", "//reboot/react:reboot-dev-reboot-react", ] @@ -135,6 +137,8 @@ frontend_env = { "REBOOT_API_NPM_PACKAGE": "$(location //rbt/v1alpha1:reboot-dev-reboot-api)", "REBOOT_NPM_PACKAGE": "$(location //reboot/nodejs:reboot-dev-reboot-" + REBOOT_VERSION + ".tgz)", "REBOOT_REACT_NPM_PACKAGE": "$(location //reboot/react:reboot-dev-reboot-react)", + "REBOOT_STD_API_PACKAGE": "$(location //rbt/std:reboot-dev-reboot-std-api)", + "REBOOT_STD_REACT_PACKAGE": "$(location //reboot/std/react:reboot-dev-reboot-std-react)", "REBOOT_WEB_NPM_PACKAGE": "$(location //reboot/web:reboot-dev-reboot-web)", "REBOOT_WHL_FILE": "$(location //reboot:reboot.dev)", } diff --git a/tests/reboot/examples/chat-room/serve_expected_output.txt b/tests/reboot/examples/chat-room/serve_expected_output.txt index a8d6f7572..2a4093dcf 100644 --- a/tests/reboot/examples/chat-room/serve_expected_output.txt +++ b/tests/reboot/examples/chat-room/serve_expected_output.txt @@ -1,5 +1,8 @@ { "messages": [ - "Hello, World!" + { + "text": "Hello, World!", + "attachmentBlobIds": [] + } ] } From eed4d75c2f5fa390b7fac05b041c741064bc3dc6 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Thu, 10 Sep 2026 08:52:58 +0000 Subject: [PATCH 07/52] `reboot`: let a library contribute plain gRPC servicers An `Application` has always taken both `servicers` and `legacy_grpc_servicers`, but a `Library` could only contribute the first, so a library offering an interface that predates Reboot -- or one shared with something that does not speak Reboot -- had no way to bring it along. Its users had to be told to pass the servicer themselves, which is exactly the coupling a library exists to remove. `AbstractLibrary.legacy_grpc_servicers()` now sits beside `servicers()`, defaulting to none so that existing libraries are unaffected, and `Application` collects it the same way. Both lists go through the same validation as before, so a Reboot servicer offered here is still refused with the message pointing at `servicers()`. --- reboot/aio/applications.py | 6 ++++++ reboot/aio/libraries.py | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/reboot/aio/applications.py b/reboot/aio/applications.py index 74911a24b..d65b52f27 100644 --- a/reboot/aio/applications.py +++ b/reboot/aio/applications.py @@ -353,6 +353,12 @@ def __init__( else: servicers = library_servicers + legacy_grpc_servicers = legacy_grpc_servicers or [] + legacy_grpc_servicers.extend( + servicer for library in libraries + for servicer in library.legacy_grpc_servicers() + ) + if servicers is not None and len(servicers) == 0: raise ValueError("'servicers' can't be an empty list") diff --git a/reboot/aio/libraries.py b/reboot/aio/libraries.py index df9031ac2..fb7a822b4 100644 --- a/reboot/aio/libraries.py +++ b/reboot/aio/libraries.py @@ -26,6 +26,14 @@ def servicers(self) -> Sequence[type[Servicer]]: """ raise NotImplementedError + def legacy_grpc_servicers(self) -> Sequence[type]: + """ + Return the list of plain gRPC servicers for this library, for a + library that offers an interface predating Reboot or shared + with something that does not speak Reboot. + """ + return [] + async def initialize(self, context: InitializeContext) -> None: """ A function to allow libraries to run initialize steps after the From 620067e70d03c8fce0f12de2dd6a628341f33cd6 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Fri, 11 Sep 2026 19:56:17 +0000 Subject: [PATCH 08/52] `reboot`: remove an untrusted caller ID on every route, not one Envoy removes `x-reboot-caller-id` from traffic whose caller IDs it does not trust, which is what lets `is_app_internal` authorizers believe the header. The removal was attached to a single route: the one matching `content-type` exactly `application/grpc`. A request arriving as `application/grpc+proto`, which gRPC permits and grpc-core sends, misses that match and falls through to one of the per-method prefix routes, which carried no removal at all -- nor did the `/` route, nor the websocket and HTTP routes. The header arrives intact, and an application ID is not a secret: it is derived from the application's name. What a caller may claim about itself should not depend on which route their request happens to match, so the removal now belongs to the route configuration, which covers every route in it. This does not arise on Reboot Cloud, where the public listener is configured to trust caller IDs because the proxies in front of it ensure they are truthful. It does arise under `rbt serve run`. While here, servers offer themselves on loopback rather than on every local network address. Envoy reaches them from the same host, so loopback is enough, except in the two cases where the caller is outside it: Envoy running in a Docker container, and Reboot Cloud, where the platform reaches a server from outside its pod. Binding every address is not itself much of an exposure -- anything on the machine can already read the application's database off disk -- but there is no reason to offer what nothing needs. --- reboot/controller/server_managers.py | 14 +- reboot/routing/envoy_config.py | 16 +- tests/reboot/routing/BUILD.bazel | 14 ++ tests/reboot/routing/envoy_config_test.py | 91 ++++++++++++ tests/reboot/server/local_envoy_test.py | 171 +++++++++++++++++++++- 5 files changed, 297 insertions(+), 9 deletions(-) create mode 100644 tests/reboot/routing/envoy_config_test.py diff --git a/reboot/controller/server_managers.py b/reboot/controller/server_managers.py index b5eb9fffb..affb5c9cc 100644 --- a/reboot/controller/server_managers.py +++ b/reboot/controller/server_managers.py @@ -47,6 +47,8 @@ ENVVAR_RBT_EFFECT_VALIDATION, ENVVAR_RBT_NODEJS, EVERY_LOCAL_NETWORK_ADDRESS, + ONLY_LOCALHOST_NETWORK_ADDRESS, + LocalEnvoyMode, ) from typing import Awaitable, Callable, Optional, Sequence @@ -794,7 +796,17 @@ async def _start_server( async def launch(): assert self._revision is not None - host = EVERY_LOCAL_NETWORK_ADDRESS + # Loopback is as much as a server needs to offer: what + # reaches it is Envoy, on the same host. Two cases need + # more. Envoy in a Docker container reaches the host from + # outside it, and on Reboot Cloud what reaches a server is + # not on its host at all. + host = ONLY_LOCALHOST_NETWORK_ADDRESS + if on_cloud() or ( + self._revision.local_envoy and + LocalEnvoyFactory.pick_mode() is LocalEnvoyMode.DOCKER + ): + host = EVERY_LOCAL_NETWORK_ADDRESS if not self._revision.in_process: return await self._launch_subprocess_server( diff --git a/reboot/routing/envoy_config.py b/reboot/routing/envoy_config.py index 79d1b1ed8..145d9510c 100644 --- a/reboot/routing/envoy_config.py +++ b/reboot/routing/envoy_config.py @@ -434,7 +434,6 @@ def _routes_for_server( server: ServerInfo, kind: ClusterKind, file_descriptor_set: FileDescriptorSet, - trust_caller_id: bool, ) -> list[route_components_pb2.Route]: # Every server gets routes to the websocket port, the gRPC # port, and the HTTP "catchall" port as described below. @@ -480,11 +479,6 @@ def _routes_for_server( max_stream_duration=route_components_pb2.RouteAction. MaxStreamDuration(grpc_timeout_header_max=ZERO_SECONDS) ), - request_headers_to_remove=( - # If we don't trust the caller ID header, remove it - # so that the upstream server can't be misled. - [CALLER_ID_HEADER] if not trust_caller_id else [] - ), ), # This route sends traffic with the 'x-reboot-server-id' # header and an exact path of '/' to the gRPC port, @@ -776,6 +770,15 @@ def _filter_http_connection_manager( codec_type=http_connection_manager_pb2.HttpConnectionManager.AUTO, route_config=route_pb2.RouteConfiguration( name="local_route", + # If we don't trust the caller ID header, remove it so that + # the upstream server can't be misled. This belongs to the + # whole route configuration rather than to any one route: + # what a caller may claim about itself cannot depend on + # which route their request happens to match, and a request + # matches whichever route it matches. + request_headers_to_remove=( + [CALLER_ID_HEADER] if not trust_caller_id else [] + ), virtual_hosts=[ route_components_pb2.VirtualHost( name="local_service", @@ -800,7 +803,6 @@ def _filter_http_connection_manager( server=server, kind=kind, file_descriptor_set=file_descriptor_set, - trust_caller_id=trust_caller_id, ) ] ), diff --git a/tests/reboot/routing/BUILD.bazel b/tests/reboot/routing/BUILD.bazel index daa1c98a5..dd1805fe5 100644 --- a/tests/reboot/routing/BUILD.bazel +++ b/tests/reboot/routing/BUILD.bazel @@ -30,6 +30,20 @@ diff_test( ], ) +py_test( + name = "envoy_config_test_py", + timeout = "short", + srcs = ["envoy_config_test.py"], + main = "envoy_config_test.py", + deps = [ + "//envoy/config/route/v3:routev3_py_proto", + "//envoy/extensions/filters/network/http_connection_manager/v3:http_connection_managerv3_py_proto", + "//reboot:helpers_py", + "//reboot/routing:envoy_config_py", + "//tests/reboot:greeter_servicers_py", + ], +) + py_test( name = "xds_server_test_py", timeout = "short", diff --git a/tests/reboot/routing/envoy_config_test.py b/tests/reboot/routing/envoy_config_test.py new file mode 100644 index 000000000..93855b34b --- /dev/null +++ b/tests/reboot/routing/envoy_config_test.py @@ -0,0 +1,91 @@ +import unittest +from envoy.config.route.v3 import route_pb2 +from envoy.extensions.filters.network.http_connection_manager.v3 import ( + http_connection_manager_pb2, +) +from google.protobuf.descriptor_pb2 import FileDescriptorSet +from pathlib import Path +from reboot.aio.headers import CALLER_ID_HEADER +from reboot.aio.types import ApplicationId +from reboot.helpers import add_file_descriptor_to_file_descriptor_set +from reboot.routing.envoy_config import ServerAddress, ServerInfo, listeners +from tests.reboot import greeter_pb2 + + +def _route_configuration( + listener, +) -> route_pb2.RouteConfiguration: + """The route configuration a listener serves.""" + filters = listener.filter_chains[0].filters + assert len(filters) == 1, filters + manager = http_connection_manager_pb2.HttpConnectionManager() + filters[0].typed_config.Unpack(manager) + return manager.route_config + + +class TestCallerIdRemoval(unittest.TestCase): + """An `is_app_internal` authorizer believes `x-reboot-caller-id` + because Envoy removes it from traffic whose caller IDs it does not + trust. That removal used to be attached to a single route, so a + request that matched any other one kept the header it arrived + with.""" + + def _listeners(self): + file_descriptor_set = FileDescriptorSet() + add_file_descriptor_to_file_descriptor_set( + return_set=file_descriptor_set, + file_descriptor=greeter_pb2.DESCRIPTOR, + routable_service_names=None, + ) + return listeners( + application_id=ApplicationId("testing"), + servers=[ + ServerInfo( + server_id="testing-c123456", + address=ServerAddress( + host="127.0.0.1", + grpc_port=1234, + websocket_port=1235, + http_port=1236, + ), + shards=[], + on_this_replica=True, + ), + ], + file_descriptor_set=file_descriptor_set, + trusted_host="127.0.0.1", + trusted_port=9992, + public_port=9991, + use_tls=False, + certificate_path=Path("certificate.pem"), + key_path=Path("key.pem"), + allowed_origins=None, + ) + + def test_an_untrusted_listener_removes_it_from_every_route(self) -> None: + by_name = {listener.name: listener for listener in self._listeners()} + public = _route_configuration(by_name["public"]) + + # Whatever route a request matches, it cannot bring its own + # caller ID in through the public port. + self.assertIn(CALLER_ID_HEADER, public.request_headers_to_remove) + self.assertGreater(len(public.virtual_hosts[0].routes), 1) + for route in public.virtual_hosts[0].routes: + self.assertNotIn( + CALLER_ID_HEADER, + route.request_headers_to_remove, + "the removal belongs to the route configuration, so that " + "it cannot be missing from a route added later", + ) + + def test_the_trusted_listener_keeps_it(self) -> None: + by_name = {listener.name: listener for listener in self._listeners()} + trusted = _route_configuration(by_name["trusted"]) + + # The trusted port is how the application reaches itself, and + # what it says about itself there is the whole point. + self.assertNotIn(CALLER_ID_HEADER, trusted.request_headers_to_remove) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/reboot/server/local_envoy_test.py b/tests/reboot/server/local_envoy_test.py index 9837d1fa9..c00521f92 100644 --- a/tests/reboot/server/local_envoy_test.py +++ b/tests/reboot/server/local_envoy_test.py @@ -14,7 +14,7 @@ from reboot.aio.auth.authorizers import allow from reboot.aio.contexts import ReaderContext, WriterContext from reboot.aio.external import ExternalContext -from reboot.aio.headers import SERVER_ID_HEADER +from reboot.aio.headers import CALLER_ID_HEADER, SERVER_ID_HEADER from reboot.aio.interceptors import LegacyGrpcContext from reboot.aio.tests import Reboot, temporary_environ from reboot.aio.types import StateRef, StateTypeName @@ -109,6 +109,105 @@ async def LegacyCall( return GeneralResponse(content=content) +class CallerIdServicer(LegacyGeneralServicer): + """Answers with the caller ID the request arrived carrying, so + that a test can see what a route let through.""" + + async def LegacyCall( + self, + request: GeneralRequest, + context: LegacyGrpcContext, + ) -> GeneralResponse: + content = Struct() + content[CALLER_ID_HEADER] = next( + ( + value for key, value in context.invocation_metadata() + if key == CALLER_ID_HEADER + ), + "", + ) + return GeneralResponse(content=content) + + +# A gRPC request carries a one-byte compression flag and a four-byte +# length before its message; an empty `GeneralRequest` is all header. +_EMPTY_GRPC_FRAME = b"\x00\x00\x00\x00\x00" + + +def _caller_id_seen_by( + endpoint: str, + *, + content_type: str, + caller_id: str, +) -> str: + """Calls `LegacyCall` over a hand-written HTTP/2 request, so that + its `content-type` can be one a client library would not send, and + answers the caller ID that reached the servicer. + + The empty string means none did.""" + host, _, port = endpoint.rpartition(":") + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((host, int(port))) + try: + connection = h2.connection.H2Connection( + config=h2.config.H2Configuration(client_side=True) + ) + connection.initiate_connection() + sock.sendall(connection.data_to_send()) + + stream_id = connection.get_next_available_stream_id() + connection.send_headers( + stream_id, + [ + (":method", "POST"), + (":path", "/tests.reboot.LegacyGeneral/LegacyCall"), + (":authority", endpoint), + (":scheme", "http"), + ("content-type", content_type), + ("te", "trailers"), + (CALLER_ID_HEADER, caller_id), + ], + ) + connection.send_data(stream_id, _EMPTY_GRPC_FRAME, end_stream=True) + sock.sendall(connection.data_to_send()) + + headers: dict = {} + trailers: dict = {} + data = b"" + while True: + received = sock.recv(4096) + if not received: + break + done = False + for event in connection.receive_data(received): + if isinstance(event, h2.events.ResponseReceived): + headers = dict(event.headers) + elif isinstance(event, h2.events.DataReceived): + data += event.data + elif isinstance(event, h2.events.TrailersReceived): + trailers = dict(event.headers) + elif isinstance(event, h2.events.StreamEnded): + done = True + sock.sendall(connection.data_to_send()) + if done: + break + finally: + sock.close() + + status = headers.get(b":status", b"") + grpc_status = headers.get(b"grpc-status") or trailers.get(b"grpc-status") + assert status == b"200" and grpc_status in (None, b"0"), ( + f"the call did not reach the servicer: HTTP {status!r}, " + f"grpc-status {grpc_status!r}, " + f"grpc-message {headers.get(b'grpc-message')!r}" + ) + + response = GeneralResponse() + response.ParseFromString(data[len(_EMPTY_GRPC_FRAME):]) + return response.content[CALLER_ID_HEADER] + + def _reader_path(state_id: str) -> str: """The path a JS client puts on the wire to call `tests.reboot.General`'s `Reader` on `state_id`.""" @@ -153,6 +252,76 @@ async def _post_exact_path(url: str, path: str) -> tuple[int, bytes]: await writer.wait_closed() +class CallerIdTestCase(unittest.IsolatedAsyncioTestCase): + """An `is_app_internal` authorizer believes `x-reboot-caller-id`, + so what a caller may claim about itself is decided entirely by + whether Envoy lets the header through.""" + + FORGED = "application_id=a-forged-application" + + async def _up(self) -> Reboot: + temporary_environ(self, {ENVVAR_LOCAL_ENVOY_DEBUG: 'true'}) + + rbt = Reboot() + await rbt.start() + self.addAsyncCleanup(rbt.stop) + + await rbt.up( + Application( + servicers=[IdentifierServicer], + legacy_grpc_servicers=[CallerIdServicer], + ), + local_envoy=True, + local_envoy_tls=False, + servers=1, + ) + return rbt + + async def test_the_public_port_removes_a_forged_caller_id(self) -> None: + # `application/grpc+proto` is a content type gRPC permits, and + # it is not the one the route carrying the removal used to + # match exactly -- so this request used to fall through to a + # per-method route and arrive with its caller ID intact. + rbt = await self._up() + + seen = await asyncio.to_thread( + _caller_id_seen_by, + rbt.url().removeprefix("http://"), + content_type="application/grpc+proto", + caller_id=self.FORGED, + ) + + self.assertEqual("", seen) + + async def test_the_public_port_removes_it_from_a_plain_grpc_call( + self, + ) -> None: + rbt = await self._up() + + seen = await asyncio.to_thread( + _caller_id_seen_by, + rbt.url().removeprefix("http://"), + content_type="application/grpc", + caller_id=self.FORGED, + ) + + self.assertEqual("", seen) + + async def test_the_trusted_port_keeps_it(self) -> None: + # The application reaches itself here, and what it says about + # itself is the whole point of the port. + rbt = await self._up() + + seen = await asyncio.to_thread( + _caller_id_seen_by, + f"localhost:{rbt.envoy_trusted_port()}", + content_type="application/grpc", + caller_id=self.FORGED, + ) + + self.assertEqual(self.FORGED, seen) + + class LocalEnvoyTestCase(unittest.IsolatedAsyncioTestCase): async def test_server_filter(self): From b348275a70f6f4990286eea3018f9f337a6bdc2f Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:35:59 +0000 Subject: [PATCH 09/52] `reboot/std`: give the filesystem store's metadata a type `meta.json` was read back as a bare `dict` and passed around as one, so every reader spelled its keys out by hand -- `meta.get("committed", False)`, `meta["upload_id"]`, `part["size"]` -- and a mistake in any of them was a runtime `KeyError` rather than something `mypy` could catch. The shape was also only discoverable by reading whichever writer happened to produce it. `BlobMetadata` and `PartRecord` now carry it, and `from_json` / `to_json` confine the `dict` to the one boundary where JSON makes it the right shape. The on-disk format is unchanged: absent fields are still omitted rather than written as `null`, so a store written by an earlier build reads back identically. --- reboot/std/blob/v1/_http.py | 22 ++--- reboot/std/blob/v1/_store.py | 107 +++++++++++++++++++------ tests/reboot/std/blob/v1/blob_tests.py | 9 ++- 3 files changed, 103 insertions(+), 35 deletions(-) diff --git a/reboot/std/blob/v1/_http.py b/reboot/std/blob/v1/_http.py index c9926673e..626bd5908 100644 --- a/reboot/std/blob/v1/_http.py +++ b/reboot/std/blob/v1/_http.py @@ -120,7 +120,7 @@ async def put_part(request: Request) -> Response: # part-PUT URL minted just before commit must not still be # usable to tamper with the bytes afterwards. meta = store.read_meta(blob) - if meta is not None and meta.get("committed", False): + if meta is not None and meta.committed: return Response(status_code=409, content="Blob already committed") # Write somewhere else and publish with a rename, rather than @@ -164,7 +164,7 @@ async def put_part(request: Request) -> Response: # belongs to has been committed. async with store.lock_for(blob): meta = store.read_meta(blob) - if meta is not None and meta.get("committed", False): + if meta is not None and meta.committed: os.unlink(temporary) return Response( status_code=409, content="Blob already committed" @@ -196,16 +196,18 @@ async def get_blob(request: Request) -> Response: return Response(status_code=403, content="Invalid signature") meta = store.read_meta(blob) - if meta is None or not meta.get("committed", False): + if meta is None or not meta.committed: return Response(status_code=404, content="No such blob") - upload_id = meta["upload_id"] - parts = meta["parts"] - total_size = sum(part["size"] for part in parts) + # Written in the same atomic update as `committed`. + assert meta.upload_id is not None and meta.etag is not None + upload_id = meta.upload_id + parts = meta.parts + total_size = sum(part.size for part in parts) async def stream(): - for part in sorted(parts, key=lambda part: part["number"]): - path = store.part_path(blob, upload_id, part["number"]) + for part in sorted(parts, key=lambda part: part.number): + path = store.part_path(blob, upload_id, part.number) # Read off the event loop: this generator is driven by # it, and a part is megabytes, so reading inline would # stall every other request this worker is serving. @@ -219,13 +221,13 @@ async def stream(): finally: await asyncio.to_thread(file.close) - media_type, safety_headers = download_headers(meta["content_type"]) + media_type, safety_headers = download_headers(meta.content_type) return StreamingResponse( stream(), media_type=media_type, headers={ "Content-Length": str(total_size), - "ETag": f'"{meta["etag"]}"', + "ETag": f'"{meta.etag}"', "Accept-Ranges": "none", **safety_headers, }, diff --git a/reboot/std/blob/v1/_store.py b/reboot/std/blob/v1/_store.py index 3d56dfe64..60862fcc7 100644 --- a/reboot/std/blob/v1/_store.py +++ b/reboot/std/blob/v1/_store.py @@ -20,7 +20,7 @@ import time from dataclasses import dataclass from reboot.crypto import root_keys -from typing import Optional, Sequence +from typing import Any, Optional, Sequence from uuid import uuid4 # The part size clients should use. Every part except the last must be @@ -68,6 +68,62 @@ class UploadedPart: size: int +@dataclass(frozen=True) +class PartRecord: + """One part of a committed blob: its number, and the size its + bytes on disk were found to have at completion.""" + number: int + size: int + + +@dataclass(frozen=True) +class BlobMetadata: + """A blob's entry in the store, as `meta.json` holds it. + + `upload_id`, `etag` and `parts` are written in the same atomic + update as `committed`, so a committed blob carries all three.""" + content_type: str + committed: bool + upload_id: Optional[str] = None + etag: Optional[str] = None + parts: tuple[PartRecord, ...] = () + + @classmethod + def from_json(cls, data: dict[str, Any]) -> "BlobMetadata": + return cls( + content_type=data["content_type"], + committed=data.get("committed", False), + upload_id=data.get("upload_id"), + etag=data.get("etag"), + parts=tuple( + PartRecord(number=part["number"], size=part["size"]) + for part in data.get("parts", ()) + ), + ) + + def to_json(self) -> dict[str, Any]: + """The on-disk form. A `dict` is the right shape at this one + boundary and nowhere else; absent fields are omitted rather + than written as `null`, so the file stays byte-comparable with + what earlier versions of this store wrote.""" + data: dict[str, Any] = { + "content_type": self.content_type, + "committed": self.committed, + } + if self.upload_id is not None: + data["upload_id"] = self.upload_id + if self.etag is not None: + data["etag"] = self.etag + if self.parts: + data["parts"] = [ + { + "number": part.number, + "size": part.size + } for part in self.parts + ] + return data + + def _encode_blob_id(blob_id: str) -> str: """Encodes a blob ID into a string safe for use as both a directory name and a URL path segment.""" @@ -184,21 +240,25 @@ def part_path( def _meta_path(self, encoded_blob_id: str) -> str: return os.path.join(self.blob_directory(encoded_blob_id), "meta.json") - def read_meta(self, encoded_blob_id: str) -> Optional[dict]: + def read_meta(self, encoded_blob_id: str) -> Optional[BlobMetadata]: try: with open(self._meta_path(encoded_blob_id), "r") as f: - return json.load(f) + return BlobMetadata.from_json(json.load(f)) except FileNotFoundError: return None - def _write_meta(self, encoded_blob_id: str, meta: dict) -> None: + def _write_meta( + self, + encoded_blob_id: str, + meta: BlobMetadata, + ) -> None: # Write to a temp file and atomically rename, so a crash # mid-write can never leave a torn `meta.json` that a # concurrent `read_meta` would fail to parse. path = self._meta_path(encoded_blob_id) temp_path = f"{path}.{uuid4().hex}.tmp" with open(temp_path, "w") as f: - json.dump(meta, f) + json.dump(meta.to_json(), f) f.flush() os.fsync(f.fileno()) os.replace(temp_path, path) @@ -213,11 +273,10 @@ def sync(): # orphaning it under a fresh upload ID. existing = self.read_meta(encoded) if ( - existing is not None and - not existing.get("committed", False) and - "upload_id" in existing + existing is not None and not existing.committed and + existing.upload_id is not None ): - upload_id = existing["upload_id"] + upload_id = existing.upload_id reuse = True else: upload_id = uuid4().hex @@ -231,11 +290,11 @@ def sync(): if not reuse: self._write_meta( encoded, - { - "content_type": content_type, - "committed": False, - "upload_id": upload_id, - }, + BlobMetadata( + content_type=content_type, + committed=False, + upload_id=upload_id, + ), ) return upload_id @@ -269,7 +328,7 @@ async def complete( def sync(): encoded = _encode_blob_id(blob_id) digests = [] - manifest = [] + manifest: list[PartRecord] = [] total_size = 0 last_part_number = max(part.number for part in parts) for part in sorted(parts, key=lambda part: part.number): @@ -309,7 +368,9 @@ def sync(): ) total_size += size digests.append(digest.digest()) - manifest.append({"number": part.number, "size": size}) + manifest.append( + PartRecord(number=part.number, size=size) + ) # Verify the *real* total against `max_size` (not the # already-checked reported sizes) as defense in depth. @@ -326,13 +387,13 @@ def sync(): ) self._write_meta( encoded, - { - "content_type": content_type, - "committed": True, - "etag": etag, - "upload_id": upload_id, - "parts": manifest, - }, + BlobMetadata( + content_type=content_type, + committed=True, + upload_id=upload_id, + etag=etag, + parts=tuple(manifest), + ), ) return etag diff --git a/tests/reboot/std/blob/v1/blob_tests.py b/tests/reboot/std/blob/v1/blob_tests.py index bbc2ec119..d3ee5049e 100644 --- a/tests/reboot/std/blob/v1/blob_tests.py +++ b/tests/reboot/std/blob/v1/blob_tests.py @@ -16,6 +16,7 @@ from reboot.aio.tests import Reboot from reboot.std.blob.v1._store import ( DEFAULT_PART_SIZE_BYTES, + BlobMetadata, BlobStoreError, FilesystemBlobStore, UploadedPart, @@ -386,10 +387,14 @@ async def test_a_part_put_cannot_land_inside_completion( may_record = threading.Event() write_meta = FilesystemBlobStore._write_meta - def paused_write_meta(self, encoded_blob_id: str, meta) -> None: + def paused_write_meta( + self, + encoded_blob_id: str, + meta: BlobMetadata, + ) -> None: # Only completion records a committed blob; `begin_upload` # writes metadata too, and must not be paused. - if meta.get("committed", False): + if meta.committed: reached_recording.set() may_record.wait(timeout=_RACE_TIMEOUT_SECONDS) write_meta(self, encoded_blob_id, meta) From fb743523d1c088138e8b98ac36cdefefad2b3062 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:38:32 +0000 Subject: [PATCH 10/52] `reboot/std`: stop writing part uploads from the event loop The download handler took care to read part files off the event loop, but the upload handler beside it wrote them from it: `f.write(chunk)` and `os.fsync()` ran inline while a part -- megabytes of it -- came off the wire, stalling every other request the worker was serving. `aiofiles`, already a dependency of this repo, now backs both byte paths. The download loop keeps the same one-hop-per-chunk cost it already paid and just says so more directly; the upload loop stops blocking. `fsync` stays an explicit `asyncio.to_thread`, since `aiofiles` does not wrap it. Deliberately not applied to `_store.py`. Its `asyncio.to_thread(sync)` calls each carry a whole operation -- `complete()` reads every part and digests it -- so one hop is amortized across all of that work. Per-call `await`s there would turn a single hop into thousands and break up sequences that are easier to reason about whole. --- reboot/std/blob/v1/BUILD.bazel | 1 + reboot/std/blob/v1/_http.py | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/reboot/std/blob/v1/BUILD.bazel b/reboot/std/blob/v1/BUILD.bazel index 2d35f4a20..ddd67e723 100644 --- a/reboot/std/blob/v1/BUILD.bazel +++ b/reboot/std/blob/v1/BUILD.bazel @@ -24,6 +24,7 @@ py_library( "@com_github_reboot_dev_reboot//log:log_py", "@com_github_reboot_dev_reboot//rbt/std/blob/v1:blob_py_reboot", "@com_github_reboot_dev_reboot//rbt/std/blob/v1:data_plane_py_reboot", + requirement("aiofiles"), requirement("aiohttp"), requirement("grpcio"), requirement("starlette"), diff --git a/reboot/std/blob/v1/_http.py b/reboot/std/blob/v1/_http.py index 626bd5908..116abf74e 100644 --- a/reboot/std/blob/v1/_http.py +++ b/reboot/std/blob/v1/_http.py @@ -13,6 +13,7 @@ the application. """ +import aiofiles import asyncio import hashlib import hmac @@ -133,15 +134,21 @@ async def put_part(request: Request) -> Response: digest = hashlib.md5() size = 0 try: - with open(temporary, "wb") as f: + # A part is megabytes, so the writes go off the event loop + # for the same reason the download below reads off it: this + # handler is driven by the loop, and writing inline would + # stall every other request this worker is serving. + async with aiofiles.open(temporary, "wb") as f: async for chunk in request.stream(): if size + len(chunk) > store.part_size: raise _PartTooLarge() digest.update(chunk) size += len(chunk) - f.write(chunk) - f.flush() - os.fsync(f.fileno()) + await f.write(chunk) + await f.flush() + # `aiofiles` has no `fsync`; `fileno()` is proxied + # straight through, so the descriptor is the real one. + await asyncio.to_thread(os.fsync, f.fileno()) except _PartTooLarge: os.unlink(temporary) return Response( @@ -211,15 +218,9 @@ async def stream(): # Read off the event loop: this generator is driven by # it, and a part is megabytes, so reading inline would # stall every other request this worker is serving. - file = await asyncio.to_thread(open, path, "rb") - try: - while chunk := await asyncio.to_thread( - file.read, - _STREAM_CHUNK_SIZE, - ): + async with aiofiles.open(path, "rb") as file: + while chunk := await file.read(_STREAM_CHUNK_SIZE): yield chunk - finally: - await asyncio.to_thread(file.close) media_type, safety_headers = download_headers(meta.content_type) return StreamingResponse( From 562687dc66f8057b7625a7bdc70aca2dae015906 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Thu, 10 Sep 2026 08:27:16 +0000 Subject: [PATCH 11/52] `reboot/std`: make the filesystem blob data plane part of the application The data plane was a program of its own: a gRPC server, an HTTP server, and a `meta.json` per blob, started as a subprocess and reached over localhost. The application then reverse-proxied byte traffic to it, so every uploaded and downloaded megabyte went through the application's Python on its way somewhere else in the same process tree. Being alone was also the only thing that made it correct: one `asyncio.Lock` ordered a part's publication against a commit, which holds exactly while one process serves every request. It is now three ordinary parts of the application that uses it. The `BlobDataPlane` gRPC service is a legacy gRPC servicer, so the control plane still speaks the one interface every data plane implements and Reboot routes to it by name -- no address to configure, and none that could be, since the servers hosting it are not running when an application's environment is composed. Its byte endpoints are routes on the application's own HTTP server, so bytes arrive on the origin they were always going to, with no hop in between. And its metadata is `StoredBlob`, a state machine. That last one is what earns the change rather than merely tidying it. A part's bytes are written by whichever of a replica's servers served the upload, and whether that part is *in* the object is now decided in one place all of them share, so the answer no longer depends on a lock local to one process. A part is written under a name nothing reads and published only once `StoredBlob` agrees it belongs; one that arrives after the commit is refused, and its bytes are dropped rather than left on top of what the recorded ETag describes. Completion no longer re-reads the parts either: it derives the object's ETag from the digests taken while the bytes were being written, so there is no window in which a part could change under it. Starting it moves with it. It was started from three places -- `rbt dev run`, `rbt serve run` and the test harness -- each with its own idea of where bytes go, its own "unless already configured" check, and its own teardown, and the two CLI paths needed a ready file, a port-reporting protocol and a separate copy of the crypto root keys just to sign URLs with. A data plane inside the application is started by the application, once, from `Application.run()`, which already knows where the run's state goes -- so blobs land beside it without anyone computing a path, and the server subprocesses `server_managers` forks never start one because they leave `Application.run()` earlier. Being started there is also what puts the two checks where the decision to store bytes locally is actually made. Reboot Cloud sets a data-plane URL of its own, so the check against it stays honest in one line rather than three. And the filesystem data plane keeps bytes on the disk of the replica running it, which a second replica cannot see: an application spread across replicas would serve a download or a 404 depending on where the request landed, and an upload's parts could go somewhere completion never looks. `LocalConfigExtractor` already parses `REBOOT_REPLICA_CONFIG` to plan placement, so it now offers the count, and anything above one is refused. Servers are not counted and do not matter: every server of a replica shares that replica's disk, so `rbt dev run`'s two and `rbt serve run`'s one per core are as fine as they always were. Cloud's own check does not subsume this one -- a customer replicating an application themselves is exactly the case that would otherwise fail quietly. `_proxy.py` goes, and with it the forwarded-path machinery it existed for: a data plane inside the application has nothing to forward to. The standalone server goes. Splitting `data_plane_py` out of `blob_py` is what makes the runtime's side of this legal: the data plane stores and serves bytes and does not know what a `Blob` is, so it can sit below `//reboot/aio` where `blob_py` -- which does know -- cannot. --- rbt/std/blob/v1/BUILD.bazel | 22 + rbt/std/blob/v1/filesystem.proto | 203 +++++++ reboot/aio/BUILD.bazel | 1 - reboot/aio/applications.py | 10 + reboot/aio/http.py | 9 + reboot/aio/reboot.py | 10 + reboot/aio/tests.py | 55 +- reboot/cli/commands/BUILD.bazel | 2 - reboot/cli/commands/dev.py | 57 -- reboot/cli/commands/serve.py | 35 +- reboot/cli/common/BUILD.bazel | 11 - reboot/cli/common/blob_data_plane.py | 141 ----- reboot/controller/BUILD.bazel | 15 +- reboot/controller/config_extractor.py | 19 +- reboot/controller/replicas.py | 31 + reboot/controller/server_managers.py | 13 +- reboot/std/blob/v1/BUILD.bazel | 10 +- reboot/std/blob/v1/_data_plane.py | 187 ++++-- reboot/std/blob/v1/_filesystem_data_plane.py | 366 ++++++++++++ reboot/std/blob/v1/_filesystem_server.py | 334 ----------- reboot/std/blob/v1/_http.py | 256 ++++----- reboot/std/blob/v1/_proxy.py | 161 ------ reboot/std/blob/v1/_store.py | 567 ++++++++++--------- reboot/std/blob/v1/_stored_blob.py | 145 +++++ reboot/std/blob/v1/blob.py | 192 ++++--- reboot/std/blob/v1/index.ts | 9 +- tests/reboot/std/blob/v1/blob_tests.py | 213 +++---- 27 files changed, 1605 insertions(+), 1469 deletions(-) create mode 100644 rbt/std/blob/v1/filesystem.proto delete mode 100644 reboot/cli/common/blob_data_plane.py create mode 100644 reboot/controller/replicas.py create mode 100644 reboot/std/blob/v1/_filesystem_data_plane.py delete mode 100644 reboot/std/blob/v1/_filesystem_server.py delete mode 100644 reboot/std/blob/v1/_proxy.py create mode 100644 reboot/std/blob/v1/_stored_blob.py diff --git a/rbt/std/blob/v1/BUILD.bazel b/rbt/std/blob/v1/BUILD.bazel index 82dfa257c..1562c3ca0 100644 --- a/rbt/std/blob/v1/BUILD.bazel +++ b/rbt/std/blob/v1/BUILD.bazel @@ -41,6 +41,28 @@ py_reboot_library( visibility = ["//visibility:public"], ) +# The filesystem data plane's own metadata state. Python-only: it is +# an implementation detail of one data plane, not part of any client's +# API. +proto_library( + name = "filesystem_proto", + srcs = [ + ":filesystem.proto", + ], + visibility = ["//visibility:public"], + deps = [ + "@com_github_reboot_dev_reboot//rbt/v1alpha1:options_proto", + "@com_google_protobuf//:descriptor_proto", + ], +) + +py_reboot_library( + name = "filesystem_py_reboot", + proto = "filesystem.proto", + proto_library = ":filesystem_proto", + visibility = ["//visibility:public"], +) + py_reboot_library( name = "blob_py_reboot", proto = "blob.proto", diff --git a/rbt/std/blob/v1/filesystem.proto b/rbt/std/blob/v1/filesystem.proto new file mode 100644 index 000000000..3c849895b --- /dev/null +++ b/rbt/std/blob/v1/filesystem.proto @@ -0,0 +1,203 @@ +// The filesystem blob data plane's own metadata. +// +// `data_plane.proto` defines the interface a data plane implements; +// this is the state behind one particular implementation, the +// filesystem data plane that Reboot applications get locally when no +// other data plane is provided. Bytes live on disk as part files. +// What they add up to -- which parts exist, how big they are, whether +// the object has been committed -- lives here. +// +// It lives in Reboot state rather than beside the bytes because a +// replica runs several servers, each hosting the data plane, and every +// one of them may serve a part upload or a download for the same blob. +// Publishing a part and committing an object have to be ordered +// against each other across all of them, which a state machine does +// and a file does not. + +syntax = "proto3"; + +package rbt.std.blob.v1; + +import "rbt/v1alpha1/options.proto"; + +// One part of an object, as its bytes on disk were found to be when +// the part was written. +message StoredPart { + // 1-based part number, following S3 multipart numbering. + uint32 number = 1; + + // The size of the part's bytes. + uint64 size = 2; + + // The MD5 of the part's bytes, computed while writing them rather + // than taken from the client, so that it describes what is on disk. + // An ETag, which is all S3 promises it to be: two different parts + // may share one, so it does not identify stored bytes. + string etag = 3; + + // Which of the files written for this part number holds these + // bytes. Minted per write, so that a part uploaded a second time + // never lands on the first one's bytes -- not even when the two + // have the same ETag. + string storage_id = 4; +} + +// One blob's bytes, as the filesystem data plane holds them. Keyed by +// the same blob ID the `Blob` control plane uses, so that the two can +// be reasoned about together without a second identifier. +message StoredBlob { + option (rbt.v1alpha1.state) = { + trusted_effects: true, + }; + + // Whether the object is finished. A committed object's bytes are + // immutable: a part arriving afterwards is refused rather than + // allowed to change what the recorded ETag describes. + bool committed = 1; + + // MIME type to serve the bytes with. + string content_type = 2; + + // The upload session the parts belong to. Absent before the first + // `BeginUpload`. + optional string upload_id = 3; + + // The object's ETag, derived from its parts' own. Absent until + // committed. + optional string etag = 4; + + // Before `Commit`, every part written so far; after it, exactly + // the parts the object is made of. A part on disk that is not + // listed here belongs to nothing, which is what lets both a part + // that lost the race against `Commit` and one the client chose not + // to commit be left where they fall. + repeated StoredPart parts = 5; +} + +//////////////////////////////////////////////////////////////////////// + +message StoredBlobBeginUploadRequest { + // MIME type to serve the bytes with. + string content_type = 1; +} + +message StoredBlobBeginUploadResponse { + // The session to write parts under, whether freshly minted or the + // one an earlier call already established. + string upload_id = 1; +} + +message StoredBlobPublishPartRequest { + // The session the part was written under. A part written under a + // session that is no longer the current one belongs to an abandoned + // upload, and is refused. + string upload_id = 1; + + // The part, as its bytes on disk were found to be. + StoredPart part = 2; +} + +message StoredBlobPublishPartResponse { + // Whether the part became part of the object. False when the object + // was committed, or the session abandoned, before this part arrived: + // its bytes are not in the object and the caller should drop them. + // Not an error, because losing that race is a normal outcome of an + // upload that ran alongside a commit. + bool published = 1; + + // The write this one displaced, when it displaced one: a part + // uploaded twice. Nothing is made of those bytes any more, so the + // caller removes them rather than leaving a file per attempt for an + // upload that may never be committed. + optional string superseded_storage_id = 2; +} + +message StoredBlobCommitRequest { + // The session being committed. + string upload_id = 1; + + // MIME type to serve the bytes with. + string content_type = 2; + + // The object's ETag, derived from the parts below. + string etag = 3; + + // Exactly the parts the object is made of, ordered by part number. + // The client chooses these by reporting them, so a part that was + // written but never reported is not in the object -- matching S3, + // where completion carries the manifest rather than inferring it. + repeated StoredPart parts = 4; +} + +message StoredBlobCommitResponse { + // Whether the object was finished. False when a part was uploaded + // again between this manifest being read and being committed: the + // bytes it names are gone, so the upload has to be reported and + // committed again. + bool committed = 1; +} + +message StoredBlobMetadataRequest {} + +message StoredBlobMetadataResponse { + // Absent when nothing has ever been stored under this blob ID, which + // a caller must distinguish from an uncommitted object: one may + // still receive parts, the other may not. + optional StoredBlob blob = 1; +} + +message StoredBlobForgetRequest {} + +message StoredBlobForgetResponse {} + +//////////////////////////////////////////////////////////////////////// + +service StoredBlobMethods { + // Establishes the upload session parts are written under, and + // returns it. The control plane retries this inside a workflow, so + // the caller asks for it once per workflow: a retry replays the + // session that was already established rather than minting a + // second one and orphaning the parts written under the first. + rpc BeginUpload(StoredBlobBeginUploadRequest) + returns (StoredBlobBeginUploadResponse) { + option (rbt.v1alpha1.method) = { + writer: { constructor: {} }, + }; + } + + // Records a part as belonging to the object, or refuses it because + // the object is committed or the session abandoned. This is the + // whole reason this metadata is a state: a part's bytes are written + // by whichever server served the upload, and this is where those + // writes are ordered against `Commit` across all of them. + rpc PublishPart(StoredBlobPublishPartRequest) + returns (StoredBlobPublishPartResponse) { + option (rbt.v1alpha1.method) = { + writer: {}, + }; + } + + // Finishes the object, fixing its parts and its ETag. A part not + // published by now is not in it. + rpc Commit(StoredBlobCommitRequest) returns (StoredBlobCommitResponse) { + option (rbt.v1alpha1.method) = { + writer: {}, + }; + } + + // The metadata, reporting absence rather than raising, since a blob + // that was never begun and one that is merely uncommitted lead to + // different answers. + rpc Metadata(StoredBlobMetadataRequest) returns (StoredBlobMetadataResponse) { + option (rbt.v1alpha1.method) = { + reader: {}, + }; + } + + // Forgets the object, for a blob whose bytes are being deleted. + rpc Forget(StoredBlobForgetRequest) returns (StoredBlobForgetResponse) { + option (rbt.v1alpha1.method) = { + writer: {}, + }; + } +} diff --git a/reboot/aio/BUILD.bazel b/reboot/aio/BUILD.bazel index 68503ea5a..cd2321c56 100644 --- a/reboot/aio/BUILD.bazel +++ b/reboot/aio/BUILD.bazel @@ -525,7 +525,6 @@ py_library( "//reboot/aio/auth:oauth_providers_py", "//reboot/aio/auth:oauth_py", "//reboot/aio/auth:oauth_server_py", - "//reboot/std/blob/v1:blob_py", ], ) diff --git a/reboot/aio/applications.py b/reboot/aio/applications.py index d65b52f27..be42ed617 100644 --- a/reboot/aio/applications.py +++ b/reboot/aio/applications.py @@ -1121,6 +1121,16 @@ async def run(self) -> NoReturn: # `rbt` CLI that spawned us. check_expected_version() + # Published so that everything that keeps state beside the + # database -- the libraries set up below, and the servers this + # process spawns, which inherit its environment -- derives the + # same directory from the environment, whether `rbt` named one + # or a temporary one was picked for an unnamed run. + if self._rbt is not None and self._rbt.state_directory is not None: + os.environ[ENVVAR_RBT_STATE_DIRECTORY] = str( + self._rbt.state_directory + ) + # Before running, do any pre-run library set up. for library in self.libraries: await library.pre_run(self) diff --git a/reboot/aio/http.py b/reboot/aio/http.py index 3c1ae418a..2f2d020eb 100644 --- a/reboot/aio/http.py +++ b/reboot/aio/http.py @@ -299,6 +299,15 @@ async def external_context_middleware(request: Request, call_next): request.state.reboot_external_context = ( external_context_from_request(request) ) + # Offered rather than applied, for a handler that can + # establish a caller's right itself and only then wants to + # act on the application's behalf. Reaching for this is a + # handler saying it has done that; the route-level + # `app_internal=True` above, which grants the same thing + # on the strength of a path alone, cannot make that check. + request.state.reboot_app_internal_context = ( + app_internal_external_context_from_request + ) return await call_next(request) diff --git a/reboot/aio/reboot.py b/reboot/aio/reboot.py index fb4ce096d..a5dda5660 100644 --- a/reboot/aio/reboot.py +++ b/reboot/aio/reboot.py @@ -171,6 +171,10 @@ def __init__( assert database_address is not None + # Where this instance keeps state on local disk; `None` when an + # external database holds it instead. + self._state_directory = state_directory + self._application_metadata = ApplicationMetadata( application_id=self._application_id, database_address=database_address, @@ -193,6 +197,12 @@ def __init__( '127.0.0.1:0' ) + @property + def state_directory(self) -> Optional[Path]: + """Where this instance keeps state on local disk, or `None` when + an external database holds it instead.""" + return self._state_directory + async def start(self): # Monitor the "parent" event loop; this is in addition to # similar monitoring that gets set up for the server processes. diff --git a/reboot/aio/tests.py b/reboot/aio/tests.py index ba68e62d9..2d3a2f4e1 100644 --- a/reboot/aio/tests.py +++ b/reboot/aio/tests.py @@ -2,7 +2,6 @@ import os import reboot.aio.reboot import secrets -import tempfile import unittest from reboot.aio.applications import Application, NodeApplication from reboot.aio.auth.oauth import OAuth @@ -23,12 +22,11 @@ from reboot.aio.servicers import Servicer from reboot.run_environments import in_nodejs, on_cloud, running_rbt_serve from reboot.settings import ( + ENVVAR_RBT_STATE_DIRECTORY, ENVVAR_REBOOT_CRYPTO_ROOT_KEYS, ENVVAR_REBOOT_ENABLE_EVENT_LOOP_BLOCKED_WATCHDOG, ENVVAR_REBOOT_IN_TEST, ) -from reboot.std.blob.v1._data_plane import ENVVAR_BLOB_DATA_PLANE_URL -from reboot.std.blob.v1._filesystem_server import FilesystemDataPlane from typing import ( Any, Awaitable, @@ -166,47 +164,6 @@ def __init__(self) -> None: os.environ[ENVVAR_REBOOT_IN_TEST] = 'true' # The application under test, or `None` before one is started. self._application: Optional[Application] = None - self._blob_data_plane: Optional[FilesystemDataPlane] = None - self._blob_data_plane_directory: Optional[tempfile.TemporaryDirectory - ] = None - - async def start(self): - result = await super().start() - # Run a filesystem blob data plane for the duration of the - # test, so that applications using `reboot.std.blob` work in - # unit tests exactly as they do under `rbt dev run` (which - # spawns the same data plane). An already-configured data plane - # is honored, mirroring the CLI — including one set up by - # another live `Reboot` instance in this process, which then - # must outlive this instance's use of it. - if not os.environ.get(ENVVAR_BLOB_DATA_PLANE_URL): - self._blob_data_plane_directory = tempfile.TemporaryDirectory( - prefix="reboot-test-blobs-" - ) - self._blob_data_plane = await FilesystemDataPlane.start( - directory=self._blob_data_plane_directory.name, - ) - os.environ[ENVVAR_BLOB_DATA_PLANE_URL] = ( - self._blob_data_plane.url - ) - return result - - async def stop(self) -> None: - try: - await super().stop() - finally: - if self._blob_data_plane is not None: - await self._blob_data_plane.stop() - # Only clear the env var if it still points at our data - # plane; another `Reboot` instance may have replaced it - # with its own in the meantime. - if os.environ.get(ENVVAR_BLOB_DATA_PLANE_URL - ) == (self._blob_data_plane.url): - os.environ.pop(ENVVAR_BLOB_DATA_PLANE_URL, None) - self._blob_data_plane = None - if self._blob_data_plane_directory is not None: - self._blob_data_plane_directory.cleanup() - self._blob_data_plane_directory = None async def make_valid_oauth_access_token( self, @@ -467,6 +424,16 @@ async def up( # Should only have `application`, `local_envoy`, # `local_envoy_port`, `servers`, `effect_validation`. + # Published for the libraries set up below and the servers + # brought up after them, just as `Application.run()` does for a + # real run. Published at `up` rather than at construction + # because an `Application` a test constructs under a faked + # `RBT_DEV` builds a `Reboot` of its own from this variable, + # and that one needs a directory of its own rather than this + # harness's database, which is open in this same process. + if self.state_directory is not None: + os.environ[ENVVAR_RBT_STATE_DIRECTORY] = str(self.state_directory) + # Do any pre-run library set up, just like `Application.run()` # does; e.g. a library may register HTTP routes. Libraries must # tolerate being `pre_run` more than once, since a test may diff --git a/reboot/cli/commands/BUILD.bazel b/reboot/cli/commands/BUILD.bazel index f5b91e00e..babc364a4 100644 --- a/reboot/cli/commands/BUILD.bazel +++ b/reboot/cli/commands/BUILD.bazel @@ -38,7 +38,6 @@ py_library( "//rbt/std/presence/v1:presence_py_reboot", "//reboot/aio:aborted_py", "//reboot/aio:external_py", - "//reboot/cli/common:blob_data_plane_py", "//reboot/cli/common:dev_extra_py", "//reboot/cli/common:directories_py", "//reboot/cli/common:frontend_py", @@ -124,7 +123,6 @@ py_library( deps = [ requirement("aiofiles"), ":dev_py", - "//reboot/cli/common:blob_data_plane_py", "//reboot/cli/common:detect_cores_py", "//reboot/cli/common:directories_py", "//reboot/cli/common:frontend_py", diff --git a/reboot/cli/commands/dev.py b/reboot/cli/commands/dev.py index 0bd978c03..2c6241a25 100644 --- a/reboot/cli/commands/dev.py +++ b/reboot/cli/commands/dev.py @@ -34,10 +34,6 @@ # We import the whole `terminal` module (as opposed to the methods it contains) # to allow us to mock these methods out in tests. from reboot.cli.common import terminal -from reboot.cli.common.blob_data_plane import ( - BLOBS_SUBDIRECTORY, - start_filesystem_data_plane, -) from reboot.cli.common.dev_extra import dev_extra_installed, missing_dev_extra from reboot.cli.common.directories import ( add_working_directory_options, @@ -95,7 +91,6 @@ RBT_APPLICATION_EXIT_CODE_BACKWARDS_INCOMPATIBILITY, LocalEnvoyMode, ) -from reboot.std.blob.v1._data_plane import ENVVAR_BLOB_DATA_PLANE_URL from reboot.version import REBOOT_VERSION from typing import Any, Awaitable, Callable, Optional, TextIO, TypeVar @@ -1673,58 +1668,6 @@ def crypto_root_keys() -> str: ) ) - # Honor a blob data plane configured via `--env-file`/`--env` (they - # are otherwise only composed into the application's environment at - # launch, below) so that no local one is spawned; mirror - # `compose_env`'s precedence: `--env` wins over `--env-file`. - if args.env_file is not None and os.path.isfile(args.env_file): - data_plane_url = _load_env_file(args.env_file - ).get(ENVVAR_BLOB_DATA_PLANE_URL) - if data_plane_url is not None: - env[ENVVAR_BLOB_DATA_PLANE_URL] = data_plane_url - for (key, value) in args.env or []: - if key == ENVVAR_BLOB_DATA_PLANE_URL: - env[key] = value - - # Start the filesystem blob data plane (unless a data plane is - # already configured) and point the application at it. Bytes live - # beside the rest of the run's state, and are reclaimed the same - # way it is: `rbt dev expunge --application-name=...` removes - # both for a named application, while an anonymous run keeps - # both under `.rbt/dev` until that directory is removed by hand. - blobs_directory = os.path.join( - env.get( - ENVVAR_RBT_STATE_DIRECTORY, - str(dot_rbt_dev_directory(args, parser)), - ), - BLOBS_SUBDIRECTORY, - ) - # The data plane derives its URL-signing key from the - # cryptographic root keys, so it needs them in its environment. - # They go in a copy rather than in `env`: `compose_env()` mints - # fresh keys for the *application* on every restart, and seeding - # `env` would satisfy its `not in composed` guard and freeze the - # application's keys for the whole run. The data plane both signs - # and verifies its own URLs, so it only needs a key that is stable - # for its own lifetime. - data_plane_env = env.copy() - data_plane_env.setdefault( - ENVVAR_REBOOT_CRYPTO_ROOT_KEYS, - crypto_root_keys(), - ) - await start_filesystem_data_plane( - data_plane_env, - blobs_directory, - subprocesses, - background_command_tasks, - ) - # `start_filesystem_data_plane` reports the URL it bound by - # writing it into the environment it was given. - if ENVVAR_BLOB_DATA_PLANE_URL in data_plane_env: - env[ENVVAR_BLOB_DATA_PLANE_URL] = ( - data_plane_env[ENVVAR_BLOB_DATA_PLANE_URL] - ) - if tracing == Tracing.JAEGER: # TODO: dynamic port. See comment in `_run_jaeger()`. env[OTEL_EXPORTER_OTLP_TRACES_ENDPOINT] = "localhost:4317" diff --git a/reboot/cli/commands/serve.py b/reboot/cli/commands/serve.py index 58a58ad92..c1ac74f1e 100644 --- a/reboot/cli/commands/serve.py +++ b/reboot/cli/commands/serve.py @@ -1,6 +1,5 @@ import aiofiles.os import argparse -import asyncio import math import os import sys @@ -14,10 +13,6 @@ try_and_become_child_subreaper_on_linux, ) from reboot.cli.common import terminal -from reboot.cli.common.blob_data_plane import ( - BLOBS_SUBDIRECTORY, - start_filesystem_data_plane, -) from reboot.cli.common.detect_cores import detect_cores from reboot.cli.common.directories import ( add_working_directory_options, @@ -330,24 +325,6 @@ async def serve_run( for (key, value) in args.env or []: env[key] = value - # Start the filesystem blob data plane (unless a data plane is - # already configured, e.g. by Reboot Cloud provisioning or via - # `--env`) and point the application at it. Bytes live under the - # state directory alongside the rest of the application's state. - # Done after `--env` is applied so an explicitly-configured data - # plane is honored rather than spawning a redundant local one. - blobs_directory = os.path.join( - env.get(ENVVAR_RBT_STATE_DIRECTORY, os.getcwd()), - BLOBS_SUBDIRECTORY, - ) - data_plane_tasks: list[asyncio.Task] = [] - await start_filesystem_data_plane( - env, - blobs_directory, - subprocesses, - data_plane_tasks, - ) - # If 'PYTHONPATH' is not explicitly set, we'll set it to the # specified generated code directory plus each proto directory. # We want to get the directories from 'rbt generate' flags, @@ -404,16 +381,8 @@ async def serve_run( application if not auto_transpilation else str(bundle), ] - try: - async with subprocesses.exec(*args, env=env) as process: - return await process.wait() - finally: - for data_plane_task in data_plane_tasks: - data_plane_task.cancel() - try: - await data_plane_task - except asyncio.CancelledError: - pass + async with subprocesses.exec(*args, env=env) as process: + return await process.wait() async def handle_serve_subcommand( diff --git a/reboot/cli/common/BUILD.bazel b/reboot/cli/common/BUILD.bazel index 1a1a42aaf..940497373 100644 --- a/reboot/cli/common/BUILD.bazel +++ b/reboot/cli/common/BUILD.bazel @@ -100,17 +100,6 @@ py_library( ], ) -py_library( - name = "blob_data_plane_py", - srcs = ["blob_data_plane.py"], - srcs_version = "PY3", - visibility = ["//visibility:public"], - deps = [ - ":subprocesses_py", - "@com_github_reboot_dev_reboot//reboot/std/blob/v1:blob_py", - ], -) - py_library( name = "monkeys_py", srcs = ["monkeys.py"], diff --git a/reboot/cli/common/blob_data_plane.py b/reboot/cli/common/blob_data_plane.py deleted file mode 100644 index 9e0c21356..000000000 --- a/reboot/cli/common/blob_data_plane.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Spawning the filesystem blob data plane for local runs. - -`rbt dev run` and `rbt serve run` start the open-source filesystem blob -data-plane server (`reboot.std.blob.v1._filesystem_server`) as a -background subprocess whenever `REBOOT_BLOB_DATA_PLANE_URL` is not -already set, and point the application at it on localhost. In Reboot -Cloud the variable is set by provisioning (to the app's facilitator), -so nothing is spawned. The server is started unconditionally when the -variable is unset — it is cheap and idle unless the application -actually uses blobs. - -The server chooses its own ports and writes them to a "ready file" -once both its gRPC and HTTP endpoints are listening; we wait for that -file before pointing the application at it. That both avoids the -port-allocation race of picking a port in the parent and guarantees -the data plane is fully serving before the application can reach it. -""" - -import asyncio -import os -import shutil -import sys -import tempfile -from reboot.cli.common.subprocesses import Subprocesses -from reboot.std.blob.v1._data_plane import ENVVAR_BLOB_DATA_PLANE_URL -from typing import Optional - -# The filesystem data plane binds loopback only (its gRPC control -# surface is unauthenticated); the application reaches it here. -_LOOPBACK_HOST = "127.0.0.1" - -# Where blob bytes live, under the application's state directory (so -# `rbt dev expunge`, which removes the state directory, removes blobs -# too). -BLOBS_SUBDIRECTORY = "blobs" - -# How long to wait for the spawned server to report its ports before -# giving up. -_READY_TIMEOUT_SECONDS = 30 -_READY_POLL_INTERVAL_SECONDS = 0.1 - - -async def _run_and_await_ready( - subprocesses: Subprocesses, - argv: list[str], - env: dict[str, str], - ready_file: str, - ready: asyncio.Future[tuple[int, int]], -) -> None: - """Runs the data-plane server until cancelled, resolving `ready` - with its `(grpc_port, http_port)` once the server writes them.""" - async with subprocesses.exec(*argv, env=env) as process: - waited = 0.0 - while not ready.done(): - if process.returncode is not None: - ready.set_exception( - RuntimeError( - "The filesystem blob data-plane server exited " - f"before becoming ready (code {process.returncode})." - ) - ) - break - ports = _read_ready_file(ready_file) - if ports is not None: - ready.set_result(ports) - break - if waited >= _READY_TIMEOUT_SECONDS: - ready.set_exception( - RuntimeError( - "Timed out waiting for the filesystem blob " - "data-plane server to become ready." - ) - ) - break - await asyncio.sleep(_READY_POLL_INTERVAL_SECONDS) - waited += _READY_POLL_INTERVAL_SECONDS - await process.wait() - - -def _read_ready_file(path: str) -> Optional[tuple[int, int]]: - try: - with open(path) as f: - content = f.read().split() - except FileNotFoundError: - return None - if len(content) != 2: - return None - return int(content[0]), int(content[1]) - - -async def start_filesystem_data_plane( - env: dict[str, str], - blobs_directory: str, - subprocesses: Subprocesses, - background_command_tasks: list[asyncio.Task], -) -> None: - """Unless a data plane is already configured in `env`, spawns the - filesystem server as a background task, waits until it is fully - listening, and points `env[REBOOT_BLOB_DATA_PLANE_URL]` at it. The - server runs with `env` — which must already carry - `REBOOT_CRYPTO_ROOT_KEYS`, from which the server derives its - URL-signing key — and is cleaned up when its task is cancelled.""" - if env.get(ENVVAR_BLOB_DATA_PLANE_URL): - return - - # The ready file lives in a fresh private directory (a bare - # `mktemp` name in a shared `/tmp` would be squattable). - ready_directory = tempfile.mkdtemp(prefix="reboot-blob-data-plane-") - ready_file = os.path.join(ready_directory, "ready") - argv = [ - sys.executable, - "-m", - "reboot.std.blob.v1._filesystem_server", - "--directory", - blobs_directory, - "--ready-file", - ready_file, - ] - - loop = asyncio.get_event_loop() - ready: asyncio.Future[tuple[int, int]] = loop.create_future() - task = asyncio.create_task( - _run_and_await_ready(subprocesses, argv, env, ready_file, ready), - name="run_filesystem_data_plane(...)", - ) - background_command_tasks.append(task) - try: - grpc_port, _ = await ready - except BaseException: - # Reap the never-ready server immediately: its task would - # otherwise run until the caller's cleanup — which under - # `rbt serve run` only happens once the application exits. - task.cancel() - try: - await task - except BaseException: - pass - raise - finally: - shutil.rmtree(ready_directory, ignore_errors=True) - env[ENVVAR_BLOB_DATA_PLANE_URL] = f"{_LOOPBACK_HOST}:{grpc_port}" diff --git a/reboot/controller/BUILD.bazel b/reboot/controller/BUILD.bazel index e95a12cbc..35b8618e4 100644 --- a/reboot/controller/BUILD.bazel +++ b/reboot/controller/BUILD.bazel @@ -28,13 +28,23 @@ py_library( visibility = ["//visibility:public"], deps = [ ":application_config_py", - ":settings_py", - "@com_github_reboot_dev_reboot//rbt/v1alpha1:placement_planner_py_proto", + ":replicas_py", "@com_github_reboot_dev_reboot//reboot/aio:servicers_py", "@com_github_reboot_dev_reboot//reboot/aio:types_py", ], ) +py_library( + name = "replicas_py", + srcs = ["replicas.py"], + visibility = ["//visibility:public"], + deps = [ + ":settings_py", + "@com_github_reboot_dev_reboot//rbt/v1alpha1:placement_planner_py_proto", + requirement("protobuf"), + ], +) + py_library( name = "settings_py", srcs = ["settings.py"], @@ -57,6 +67,7 @@ py_library( requirement("aiofiles"), requirement("psutil"), ":servers_py", + ":replicas_py", ":settings_py", "@com_github_reboot_dev_reboot//reboot:naming_py", "//reboot/aio:http_py", diff --git a/reboot/controller/config_extractor.py b/reboot/controller/config_extractor.py index bcf56bb74..6972bfbb7 100644 --- a/reboot/controller/config_extractor.py +++ b/reboot/controller/config_extractor.py @@ -1,6 +1,3 @@ -import os -from google.protobuf import json_format -from rbt.v1alpha1 import placement_planner_pb2 from reboot.aio.servicers import Serviceable from reboot.aio.types import ApplicationId from reboot.controller.application_config import ( @@ -8,7 +5,7 @@ LocalApplicationConfig, application_config_spec_from_routables, ) -from reboot.controller.settings import ENVVAR_REBOOT_REPLICA_CONFIG +from reboot.controller.replicas import num_replicas from typing import Optional @@ -16,19 +13,7 @@ class LocalConfigExtractor: def __init__(self, application_id: ApplicationId): self._application_id = application_id - - replica_config_json = os.environ.get(ENVVAR_REBOOT_REPLICA_CONFIG) - if replica_config_json is None: - # The replica config is only required to be set when there - # are multiple replicas; in cases where there is only a - # single replica (e.g. `rbt dev run`) the environment - # variable may remain unset. Therefore, this situation means - # there is only one replica (namely: this process). - self._replicas = 1 - else: - replica_config = placement_planner_pb2.ReplicaConfig() - json_format.Parse(replica_config_json, replica_config) - self._replicas = len(replica_config.replicas) + self._replicas = num_replicas() def config_from_serviceables( self, diff --git a/reboot/controller/replicas.py b/reboot/controller/replicas.py new file mode 100644 index 000000000..63b498955 --- /dev/null +++ b/reboot/controller/replicas.py @@ -0,0 +1,31 @@ +"""How this application is replicated, per `REBOOT_REPLICA_CONFIG`.""" + +import os +from google.protobuf import json_format +from rbt.v1alpha1 import placement_planner_pb2 +from reboot.controller.settings import ENVVAR_REBOOT_REPLICA_CONFIG +from typing import Optional + + +def replica_config() -> Optional[placement_planner_pb2.ReplicaConfig]: + """The `ReplicaConfig` this application runs under, or `None` when + `REBOOT_REPLICA_CONFIG` is unset, which is how a single-replica run + (e.g. `rbt dev run`) is configured.""" + replica_config_json = os.environ.get(ENVVAR_REBOOT_REPLICA_CONFIG) + if replica_config_json is None: + return None + config = placement_planner_pb2.ReplicaConfig() + json_format.Parse( + replica_config_json, + config, + # For forwards-compatibility with newer fields. + ignore_unknown_fields=True, + ) + return config + + +def num_replicas() -> int: + """How many replicas this application runs across; one when no + `REBOOT_REPLICA_CONFIG` is set.""" + config = replica_config() + return 1 if config is None else len(config.replicas) diff --git a/reboot/controller/server_managers.py b/reboot/controller/server_managers.py index affb5c9cc..a87e29d6d 100644 --- a/reboot/controller/server_managers.py +++ b/reboot/controller/server_managers.py @@ -15,7 +15,6 @@ import time import traceback from dataclasses import dataclass -from google.protobuf import json_format from pathlib import Path from rbt.v1alpha1 import database_pb2, placement_planner_pb2 from reboot.aio.auth.token_verifiers import TokenVerifier @@ -30,6 +29,7 @@ from reboot.aio.servicers import Serviceable from reboot.aio.state_managers import SidecarStateManager from reboot.aio.types import ApplicationId, RoutableAddress, ServerId +from reboot.controller.replicas import replica_config from reboot.controller.servers import ServerSpec from reboot.controller.settings import ( ENVVAR_REBOOT_REPLICA_CONFIG, @@ -556,16 +556,7 @@ def __init__( self._replica_index = ( int(replica_index_str) if replica_index_str is not None else 0 ) - self._replica_config: Optional[placement_planner_pb2.ReplicaConfig - ] = None - if replica_config_json is not None: - self._replica_config = placement_planner_pb2.ReplicaConfig() - json_format.Parse( - replica_config_json, - self._replica_config, - # For forwards-compatibility with newer fields. - ignore_unknown_fields=True, - ) + self._replica_config = replica_config() def __del__(self): """Custom destructor in order to avoid the temporary directory being diff --git a/reboot/std/blob/v1/BUILD.bazel b/reboot/std/blob/v1/BUILD.bazel index ddd67e723..da761d737 100644 --- a/reboot/std/blob/v1/BUILD.bazel +++ b/reboot/std/blob/v1/BUILD.bazel @@ -7,28 +7,30 @@ py_library( srcs = [ "_content_type.py", "_data_plane.py", - "_filesystem_server.py", + "_filesystem_data_plane.py", "_http.py", - "_proxy.py", "_store.py", + "_stored_blob.py", "blob.py", ], visibility = ["//visibility:public"], deps = [ "//reboot/aio:applications_py", "//reboot/aio:contexts_py", + "//reboot/aio:external_py", "//reboot/aio:http_py", "//reboot/aio:workflows_py", "//reboot/aio/auth:authorizers_py", + "//reboot/controller:replicas_py", "//reboot/crypto:root_keys_py", "@com_github_reboot_dev_reboot//log:log_py", "@com_github_reboot_dev_reboot//rbt/std/blob/v1:blob_py_reboot", "@com_github_reboot_dev_reboot//rbt/std/blob/v1:data_plane_py_reboot", + "@com_github_reboot_dev_reboot//rbt/std/blob/v1:filesystem_py_reboot", + "@com_github_reboot_dev_reboot//reboot:run_environments_py", requirement("aiofiles"), - requirement("aiohttp"), requirement("grpcio"), requirement("starlette"), - requirement("uvicorn"), ], ) diff --git a/reboot/std/blob/v1/_data_plane.py b/reboot/std/blob/v1/_data_plane.py index 33a2c8953..6f11465f5 100644 --- a/reboot/std/blob/v1/_data_plane.py +++ b/reboot/std/blob/v1/_data_plane.py @@ -1,8 +1,20 @@ """Client-side glue for talking to a `BlobDataPlane` gRPC service.""" +import functools import grpc import os +from contextlib import asynccontextmanager from rbt.std.blob.v1.data_plane_pb2_grpc import BlobDataPlaneStub +from reboot.aio.caller_id import CallerID +from reboot.aio.contexts import Context +from reboot.aio.external import ExternalContext +from reboot.aio.headers import CALLER_ID_HEADER +from reboot.aio.internals.contextvars import get_application_id +from reboot.aio.types import ApplicationId +from reboot.controller.replicas import num_replicas +from reboot.run_environments import on_cloud +from reboot.settings import ENVVAR_RBT_STATE_DIRECTORY +from typing import AsyncIterator, Optional from urllib.parse import urlparse # The URL of the `BlobDataPlane` gRPC service. A bare `host:port`, or a @@ -10,21 +22,25 @@ # secure, anything else -> insecure). ENVVAR_BLOB_DATA_PLANE_URL = "REBOOT_BLOB_DATA_PLANE_URL" -# The path namespace a data plane's forwarded paths must live under; -# the application refuses to forward anything else, so a data plane -# can never claim application routes (see `ForwardedPath` in -# `data_plane.proto`). -FORWARDED_PATH_PREFIX = "/__/reboot/blob/" +# Where the filesystem data plane keeps bytes, relative to the +# application's state directory, so that whatever reclaims that state +# reclaims the blobs with it. +BLOBS_SUBDIRECTORY = "blobs" _SECURE_SCHEMES = ("https", "grpcs") class DataPlaneNotConfigured(RuntimeError): - """Raised when no data-plane URL is configured. Under `rbt dev - run`/`rbt serve run` and in `reboot.aio.tests.Reboot` unit tests - this never happens (they provide the filesystem data plane and set - the URL); it indicates the application was started some other way - without a data plane.""" + """Raised when there is no data plane to reach: none is named by + `REBOOT_BLOB_DATA_PLANE_URL`, and the one an application hosts + itself is only reachable from within that application.""" + + +def configured_data_plane_url() -> Optional[str]: + """The URL of the data plane this application has been pointed at + via `REBOOT_BLOB_DATA_PLANE_URL`, or `None` when it hosts one + itself.""" + return os.environ.get(ENVVAR_BLOB_DATA_PLANE_URL) or None def channel_for_url(url: str) -> grpc.aio.Channel: @@ -41,41 +57,122 @@ def channel_for_url(url: str) -> grpc.aio.Channel: return grpc.aio.insecure_channel(target) -def proxy_target_from_environment(http_port: int) -> str: - """The base URL of the data plane's HTTP byte endpoint, to which - the application proxies forwarded paths: the host the application - already reaches the data plane's gRPC service on (from - `REBOOT_BLOB_DATA_PLANE_URL`), with the `http_port` its - `Configuration` reported, over the same transport security.""" - url = os.environ.get(ENVVAR_BLOB_DATA_PLANE_URL) - if not url: - raise DataPlaneNotConfigured( - f"`{ENVVAR_BLOB_DATA_PLANE_URL}` is not set." +@asynccontextmanager +async def data_plane_stub_at(url: str) -> AsyncIterator[BlobDataPlaneStub]: + """A stub for the data plane at `url`, over a channel that lives + as long as the `with` block.""" + channel = channel_for_url(url) + try: + yield BlobDataPlaneStub(channel) + finally: + await channel.close() + + +class _NamedCallerStub(BlobDataPlaneStub): + """A `BlobDataPlaneStub` that says which application is calling. + + The data plane refuses a call that does not, because being a + service of the application makes it reachable by anyone who can + reach the application. Reboot attaches this header to its own + calls, but a legacy gRPC channel carries only what its caller puts + on it.""" + + def __init__( + self, + channel: grpc.aio.Channel, + application_id: ApplicationId, + ) -> None: + super().__init__(channel) + metadata = ( + ( + CALLER_ID_HEADER, + str(CallerID(application_id=application_id)), + ), ) - parsed = urlparse(url if "://" in url else f"grpc://{url}") - scheme = "https" if parsed.scheme in _SECURE_SCHEMES else "http" - host = parsed.hostname or "" - # `urlparse` strips the brackets off an IPv6 literal; put them - # back, since they are required in a URL authority. - if ":" in host: - host = f"[{host}]" - return f"{scheme}://{host}:{http_port}" - - -def stub_from_environment() -> BlobDataPlaneStub: - """Builds a data-plane stub from `REBOOT_BLOB_DATA_PLANE_URL`. A - fresh channel is created on the current event loop (rather than - memoized) because `grpc.aio` channels are event-loop-affine, and - an application may be brought up on a new loop. Called once per - library `pre_run`; the channel then lives as long as the - application (the `Library` has no shutdown hook on which to close - it).""" - url = os.environ.get(ENVVAR_BLOB_DATA_PLANE_URL) - if not url: + # The generated stub gives itself one multicallable per RPC; + # each is rebound with the metadata, so that every call made + # through this stub carries it. + for name, multicallable in list(vars(self).items()): + setattr( + self, + name, + functools.partial(multicallable, metadata=metadata), + ) + + +@asynccontextmanager +async def data_plane_stub( + context: Context | ExternalContext, +) -> AsyncIterator[BlobDataPlaneStub]: + """A stub for whichever data plane this application uses. + + `REBOOT_BLOB_DATA_PLANE_URL` names one that lives elsewhere, and is + honored wherever it is set. With it unset, the data plane is the + one this application hosts itself, reached over Reboot's own + routing rather than an address. + + Either way it is the same gRPC service, so a `Blob` speaks to both + on one code path.""" + url = configured_data_plane_url() + if url is not None: + async with data_plane_stub_at(url) as stub: + yield stub + return + + # The data plane this application hosts itself. Reboot routes to + # it by service name, so there is no address to configure -- and + # none could be, since the servers hosting it are not running when + # an application's environment is composed. + application_id = get_application_id() + if application_id is None: raise DataPlaneNotConfigured( - f"`{ENVVAR_BLOB_DATA_PLANE_URL}` is not set. Run the " - "application via `rbt dev run` or `rbt serve run` (which " - "start the filesystem blob data plane automatically), or " - "set the variable to your own data-plane service's URL." + "the blob data plane can only be reached from within an " + "application" + ) + async with context.legacy_grpc_channel() as channel: + yield _NamedCallerStub(channel, application_id) + + +def blobs_directory() -> str: + """Where this application keeps blob bytes. + + Beside the rest of its state, so that whatever reclaims that state + reclaims the blobs with it, and so that every server of the + application arrives at the same directory from the state directory + alone. A run that keeps no state across restarts keeps its blobs in + the same temporary directory as everything else it stores.""" + if on_cloud(): + # Reboot Cloud sets `REBOOT_BLOB_DATA_PLANE_URL` for every + # application it runs, so one only reaches here having been + # left without it. Local disk is no stand-in: it is neither + # durable nor shared between replicas, so serving from it would + # take uploads that are then silently lost. + raise RuntimeError( + f"'{ENVVAR_BLOB_DATA_PLANE_URL}' is not set. It is set " + 'automatically for applications on Reboot Cloud; please ' + 'contact your administrator.' + ) + + count = num_replicas() + if count > 1: + # Every server of one replica shares that replica's disk, so + # any number of them is fine; a second replica is not, since it + # cannot see the bytes this one stores. Refuse rather than + # serve downloads that succeed or 404 depending on which + # replica the request reached. + raise RuntimeError( + 'Blob bytes are stored on the local disk of a single ' + 'replica, so they cannot be served by an application ' + f'running across {count} replicas. Point ' + f"'{ENVVAR_BLOB_DATA_PLANE_URL}' at a data plane all " + 'replicas share.' + ) + + state_directory = os.environ.get(ENVVAR_RBT_STATE_DIRECTORY) + if state_directory is None: + raise RuntimeError( + f"'{ENVVAR_RBT_STATE_DIRECTORY}' is not set, so there is no " + "state directory to keep blob bytes beside. Set it, or point " + f"'{ENVVAR_BLOB_DATA_PLANE_URL}' at a data plane." ) - return BlobDataPlaneStub(channel_for_url(url)) + return os.path.join(state_directory, BLOBS_SUBDIRECTORY) diff --git a/reboot/std/blob/v1/_filesystem_data_plane.py b/reboot/std/blob/v1/_filesystem_data_plane.py new file mode 100644 index 000000000..2f1a71754 --- /dev/null +++ b/reboot/std/blob/v1/_filesystem_data_plane.py @@ -0,0 +1,366 @@ +"""The filesystem blob data plane. + +Implements the `BlobDataPlane` gRPC service (see `data_plane.proto`) +over bytes on local disk, for an application that has not been pointed +at a data plane elsewhere. A plain gRPC service, like any other data +plane, so that the control plane speaks one interface wherever the +bytes live and over whatever transport reaches them. + +Being a service of the application means being routed like one, and +Reboot serves legacy gRPC to whoever can reach the application -- +Envoy will even transcode HTTP to it. Nothing here may be called that +way: `GetDownloadUrl` mints a capability for a blob's bytes and +`Delete` destroys them, both without consulting the `Blob` control +plane, whose authorizer is what decides who may read or remove a blob. +So every method names its caller first, on the same footing as every +other authorizer that asks whether a call is app-internal. What that +rests on is Envoy: a listener whose caller IDs it does not trust has +`x-reboot-caller-id` removed from everything arriving on it (see +`trust_caller_id` in `reboot/routing/envoy_config.py`), so a caller ID +that survives was put there by something entitled to. + +Metadata lives in `StoredBlob`; bytes live in `FilesystemBlobStore`. +Neither holds state of its own, so any of a replica's servers can +serve any call. +""" + +import grpc +import rbt.std.blob.v1.filesystem_pb2 as filesystem_pb2 +import rbt.v1alpha1.errors_pb2 +from rbt.std.blob.v1.data_plane_pb2 import ( + ConfigurationRequest, + ConfigurationResponse, + DataPlaneBeginUploadRequest, + DataPlaneBeginUploadResponse, + DataPlaneCompleteUploadRequest, + DataPlaneCompleteUploadResponse, + DataPlaneDeleteRequest, + DataPlaneDeleteResponse, + DataPlaneGetDownloadUrlRequest, + DataPlaneGetDownloadUrlResponse, + DataPlaneGetPartUploadInstructionsRequest, + DataPlaneGetPartUploadInstructionsResponse, + DataPlanePartUploadInstruction, +) +from rbt.std.blob.v1.data_plane_pb2_grpc import BlobDataPlaneServicer +from rbt.std.blob.v1.filesystem_rbt import StoredBlob, StoredPart +from reboot.aio.caller_id import CallerID +from reboot.aio.external import ExternalContext +from reboot.aio.headers import CALLER_ID_HEADER +from reboot.aio.interceptors import LegacyGrpcContext +from reboot.aio.internals.contextvars import get_application_id +from reboot.std.blob.v1._store import ( + BlobStoreError, + FilesystemBlobStore, + _encode_blob_id, + composite_etag, +) +from typing import Optional +from uuid import NAMESPACE_URL, UUID, uuid5 + + +def _begin_upload_key(blob_id: str) -> UUID: + """The idempotency key for beginning one blob's upload. + + Derived from the blob ID rather than taken from the caller, + because the control plane both retries this inside a workflow and + re-runs it to validate that workflow's effects. "Begin the upload + for this blob" is one operation however many times it is asked + for, so the blob names it.""" + return uuid5(NAMESPACE_URL, f"reboot.std.blob.v1/begin-upload/{blob_id}") + + +class FilesystemDataPlaneServicer(BlobDataPlaneServicer): + """Serves `BlobDataPlane` from the application whose blobs it + holds. + + The store is set by `BlobLibrary` once it knows where this + application keeps them.""" + + _store: FilesystemBlobStore + + async def _authorize_caller(self, context: LegacyGrpcContext) -> None: + """Refuses anyone but this application's own code. + + This is the check `is_app_internal` makes, made by hand + because a legacy gRPC servicer has no authorizer to make it. + It reads a header, and what stands behind that header is + Envoy removing it from traffic whose caller IDs it does not + trust -- see the note at the top of this module.""" + application_id = get_application_id() + caller_id_header = dict(context.invocation_metadata() + ).get(CALLER_ID_HEADER) + if caller_id_header is None or application_id is None: + await context.abort( + grpc.StatusCode.UNAUTHENTICATED, + "the blob data plane serves only the application it " + "belongs to", + ) + raise RuntimeError("This is unreachable") + + try: + caller_id = CallerID.parse(caller_id_header) + except ValueError: + await context.abort( + grpc.StatusCode.UNAUTHENTICATED, + "cannot deduce calling application from caller ID header", + ) + raise RuntimeError("This is unreachable") + + if caller_id.application_id != application_id: + await context.abort( + grpc.StatusCode.PERMISSION_DENIED, + "the blob data plane serves only the application it " + "belongs to", + ) + raise RuntimeError("This is unreachable") + + def _context(self, grpc_context: LegacyGrpcContext) -> ExternalContext: + """A context for reaching `StoredBlob` on behalf of a call + `_authorize_caller` has admitted.""" + return grpc_context.external_context(name="blob data plane") + + async def Configuration( + self, + request: ConfigurationRequest, + grpc_context: LegacyGrpcContext, + ) -> ConfigurationResponse: + await self._authorize_caller(grpc_context) + return ConfigurationResponse(part_size=self._store.part_size) + + async def BeginUpload( + self, + request: DataPlaneBeginUploadRequest, + grpc_context: LegacyGrpcContext, + ) -> DataPlaneBeginUploadResponse: + await self._authorize_caller(grpc_context) + context = self._context(grpc_context) + _, response = await StoredBlob.idempotently( + key=_begin_upload_key(request.blob_id), + ).BeginUpload( + context, + request.blob_id, + content_type=request.content_type, + ) + # After the session exists in state, so a directory is never + # left behind for a session nothing knows about. + await self._store.make_upload_directory( + request.blob_id, + response.upload_id, + ) + return DataPlaneBeginUploadResponse(upload_id=response.upload_id) + + async def GetPartUploadInstructions( + self, + request: DataPlaneGetPartUploadInstructionsRequest, + grpc_context: LegacyGrpcContext, + ) -> DataPlaneGetPartUploadInstructionsResponse: + await self._authorize_caller(grpc_context) + instructions = [ + DataPlanePartUploadInstruction( + part_number=part_number, + url=self._store.part_put_url( + request.blob_id, + request.upload_id, + part_number, + ), + ) for part_number in request.part_numbers + ] + return DataPlaneGetPartUploadInstructionsResponse( + instructions=instructions + ) + + async def CompleteUpload( + self, + request: DataPlaneCompleteUploadRequest, + grpc_context: LegacyGrpcContext, + ) -> DataPlaneCompleteUploadResponse: + await self._authorize_caller(grpc_context) + try: + etag = await self._complete(request, self._context(grpc_context)) + return DataPlaneCompleteUploadResponse(etag=etag) + except BlobStoreError as error: + # A permanent failure: report it so the control plane can + # surface it and let the client re-upload. Transient + # failures raise other exceptions, which the control + # plane's workflow retries. + return DataPlaneCompleteUploadResponse(error=str(error)) + + async def _stored( + self, + blob_id: str, + context: ExternalContext, + ) -> Optional[filesystem_pb2.StoredBlob]: + """The metadata stored for a blob, or `None` when none is. + + A blob whose upload never began has no state at all, which the + framework reports by refusing the read rather than by + answering with an absent one.""" + try: + metadata = await StoredBlob.ref(blob_id).metadata(context) + except StoredBlob.MetadataAborted as aborted: + if isinstance( + aborted.error, + rbt.v1alpha1.errors_pb2.StateNotConstructed, + ): + return None + raise + return metadata.blob if metadata.HasField("blob") else None + + async def _complete( + self, + request: DataPlaneCompleteUploadRequest, + context: ExternalContext, + ) -> str: + stored = await self._stored(request.blob_id, context) + if stored is None: + raise BlobStoreError("no upload was ever begun for this blob") + if stored.committed: + # A retried `CompleteUpload`. The object is finished and + # its ETag is what it was -- but reclaiming may not have + # run, or not finished, so it runs again from what was + # committed. + await self._store.reclaim( + _encode_blob_id(request.blob_id), + stored.upload_id, + [(part.number, part.storage_id) for part in stored.parts], + ) + return stored.etag + if stored.upload_id != request.upload_id: + # The parts that would be committed were written under a + # different session than the one being completed, so they + # are not the parts this verified. + raise BlobStoreError( + "the upload session being completed is not the one this " + "blob's parts were written under" + ) + + published = {part.number: part for part in stored.parts} + reported = {part.number: part for part in request.parts} + if len(reported) == 0: + raise BlobStoreError("no parts were reported") + + last_part_number = max(reported) + for number in sorted(reported): + part = published.get(number) + if part is None: + raise BlobStoreError(f"part {number} was never uploaded") + if part.etag != reported[number].etag.strip('"'): + raise BlobStoreError( + f"part {number} ETag mismatch: the uploaded bytes do " + "not match what was reported via `PartUploaded`" + ) + if part.size != reported[number].size: + raise BlobStoreError( + f"part {number} size mismatch: uploaded {part.size} " + f"bytes but {reported[number].size} were reported via " + "`PartUploaded`" + ) + if ( + number != last_part_number and + part.size != self._store.part_size + ): + # S3 rejects a short middle part with `EntityTooSmall`; + # reject it here too, so that an upload which cannot + # commit against the S3 store cannot commit against + # this one either. + raise BlobStoreError( + f"part {number} is {part.size} bytes, but every part " + f"except the last must be exactly " + f"{self._store.part_size} bytes" + ) + + total_size = sum(published[number].size for number in reported) + max_size: Optional[int] = ( + request.max_size if request.HasField("max_size") else None + ) + # Checked against what the parts were found to hold, not + # against the sizes that were reported alongside them. + if max_size is not None and total_size > max_size: + raise BlobStoreError( + f"uploaded {total_size} bytes exceeds the maximum of " + f"{max_size}" + ) + + manifest = [ + StoredPart( + number=number, + size=published[number].size, + etag=published[number].etag, + storage_id=published[number].storage_id, + ) for number in sorted(reported) + ] + etag = composite_etag([part.etag for part in manifest]) + committed = await StoredBlob.ref(request.blob_id).always().commit( + context, + upload_id=request.upload_id, + content_type=request.content_type, + etag=etag, + parts=manifest, + ) + if not committed.committed: + raise BlobStoreError( + "a part was uploaded again while this upload was being " + "completed; report the parts and commit again" + ) + # The manifest is fixed, so anything else this session wrote + # -- a part uploaded and never reported, a version of a part + # that lost -- belongs to nothing and is safe to remove. Done + # after the commit, so a failure here leaves files behind + # rather than taking away bytes the object is made of; a retry + # reclaims them above. + # `commit` accepted this manifest, and refuses one whose parts + # have been superseded, so it is exactly what was recorded. + await self._store.reclaim( + _encode_blob_id(request.blob_id), + request.upload_id, + [(part.number, part.storage_id) for part in manifest], + ) + return etag + + async def GetDownloadUrl( + self, + request: DataPlaneGetDownloadUrlRequest, + grpc_context: LegacyGrpcContext, + ) -> DataPlaneGetDownloadUrlResponse: + await self._authorize_caller(grpc_context) + url, ttl_seconds = self._store.download_url( + request.blob_id, + request.ttl_seconds if request.HasField("ttl_seconds") else None, + ) + return DataPlaneGetDownloadUrlResponse( + url=url, + ttl_seconds=ttl_seconds, + ) + + async def Delete( + self, + request: DataPlaneDeleteRequest, + grpc_context: LegacyGrpcContext, + ) -> DataPlaneDeleteResponse: + await self._authorize_caller(grpc_context) + context = self._context(grpc_context) + # A part lives inside the blob's own directory, so removing the + # directory removes any unfinished upload with it, whatever + # `upload_ids` says. + # Forgotten before the bytes go, so that nothing reads a + # manifest naming bytes that are already gone: between the two + # a download would answer `200` and then run out of file. + try: + await StoredBlob.ref(request.blob_id).always().forget(context) + except StoredBlob.ForgetAborted as aborted: + if isinstance( + aborted.error, + rbt.v1alpha1.errors_pb2.StateNotConstructed, + ): + # Nothing was ever stored for this blob, so there is + # nothing to forget and deleting it has succeeded. + pass + else: + raise + await self._store.delete(request.blob_id) + return DataPlaneDeleteResponse() + + +def legacy_grpc_servicers() -> list[type]: + return [FilesystemDataPlaneServicer] diff --git a/reboot/std/blob/v1/_filesystem_server.py b/reboot/std/blob/v1/_filesystem_server.py deleted file mode 100644 index eccb12a71..000000000 --- a/reboot/std/blob/v1/_filesystem_server.py +++ /dev/null @@ -1,334 +0,0 @@ -"""The filesystem blob data-plane server. - -Implements the `BlobDataPlane` gRPC service backed by the local -filesystem, plus the HTTP byte endpoint its minted URLs point at. It -runs in two ways: `rbt dev run` and `rbt serve run` spawn it as a -standalone program (via `main`) whenever `REBOOT_BLOB_DATA_PLANE_URL` -is not already set, and the `reboot.aio.tests.Reboot` test harness -runs it in-process (via `FilesystemDataPlane.start`), so that blob -storage works out of the box in every local run mode. - -Its URLs are application-relative proxy paths: its `Configuration` -asks the application to forward the blob path namespace to it, so the -application reverse-proxies byte `PUT`/`GET` here rather than exposing -this server on its own port (see `_proxy.py`). -""" - -import argparse -import asyncio -import contextlib -import grpc -import os -import uvicorn # type: ignore[import] -from rbt.std.blob.v1.data_plane_pb2 import ( - HTTP_METHOD_GET, - HTTP_METHOD_PUT, - ConfigurationResponse, - DataPlaneBeginUploadResponse, - DataPlaneCompleteUploadResponse, - DataPlaneDeleteResponse, - DataPlaneGetDownloadUrlResponse, - DataPlaneGetPartUploadInstructionsResponse, - DataPlanePartUploadInstruction, - ForwardedPath, -) -from rbt.std.blob.v1.data_plane_pb2_grpc import ( - BlobDataPlaneServicer, - add_BlobDataPlaneServicer_to_server, -) -from reboot.std.blob.v1._http import build_http_app -from reboot.std.blob.v1._store import ( - DEFAULT_PART_SIZE_BYTES, - HTTP_PATH_PREFIX, - BlobStoreError, - FilesystemBlobStore, - UploadedPart, -) -from typing import Generator, Optional -from uuid import uuid4 - -# The filesystem server binds loopback only: it has no authentication -# (its URLs are HMAC-signed, but the gRPC control surface is not), so -# it must never be reachable off the host. The application reaches it -# over localhost and proxies client byte traffic to it. -LOOPBACK_HOST = "127.0.0.1" - - -class FilesystemDataPlaneServicer(BlobDataPlaneServicer): - """Implements `BlobDataPlane` over a `FilesystemBlobStore`.""" - - def __init__( - self, - store: FilesystemBlobStore, - http_port: int, - ): - self._store = store - self._http_port = http_port - - async def Configuration(self, request, context): - # This server's URLs are application-relative under - # `HTTP_PATH_PREFIX`; ask the application to forward that - # namespace to the HTTP byte endpoint. - return ConfigurationResponse( - part_size=self._store.part_size, - forwarded_paths=[ - ForwardedPath( - method=HTTP_METHOD_GET, - path_prefix=HTTP_PATH_PREFIX + "/", - ), - ForwardedPath( - method=HTTP_METHOD_PUT, - path_prefix=HTTP_PATH_PREFIX + "/", - ), - ], - http_port=self._http_port, - ) - - async def BeginUpload(self, request, context): - upload_id = await self._store.begin_upload( - request.blob_id, - request.content_type, - ) - return DataPlaneBeginUploadResponse(upload_id=upload_id) - - async def GetPartUploadInstructions(self, request, context): - instructions = [ - DataPlanePartUploadInstruction( - part_number=part_number, - url=self._store.part_put_url( - request.blob_id, - request.upload_id, - part_number, - ), - ) for part_number in request.part_numbers - ] - return DataPlaneGetPartUploadInstructionsResponse( - instructions=instructions - ) - - async def CompleteUpload(self, request, context): - try: - etag = await self._store.complete( - request.blob_id, - request.upload_id, - request.content_type, - [ - UploadedPart( - number=part.number, etag=part.etag, size=part.size - ) for part in request.parts - ], - max_size=( - request.max_size if request.HasField("max_size") else None - ), - ) - return DataPlaneCompleteUploadResponse(etag=etag) - except BlobStoreError as error: - # A permanent failure: report it so the control plane can - # surface it and let the client re-upload. Transient - # failures raise other exceptions, which become gRPC errors - # so the control-plane workflow retries. - return DataPlaneCompleteUploadResponse(error=str(error)) - - async def GetDownloadUrl(self, request, context): - url, ttl_seconds = self._store.download_url( - request.blob_id, - request.ttl_seconds if request.HasField("ttl_seconds") else None, - ) - return DataPlaneGetDownloadUrlResponse( - url=url, - ttl_seconds=ttl_seconds, - ) - - async def Delete(self, request, context): - await self._store.delete(request.blob_id) - return DataPlaneDeleteResponse() - - -class FilesystemDataPlane: - """A running filesystem blob data plane: the `BlobDataPlane` gRPC - service plus the HTTP byte endpoint its minted URLs point at, both - bound to loopback. Construct via `start()`.""" - - def __init__( - self, - *, - grpc_server: grpc.aio.Server, - grpc_port: int, - http_server, - http_task: asyncio.Task, - http_port: int, - ): - self._grpc_server = grpc_server - self._grpc_port = grpc_port - self._http_server = http_server - self._http_task = http_task - self._http_port = http_port - - @classmethod - async def start( - cls, - *, - directory: str, - part_size: int = DEFAULT_PART_SIZE_BYTES, - grpc_port: int = 0, - http_port: int = 0, - ) -> "FilesystemDataPlane": - """Starts serving, with bytes stored under `directory`. Ports - default to 0 (an ephemeral port chosen by the OS). The HTTP - byte endpoint is brought up before the gRPC surface, so that by - the time `Configuration` is reachable the `http_port` it - reports is already serving.""" - store = FilesystemBlobStore(directory, part_size=part_size) - - class Server(uvicorn.Server): - """We need to override the installation of signal handlers as - Reboot is already handling this itself. - """ - - @contextlib.contextmanager - def capture_signals(self) -> Generator[None, None, None]: - # Do nothing - yield - - http_server = Server( - uvicorn.Config( - build_http_app(store), - host=LOOPBACK_HOST, - port=http_port, - log_level="warning", - ) - ) - http_task = asyncio.create_task(http_server.serve()) - while not http_server.started: - if http_task.done(): - # Startup failed (e.g. the requested port is in use); - # surface the underlying error. - http_task.result() - raise RuntimeError( - "The blob data plane's HTTP server exited during " - "startup" - ) - await asyncio.sleep(0.01) - actual_http_port = http_server.servers[0].sockets[0].getsockname()[1] - - # From here on the HTTP server is live; shut it down if the - # gRPC surface fails to come up, so a failed `start` leaves - # nothing running. - try: - grpc_server = grpc.aio.server() - add_BlobDataPlaneServicer_to_server( - FilesystemDataPlaneServicer(store, actual_http_port), - grpc_server, - ) - actual_grpc_port = grpc_server.add_insecure_port( - f"{LOOPBACK_HOST}:{grpc_port}" - ) - if actual_grpc_port == 0: - raise RuntimeError( - "The blob data plane's gRPC server could not bind " - f"port {grpc_port}" - ) - await grpc_server.start() - except BaseException: - http_server.should_exit = True - await http_task - raise - - return cls( - grpc_server=grpc_server, - grpc_port=actual_grpc_port, - http_server=http_server, - http_task=http_task, - http_port=actual_http_port, - ) - - @property - def grpc_port(self) -> int: - return self._grpc_port - - @property - def http_port(self) -> int: - return self._http_port - - @property - def url(self) -> str: - """The gRPC address to put in `REBOOT_BLOB_DATA_PLANE_URL`.""" - return f"{LOOPBACK_HOST}:{self._grpc_port}" - - async def stop(self) -> None: - await self._grpc_server.stop(grace=None) - self._http_server.should_exit = True - await self._http_task - - async def wait(self) -> None: - """Blocks until the servers terminate (they don't, absent - `stop()`; this is how the standalone program serves forever).""" - await asyncio.gather( - self._grpc_server.wait_for_termination(), - self._http_task, - ) - - -def _write_ready_file(path: str, grpc_port: int, http_port: int) -> None: - """Atomically writes the two chosen ports so the spawning process - learns them only once both servers are listening.""" - temp_path = f"{path}.{uuid4().hex}.tmp" - with open(temp_path, "w") as f: - f.write(f"{grpc_port}\n{http_port}\n") - f.flush() - os.fsync(f.fileno()) - os.replace(temp_path, path) - - -async def serve( - directory: str, - grpc_port: int = 0, - http_port: int = 0, - part_size: int = DEFAULT_PART_SIZE_BYTES, - ready_file: Optional[str] = None, -) -> None: - """Runs the data plane until terminated. When `ready_file` is - given, the actually-bound ports are written to it once both - endpoints are listening, for the spawning process to read.""" - data_plane = await FilesystemDataPlane.start( - directory=directory, - part_size=part_size, - grpc_port=grpc_port, - http_port=http_port, - ) - if ready_file is not None: - _write_ready_file( - ready_file, - data_plane.grpc_port, - data_plane.http_port, - ) - await data_plane.wait() - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Filesystem blob data-plane server." - ) - parser.add_argument("--directory", required=True) - parser.add_argument("--grpc-port", type=int, default=0) - parser.add_argument("--http-port", type=int, default=0) - parser.add_argument( - "--part-size", - type=int, - default=DEFAULT_PART_SIZE_BYTES, - ) - parser.add_argument("--ready-file", default=None) - args = parser.parse_args() - asyncio.run( - serve( - args.directory, - args.grpc_port, - args.http_port, - part_size=args.part_size, - ready_file=args.ready_file, - ) - ) - - -if __name__ == "__main__": - main() diff --git a/reboot/std/blob/v1/_http.py b/reboot/std/blob/v1/_http.py index 116abf74e..cc146d064 100644 --- a/reboot/std/blob/v1/_http.py +++ b/reboot/std/blob/v1/_http.py @@ -1,56 +1,53 @@ -"""The HTTP byte endpoint of the filesystem blob data-plane server. +"""The byte endpoints of the filesystem blob data plane. Serves `PUT` (part upload) and `GET` (download) under -`/__/reboot/blob/`. The filesystem server (`_filesystem_server.py`) -runs this on localhost; the application's `Blob` library reverse- -proxies to it (see `_proxy.py`), so the bytes never leave a single -origin even though they live in a separate process. - -These handlers are self-authorizing: every URL carries an expiring -HMAC signature minted by the data plane, so the handlers never call -back into Reboot state. They touch only the store's directory, -mirroring how a presigned S3 URL is served by S3 without consulting -the application. +`/__/reboot/blob/`, on the application's own HTTP server: the data +plane lives inside the application, so the bytes arrive on the same +origin as everything else with no hop in between. + +Every URL carries an expiring HMAC signature minted by the data plane, +and that signature is the caller's whole capability -- these routes are +reachable by anyone. It is therefore checked before anything else +happens, in particular before any of the request's own input reaches +`StoredBlob`. That ordering is what makes it safe for these routes to +hold an app-internal context (see the DANGER note on +`reboot.aio.http`): by the time one is used, the request has proven it +holds a URL this data plane minted for exactly this blob, session and +part. """ -import aiofiles -import asyncio -import hashlib +import base64 import hmac -import os import re import time +from rbt.std.blob.v1.filesystem_rbt import StoredBlob, StoredPart +from reboot.aio.http import PythonWebFramework from reboot.std.blob.v1._content_type import download_headers from reboot.std.blob.v1._store import ( - HTTP_PATH_PREFIX, + BLOB_PATH, MAX_PARTS, + PART_PATH, FilesystemBlobStore, + PartTooLarge, ) -from starlette.applications import Starlette from starlette.requests import Request from starlette.responses import Response, StreamingResponse -from starlette.routing import Route -from typing import Optional -from uuid import uuid4 - -_STREAM_CHUNK_SIZE = 1024 * 1024 +from typing import AsyncIterator, Callable, Coroutine, Optional # Path parameters are also filesystem path components; restrict them # to the alphabets the store actually produces (URL-safe base64 blob # IDs, hex upload IDs) as defense in depth against traversal — even -# though a forged path could never carry a valid signature. -_ENCODED_BLOB_ID_PATTERN = re.compile(r"^[A-Za-z0-9_-]+={0,2}$") -_UPLOAD_ID_PATTERN = re.compile(r"^[0-9a-f]{32}$") +# though a forged path could never carry a valid signature. Matched +# with `fullmatch`: `$` would also accept a trailing newline, which is +# less than the "only this alphabet" these are here to promise. +_ENCODED_BLOB_ID_PATTERN = re.compile(r"[A-Za-z0-9_-]+={0,2}") +_UPLOAD_ID_PATTERN = re.compile(r"[0-9a-f]{32}") # Enough digits for any epoch second this will ever mint, and # far below the length `int()` refuses. _MAX_EXPIRATION_DIGITS = 20 -class _PartTooLarge(Exception): - pass - - def _signature_matches(expected: str, actual: str) -> bool: # Compared as bytes: `compare_digest` refuses `str` arguments # that are not ASCII, and `actual` is a query parameter, so a @@ -84,73 +81,54 @@ def _unexpired_expiration(request: Request) -> Optional[int]: return parsed -def _make_put_part(store: FilesystemBlobStore): +def _blob_id(encoded_blob_id: str) -> str: + """The blob ID a URL's encoded path segment names.""" + return base64.urlsafe_b64decode(encoded_blob_id.encode()).decode() + + +def _make_put_part( + store: FilesystemBlobStore, +) -> Callable[[Request], Coroutine[None, None, Response]]: async def put_part(request: Request) -> Response: - blob = request.path_params["blob"] - upload = request.path_params["upload"] + blob = request.query_params.get("blob", "") + upload = request.query_params.get("upload", "") try: - part = int(request.path_params["part"]) + part_number = int(request.query_params.get("part", "")) except ValueError: return Response(status_code=400, content="Invalid part number") - if part < 1 or part > MAX_PARTS: + if part_number < 1 or part_number > MAX_PARTS: return Response(status_code=400, content="Invalid part number") if ( - not _ENCODED_BLOB_ID_PATTERN.match(blob) or - not _UPLOAD_ID_PATTERN.match(upload) + not _ENCODED_BLOB_ID_PATTERN.fullmatch(blob) or + not _UPLOAD_ID_PATTERN.fullmatch(upload) ): return Response(status_code=400, content="Invalid blob ID") expiration = _unexpired_expiration(request) if expiration is None: return Response(status_code=403, content="URL expired") - expected = store.signature_for_put(blob, upload, part, expiration) + expected = store.signature_for_put( + blob, upload, part_number, expiration + ) if not _signature_matches( expected, request.query_params.get("sig", "") ): return Response(status_code=403, content="Invalid signature") - path = store.part_path(blob, upload, part) - # The upload directory is created by `begin_upload`; a missing - # directory means the blob was never created (or was deleted). - if not os.path.isdir(os.path.dirname(path)): + # Everything below acts for a caller that has proven it holds a + # URL this data plane minted. + if not await store.upload_directory_exists(blob, upload): return Response(status_code=404, content="No such upload") - # Refuse to mutate a committed blob's bytes: the part files - # *are* the committed object's on-disk representation, so a - # part-PUT URL minted just before commit must not still be - # usable to tamper with the bytes afterwards. - meta = store.read_meta(blob) - if meta is not None and meta.committed: - return Response(status_code=409, content="Blob already committed") - - # Write somewhere else and publish with a rename, rather than - # writing `path` in place: completion reads the part files to - # validate them, and a part being rewritten underneath it - # would leave a committed blob whose bytes no longer match the - # ETag it recorded. A rename is atomic, so completion sees - # either the whole old part or the whole new one. - temporary = f"{path}.{uuid4().hex}.partial" - digest = hashlib.md5() - size = 0 try: - # A part is megabytes, so the writes go off the event loop - # for the same reason the download below reads off it: this - # handler is driven by the loop, and writing inline would - # stall every other request this worker is serving. - async with aiofiles.open(temporary, "wb") as f: - async for chunk in request.stream(): - if size + len(chunk) > store.part_size: - raise _PartTooLarge() - digest.update(chunk) - size += len(chunk) - await f.write(chunk) - await f.flush() - # `aiofiles` has no `fsync`; `fileno()` is proxied - # straight through, so the descriptor is the real one. - await asyncio.to_thread(os.fsync, f.fileno()) - except _PartTooLarge: - os.unlink(temporary) + staged = await store.stage_part( + blob, + upload, + part_number, + request.stream(), + ) + except PartTooLarge: return Response( status_code=413, content=( @@ -158,40 +136,57 @@ async def put_part(request: Request) -> Response: f"{store.part_size} bytes" ), ) - except BaseException: - # Never leave a partial file behind to be mistaken for a - # part. - if os.path.exists(temporary): - os.unlink(temporary) - raise - - # Publish under the blob's lock, and re-read the metadata - # inside it: completion may have run while these bytes were - # being uploaded, and a part must not appear after the blob it - # belongs to has been committed. - async with store.lock_for(blob): - meta = store.read_meta(blob) - if meta is not None and meta.committed: - os.unlink(temporary) - return Response( - status_code=409, content="Blob already committed" - ) - os.replace(temporary, path) + + # The bytes are on disk under a name of their own; whether + # the object is made of them is this call's to decide, and it + # decides for every server that might be serving this blob. + # Refused bytes go, which is safe because the file's name + # belongs to this write alone: no manifest can point at it + # unless this very call's claim succeeded. Anything that + # outlives an interrupted request is reclaimed at commit, and + # with the blob's directory on `Delete`. + # An app-internal context, which is what reaches `StoredBlob`, + # taken only now that the signature has verified this request + # holds a URL this data plane minted. + published = await StoredBlob.ref(_blob_id(blob)).always().publish_part( + request.state.reboot_app_internal_context(request), + upload_id=upload, + part=StoredPart( + number=staged.part.number, + size=staged.part.size, + etag=staged.part.etag, + storage_id=staged.part.storage_id, + ), + ) + if not published.published: + await store.discard_part(staged) + return Response(status_code=409, content="Blob already committed") + + if published.HasField("superseded_storage_id"): + # This part had been uploaded before. Nothing is made of + # the earlier bytes now, and the manifest that could still + # name them is refused at commit, so they are removed + # rather than left to accumulate a file per attempt. + await store.discard_storage_id( + blob, upload, part_number, published.superseded_storage_id + ) # Match S3: the ETag response header is the part's MD5, quoted. return Response( status_code=200, - headers={"ETag": f'"{digest.hexdigest()}"'}, + headers={"ETag": f'"{staged.part.etag}"'}, ) return put_part -def _make_get_blob(store: FilesystemBlobStore): +def _make_get_blob( + store: FilesystemBlobStore, +) -> Callable[[Request], Coroutine[None, None, Response]]: async def get_blob(request: Request) -> Response: - blob = request.path_params["blob"] - if not _ENCODED_BLOB_ID_PATTERN.match(blob): + blob = request.query_params.get("blob", "") + if not _ENCODED_BLOB_ID_PATTERN.fullmatch(blob): return Response(status_code=400, content="Invalid blob ID") expiration = _unexpired_expiration(request) if expiration is None: @@ -202,33 +197,33 @@ async def get_blob(request: Request) -> Response: ): return Response(status_code=403, content="Invalid signature") - meta = store.read_meta(blob) - if meta is None or not meta.committed: + # As in `put_part`: an app-internal context, taken only below a + # verified signature. + metadata = await StoredBlob.ref(_blob_id(blob)).metadata( + request.state.reboot_app_internal_context(request) + ) + stored = metadata.blob if metadata.HasField("blob") else None + if stored is None or not stored.committed: return Response(status_code=404, content="No such blob") - # Written in the same atomic update as `committed`. - assert meta.upload_id is not None and meta.etag is not None - upload_id = meta.upload_id - parts = meta.parts + upload_id = stored.upload_id + parts = sorted(stored.parts, key=lambda part: part.number) total_size = sum(part.size for part in parts) - async def stream(): - for part in sorted(parts, key=lambda part: part.number): - path = store.part_path(blob, upload_id, part.number) - # Read off the event loop: this generator is driven by - # it, and a part is megabytes, so reading inline would - # stall every other request this worker is serving. - async with aiofiles.open(path, "rb") as file: - while chunk := await file.read(_STREAM_CHUNK_SIZE): - yield chunk - - media_type, safety_headers = download_headers(meta.content_type) + async def stream() -> AsyncIterator[bytes]: + for part in parts: + async for chunk in store.read_part( + blob, upload_id, part.number, part.storage_id + ): + yield chunk + + media_type, safety_headers = download_headers(stored.content_type) return StreamingResponse( stream(), media_type=media_type, headers={ "Content-Length": str(total_size), - "ETag": f'"{meta.etag}"', + "ETag": f'"{stored.etag}"', "Accept-Ranges": "none", **safety_headers, }, @@ -237,19 +232,16 @@ async def stream(): return get_blob -def build_http_app(store: FilesystemBlobStore) -> Starlette: - """Builds the Starlette app serving `store`'s byte `PUT`/`GET`.""" - return Starlette( - routes=[ - Route( - HTTP_PATH_PREFIX + "/{blob}/{upload}/parts/{part}", - _make_put_part(store), - methods=["PUT"], - ), - Route( - HTTP_PATH_PREFIX + "/{blob}", - _make_get_blob(store), - methods=["GET"], - ), - ], - ) +def mount_byte_routes( + http: PythonWebFramework.HTTP, + store: FilesystemBlobStore, +) -> None: + """Registers the data plane's byte endpoints on the application's + own HTTP server. + + Registered like any other route, with no privilege of their own: + what reaches `StoredBlob` is a context each handler takes for + itself once a signature has verified, which is the only point at + which it has established anything about its caller.""" + http.put(PART_PATH)(_make_put_part(store)) + http.get(BLOB_PATH)(_make_get_blob(store)) diff --git a/reboot/std/blob/v1/_proxy.py b/reboot/std/blob/v1/_proxy.py deleted file mode 100644 index 8ea44d113..000000000 --- a/reboot/std/blob/v1/_proxy.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Application-side reverse proxy for a blob data plane's forwarded -paths. - -A data plane whose URLs are not directly reachable by clients (e.g. -the filesystem server, which binds localhost only) asks, via its -`Configuration`, for path namespaces to be forwarded to it. The `Blob` -library then registers these routes on the application's own HTTP -server, forwarding the requests to the data plane's HTTP endpoint. -That keeps the data plane off any externally-exposed port: a single -application origin/tunnel serves both the control plane and the -bytes. - -The proxy is deliberately dumb: it forwards the request path and query -verbatim and never inspects them. The data plane minted the URL and -validates its own signature, so the proxy adds no trust. The one thing -it enforces is the namespace: every forwarded path must live under -`FORWARDED_PATH_PREFIX`, so a data plane can never claim application -routes. -""" - -import aiohttp -from rbt.std.blob.v1.data_plane_pb2 import ( - HTTP_METHOD_GET, - HTTP_METHOD_PUT, - ForwardedPath, - HttpMethod, -) -from reboot.aio.http import PythonWebFramework -from reboot.std.blob.v1._content_type import download_headers -from reboot.std.blob.v1._data_plane import FORWARDED_PATH_PREFIX -from starlette.requests import Request -from starlette.responses import Response, StreamingResponse -from typing import Iterable - -_STREAM_CHUNK_SIZE = 1024 * 1024 - -# Response headers worth carrying back from the data plane on a -# download; others are hop-by-hop or recomputed by the framework. -_FORWARDED_GET_HEADERS = ( - "Content-Type", - "Content-Length", - "ETag", - "Accept-Ranges", - # Without this the data plane's own `nosniff` would be dropped on - # the way through, which is exactly where it matters: these bytes - # reach the browser on the application's origin. - "X-Content-Type-Options", -) - - -def mount_proxy_routes( - http: PythonWebFramework.HTTP, - proxy_target_url: str, - forwarded_paths: Iterable[ForwardedPath], -) -> None: - """Registers a reverse-proxy route for each of the data plane's - `forwarded_paths`, forwarding to `proxy_target_url`. Refuses paths - outside `FORWARDED_PATH_PREFIX` and unknown methods.""" - - target = proxy_target_url.rstrip("/") - - def _forward_url(request: Request) -> str: - url = target + request.url.path - if request.url.query: - url += "?" + request.url.query - return url - - async def put_forward(rest: str, request: Request) -> Response: - session = aiohttp.ClientSession() - try: - upstream = await session.put( - _forward_url(request), - data=request.stream(), - ) - body = await upstream.read() - headers = {} - if "ETag" in upstream.headers: - headers["ETag"] = upstream.headers["ETag"] - return Response( - content=body, - status_code=upstream.status, - headers=headers, - ) - finally: - await session.close() - - async def get_forward(rest: str, request: Request) -> Response: - session = aiohttp.ClientSession() - # Close the session on every non-streaming path (connection - # error, non-200); on the streaming path the generator's - # `finally` closes it once the body is fully read or the client - # disconnects. - try: - upstream = await session.get(_forward_url(request)) - except Exception: - await session.close() - raise - - if upstream.status != 200: - try: - body = await upstream.read() - finally: - await session.close() - return Response(content=body, status_code=upstream.status) - - headers = { - name: upstream.headers[name] - for name in _FORWARDED_GET_HEADERS - if name in upstream.headers - } - # Applied again on the way out, not just where the bytes - # are stored: a data plane that is not ours -- S3, say -- - # returns whatever content type it was given at upload, - # and this hop is what puts it on the application's - # origin. - media_type, safety_headers = download_headers( - upstream.headers.get("Content-Type", "") - ) - headers.pop("Content-Type", None) - headers.update(safety_headers) - - async def stream(): - try: - async for chunk in upstream.content.iter_chunked( - _STREAM_CHUNK_SIZE - ): - yield chunk - finally: - await session.close() - - return StreamingResponse( - stream(), - status_code=200, - media_type=media_type, - headers=headers, - ) - - mounted: set[tuple[int, str]] = set() - for forwarded in forwarded_paths: - prefix = forwarded.path_prefix - if not prefix.startswith(FORWARDED_PATH_PREFIX): - raise ValueError( - f"Blob data plane requested forwarding of '{prefix}', " - f"which is outside `{FORWARDED_PATH_PREFIX}`; refusing" - ) - if (forwarded.method, prefix) in mounted: - continue - mounted.add((forwarded.method, prefix)) - # `{rest:path}` matches anything, including `/`s and the empty - # string, so the route covers exactly "path starts with - # `prefix`". - route = prefix + "{rest:path}" - if forwarded.method == HTTP_METHOD_GET: - http.get(route)(get_forward) - elif forwarded.method == HTTP_METHOD_PUT: - http.put(route)(put_forward) - else: - raise ValueError( - "Blob data plane requested forwarding with unsupported " - f"method {HttpMethod.Name(forwarded.method)}" - ) diff --git a/reboot/std/blob/v1/_store.py b/reboot/std/blob/v1/_store.py index 60862fcc7..ef90425ba 100644 --- a/reboot/std/blob/v1/_store.py +++ b/reboot/std/blob/v1/_store.py @@ -1,26 +1,33 @@ """The filesystem blob store: bytes storage for the open-source blob data plane. -A blob's *bytes* live here; all its metadata lives in the `Blob` state -machine (the control plane), which talks to the data plane only over -the `BlobDataPlane` gRPC interface (see `data_plane.proto` — that -interface, not this module, is the contract a data plane implements). +A blob's *bytes* live here; all its metadata lives in state machines. +The `Blob` control plane (see `blob.proto`) holds what the application +knows about a blob, and `StoredBlob` (see `filesystem.proto`) holds +what this store knows about its parts. Nothing about an object is +recorded on disk beside the bytes, so any of a replica's servers can +serve an upload or a download for one blob while agreeing with the +others on nothing but the directory. + The store mimics S3's multipart-upload semantics (numbered parts, per-part MD5 ETags, ETag-validating completion) so that clients drive one protocol regardless of which data plane serves them. """ +from __future__ import annotations + +import aiofiles +import aiofiles.os import asyncio import base64 import hashlib import hmac -import json import os import shutil import time from dataclasses import dataclass from reboot.crypto import root_keys -from typing import Any, Optional, Sequence +from typing import AsyncIterator, Optional, Sequence from uuid import uuid4 # The part size clients should use. Every part except the last must be @@ -41,23 +48,30 @@ # whatever their signing scheme allows. _MAX_URL_TTL_SECONDS = 7 * 24 * 60 * 60 -# The URL path prefix under which blob bytes are `PUT` and `GET`: the -# filesystem data-plane server serves it, and the application's proxy -# routes (see `_proxy.py`) forward it. -HTTP_PATH_PREFIX = "/__/reboot/blob" +# The paths under which blob bytes are `PUT` and `GET` on the +# application's own HTTP server (see `_http.py`). The blob, session +# and part travel in the query rather than the path; all of them are +# covered by the URL's signature either way. +PART_PATH = "/__/reboot/blob/part" +BLOB_PATH = "/__/reboot/blob" # HKDF `info` (domain separator) for the filesystem store's URL-signing # key. _SIGNING_INFO = b"reboot.std.blob.url-signing" +_STREAM_CHUNK_BYTES = 1024 * 1024 + class BlobStoreError(Exception): - """A permanent storage failure (e.g. a part ETag mismatch at - completion time), reported to the control plane as a - `CompleteUpload` `error` so the client can re-upload. Transient - failures (e.g. network errors) are raised as their original - exception types instead, becoming gRPC errors that the control - plane's workflow retries.""" + """A permanent storage failure (e.g. a part missing at completion + time), reported to the control plane as a `CompleteUpload` `error` + so the client can re-upload. Transient failures (e.g. network + errors) are raised as their original exception types instead, + becoming gRPC errors that the control plane's workflow retries.""" + + +class PartTooLarge(Exception): + """A part's bytes exceeded the store's part size.""" @dataclass(frozen=True) @@ -69,59 +83,24 @@ class UploadedPart: @dataclass(frozen=True) -class PartRecord: - """One part of a committed blob: its number, and the size its - bytes on disk were found to have at completion.""" +class WrittenPart: + """One part of an upload, as this store found its bytes to be.""" number: int + etag: str size: int + storage_id: str @dataclass(frozen=True) -class BlobMetadata: - """A blob's entry in the store, as `meta.json` holds it. +class StagedPart: + """A part whose bytes are on disk under their final name but not + yet claimed by the object. - `upload_id`, `etag` and `parts` are written in the same atomic - update as `committed`, so a committed blob carries all three.""" - content_type: str - committed: bool - upload_id: Optional[str] = None - etag: Optional[str] = None - parts: tuple[PartRecord, ...] = () - - @classmethod - def from_json(cls, data: dict[str, Any]) -> "BlobMetadata": - return cls( - content_type=data["content_type"], - committed=data.get("committed", False), - upload_id=data.get("upload_id"), - etag=data.get("etag"), - parts=tuple( - PartRecord(number=part["number"], size=part["size"]) - for part in data.get("parts", ()) - ), - ) - - def to_json(self) -> dict[str, Any]: - """The on-disk form. A `dict` is the right shape at this one - boundary and nowhere else; absent fields are omitted rather - than written as `null`, so the file stays byte-comparable with - what earlier versions of this store wrote.""" - data: dict[str, Any] = { - "content_type": self.content_type, - "committed": self.committed, - } - if self.upload_id is not None: - data["upload_id"] = self.upload_id - if self.etag is not None: - data["etag"] = self.etag - if self.parts: - data["parts"] = [ - { - "number": part.number, - "size": part.size - } for part in self.parts - ] - return data + That name is this write's alone, so publishing a part can never + land on another's bytes and the manifest decides which of a part + number's files the object is made of.""" + part: WrittenPart + path: str def _encode_blob_id(blob_id: str) -> str: @@ -130,58 +109,96 @@ def _encode_blob_id(blob_id: str) -> str: return base64.urlsafe_b64encode(blob_id.encode()).decode() -def _fsync_path(path: str) -> None: - fd = os.open(path, os.O_RDONLY) +async def _unlink_if_present(path: str) -> None: try: - os.fsync(fd) - finally: - os.close(fd) + await aiofiles.os.unlink(path) + except FileNotFoundError: + pass + + +async def _fsync_directory(path: str) -> None: + """Persists a directory's entries. In a thread, since `aiofiles` + has no `fsync`.""" + + def sync() -> None: + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + await asyncio.to_thread(sync) + + +async def _publish(temporary: str, path: str) -> None: + """Renames a part's bytes into place and makes the rename itself + durable. + + Without the directory `fsync` the rename can still be in the page + cache when the part is recorded as published, and a crash there + leaves the manifest naming a file that does not exist -- which + nothing downstream re-checks, since completion works from the + digests rather than the bytes.""" + await aiofiles.os.replace(temporary, path) + await _fsync_directory(os.path.dirname(path)) + + +def composite_etag(etags: Sequence[str]) -> str: + """An object's ETag, S3-style: the MD5 of its parts' concatenated + MD5 digests, suffixed with the part count.""" + digests = b"".join(bytes.fromhex(etag) for etag in etags) + return hashlib.md5(digests).hexdigest() + f"-{len(etags)}" class FilesystemBlobStore: """Stores blob bytes as part files on the local filesystem, served - over HTTP by the filesystem data-plane server (see `_http.py`). + over HTTP by the application (see `_http.py`). Layout, under `directory`: {encoded_blob_id}/ - meta.json Content type; part manifest and - composite ETag once committed. {upload_id}/ - part.{number:08d} One file per uploaded part. - - Parts are written once under a random `upload_id` directory (so no - temp-file-and-rename protocol is needed) and fsynced before the - data plane returns their ETag. The part files remain the committed - object's on-disk representation: downloads stream them in part - order, so completion never rewrites bytes. + part.{number:08d}.{storage_id} One file per write of a + part; the manifest says + which of them the object + is made of. + + Parts are written under a random `upload_id` directory and fsynced + before the store reports what they hold. The part files remain the + committed object's on-disk representation: downloads stream them in + part order, so completion never rewrites bytes. """ def __init__( self, directory: str, part_size: int = DEFAULT_PART_SIZE_BYTES, - ): + ) -> None: self._directory = directory self._part_size = part_size - self._locks: dict[str, asyncio.Lock] = {} - os.makedirs(directory, exist_ok=True) + + @classmethod + async def create( + cls, + directory: str, + part_size: int = DEFAULT_PART_SIZE_BYTES, + ) -> FilesystemBlobStore: + """A store over `directory`, which is created if it does not + exist and made durable before anything is written into it.""" + await aiofiles.os.makedirs(directory, exist_ok=True) + # The store's own entry in its parent, not just its contents: + # fsyncing a directory persists what is in it, not its name, + # so without this a crash can take the whole store away along + # with every committed upload inside it. + parent = os.path.dirname(os.path.normpath(directory)) + if parent: + await _fsync_directory(parent) + return cls(directory, part_size) @property def directory(self) -> str: return self._directory - def lock_for(self, encoded_blob_id: str) -> asyncio.Lock: - """Serializes one blob's completion against the part `PUT`s - that publish its bytes. - - Both run in this one process -- the server hosts the gRPC - service and the byte endpoint together -- so an in-process - lock is enough to make completion see a fixed set of parts. - Held only across a `PUT`'s final rename, not across the upload - itself, so parts still upload concurrently.""" - return self._locks.setdefault(encoded_blob_id, asyncio.Lock()) - @property def part_size(self) -> int: return self._part_size @@ -230,75 +247,41 @@ def part_path( encoded_blob_id: str, upload_id: str, part_number: int, + storage_id: str, ) -> str: + """Where one part's bytes live. + + Named by a value minted for the write that produced them as + well as by the part number, so that the name is immutable: + re-uploading a part writes a second file rather than replacing + the first, and what the manifest records is what a download + reads. Part numbers still order the object; storage IDs only + keep one part's writes apart. Deliberately not the ETag: MD5 + is what the protocol calls for, and two different parts can + share one.""" return os.path.join( self.blob_directory(encoded_blob_id), upload_id, - f"part.{part_number:08d}", + f"part.{part_number:08d}.{storage_id}", ) - def _meta_path(self, encoded_blob_id: str) -> str: - return os.path.join(self.blob_directory(encoded_blob_id), "meta.json") - - def read_meta(self, encoded_blob_id: str) -> Optional[BlobMetadata]: - try: - with open(self._meta_path(encoded_blob_id), "r") as f: - return BlobMetadata.from_json(json.load(f)) - except FileNotFoundError: - return None - - def _write_meta( + async def make_upload_directory( self, - encoded_blob_id: str, - meta: BlobMetadata, + blob_id: str, + upload_id: str, ) -> None: - # Write to a temp file and atomically rename, so a crash - # mid-write can never leave a torn `meta.json` that a - # concurrent `read_meta` would fail to parse. - path = self._meta_path(encoded_blob_id) - temp_path = f"{path}.{uuid4().hex}.tmp" - with open(temp_path, "w") as f: - json.dump(meta.to_json(), f) - f.flush() - os.fsync(f.fileno()) - os.replace(temp_path, path) - _fsync_path(os.path.dirname(path)) - - async def begin_upload(self, blob_id: str, content_type: str) -> str: + """Prepares the directory a session's parts are written into.""" encoded = _encode_blob_id(blob_id) - - def sync(): - # Idempotent by blob ID: if an uncommitted session already - # exists (a retried `BeginUpload`), reuse it rather than - # orphaning it under a fresh upload ID. - existing = self.read_meta(encoded) - if ( - existing is not None and not existing.committed and - existing.upload_id is not None - ): - upload_id = existing.upload_id - reuse = True - else: - upload_id = uuid4().hex - reuse = False - # Create the upload directory (and, with it, the blob - # directory) before writing `meta.json` into the latter. - os.makedirs( - os.path.join(self.blob_directory(encoded), upload_id), - exist_ok=True, - ) - if not reuse: - self._write_meta( - encoded, - BlobMetadata( - content_type=content_type, - committed=False, - upload_id=upload_id, - ), - ) - return upload_id - - return await asyncio.to_thread(sync) + await aiofiles.os.makedirs( + os.path.join(self.blob_directory(encoded), upload_id), + exist_ok=True, + ) + # Each directory is made durable before anything is written + # into it: a part file fsynced into a directory entry that a + # crash then loses is a manifest naming bytes that are not + # there. + await _fsync_directory(self._directory) + await _fsync_directory(self.blob_directory(encoded)) def part_put_url( self, @@ -312,102 +295,10 @@ def part_put_url( encoded, upload_id, part_number, expiration ) return ( - f"{HTTP_PATH_PREFIX}/{encoded}/{upload_id}/parts/{part_number}" - f"?exp={expiration}&sig={signature}" + f"{PART_PATH}?blob={encoded}&upload={upload_id}" + f"&part={part_number}&exp={expiration}&sig={signature}" ) - async def complete( - self, - blob_id: str, - upload_id: str, - content_type: str, - parts: list[UploadedPart], - max_size: Optional[int] = None, - ) -> str: - - def sync(): - encoded = _encode_blob_id(blob_id) - digests = [] - manifest: list[PartRecord] = [] - total_size = 0 - last_part_number = max(part.number for part in parts) - for part in sorted(parts, key=lambda part: part.number): - path = self.part_path(encoded, upload_id, part.number) - digest = hashlib.md5() - size = 0 - try: - with open(path, "rb") as f: - while chunk := f.read(1024 * 1024): - digest.update(chunk) - size += len(chunk) - except FileNotFoundError: - raise BlobStoreError( - f"part {part.number} was never uploaded" - ) - if digest.hexdigest() != part.etag.strip('"'): - raise BlobStoreError( - f"part {part.number} ETag mismatch: the uploaded " - "bytes do not match what was reported via " - "`PartUploaded`" - ) - if size != part.size: - raise BlobStoreError( - f"part {part.number} size mismatch: uploaded " - f"{size} bytes but {part.size} were reported via " - "`PartUploaded`" - ) - if part.number != last_part_number and size != self.part_size: - # S3 rejects a short middle part with - # `EntityTooSmall`; reject it here too, so that an - # upload which cannot commit against the S3 store - # cannot commit against this one either. - raise BlobStoreError( - f"part {part.number} is {size} bytes, but every " - f"part except the last must be exactly " - f"{self.part_size} bytes" - ) - total_size += size - digests.append(digest.digest()) - manifest.append( - PartRecord(number=part.number, size=size) - ) - - # Verify the *real* total against `max_size` (not the - # already-checked reported sizes) as defense in depth. - if max_size is not None and total_size > max_size: - raise BlobStoreError( - f"uploaded {total_size} bytes exceeds the maximum of " - f"{max_size}" - ) - - # Composite ETag, S3-style: the MD5 of the concatenated - # part MD5 digests, suffixed with the part count. - etag = ( - hashlib.md5(b"".join(digests)).hexdigest() + f"-{len(digests)}" - ) - self._write_meta( - encoded, - BlobMetadata( - content_type=content_type, - committed=True, - upload_id=upload_id, - etag=etag, - parts=tuple(manifest), - ), - ) - return etag - - # Completion validates the bytes on disk and then records - # the ETag it computed from them. A part `PUT` landing in - # between would leave a committed blob whose bytes no - # longer match its recorded ETag, so hold the blob's lock - # across the whole of it. (S3 gets this for free: a - # concurrent `UploadPart` changes the part's ETag and - # `CompleteMultipartUpload` then fails with - # `InvalidPart`.) - async with self.lock_for(_encode_blob_id(blob_id)): - return await asyncio.to_thread(sync) - def download_url( self, blob_id: str, @@ -422,28 +313,174 @@ def download_url( ) expiration = int(time.time()) + ttl signature = self.signature_for_get(encoded, expiration) - url = ( - f"{HTTP_PATH_PREFIX}/{encoded}?exp={expiration}&sig={signature}" - ) + url = (f"{BLOB_PATH}?blob={encoded}&exp={expiration}&sig={signature}") return url, ttl - async def delete( + async def stage_part( self, - blob_id: str, - upload_ids: Sequence[str] = (), + encoded_blob_id: str, + upload_id: str, + part_number: int, + chunks: AsyncIterator[bytes], + ) -> StagedPart: + """Writes one part's bytes under a name nothing reads, and + reports what they turned out to be, digesting them on the way + through so that the ETag describes what landed rather than + what was claimed. + + The bytes land under a name minted for this write, which no + other write occupies; whether the object is *made of* them is + `StoredBlob`'s to say. Raises `PartTooLarge`, having kept + nothing, if the bytes exceed the part size.""" + storage_id = uuid4().hex + temporary = os.path.join( + self.blob_directory(encoded_blob_id), + upload_id, + f"part.{part_number:08d}.{storage_id}.partial", + ) + digest = hashlib.md5() + size = 0 + try: + # A part is megabytes, so the writes go off the event loop + # for the same reason the download reads off it: this runs + # on the loop, and writing inline would stall every other + # request this server is handling. + async with aiofiles.open(temporary, "wb") as file: + async for chunk in chunks: + if size + len(chunk) > self._part_size: + raise PartTooLarge() + digest.update(chunk) + size += len(chunk) + await file.write(chunk) + await file.flush() + # `aiofiles` has no `fsync`; `fileno()` is proxied + # straight through, so the descriptor is the real one. + await asyncio.to_thread(os.fsync, file.fileno()) + except BaseException: + # Never leave a partial file behind to be mistaken for a + # part. + await _unlink_if_present(temporary) + raise + + path = self.part_path( + encoded_blob_id, upload_id, part_number, storage_id + ) + # Renamed into place rather than written there, so a download + # never catches a part half-written. Safe to do before the + # manifest claims these bytes, because the name is theirs + # alone: at worst they are left unclaimed. + await _publish(temporary, path) + return StagedPart( + part=WrittenPart( + number=part_number, + etag=digest.hexdigest(), + size=size, + storage_id=storage_id, + ), + path=path, + ) + + async def discard_storage_id( + self, + encoded_blob_id: str, + upload_id: str, + part_number: int, + storage_id: str, ) -> None: - # `upload_ids` is not needed here: a part lives inside the - # blob's own directory, so removing the directory removes any - # unfinished upload with it. - encoded = _encode_blob_id(blob_id) + """Drops one write of a part by name.""" + await _unlink_if_present( + self.part_path( + encoded_blob_id, upload_id, part_number, storage_id + ), + ) + + async def discard_part(self, staged: StagedPart) -> None: + """Drops a part's bytes, for one the object turned out not to + be made of. - def sync(): - # Only a blob that is already gone is ignored: any - # other failure must reach the caller, or `PerformRemove` - # would report bytes deleted that are still on disk. - try: - shutil.rmtree(self.blob_directory(encoded)) - except FileNotFoundError: - pass + Safe because the name belongs to this write alone: no manifest + can be pointing at it unless this write's own `PublishPart` + succeeded.""" + await _unlink_if_present(staged.path) - await asyncio.to_thread(sync) + async def reclaim( + self, + encoded_blob_id: str, + upload_id: str, + keep: Sequence[tuple[int, str]], + ) -> None: + """Removes every part file of a session except the ones the + object is made of. + + Called once the manifest is fixed, which is the first moment + it is known which versions of a part are not in the object: + a part re-uploaded with different bytes leaves its earlier + version behind, and a client that uploads more parts than it + commits leaves those. Until then the extra files are what + makes re-uploading a part safe, so they cannot be reclaimed + eagerly.""" + directory = os.path.join( + self.blob_directory(encoded_blob_id), upload_id + ) + wanted = { + os.path.basename( + self.part_path(encoded_blob_id, upload_id, number, storage_id) + ) for number, storage_id in keep + } + + try: + names = await aiofiles.os.listdir(directory) + except FileNotFoundError: + return + for name in names: + if name in wanted: + continue + if name.endswith(".partial"): + # A write still in flight. Its own writer removes it if + # it fails and renames it if it succeeds; taking it + # here would turn that writer's refusal into a failure + # to rename. + continue + await _unlink_if_present(os.path.join(directory, name)) + + async def read_part( + self, + encoded_blob_id: str, + upload_id: str, + part_number: int, + storage_id: str, + ) -> AsyncIterator[bytes]: + """Streams one part's bytes: the write the object's manifest + recorded, and no later write of that part.""" + path = self.part_path( + encoded_blob_id, upload_id, part_number, storage_id + ) + # Read off the event loop: this generator is driven by it, and + # a part is megabytes, so reading inline would stall every + # other request this server is handling. + async with aiofiles.open(path, "rb") as file: + while chunk := await file.read(_STREAM_CHUNK_BYTES): + yield chunk + + async def upload_directory_exists( + self, + encoded_blob_id: str, + upload_id: str, + ) -> bool: + return await aiofiles.os.path.isdir( + os.path.join(self.blob_directory(encoded_blob_id), upload_id) + ) + + async def delete(self, blob_id: str) -> None: + """Removes every byte this store holds for a blob.""" + encoded = _encode_blob_id(blob_id) + # Only a blob that is already gone is ignored: any other failure + # must reach the caller, or `PerformRemove` would report bytes + # deleted that are still on disk. In a thread because + # `aiofiles` has no `rmtree`. + try: + await asyncio.to_thread( + shutil.rmtree, self.blob_directory(encoded) + ) + except FileNotFoundError: + pass diff --git a/reboot/std/blob/v1/_stored_blob.py b/reboot/std/blob/v1/_stored_blob.py new file mode 100644 index 000000000..a324b5a50 --- /dev/null +++ b/reboot/std/blob/v1/_stored_blob.py @@ -0,0 +1,145 @@ +"""The filesystem data plane's metadata, as a Reboot state machine. + +Bytes live on disk (see `_store.py`); which parts make up an object, +and whether that object is finished, live here. The two are ordered +against each other by this state and nothing else: a part's bytes are +written by whichever of a replica's servers served the upload, so the +decision of whether that part is *in* the object has to be made +somewhere all of them agree, which is here. +""" + +from rbt.std.blob.v1.filesystem_rbt import ( + StoredBlob, + StoredBlobBeginUploadRequest, + StoredBlobBeginUploadResponse, + StoredBlobCommitRequest, + StoredBlobCommitResponse, + StoredBlobForgetRequest, + StoredBlobForgetResponse, + StoredBlobMetadataRequest, + StoredBlobMetadataResponse, + StoredBlobPublishPartRequest, + StoredBlobPublishPartResponse, +) +from reboot.aio.auth.authorizers import allow_if, is_app_internal +from reboot.aio.contexts import ReaderContext, WriterContext +from typing import Optional +from uuid import uuid4 + + +class StoredBlobServicer(StoredBlob.Servicer): + + def authorizer(self) -> StoredBlob.Authorizer: + # Nothing here is reachable by an end user. The data plane's + # gRPC surface and its byte endpoints are the only callers, and + # both are inside this application; a client's capability is + # the signed URL it was given, not access to this state. + return StoredBlob.Authorizer( + begin_upload=allow_if(any=[is_app_internal]), + publish_part=allow_if(any=[is_app_internal]), + commit=allow_if(any=[is_app_internal]), + metadata=allow_if(any=[is_app_internal]), + forget=allow_if(any=[is_app_internal]), + ) + + async def begin_upload( + self, + context: WriterContext, + request: StoredBlobBeginUploadRequest, + ) -> StoredBlobBeginUploadResponse: + self.state.committed = False + self.state.content_type = request.content_type + self.state.upload_id = uuid4().hex + self.state.ClearField("etag") + del self.state.parts[:] + return StoredBlobBeginUploadResponse(upload_id=self.state.upload_id) + + async def publish_part( + self, + context: WriterContext, + request: StoredBlobPublishPartRequest, + ) -> StoredBlobPublishPartResponse: + if self.state.committed: + return StoredBlobPublishPartResponse(published=False) + if ( + not self.state.HasField("upload_id") or + self.state.upload_id != request.upload_id + ): + return StoredBlobPublishPartResponse(published=False) + + # A part number may arrive more than once: an upload the + # client retried, or two attempts racing. Whichever lands last + # is the one the object is made of -- each wrote its own file, + # so this is the only place the choice is made, and the one it + # displaces is named back so its bytes can go. + superseded: Optional[str] = None + for index, part in enumerate(self.state.parts): + if part.number == request.part.number: + superseded = part.storage_id + self.state.parts[index].CopyFrom(request.part) + break + else: + self.state.parts.append(request.part) + self.state.parts.sort(key=lambda part: part.number) + + return StoredBlobPublishPartResponse( + published=True, + superseded_storage_id=superseded, + ) + + async def commit( + self, + context: WriterContext, + request: StoredBlobCommitRequest, + ) -> StoredBlobCommitResponse: + if self.state.committed: + # `CompleteUpload` is retried by a workflow, so arriving at + # an object that is already finished is success, not a + # conflict. + return StoredBlobCommitResponse(committed=True) + + published = {part.number: part for part in self.state.parts} + for part in request.parts: + current = published.get(part.number) + if current is None or current.storage_id != part.storage_id: + # A part was uploaded again between this manifest being + # read and being committed, so the bytes it names are + # not the object's any more -- and the request that + # replaced them has taken them away. + return StoredBlobCommitResponse(committed=False) + + self.state.committed = True + self.state.content_type = request.content_type + self.state.upload_id = request.upload_id + self.state.etag = request.etag + # Replaced, not added to: what the object is made of is the + # manifest given here. A part written under this session but + # left out of it -- uploaded and never reported, or a version + # of a part superseded by another -- belongs to nothing, and + # is not what a download reads or what the ETag describes. + del self.state.parts[:] + self.state.parts.extend(request.parts) + return StoredBlobCommitResponse(committed=True) + + async def metadata( + self, + context: ReaderContext, + request: StoredBlobMetadataRequest, + ) -> StoredBlobMetadataResponse: + return StoredBlobMetadataResponse(blob=self.state) + + async def forget( + self, + context: WriterContext, + request: StoredBlobForgetRequest, + ) -> StoredBlobForgetResponse: + self.state.committed = False + self.state.content_type = "" + self.state.ClearField("upload_id") + self.state.ClearField("etag") + del self.state.parts[:] + return StoredBlobForgetResponse() + + +def servicers() -> list[type[StoredBlob.Servicer]]: + return [StoredBlobServicer] diff --git a/reboot/std/blob/v1/blob.py b/reboot/std/blob/v1/blob.py index 6a7349b7e..930695229 100644 --- a/reboot/std/blob/v1/blob.py +++ b/reboot/std/blob/v1/blob.py @@ -78,20 +78,26 @@ DataPlaneGetPartUploadInstructionsRequest, DataPlaneUploadedPart, ) -from rbt.std.blob.v1.data_plane_pb2_grpc import BlobDataPlaneStub from reboot.aio.applications import Application, Library from reboot.aio.auth.authorizers import Authorizer, allow_if, is_app_internal from reboot.aio.backoff import Backoff from reboot.aio.contexts import ReaderContext, WorkflowContext, WriterContext from reboot.aio.http import PythonWebFramework +from reboot.aio.servicers import Servicer from reboot.aio.workflows import at_least_once_per_workflow from reboot.std.blob.v1._data_plane import ( ENVVAR_BLOB_DATA_PLANE_URL, - proxy_target_from_environment, - stub_from_environment, + blobs_directory, + configured_data_plane_url, + data_plane_stub, + data_plane_stub_at, ) -from reboot.std.blob.v1._proxy import mount_proxy_routes -from reboot.std.blob.v1._store import MAX_PARTS +from reboot.std.blob.v1._filesystem_data_plane import ( + FilesystemDataPlaneServicer, +) +from reboot.std.blob.v1._http import mount_byte_routes +from reboot.std.blob.v1._store import MAX_PARTS, FilesystemBlobStore +from reboot.std.blob.v1._stored_blob import StoredBlobServicer from typing import Optional logger = log.log.get_logger(__name__) @@ -114,7 +120,7 @@ # cannot smuggle arbitrary content into a value a data plane later # relies on to finalize the object. Any narrower format belongs to the # store that produces it. -_PART_ETAG_PATTERN = re.compile(r'^[!#-~]{1,128}$') +_PART_ETAG_PATTERN = re.compile(r'[!#-~]{1,128}') def _size_ceiling(state: Blob.State) -> Optional[int]: @@ -187,9 +193,8 @@ def _downloader_or_open( class BlobServicer(Blob.Servicer): - # The data-plane gRPC stub and the part size it reported, both set - # by `BlobLibrary` once it has connected to the data plane. - _data_plane: BlobDataPlaneStub + # The part size the data plane reported, set by `BlobLibrary` + # once it has asked. _part_size: int def authorizer(self) -> Blob.Authorizer: @@ -263,13 +268,14 @@ async def begin_upload( state = await Blob.ref().read(context) async def provision() -> str: - response = await cls._data_plane.BeginUpload( - DataPlaneBeginUploadRequest( - blob_id=context.state_id, - content_type=state.content_type, + async with data_plane_stub(context) as data_plane: + response = await data_plane.BeginUpload( + DataPlaneBeginUploadRequest( + blob_id=context.state_id, + content_type=state.content_type, + ) ) - ) - return response.upload_id + return response.upload_id upload_id = await at_least_once_per_workflow( "provision upload session", context, provision @@ -296,12 +302,13 @@ async def record(state: Blob.State) -> None: await Blob.ref().write(context, record) if removed: - await cls._data_plane.Delete( - DataPlaneDeleteRequest( - blob_id=context.state_id, - upload_ids=[upload_id], + async with data_plane_stub(context) as data_plane: + await data_plane.Delete( + DataPlaneDeleteRequest( + blob_id=context.state_id, + upload_ids=[upload_id], + ) ) - ) return BeginUploadResponse() @@ -340,13 +347,14 @@ async def get_part_upload_instructions( number for number in request.part_numbers if 1 <= number <= max_part_number ] - response = await self._data_plane.GetPartUploadInstructions( - DataPlaneGetPartUploadInstructionsRequest( - blob_id=context.state_id, - upload_id=self.state.upload_id, - part_numbers=part_numbers, + async with data_plane_stub(context) as data_plane: + response = await data_plane.GetPartUploadInstructions( + DataPlaneGetPartUploadInstructionsRequest( + blob_id=context.state_id, + upload_id=self.state.upload_id, + part_numbers=part_numbers, + ) ) - ) instructions = [ PartUploadInstruction( part_number=instruction.part_number, @@ -375,11 +383,9 @@ async def part_uploaded( if request.part_number < 1 or request.part_number > MAX_PARTS: raise Blob.PartUploadedAborted(IncompleteParts()) - # Validate the ETag as an MD5 hex digest, as the data-plane - # contract requires, so a client can't smuggle arbitrary - # content into the value a data plane later relies on to - # finalize the object. - if not _PART_ETAG_PATTERN.match(request.etag): + # Check that the ETag is safe to carry; see + # `_PART_ETAG_PATTERN` for why that is all this can check. + if not _PART_ETAG_PATTERN.fullmatch(request.etag): raise Blob.PartUploadedAborted(IncompleteParts()) part = BlobPart( @@ -466,7 +472,8 @@ async def attempt() -> tuple: # mismatch): report it back onto the blob so the client can # re-upload and re-commit. A gRPC error is transient and # propagates, so the workflow retries. - response = await cls._data_plane.CompleteUpload(complete_request) + async with data_plane_stub(context) as data_plane: + response = await data_plane.CompleteUpload(complete_request) if response.HasField("error"): return ("failed", response.error) return ("committed", response.etag) @@ -500,9 +507,10 @@ async def record(state: Blob.State) -> None: if superseded[0] and outcome == "committed": async def cleanup() -> None: - await cls._data_plane.Delete( - DataPlaneDeleteRequest(blob_id=context.state_id) - ) + async with data_plane_stub(context) as data_plane: + await data_plane.Delete( + DataPlaneDeleteRequest(blob_id=context.state_id) + ) await at_least_once_per_workflow( "cleanup orphaned bytes", context, cleanup @@ -544,7 +552,8 @@ async def get_download_url( ) if request.HasField("ttl_seconds"): download_request.ttl_seconds = request.ttl_seconds - response = await self._data_plane.GetDownloadUrl(download_request) + async with data_plane_stub(context) as data_plane: + response = await data_plane.GetDownloadUrl(download_request) return GetDownloadUrlResponse( url=response.url, ttl_seconds=response.ttl_seconds, @@ -578,12 +587,13 @@ async def perform_remove( upload_ids = [state.upload_id] if state.HasField("upload_id") else [] async def remove() -> None: - await cls._data_plane.Delete( - DataPlaneDeleteRequest( - blob_id=context.state_id, - upload_ids=upload_ids, + async with data_plane_stub(context) as data_plane: + await data_plane.Delete( + DataPlaneDeleteRequest( + blob_id=context.state_id, + upload_ids=upload_ids, + ) ) - ) await at_least_once_per_workflow("remove bytes", context, remove) @@ -618,22 +628,63 @@ async def expire_if_not_committed( class BlobLibrary(Library): name = BLOBS_LIBRARY_NAME - def __init__(self) -> None: - self._connected = False + def __init__(self, *, blobs_directory: Optional[str] = None) -> None: + self._blobs_directory = blobs_directory + self._store: Optional[FilesystemBlobStore] = None + self._prepared = False - def servicers(self) -> list[type[Blob.Servicer]]: - return [BlobServicer] + def _hosts_data_plane(self) -> bool: + """Whether this application serves its own blob bytes. It does + unless `REBOOT_BLOB_DATA_PLANE_URL` points it at a data plane + elsewhere.""" + return configured_data_plane_url() is None + + def servicers(self) -> list[type[Servicer]]: + if not self._hosts_data_plane(): + return [BlobServicer] + return [BlobServicer, StoredBlobServicer] + + def legacy_grpc_servicers(self) -> list[type]: + if not self._hosts_data_plane(): + return [] + return [FilesystemDataPlaneServicer] async def pre_run(self, application: Application) -> None: # `pre_run` may be called more than once (e.g. a test that - # `up`s an application after a `down`); connect once. - if self._connected: + # `up`s an application after a `down`); prepare once. + if self._prepared: return - stub = stub_from_environment() - BlobServicer._data_plane = stub + url = configured_data_plane_url() + if url is None: + if not isinstance(application.web_framework, PythonWebFramework): + # Better to fail here than to hand out URLs that will + # 404: without the byte routes this application cannot + # serve any bytes it stores. + raise RuntimeError( + "Serving blob bytes needs HTTP routes, which only " + "Python applications currently support; configure a " + "data plane that serves its own URLs via " + f"`{ENVVAR_BLOB_DATA_PLANE_URL}`." + ) + store = await FilesystemBlobStore.create( + self._blobs_directory or blobs_directory() + ) + self._store = store + FilesystemDataPlaneServicer._store = store + mount_byte_routes(application.http, store) + # Known without asking, since this application is the data + # plane; `Configuration` still reports it, for a client + # that does not know which data plane it is talking to. + BlobServicer._part_size = store.part_size + self._prepared = True + return - configuration = await self._configuration(stub) + # A data plane elsewhere has to be asked. Asked here, in + # `pre_run`, because this runs in every server process, and + # `_part_size` is per-process state that each of them serves + # `GetPartUploadInstructions` from. + configuration = await self._configuration(url) if configuration.part_size == 0: # Fail here rather than let a zero reach the browser # uploader, which divides the blob's size by it. @@ -642,45 +693,22 @@ async def pre_run(self, application: Application) -> None: f"check the service at `{ENVVAR_BLOB_DATA_PLANE_URL}`" ) BlobServicer._part_size = configuration.part_size + self._prepared = True - # A data plane whose URLs are not directly reachable (e.g. the - # localhost filesystem server) asks for paths to be forwarded - # to it, and the application proxies those to its HTTP - # endpoint; one that serves its own URLs (e.g. S3) asks for - # none and needs no application routes. - if configuration.forwarded_paths: - if not isinstance(application.web_framework, PythonWebFramework): - # Better to fail fast here than to hand out URLs that - # will 404: without the proxy routes, a forwarded-path - # data plane cannot serve any bytes. - raise RuntimeError( - "This blob data plane needs paths forwarded to it, " - "which only Python applications currently support; " - "configure a data plane whose URLs are directly " - "reachable by clients via " - f"`{ENVVAR_BLOB_DATA_PLANE_URL}`." - ) - mount_proxy_routes( - application.http, - proxy_target_from_environment(configuration.http_port), - configuration.forwarded_paths, - ) - - self._connected = True - - async def _configuration( - self, - stub: BlobDataPlaneStub, - ) -> ConfigurationResponse: - # The data plane is normally already running, but tolerate a - # startup race by retrying while it becomes reachable. + async def _configuration(self, url: str) -> ConfigurationResponse: + """Asks the data plane at `url` how it is configured. The data + plane is normally already running, but a startup race is + tolerated by retrying while it becomes reachable.""" backoff = Backoff( max_backoff_seconds=_CONFIGURATION_MAX_BACKOFF_SECONDS, ) deadline = time.monotonic() + _CONFIGURATION_RETRY_SECONDS while True: try: - return await stub.Configuration(ConfigurationRequest()) + async with data_plane_stub_at(url) as data_plane: + return await data_plane.Configuration( + ConfigurationRequest() + ) except AioRpcError as error: if time.monotonic() >= deadline: raise RuntimeError( diff --git a/reboot/std/blob/v1/index.ts b/reboot/std/blob/v1/index.ts index 1e0e369a4..0e568203f 100644 --- a/reboot/std/blob/v1/index.ts +++ b/reboot/std/blob/v1/index.ts @@ -5,11 +5,10 @@ export * from "@reboot-dev/reboot-std-api/blob/v1/blob_rbt.js"; // The servicers are implemented in Python (the data-plane client // lives there); Node.js applications host them as "native" servicers. // -// NOTE: the application-side routes that proxy bytes to a data plane -// requesting forwarded paths (such as the local filesystem one) are -// currently only registered by Python applications; Node.js -// applications need a data plane whose URLs are directly reachable by -// clients (no forwarded paths, e.g. Reboot Cloud's). +// NOTE: the HTTP routes that serve the filesystem data plane's bytes +// are currently only registered by Python applications; a Node.js +// application needs a data plane that serves its own URLs, named by +// `REBOOT_BLOB_DATA_PLANE_URL`. export default { servicers: (): NativeServicer[] => { return [ diff --git a/tests/reboot/std/blob/v1/blob_tests.py b/tests/reboot/std/blob/v1/blob_tests.py index d3ee5049e..f60ddaa9f 100644 --- a/tests/reboot/std/blob/v1/blob_tests.py +++ b/tests/reboot/std/blob/v1/blob_tests.py @@ -1,8 +1,6 @@ import aiohttp import asyncio import hashlib -import tempfile -import threading import unittest from rbt.std.blob.v1.blob_rbt import ( Blob, @@ -14,16 +12,8 @@ ) from reboot.aio.applications import Application from reboot.aio.tests import Reboot -from reboot.std.blob.v1._store import ( - DEFAULT_PART_SIZE_BYTES, - BlobMetadata, - BlobStoreError, - FilesystemBlobStore, - UploadedPart, - _encode_blob_id, -) +from reboot.std.blob.v1._store import DEFAULT_PART_SIZE_BYTES from reboot.std.blob.v1.blob import blob_library -from unittest import mock # How long the completion/`PUT` handshake waits before giving up, # generous because it only ever elapses when the test is already @@ -315,55 +305,79 @@ async def test_get_download_url_requires_committed(self) -> None: await blob.get_download_url(self.context) self.assertIsInstance(raised.exception.error, NotCommitted) - async def test_store_complete_enforces_max_size(self) -> None: - # `complete()` verifies the *real* total size against - # `max_size` independently of the control plane's own - # (reported-size) check, as defense in depth against untruthful - # reported sizes. A store-level test, on its own store. - with tempfile.TemporaryDirectory() as directory: - store = FilesystemBlobStore(directory) - blob_id = "max-size-blob" - upload_id = await store.begin_upload(blob_id, "text/plain") - encoded = _encode_blob_id(blob_id) - data = b"x" * 100 - with open(store.part_path(encoded, upload_id, 1), "wb") as f: - f.write(data) - part = UploadedPart( - number=1, - etag=hashlib.md5(data).hexdigest(), - size=len(data), - ) - with self.assertRaises(BlobStoreError): - await store.complete( - blob_id, - upload_id, - "text/plain", - [part], - max_size=50, - ) - # Within the bound, it succeeds. - etag = await store.complete( - blob_id, - upload_id, - "text/plain", - [part], - max_size=1000, - ) - self.assertTrue(etag.endswith("-1")) + async def test_commit_refuses_a_part_that_was_misreported( + self, + ) -> None: + # The data plane records what each part's bytes turned out to + # be, and completion compares that against what the client + # said it uploaded. A client that under-reports a part's size + # -- the shape of an attempt to slip past `max_size` -- is + # refused on the strength of the bytes rather than the claim. + data = b"x" * 100 + blob, _ = await Blob.create( + self.context, + content_type="text/plain", + max_size=1000, + ) + instructions = await self._instructions(blob, [1]) + etag = await self._put(instructions.instructions[0].url, data) + await blob.part_uploaded( + self.context, + part_number=1, + etag=etag, + size=len(data) - 1, + ) + await blob.commit(self.context) + + info = await self._wait_until_status( + blob, {Blob.State.UPLOADING, Blob.State.COMMITTED} + ) + self.assertEqual(Blob.State.UPLOADING, info.status) + self.assertIn("size mismatch", info.commit_error) + + async def test_replaying_a_part_with_the_same_bytes_after_commit( + self, + ) -> None: + # The same bytes, not merely different ones: a part's file is + # named by a value minted for the write that produced it, so + # even an identical replay lands somewhere of its own. Naming + # it after the bytes instead -- by their ETag, say, which is + # only an MD5 -- would put this replay on top of the committed + # part and then delete it when the replay was refused. + data = b"identical bytes" + + blob, _ = await Blob.create( + self.context, + content_type="text/plain", + size=len(data), + ) + instructions = await self._instructions(blob, [1]) + url = instructions.instructions[0].url + etag = await self._put(url, data) + await blob.part_uploaded( + self.context, + part_number=1, + etag=etag, + size=len(data), + ) + await blob.commit(self.context) + await self._wait_until_status(blob, {Blob.State.COMMITTED}) - async def test_a_part_put_cannot_land_inside_completion( + self.assertEqual(409, await self._put_returning_status(url, data)) + + downloaded, _ = await self._download(blob) + self.assertEqual(data, downloaded) + + async def test_a_part_published_after_commit_is_refused( self, ) -> None: - # The race the blob's lock exists to close, driven to the - # exact interleaving rather than raced for: completion is - # paused after it has read the parts and computed their ETag - # but before it records either, and a part `PUT` on a - # still-valid signed URL is issued into that window. - # - # `_write_meta` runs on a worker thread (completion's body - # goes through `asyncio.to_thread`), so the handshake is - # `threading.Event`, not `asyncio.Event`: the thread cannot - # await one, and setting one from off the loop is not safe. + # The ordering `StoredBlob` exists to impose. A part `PUT` on a + # still-valid signed URL, arriving once the object is + # committed, must not become part of it -- and must not + # replace bytes the recorded ETag already describes. Whichever + # server serves that upload asks the state, and the state has + # already decided what the object is made of, so this holds + # however many servers a replica runs. data = b"original bytes" replacement = b"REPLACED bytes" self.assertEqual(len(data), len(replacement)) @@ -382,71 +396,17 @@ async def test_a_part_put_cannot_land_inside_completion( etag=etag, size=len(data), ) + await blob.commit(self.context) + await self._wait_until_status(blob, {Blob.State.COMMITTED}) - reached_recording = threading.Event() - may_record = threading.Event() - write_meta = FilesystemBlobStore._write_meta - - def paused_write_meta( - self, - encoded_blob_id: str, - meta: BlobMetadata, - ) -> None: - # Only completion records a committed blob; `begin_upload` - # writes metadata too, and must not be paused. - if meta.committed: - reached_recording.set() - may_record.wait(timeout=_RACE_TIMEOUT_SECONDS) - write_meta(self, encoded_blob_id, meta) - - with mock.patch.object( - FilesystemBlobStore, - "_write_meta", - paused_write_meta, - ): - await blob.commit(self.context) - self.assertTrue( - await asyncio.to_thread( - reached_recording.wait, - _RACE_TIMEOUT_SECONDS, - ), - "completion never reached the point where it records " - "what it read", - ) - - # Deliberately not awaited yet: while completion holds the - # blob's lock this `PUT` cannot finish, so awaiting it - # before releasing completion would deadlock the test - # rather than test anything. - overwrite = asyncio.ensure_future( - self._put_returning_status(url, replacement) - ) - # Long enough for the `PUT` to reach the lock and block on - # it -- or, unlocked, to publish its bytes and return. - await asyncio.sleep(0.5) - may_record.set() - - # Bounded: a `PUT` that did land inside completion leaves the - # bytes disagreeing with the ETag completion computed, the - # commit fails its own digest check, and the blob never - # commits. Without a bound that is a test that hangs instead - # of a test that reports what broke. - try: - await asyncio.wait_for( - self._wait_until_status(blob, {Blob.State.COMMITTED}), - timeout=_RACE_TIMEOUT_SECONDS, - ) - except asyncio.TimeoutError: - self.fail( - "the blob never committed: a part `PUT` published " - "bytes inside completion, so the ETag completion " - "computed no longer described them" - ) - status = await overwrite + self.assertEqual( + 409, + await self._put_returning_status(url, replacement), + ) - # Serialized behind completion, the `PUT` finds the blob - # committed and refuses rather than publishing. - self.assertEqual(409, status) + # And what downloads is what was committed. + downloaded, _ = await self._download(blob) + self.assertEqual(data, downloaded) # The bytes served are the bytes completion actually read, so # they still match the ETag and length it recorded. @@ -604,8 +564,8 @@ async def test_part_uploaded_rejects_bad_input(self) -> None: etag="0" * 32, size=1, ) - # A non-MD5-hex ETag is rejected (it would corrupt the S3 - # completion XML). + # An ETag carrying characters that are not safe to hand on + # is rejected (this one would corrupt the S3 completion XML). with self.assertRaises(Blob.PartUploadedAborted): await blob.part_uploaded( self.context, @@ -613,6 +573,15 @@ async def test_part_uploaded_rejects_bad_input(self) -> None: etag='">', size=1, ) + # Including a trailing newline, which an anchored `match` + # would admit: Python's `$` also matches just before one. + with self.assertRaises(Blob.PartUploadedAborted): + await blob.part_uploaded( + self.context, + part_number=1, + etag="0" * 32 + "\n", + size=1, + ) async def test_uploader_gating(self) -> None: # A blob with an uploader: external callers that are not the From ffefd37d86b3c7bd2b70c65096f2ce5c66642ef4 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:15:06 +0000 Subject: [PATCH 12/52] `reboot/std`: say absent rather than empty in the `Blob` API Review of the `Blob` API found three places where a field's default value was carrying meaning. `uploader_id` used the empty string to mean "anyone who knows this blob's ID may upload", on the state, on `Create` and on `Info`, which asks a reader to know that an empty string is a decision rather than an omission. All three are `optional` now, and the authorizer asks whether the field is there rather than whether it is empty. `downloader_ids` is `downloaders`, since it holds a `Downloaders` whose own field is already `user_ids`: `downloaders.user_ids` says once what `downloader_ids.user_ids` said twice. And `DataPlaneCompleteUploadResponse` carries a `oneof`. Completion either finished the object or did not, and the pair of an `etag` that was "empty when `error` is set" alongside an `optional error` left both of those states expressible at once. --- rbt/std/blob/v1/blob.proto | 24 +++++++------ rbt/std/blob/v1/data_plane.proto | 18 ++++++---- .../backend/src/chat_room_servicer.py | 2 +- reboot/std/blob/v1/blob.py | 36 ++++++++++--------- tests/reboot/std/blob/v1/blob_tests.py | 12 +++---- 5 files changed, 51 insertions(+), 41 deletions(-) diff --git a/rbt/std/blob/v1/blob.proto b/rbt/std/blob/v1/blob.proto index cb03817f1..66b6ddf78 100644 --- a/rbt/std/blob/v1/blob.proto +++ b/rbt/std/blob/v1/blob.proto @@ -102,10 +102,10 @@ message Blob { uint64 max_size = 4; } - // The ID of the user that may upload into this blob. Empty means + // The ID of the user that may upload into this blob. Absent means // anyone who knows this blob's ID may upload; see the authorizer // note in `reboot.std.blob.v1.blob`. - string uploader_id = 5; + optional string uploader_id = 5; // The IDs of the users who may download this blob. When unset // (omitted at `Create`), anyone who knows the blob's ID may @@ -113,7 +113,7 @@ message Blob { // means no one but app-internal callers). The uploader is NOT // implicitly a downloader. See the authorizer note in // `reboot.std.blob.v1.blob`. - optional Downloaders downloader_ids = 10; + optional Downloaders downloaders = 10; // Data-plane upload session ID, set by the `BeginUpload` workflow. // `GetPartUploadInstructions` reports `ready: false` until this is set. @@ -159,9 +159,9 @@ message CreateRequest { uint64 max_size = 3; } - // The ID of the user that may upload into this blob. Leave empty to + // The ID of the user that may upload into this blob. Omit it to // allow anyone who knows this blob's ID to upload. - string uploader_id = 4; + optional string uploader_id = 4; // The IDs of the users who may download this blob. Omit the field // entirely to let anyone who knows the blob's ID download it; set it @@ -169,7 +169,7 @@ message CreateRequest { // The uploader is NOT implicitly a downloader — omit them from the // list if they shouldn't be able to download (e.g. a drop-box). Can // be changed later with `SetDownloaders`. - optional Downloaders downloader_ids = 5; + optional Downloaders downloaders = 5; } message CreateResponse {} @@ -181,7 +181,7 @@ message SetDownloadersRequest { // Omit the field entirely to remove any restriction (anyone who // knows the blob's ID may download); set it (even to an empty list) // to restrict downloads to the listed users. - optional Downloaders downloader_ids = 1; + optional Downloaders downloaders = 1; } message SetDownloadersResponse {} @@ -263,7 +263,9 @@ message InfoResponse { uint64 max_size = 4; } - string uploader_id = 5; + // Mirrors `Blob.uploader_id`: absent means anyone who knows this + // blob's ID may upload. + optional string uploader_id = 5; // Total bytes across the parts reported uploaded so far. Together // with `size` this gives upload progress, observable reactively by @@ -322,7 +324,7 @@ service BlobMethods { // backend. Only application code may call this; it is the // application's chance to enforce quota and size policy (directly, // or by setting `max_size`), to record which user may upload - // (`uploader_id`), and to restrict who may download (`downloader_ids`). + // (`uploader_id`), and to restrict who may download (`downloaders`). rpc Create(CreateRequest) returns (CreateResponse) { option (rbt.v1alpha1.method) = { writer: { constructor: {} }, @@ -330,7 +332,7 @@ service BlobMethods { } // Replaces the blob's download allow-list. Application-mediated - // (app-internal only), like `Create`. Omit `downloader_ids` in the + // (app-internal only), like `Create`. Omit `downloaders` in the // request to remove any restriction (anyone who knows the ID may // download again); set it to restrict downloads to the listed // users. @@ -403,7 +405,7 @@ service BlobMethods { // Metadata and upload progress. Reactive: watch it to render a // progress bar. Visible to anyone who may upload or download the - // blob: the `uploader_id` and any listed `downloader_ids`, plus anyone + // blob: the `uploader_id` and any listed `downloaders`, plus anyone // who knows the ID whenever either side is left open. rpc Info(InfoRequest) returns (InfoResponse) { option (rbt.v1alpha1.method) = { diff --git a/rbt/std/blob/v1/data_plane.proto b/rbt/std/blob/v1/data_plane.proto index 25dc6223d..0f8ccd170 100644 --- a/rbt/std/blob/v1/data_plane.proto +++ b/rbt/std/blob/v1/data_plane.proto @@ -154,13 +154,17 @@ message DataPlaneCompleteUploadRequest { } message DataPlaneCompleteUploadResponse { - // The committed object's composite ETag. Empty when `error` is set. - string etag = 1; - - // A permanent-failure reason, if completion failed in a way the - // client can fix by re-uploading. When set, `etag` is empty and the - // control plane reverts the blob to UPLOADING with this message. - optional string error = 2; + // Completion either finished the object or did not; there is no + // answer that is both. + oneof outcome { + // The committed object's composite ETag. + string etag = 1; + + // A permanent-failure reason: completion failed in a way the + // client can fix by re-uploading. The control plane reverts the + // blob to UPLOADING with this message. + string error = 2; + } } //////////////////////////////////////////////////////////////////////// diff --git a/reboot/examples/chat-room/backend/src/chat_room_servicer.py b/reboot/examples/chat-room/backend/src/chat_room_servicer.py index 25c3f7480..c9883a8f1 100644 --- a/reboot/examples/chat-room/backend/src/chat_room_servicer.py +++ b/reboot/examples/chat-room/backend/src/chat_room_servicer.py @@ -37,7 +37,7 @@ async def send( # enforced; the blobs' random IDs then act as upload/download # capabilities. This example has no end-user authentication, # so `uploader_id` is left empty: anyone who knows a blob's ID - # may upload into it. `downloader_ids` is likewise omitted, so + # may upload into it. `downloaders` is likewise omitted, so # anyone who knows a blob's ID may download it too. attachment_blob_ids = [] for attachment in request.attachments: diff --git a/reboot/std/blob/v1/blob.py b/reboot/std/blob/v1/blob.py index 930695229..d2b958a71 100644 --- a/reboot/std/blob/v1/blob.py +++ b/reboot/std/blob/v1/blob.py @@ -17,15 +17,15 @@ blob's framework-generated random ID then acts as a capability. Upload-side calls (`GetPartUploadInstructions`, `PartUploaded`, `Commit`) and `Remove` are restricted to the `uploader_id` recorded at `Create` — -unless `uploader_id` is left empty, which deliberately allows anyone +unless `uploader_id` is omitted, which deliberately allows anyone who knows the blob's ID to upload (for applications without end-user authentication). Downloads (`GetDownloadUrl`) are open to anyone who knows the ID by default, but if `Create` (or a later `SetDownloaders`) -records a `downloader_ids` allow-list only the listed users may download; +records a `downloaders` allow-list only the listed users may download; an empty list restricts downloads to app-internal callers, and the uploader is *not* implicitly a downloader. `Info` (metadata and upload progress, watchable reactively) is visible to anyone who may upload or -download the blob: the `uploader_id` and listed `downloader_ids`, plus +download the blob: the `uploader_id` and listed `downloaders`, plus anyone who knows the ID whenever either side is left open. """ @@ -143,7 +143,7 @@ def _uploader_or_open( ) -> Authorizer.Decision: """Allow app-internal callers and the blob's recorded uploader to make upload-side calls, or anyone when no uploader was recorded. An - empty `uploader_id` means the blob was created without end-user + absent `uploader_id` means the blob was created without end-user authentication, so anyone who knows the blob's ID may upload into it. This handles the app-internal case itself (rather than composing `is_app_internal` via `any=[...]`) so that an @@ -153,7 +153,7 @@ def _uploader_or_open( return rbt.v1alpha1.errors_pb2.Ok() if state is None: return rbt.v1alpha1.errors_pb2.PermissionDenied() - if state.uploader_id == "": + if not state.HasField("uploader_id"): return rbt.v1alpha1.errors_pb2.Ok() if context.auth is None or context.auth.user_id is None: return rbt.v1alpha1.errors_pb2.Unauthenticated() @@ -170,7 +170,7 @@ def _downloader_or_open( **kwargs, ) -> Authorizer.Decision: """Allow app-internal callers, and restrict `GetDownloadUrl` to the - blob's download allow-list. When no `downloader_ids` list was + blob's download allow-list. When no `downloaders` list was recorded (the field is unset) anyone who knows the blob's ID may download; when one was recorded only the listed users may (an empty list means no @@ -182,11 +182,11 @@ def _downloader_or_open( return rbt.v1alpha1.errors_pb2.Ok() if state is None: return rbt.v1alpha1.errors_pb2.PermissionDenied() - if not state.HasField("downloader_ids"): + if not state.HasField("downloaders"): return rbt.v1alpha1.errors_pb2.Ok() if context.auth is None or context.auth.user_id is None: return rbt.v1alpha1.errors_pb2.Unauthenticated() - if context.auth.user_id in state.downloader_ids.user_ids: + if context.auth.user_id in state.downloaders.user_ids: return rbt.v1alpha1.errors_pb2.Ok() return rbt.v1alpha1.errors_pb2.PermissionDenied() @@ -225,9 +225,10 @@ async def create( ) -> CreateResponse: self.state.status = Blob.State.UPLOADING self.state.content_type = request.content_type - self.state.uploader_id = request.uploader_id - if request.HasField("downloader_ids"): - self.state.downloader_ids.CopyFrom(request.downloader_ids) + if request.HasField("uploader_id"): + self.state.uploader_id = request.uploader_id + if request.HasField("downloaders"): + self.state.downloaders.CopyFrom(request.downloaders) if request.HasField("size"): self.state.size = request.size if request.HasField("max_size"): @@ -249,14 +250,14 @@ async def set_downloaders( context: WriterContext, request: SetDownloadersRequest, ) -> SetDownloadersResponse: - # Replace semantics: a present `downloader_ids` (even empty) + # Replace semantics: a present `downloaders` (even empty) # restricts downloads to the listed users; an omitted one # removes any restriction so anyone who knows the ID may # download again. - if request.HasField("downloader_ids"): - self.state.downloader_ids.CopyFrom(request.downloader_ids) + if request.HasField("downloaders"): + self.state.downloaders.CopyFrom(request.downloaders) else: - self.state.ClearField("downloader_ids") + self.state.ClearField("downloaders") return SetDownloadersResponse() @classmethod @@ -526,7 +527,10 @@ async def info( response = InfoResponse( status=self.state.status, content_type=self.state.content_type, - uploader_id=self.state.uploader_id, + uploader_id=( + self.state.uploader_id + if self.state.HasField("uploader_id") else None + ), bytes_uploaded=sum(part.size for part in self.state.parts), parts=self.state.parts, ) diff --git a/tests/reboot/std/blob/v1/blob_tests.py b/tests/reboot/std/blob/v1/blob_tests.py index f60ddaa9f..7f218a742 100644 --- a/tests/reboot/std/blob/v1/blob_tests.py +++ b/tests/reboot/std/blob/v1/blob_tests.py @@ -636,7 +636,7 @@ async def test_download_gating(self) -> None: blob, _ = await Blob.create( self.context, content_type="text/plain", - downloader_ids=Downloaders(user_ids=["bob"]), + downloaders=Downloaders(user_ids=["bob"]), ) await self._upload(blob, data) await blob.commit(self.context) @@ -684,13 +684,13 @@ async def test_set_downloaders(self) -> None: # Restrict downloads to a user the external caller is not. await blob.set_downloaders( self.context, - downloader_ids=Downloaders(user_ids=["bob"]), + downloaders=Downloaders(user_ids=["bob"]), ) with self.assertRaises(Blob.GetDownloadUrlAborted): await Blob.ref(blob.state_id ).get_download_url(self.external_context) - # Remove the restriction again by omitting `downloader_ids`. + # Remove the restriction again by omitting `downloaders`. await blob.set_downloaders(self.context) response = await Blob.ref(blob.state_id ).get_download_url(self.external_context) @@ -705,7 +705,7 @@ async def test_info_gating(self) -> None: self.context, content_type="text/plain", uploader_id="alice", - downloader_ids=Downloaders(user_ids=["bob"]), + downloaders=Downloaders(user_ids=["bob"]), ) with self.assertRaises(Blob.InfoAborted): await Blob.ref(locked.state_id).info(self.external_context) @@ -716,12 +716,12 @@ async def test_info_gating(self) -> None: upload_open, _ = await Blob.create( self.context, content_type="text/plain", - downloader_ids=Downloaders(user_ids=["bob"]), + downloaders=Downloaders(user_ids=["bob"]), ) info = await Blob.ref(upload_open.state_id).info(self.external_context) self.assertEqual(info.status, Blob.State.UPLOADING) - # Open download side (omitted `downloader_ids`): anyone who may + # Open download side (omitted `downloaders`): anyone who may # download may read `Info`, even with a specific uploader. download_open, _ = await Blob.create( self.context, From 0589391e87d1aa8b0bc8eaf01af8fd8c7b9426ba Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:49:08 +0000 Subject: [PATCH 13/52] `tests`: wait on blob state reactively rather than by polling Both of the blob tests' waiting helpers read in a loop with a 50ms sleep between attempts: one for the upload session the `BeginUpload` workflow provisions, one for a blob reaching a status. Reviewing them pointed out what the repo already asks for -- `reactively()`, which returns when the state changes instead of on the next tick. Neither loop takes a deadline, before or after: the test's own timeout already covers a condition that never becomes true, and a second, lower-level one would only add a way to fail under load. --- tests/reboot/std/blob/v1/blob_tests.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/reboot/std/blob/v1/blob_tests.py b/tests/reboot/std/blob/v1/blob_tests.py index 7f218a742..f1b785c6f 100644 --- a/tests/reboot/std/blob/v1/blob_tests.py +++ b/tests/reboot/std/blob/v1/blob_tests.py @@ -1,5 +1,4 @@ import aiohttp -import asyncio import hashlib import unittest from rbt.std.blob.v1.blob_rbt import ( @@ -50,14 +49,13 @@ async def asyncTearDown(self) -> None: async def _instructions(self, blob, part_numbers: list[int]): """Fetches upload instructions, waiting for the `BeginUpload` workflow to have provisioned the upload session.""" - while True: - response = await blob.get_part_upload_instructions( - self.context, - part_numbers=part_numbers, - ) + async for response in blob.reactively().get_part_upload_instructions( + self.context, + part_numbers=part_numbers, + ): if response.ready: return response - await asyncio.sleep(0.05) + raise AssertionError("Reacting to a blob ended without a session") async def _part_numbers(self, blob, part_numbers: list[int]) -> list[int]: """The part numbers that upload instructions were minted for, @@ -108,11 +106,10 @@ async def _upload(self, blob, data: bytes) -> None: ) async def _wait_until_status(self, blob, statuses) -> InfoResponse: - while True: - info = await blob.info(self.context) + async for info in blob.reactively().info(self.context): if info.status in statuses: return info - await asyncio.sleep(0.05) + raise AssertionError("Reacting to a blob ended before its status did") async def _download(self, blob) -> tuple[bytes, str]: """Downloads the blob's bytes via its download URL, returning From 700f72d4c3a9807f076be39de3e59339a5e466b8 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Fri, 11 Sep 2026 19:56:30 +0000 Subject: [PATCH 14/52] `documentation`: address review comments The `Application` snippet carried three lines about blob data planes, which review found dense for a page whose subject is `Application` -- the TypeScript tab beside it shows the same construction in four lines. The example says why the library is there and no more. What the comment was the only home for has somewhere better to be. The page never said what `libraries` is at all, though the snippet has always passed one, so it says so now, and the data plane and `REBOOT_BLOB_DATA_PLANE_URL` are described there in prose rather than inside the code the reader is meant to be reading. --- documentation/docs/implement/application.mdx | 24 +++++++++++++++---- reboot/examples/chat-room/backend/src/main.py | 4 +--- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/documentation/docs/implement/application.mdx b/documentation/docs/implement/application.mdx index 24e3d6162..f46ba9012 100644 --- a/documentation/docs/implement/application.mdx +++ b/documentation/docs/implement/application.mdx @@ -13,16 +13,14 @@ Your entrypoint will construct an `Application` and then `run` it: +(CODE:src=../../../reboot/examples/chat-room/backend/src/main.py&lines=26-32) --> ```py async def main(): await Application( servicers=[ChatRoomServicer], - # Message attachments are stored as blobs; `rbt dev run` and - # `rbt serve run` provide a local filesystem data plane (see - # `REBOOT_BLOB_DATA_PLANE_URL` for using a custom one). + # Message attachments are stored as blobs. libraries=[blob_library()], initialize=initialize, ).run() @@ -47,6 +45,24 @@ new Application({ +### `libraries` + +The optional `libraries` argument brings in [library +services](/library_services/overview): state machines that Reboot +implements for you, which your application serves alongside its own. +The example above uses `blob_library()`, which gives the application +the `Blob` state machine its message attachments are stored in. + +A library service may need somewhere to keep what it holds. Blobs keep +bytes, and a Python application serves them itself from a data plane +on the local filesystem, under the application's state directory, so +`rbt dev run` and `rbt serve run` need nothing more. Setting +`REBOOT_BLOB_DATA_PLANE_URL` points an application at a different data +plane instead -- on Reboot Cloud, that is how blobs come to live in +object storage. A TypeScript application cannot serve the local data +plane yet, so it needs `REBOOT_BLOB_DATA_PLANE_URL` set, and refuses to +start without it. + ### `initialize` functions The optional `initialize` argument to the `Application` constructor is a function that diff --git a/reboot/examples/chat-room/backend/src/main.py b/reboot/examples/chat-room/backend/src/main.py index 2f258ff6d..ac3360ef5 100644 --- a/reboot/examples/chat-room/backend/src/main.py +++ b/reboot/examples/chat-room/backend/src/main.py @@ -26,9 +26,7 @@ async def initialize(context: InitializeContext): async def main(): await Application( servicers=[ChatRoomServicer], - # Message attachments are stored as blobs; `rbt dev run` and - # `rbt serve run` provide a local filesystem data plane (see - # `REBOOT_BLOB_DATA_PLANE_URL` for using a custom one). + # Message attachments are stored as blobs. libraries=[blob_library()], initialize=initialize, ).run() From 7fa4e4be2d529a6c1aa84cd12c15e56c03016553 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Fri, 11 Sep 2026 21:35:02 +0000 Subject: [PATCH 15/52] `documentation`: close the `from_react` snippet's `if` Review found the extracted range one line short: it stopped at `return;` on line 200 and left behind the `}` that closes `if (aborted !== undefined) {` on line 201, so the rendered block opened a branch it never closed. Nothing was going to catch that. `markdown-autodocs` copies the range out verbatim without parsing it, and Docusaurus renders an unbalanced fenced block as the inert text it treats every fenced block as. The regenerated snippet is committed alongside the range, since changing the range alone does not rewrite the body. --- documentation/docs/call/from_react.mdx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/documentation/docs/call/from_react.mdx b/documentation/docs/call/from_react.mdx index 1f06085d1..d6b958f4f 100644 --- a/documentation/docs/call/from_react.mdx +++ b/documentation/docs/call/from_react.mdx @@ -413,7 +413,7 @@ Every call to a mutator returns both a `response` and an `aborted`; successful c [Learn more about Reboot errors and error types.](/errors) +(CODE:src=../../../reboot/examples/chat-room/frontend/web/src/App.tsx&&lines=181-201) --> ```tsx @@ -437,6 +437,7 @@ if (aborted !== undefined) { setError(`Couldn't send your message: ${aborted.message}`); } return; +} ``` From b872604160b1e5fe895e7e33dc43236d5c4d6a17 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:56:36 +0000 Subject: [PATCH 16/52] Address review comments Drop Reboot Cloud implementation details from two comments: the `data_plane_proto` comment in `rbt/std/blob/v1/BUILD.bazel` named the Cloud facilitator as the thing that hosts the data plane via `legacy_grpc_servicers`, when any Reboot application can, and the `_CONFIGURATION_RETRY_SECONDS` comment in `blob.py` named who spawns the data plane, when all that matters is that it is normally already running at the address `REBOOT_BLOB_DATA_PLANE_URL` names. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY --- rbt/std/blob/v1/BUILD.bazel | 6 +++--- reboot/std/blob/v1/blob.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rbt/std/blob/v1/BUILD.bazel b/rbt/std/blob/v1/BUILD.bazel index 1562c3ca0..da7023db3 100644 --- a/rbt/std/blob/v1/BUILD.bazel +++ b/rbt/std/blob/v1/BUILD.bazel @@ -21,9 +21,9 @@ proto_library( ) # The blob data-plane interface: a plain gRPC service (no Reboot state -# options). Built with `py_reboot_library` — which, besides the plain -# `_pb2`/`_pb2_grpc` modules the filesystem server uses directly, emits -# the `_rbt` module that lets the Cloud facilitator register it via +# options). Built with `py_reboot_library` so that, besides the plain +# `_pb2`/`_pb2_grpc` modules, it emits the `_rbt` module that lets a +# Reboot application host an implementation via # `legacy_grpc_servicers`. Python-only: the JS SDK talks to the `Blob` # control plane, never the data plane directly. proto_library( diff --git a/reboot/std/blob/v1/blob.py b/reboot/std/blob/v1/blob.py index d2b958a71..b62beb8df 100644 --- a/reboot/std/blob/v1/blob.py +++ b/reboot/std/blob/v1/blob.py @@ -103,8 +103,8 @@ logger = log.log.get_logger(__name__) # How long to keep retrying `Configuration` while the data plane -# comes up, before `pre_run` gives up. The data plane is normally -# already running (spawned by `rbt` or a ready facilitator), so this is +# comes up, before `pre_run` gives up. The data plane named by +# `REBOOT_BLOB_DATA_PLANE_URL` is normally already running, so this is # only a startup-race cushion. _CONFIGURATION_RETRY_SECONDS = 30 _CONFIGURATION_MAX_BACKOFF_SECONDS = 2 From 5a041a7cf9b2dc2bd9576a26b5c7ac49abdb154e Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Mon, 14 Sep 2026 13:26:04 +0000 Subject: [PATCH 17/52] Address review comments - `data_plane.proto`: remove `forwarded_paths`, `ForwardedPath`, `HttpMethod` and `http_port` from `Configuration`. Before this change the option survived the removal of the byte-route proxying it configured, guarded by a refusal at startup; the feature was never released, so nothing needs the option or the fallback. - `reboot.aio.tests.Reboot.up()`: run `pre_run` for a Node.js application's Python-native libraries, as `NodeApplication.run()` does. Before this change the harness skipped every library's `pre_run` under Node.js, so a library refusing to serve such an application (the blob library, without a data plane URL) was refused under `rbt dev run` but silently started in tests. - `tests/reboot/std/blob/v1/blob_tests.ts`: show that a Node.js application registering the blob library without `REBOOT_BLOB_DATA_PLANE_URL` is refused at startup, with the message naming the variable. - `.claude/rules/python-annotate-return-types.md`: record that every function gets a return type annotation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY --- .claude/rules/python-annotate-return-types.md | 27 +++++++++++ rbt/std/blob/v1/data_plane.proto | 47 +++---------------- reboot/aio/tests.py | 14 ++++-- tests/reboot/std/blob/v1/BUILD.bazel | 34 ++++++++++++++ tests/reboot/std/blob/v1/blob_tests.ts | 30 ++++++++++++ tests/reboot/std/blob/v1/package.json | 3 ++ 6 files changed, 111 insertions(+), 44 deletions(-) create mode 100644 .claude/rules/python-annotate-return-types.md create mode 100644 tests/reboot/std/blob/v1/blob_tests.ts create mode 100644 tests/reboot/std/blob/v1/package.json diff --git a/.claude/rules/python-annotate-return-types.md b/.claude/rules/python-annotate-return-types.md new file mode 100644 index 000000000..b0c918712 --- /dev/null +++ b/.claude/rules/python-annotate-return-types.md @@ -0,0 +1,27 @@ +--- +paths: + - "**/*.py" +--- + +# Annotate every function's return type + +Every Python function and method you write or touch gets a return +type annotation, `-> None` included, wherever one can be written. The +same goes for parameters: annotate them unless the type genuinely +cannot be named. Generated gRPC servicer methods are no exception — +annotate `request` with its message type and the method with its +response type. + +**Why:** A missing return annotation makes `mypy` treat the function +as returning `Any`, which silently switches off type checking for +everything the result flows into. It also leaves the reader to +reconstruct from the body what a method hands back — a method named +`_caller` that returned an `ExternalContext` went unnoticed in review +until someone asked. + +**How to apply:** Before committing, scan the diff for `def` lines +without `->`. When a function returns nothing, write `-> None`. When +the natural return type needs an import (a `_pb2` message, an +`Optional[...]`), add the import rather than leaving the annotation +off. The only acceptable omission is a signature whose type cannot be +expressed without a `# type: ignore`, and that deserves a comment. diff --git a/rbt/std/blob/v1/data_plane.proto b/rbt/std/blob/v1/data_plane.proto index 0f8ccd170..186467b56 100644 --- a/rbt/std/blob/v1/data_plane.proto +++ b/rbt/std/blob/v1/data_plane.proto @@ -5,9 +5,9 @@ // control plane never touches bytes: it calls this service to // provision uploads, finalize them, mint download URLs, and delete // objects, and the bytes themselves travel directly between the client -// and wherever the data plane keeps them (a presigned-URL store like -// S3, or — for a data plane whose `Configuration` asks for forwarded -// paths — the application's own HTTP routes proxying to it). +// and wherever the data plane keeps them, over the URLs it mints: a +// presigned-URL store like S3, or the HTTP routes of an application +// that hosts the filesystem data plane itself. // // This is a plain gRPC service, deliberately free of any Reboot // framework options, so that it can be implemented by anything and @@ -29,8 +29,7 @@ package rbt.std.blob.v1; service BlobDataPlane { // Reports what the control plane needs to know to talk to this data - // plane: the client part size, and which paths (if any) the - // application must forward to it. Read once at application startup. + // plane: the client part size. Read once at application startup. rpc Configuration(ConfigurationRequest) returns (ConfigurationResponse); // Provisions an upload session for a blob and returns its data-plane @@ -42,8 +41,8 @@ service BlobDataPlane { // Mints URLs to which the client `PUT`s each requested part's bytes // directly. A URL is either absolute (e.g. a presigned S3 URL, - // directly reachable by the client) or application-relative, under a - // path the data plane's `Configuration` asked to have forwarded. + // directly reachable by the client) or application-relative, for a + // data plane the application serves itself. rpc GetPartUploadInstructions(DataPlaneGetPartUploadInstructionsRequest) returns (DataPlaneGetPartUploadInstructionsResponse); @@ -60,8 +59,7 @@ service BlobDataPlane { // Mints a URL from which the committed object's bytes can be // downloaded: absolute (e.g. a presigned S3/CloudFront URL) or - // application-relative under a forwarded path, like - // `GetPartUploadInstructions` URLs. + // application-relative, like `GetPartUploadInstructions` URLs. rpc GetDownloadUrl(DataPlaneGetDownloadUrlRequest) returns (DataPlaneGetDownloadUrlResponse); @@ -72,41 +70,10 @@ service BlobDataPlane { message ConfigurationRequest {} -// The HTTP method of a forwarded path. -enum HttpMethod { - HTTP_METHOD_UNSPECIFIED = 0; - HTTP_METHOD_GET = 1; - HTTP_METHOD_PUT = 2; -} - -// One path namespace the application must forward to the data plane: -// requests with the given method whose path starts with `path_prefix` -// are reverse-proxied to the data plane's HTTP endpoint verbatim -// (path and query). `path_prefix` must itself start with -// `/__/reboot/blob/` — the application refuses anything else, so a -// data plane can never claim application routes. -message ForwardedPath { - HttpMethod method = 1; - string path_prefix = 2; -} - message ConfigurationResponse { // The part size clients must use; every part but the last must be // exactly this size. uint64 part_size = 1; - - // Paths the application must forward to this data plane. Empty when - // the data plane's URLs are directly reachable by clients (e.g. - // presigned S3 or CloudFront URLs) and no forwarding is needed. - repeated ForwardedPath forwarded_paths = 2; - - // The port of this data plane's HTTP byte endpoint, to which - // forwarded requests are proxied. The host is the one the - // application already reaches this gRPC service on (so the data - // plane need not know how it is addressed externally), and the - // scheme follows that connection's transport security. Only - // meaningful when `forwarded_paths` is non-empty. - uint32 http_port = 3; } //////////////////////////////////////////////////////////////////////// diff --git a/reboot/aio/tests.py b/reboot/aio/tests.py index 2d3a2f4e1..983972e92 100644 --- a/reboot/aio/tests.py +++ b/reboot/aio/tests.py @@ -3,7 +3,11 @@ import reboot.aio.reboot import secrets import unittest -from reboot.aio.applications import Application, NodeApplication +from reboot.aio.applications import ( + Application, + NodeAdaptorLibrary, + NodeApplication, +) from reboot.aio.auth.oauth import OAuth from reboot.aio.auth.oauth_providers import ( ExchangeResult, @@ -437,9 +441,11 @@ async def up( # Do any pre-run library set up, just like `Application.run()` # does; e.g. a library may register HTTP routes. Libraries must # tolerate being `pre_run` more than once, since a test may - # `up` the same `Application` after a `down`. - if not in_nodejs(): - for library in application.libraries: + # `up` the same `Application` after a `down`. A Node.js + # library gets its pre-run in TypeScript, so, as in + # `NodeApplication.run()`, only the others are run here. + for library in application.libraries: + if not isinstance(library, NodeAdaptorLibrary): await library.pre_run(application) # Check if application.http has methods or mounts (note this diff --git a/tests/reboot/std/blob/v1/BUILD.bazel b/tests/reboot/std/blob/v1/BUILD.bazel index 06c6c4a10..41875c49e 100644 --- a/tests/reboot/std/blob/v1/BUILD.bazel +++ b/tests/reboot/std/blob/v1/BUILD.bazel @@ -1,5 +1,7 @@ +load("@aspect_rules_ts//ts:defs.bzl", "ts_project") load("@rbt_pypi//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_test") +load("//reboot/nodejs:rules.bzl", "js_reboot_test") py_test( name = "blob_tests_py", @@ -14,3 +16,35 @@ py_test( requirement("aiohttp"), ], ) + +ts_project( + name = "blob_tests_ts", + srcs = [ + "blob_tests.ts", + ":package.json", + ], + declaration = True, + tsconfig = { + "compilerOptions": { + "declaration": True, + "module": "nodenext", + "moduleResolution": "nodenext", + "target": "es2020", + }, + }, + visibility = ["//visibility:public"], + deps = [ + "//:node_modules/@reboot-dev/reboot", + "//:node_modules/@reboot-dev/reboot-std", + "//:node_modules/@types/node", + ], +) + +js_reboot_test( + name = "test_blob_tests_ts", + data = [ + ":blob_tests_ts", + ], + entry_point = "blob_tests.js", + visibility = ["//visibility:public"], +) diff --git a/tests/reboot/std/blob/v1/blob_tests.ts b/tests/reboot/std/blob/v1/blob_tests.ts new file mode 100644 index 000000000..0be6eb943 --- /dev/null +++ b/tests/reboot/std/blob/v1/blob_tests.ts @@ -0,0 +1,30 @@ +import { Application, Reboot } from "@reboot-dev/reboot"; +import { blobLibrary } from "@reboot-dev/reboot-std/blob/v1"; +import { strict as assert } from "node:assert"; +import test from "node:test"; + +test("blob library in a Node.js application", async (t) => { + let rbt: Reboot; + + t.beforeEach(async () => { + rbt = new Reboot(); + await rbt.start(); + }); + + t.afterEach(async () => { + await rbt.stop(); + }); + + await t.test("refuses to start without a data plane URL", async (t) => { + // The filesystem data plane serves its bytes over HTTP routes that + // only Python applications register, so a Node.js application + // that is not pointed at a data plane elsewhere must be refused at + // startup, rather than start and mint upload URLs nothing serves. + const application = new Application({ + libraries: [blobLibrary()], + }); + await assert.rejects(rbt.up(application), (error: Error) => + error.message.includes("REBOOT_BLOB_DATA_PLANE_URL") + ); + }); +}); diff --git a/tests/reboot/std/blob/v1/package.json b/tests/reboot/std/blob/v1/package.json new file mode 100644 index 000000000..3dbc1ca59 --- /dev/null +++ b/tests/reboot/std/blob/v1/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} From e3112a7b4f9260738d137c720223c2f02f267130 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:11:51 +0000 Subject: [PATCH 18/52] `reboot/examples/chat-room`: say how `Send` holds its lock Before this change, `Send` was declared a bare `transaction: {}`, which `main` no longer accepts: since the rebase, every transaction has to declare whether it takes the lock on its own state `exclusive` or `shared`. `Send` appends the message to its own state, so it takes the lock exclusive, as the generator's own error message suggests for a transaction that writes its state. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY --- reboot/examples/chat-room/api/chat_room/v1/chat_room.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reboot/examples/chat-room/api/chat_room/v1/chat_room.proto b/reboot/examples/chat-room/api/chat_room/v1/chat_room.proto index 2ed45d946..62c7fcd8d 100644 --- a/reboot/examples/chat-room/api/chat_room/v1/chat_room.proto +++ b/reboot/examples/chat-room/api/chat_room/v1/chat_room.proto @@ -43,7 +43,7 @@ service ChatRoomMethods { // the returned blob IDs. rpc Send(SendRequest) returns (SendResponse) { option (rbt.v1alpha1.method) = { - transaction: {}, + transaction: { exclusive: {} }, errors: [ "AttachmentTooLarge" ], }; } From 1a978c39ad739a4f82906ad432ace25af7862603 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:53:57 +0000 Subject: [PATCH 19/52] `reboot/std`: move the filesystem data plane's bookkeeping into its store The S3 data plane's servicer authorizes a call and hands it to its store, because S3 keeps the metadata: the multipart session, the parts it holds, the object once completed. Before this change the filesystem data plane's servicer did that bookkeeping itself -- beginning a `StoredBlob` session, checking reported parts against written ones, committing a manifest, reclaiming what it did not name, forgetting a blob before removing its bytes -- and the byte routes drove `StoredBlob` directly as well, so three modules spoke the metadata protocol and the two servicers looked nothing alike. `FilesystemBlobStore` now drives `StoredBlob` itself and offers the surface `S3BlobStore` has: `begin_upload`, `part_put_url`, `complete`, `download_url` and `delete`, plus `publish_part` and `stored` for the byte routes. Its methods take the context they reach `StoredBlob` with, which is the one honest difference from a store whose metadata lives in an object store. The servicer keeps the caller check and one delegating call per RPC, and `_http.py` no longer imports `StoredBlob`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY --- reboot/std/blob/v1/_filesystem_data_plane.py | 222 ++---------- reboot/std/blob/v1/_http.py | 71 ++-- reboot/std/blob/v1/_store.py | 338 +++++++++++++++++-- 3 files changed, 356 insertions(+), 275 deletions(-) diff --git a/reboot/std/blob/v1/_filesystem_data_plane.py b/reboot/std/blob/v1/_filesystem_data_plane.py index 2f1a71754..4619950bf 100644 --- a/reboot/std/blob/v1/_filesystem_data_plane.py +++ b/reboot/std/blob/v1/_filesystem_data_plane.py @@ -19,14 +19,14 @@ `trust_caller_id` in `reboot/routing/envoy_config.py`), so a caller ID that survives was put there by something entitled to. -Metadata lives in `StoredBlob`; bytes live in `FilesystemBlobStore`. -Neither holds state of its own, so any of a replica's servers can -serve any call. +Everything else is the store's: `FilesystemBlobStore` keeps the +bytes and drives `StoredBlob`, the state machine that keeps the +metadata, so each call here is authorized and then handed over. +Nothing here holds state of its own, so any of a replica's servers +can serve any call. """ import grpc -import rbt.std.blob.v1.filesystem_pb2 as filesystem_pb2 -import rbt.v1alpha1.errors_pb2 from rbt.std.blob.v1.data_plane_pb2 import ( ConfigurationRequest, ConfigurationResponse, @@ -43,7 +43,6 @@ DataPlanePartUploadInstruction, ) from rbt.std.blob.v1.data_plane_pb2_grpc import BlobDataPlaneServicer -from rbt.std.blob.v1.filesystem_rbt import StoredBlob, StoredPart from reboot.aio.caller_id import CallerID from reboot.aio.external import ExternalContext from reboot.aio.headers import CALLER_ID_HEADER @@ -52,27 +51,13 @@ from reboot.std.blob.v1._store import ( BlobStoreError, FilesystemBlobStore, - _encode_blob_id, - composite_etag, + UploadedPart, ) -from typing import Optional -from uuid import NAMESPACE_URL, UUID, uuid5 - - -def _begin_upload_key(blob_id: str) -> UUID: - """The idempotency key for beginning one blob's upload. - - Derived from the blob ID rather than taken from the caller, - because the control plane both retries this inside a workflow and - re-runs it to validate that workflow's effects. "Begin the upload - for this blob" is one operation however many times it is asked - for, so the blob names it.""" - return uuid5(NAMESPACE_URL, f"reboot.std.blob.v1/begin-upload/{blob_id}") class FilesystemDataPlaneServicer(BlobDataPlaneServicer): """Serves `BlobDataPlane` from the application whose blobs it - holds. + holds: each call is authorized, then handed to the store. The store is set by `BlobLibrary` once it knows where this application keeps them.""" @@ -116,8 +101,8 @@ async def _authorize_caller(self, context: LegacyGrpcContext) -> None: raise RuntimeError("This is unreachable") def _context(self, grpc_context: LegacyGrpcContext) -> ExternalContext: - """A context for reaching `StoredBlob` on behalf of a call - `_authorize_caller` has admitted.""" + """The context the store reaches `StoredBlob` with, on behalf + of a call `_authorize_caller` has admitted.""" return grpc_context.external_context(name="blob data plane") async def Configuration( @@ -134,21 +119,12 @@ async def BeginUpload( grpc_context: LegacyGrpcContext, ) -> DataPlaneBeginUploadResponse: await self._authorize_caller(grpc_context) - context = self._context(grpc_context) - _, response = await StoredBlob.idempotently( - key=_begin_upload_key(request.blob_id), - ).BeginUpload( - context, - request.blob_id, - content_type=request.content_type, - ) - # After the session exists in state, so a directory is never - # left behind for a session nothing knows about. - await self._store.make_upload_directory( + upload_id = await self._store.begin_upload( + self._context(grpc_context), request.blob_id, - response.upload_id, + request.content_type, ) - return DataPlaneBeginUploadResponse(upload_id=response.upload_id) + return DataPlaneBeginUploadResponse(upload_id=upload_id) async def GetPartUploadInstructions( self, @@ -177,7 +153,20 @@ async def CompleteUpload( ) -> DataPlaneCompleteUploadResponse: await self._authorize_caller(grpc_context) try: - etag = await self._complete(request, self._context(grpc_context)) + etag = await self._store.complete( + self._context(grpc_context), + request.blob_id, + request.upload_id, + request.content_type, + [ + UploadedPart( + number=part.number, etag=part.etag, size=part.size + ) for part in request.parts + ], + max_size=( + request.max_size if request.HasField("max_size") else None + ), + ) return DataPlaneCompleteUploadResponse(etag=etag) except BlobStoreError as error: # A permanent failure: report it so the control plane can @@ -186,138 +175,6 @@ async def CompleteUpload( # plane's workflow retries. return DataPlaneCompleteUploadResponse(error=str(error)) - async def _stored( - self, - blob_id: str, - context: ExternalContext, - ) -> Optional[filesystem_pb2.StoredBlob]: - """The metadata stored for a blob, or `None` when none is. - - A blob whose upload never began has no state at all, which the - framework reports by refusing the read rather than by - answering with an absent one.""" - try: - metadata = await StoredBlob.ref(blob_id).metadata(context) - except StoredBlob.MetadataAborted as aborted: - if isinstance( - aborted.error, - rbt.v1alpha1.errors_pb2.StateNotConstructed, - ): - return None - raise - return metadata.blob if metadata.HasField("blob") else None - - async def _complete( - self, - request: DataPlaneCompleteUploadRequest, - context: ExternalContext, - ) -> str: - stored = await self._stored(request.blob_id, context) - if stored is None: - raise BlobStoreError("no upload was ever begun for this blob") - if stored.committed: - # A retried `CompleteUpload`. The object is finished and - # its ETag is what it was -- but reclaiming may not have - # run, or not finished, so it runs again from what was - # committed. - await self._store.reclaim( - _encode_blob_id(request.blob_id), - stored.upload_id, - [(part.number, part.storage_id) for part in stored.parts], - ) - return stored.etag - if stored.upload_id != request.upload_id: - # The parts that would be committed were written under a - # different session than the one being completed, so they - # are not the parts this verified. - raise BlobStoreError( - "the upload session being completed is not the one this " - "blob's parts were written under" - ) - - published = {part.number: part for part in stored.parts} - reported = {part.number: part for part in request.parts} - if len(reported) == 0: - raise BlobStoreError("no parts were reported") - - last_part_number = max(reported) - for number in sorted(reported): - part = published.get(number) - if part is None: - raise BlobStoreError(f"part {number} was never uploaded") - if part.etag != reported[number].etag.strip('"'): - raise BlobStoreError( - f"part {number} ETag mismatch: the uploaded bytes do " - "not match what was reported via `PartUploaded`" - ) - if part.size != reported[number].size: - raise BlobStoreError( - f"part {number} size mismatch: uploaded {part.size} " - f"bytes but {reported[number].size} were reported via " - "`PartUploaded`" - ) - if ( - number != last_part_number and - part.size != self._store.part_size - ): - # S3 rejects a short middle part with `EntityTooSmall`; - # reject it here too, so that an upload which cannot - # commit against the S3 store cannot commit against - # this one either. - raise BlobStoreError( - f"part {number} is {part.size} bytes, but every part " - f"except the last must be exactly " - f"{self._store.part_size} bytes" - ) - - total_size = sum(published[number].size for number in reported) - max_size: Optional[int] = ( - request.max_size if request.HasField("max_size") else None - ) - # Checked against what the parts were found to hold, not - # against the sizes that were reported alongside them. - if max_size is not None and total_size > max_size: - raise BlobStoreError( - f"uploaded {total_size} bytes exceeds the maximum of " - f"{max_size}" - ) - - manifest = [ - StoredPart( - number=number, - size=published[number].size, - etag=published[number].etag, - storage_id=published[number].storage_id, - ) for number in sorted(reported) - ] - etag = composite_etag([part.etag for part in manifest]) - committed = await StoredBlob.ref(request.blob_id).always().commit( - context, - upload_id=request.upload_id, - content_type=request.content_type, - etag=etag, - parts=manifest, - ) - if not committed.committed: - raise BlobStoreError( - "a part was uploaded again while this upload was being " - "completed; report the parts and commit again" - ) - # The manifest is fixed, so anything else this session wrote - # -- a part uploaded and never reported, a version of a part - # that lost -- belongs to nothing and is safe to remove. Done - # after the commit, so a failure here leaves files behind - # rather than taking away bytes the object is made of; a retry - # reclaims them above. - # `commit` accepted this manifest, and refuses one whose parts - # have been superseded, so it is exactly what was recorded. - await self._store.reclaim( - _encode_blob_id(request.blob_id), - request.upload_id, - [(part.number, part.storage_id) for part in manifest], - ) - return etag - async def GetDownloadUrl( self, request: DataPlaneGetDownloadUrlRequest, @@ -339,26 +196,11 @@ async def Delete( grpc_context: LegacyGrpcContext, ) -> DataPlaneDeleteResponse: await self._authorize_caller(grpc_context) - context = self._context(grpc_context) - # A part lives inside the blob's own directory, so removing the - # directory removes any unfinished upload with it, whatever - # `upload_ids` says. - # Forgotten before the bytes go, so that nothing reads a - # manifest naming bytes that are already gone: between the two - # a download would answer `200` and then run out of file. - try: - await StoredBlob.ref(request.blob_id).always().forget(context) - except StoredBlob.ForgetAborted as aborted: - if isinstance( - aborted.error, - rbt.v1alpha1.errors_pb2.StateNotConstructed, - ): - # Nothing was ever stored for this blob, so there is - # nothing to forget and deleting it has succeeded. - pass - else: - raise - await self._store.delete(request.blob_id) + await self._store.delete( + self._context(grpc_context), + request.blob_id, + upload_ids=list(request.upload_ids), + ) return DataPlaneDeleteResponse() diff --git a/reboot/std/blob/v1/_http.py b/reboot/std/blob/v1/_http.py index cc146d064..8bf4ef7b0 100644 --- a/reboot/std/blob/v1/_http.py +++ b/reboot/std/blob/v1/_http.py @@ -20,7 +20,6 @@ import hmac import re import time -from rbt.std.blob.v1.filesystem_rbt import StoredBlob, StoredPart from reboot.aio.http import PythonWebFramework from reboot.std.blob.v1._content_type import download_headers from reboot.std.blob.v1._store import ( @@ -32,7 +31,7 @@ ) from starlette.requests import Request from starlette.responses import Response, StreamingResponse -from typing import AsyncIterator, Callable, Coroutine, Optional +from typing import Callable, Coroutine, Optional # Path parameters are also filesystem path components; restrict them # to the alphabets the store actually produces (URL-safe base64 blob @@ -137,40 +136,18 @@ async def put_part(request: Request) -> Response: ), ) - # The bytes are on disk under a name of their own; whether - # the object is made of them is this call's to decide, and it - # decides for every server that might be serving this blob. - # Refused bytes go, which is safe because the file's name - # belongs to this write alone: no manifest can point at it - # unless this very call's claim succeeded. Anything that - # outlives an interrupted request is reclaimed at commit, and - # with the blob's directory on `Delete`. - # An app-internal context, which is what reaches `StoredBlob`, - # taken only now that the signature has verified this request - # holds a URL this data plane minted. - published = await StoredBlob.ref(_blob_id(blob)).always().publish_part( + # An app-internal context, which is what the store reaches + # `StoredBlob` with, taken only now that the signature has + # verified this request holds a URL this data plane minted. + published = await store.publish_part( request.state.reboot_app_internal_context(request), - upload_id=upload, - part=StoredPart( - number=staged.part.number, - size=staged.part.size, - etag=staged.part.etag, - storage_id=staged.part.storage_id, - ), + _blob_id(blob), + upload, + staged, ) - if not published.published: - await store.discard_part(staged) + if not published: return Response(status_code=409, content="Blob already committed") - if published.HasField("superseded_storage_id"): - # This part had been uploaded before. Nothing is made of - # the earlier bytes now, and the manifest that could still - # name them is refused at commit, so they are removed - # rather than left to accumulate a file per attempt. - await store.discard_storage_id( - blob, upload, part_number, published.superseded_storage_id - ) - # Match S3: the ETag response header is the part's MD5, quoted. return Response( status_code=200, @@ -199,30 +176,19 @@ async def get_blob(request: Request) -> Response: # As in `put_part`: an app-internal context, taken only below a # verified signature. - metadata = await StoredBlob.ref(_blob_id(blob)).metadata( - request.state.reboot_app_internal_context(request) + stored = await store.read( + request.state.reboot_app_internal_context(request), + _blob_id(blob), ) - stored = metadata.blob if metadata.HasField("blob") else None - if stored is None or not stored.committed: + if stored is None: return Response(status_code=404, content="No such blob") - upload_id = stored.upload_id - parts = sorted(stored.parts, key=lambda part: part.number) - total_size = sum(part.size for part in parts) - - async def stream() -> AsyncIterator[bytes]: - for part in parts: - async for chunk in store.read_part( - blob, upload_id, part.number, part.storage_id - ): - yield chunk - media_type, safety_headers = download_headers(stored.content_type) return StreamingResponse( - stream(), + stored.chunks, media_type=media_type, headers={ - "Content-Length": str(total_size), + "Content-Length": str(stored.size), "ETag": f'"{stored.etag}"', "Accept-Ranges": "none", **safety_headers, @@ -240,8 +206,9 @@ def mount_byte_routes( own HTTP server. Registered like any other route, with no privilege of their own: - what reaches `StoredBlob` is a context each handler takes for - itself once a signature has verified, which is the only point at - which it has established anything about its caller.""" + the context the store reaches `StoredBlob` with is one each + handler takes for itself once a signature has verified, which is + the only point at which it has established anything about its + caller.""" http.put(PART_PATH)(_make_put_part(store)) http.get(BLOB_PATH)(_make_get_blob(store)) diff --git a/reboot/std/blob/v1/_store.py b/reboot/std/blob/v1/_store.py index ef90425ba..f100c4bab 100644 --- a/reboot/std/blob/v1/_store.py +++ b/reboot/std/blob/v1/_store.py @@ -1,13 +1,23 @@ -"""The filesystem blob store: bytes storage for the open-source blob -data plane. - -A blob's *bytes* live here; all its metadata lives in state machines. -The `Blob` control plane (see `blob.proto`) holds what the application -knows about a blob, and `StoredBlob` (see `filesystem.proto`) holds -what this store knows about its parts. Nothing about an object is -recorded on disk beside the bytes, so any of a replica's servers can -serve an upload or a download for one blob while agreeing with the -others on nothing but the directory. +"""The filesystem blob store: the open-source blob data plane's +storage, and the bookkeeping that makes it one. + +A blob's *bytes* live here as part files; its metadata lives in state +machines. The `Blob` control plane (see `blob.proto`) holds what the +application knows about a blob, and `StoredBlob` (see +`filesystem.proto`) holds what this store knows about its parts: +which session they were written under, which writes the object is +made of, whether it is committed. Nothing about an object is recorded +on disk beside the bytes, so any of a replica's servers can serve an +upload or a download for one blob while agreeing with the others on +nothing but the directory -- `StoredBlob` is where their writes are +ordered against each other. + +This store drives that state machine itself, so that it offers the +same surface an object store does (`begin_upload`, `complete`, +`delete`, ...) and whoever serves it -- the `BlobDataPlane` servicer, +the byte routes -- only authorizes and delegates. Its methods take +the context they reach `StoredBlob` with; a store backed by an object +store keeps its metadata there instead and needs none. The store mimics S3's multipart-upload semantics (numbered parts, per-part MD5 ETags, ETag-validating completion) so that clients drive @@ -23,12 +33,16 @@ import hashlib import hmac import os +import rbt.std.blob.v1.filesystem_pb2 as filesystem_pb2 +import rbt.v1alpha1.errors_pb2 import shutil import time from dataclasses import dataclass +from rbt.std.blob.v1.filesystem_rbt import StoredBlob, StoredPart +from reboot.aio.external import ExternalContext from reboot.crypto import root_keys from typing import AsyncIterator, Optional, Sequence -from uuid import uuid4 +from uuid import NAMESPACE_URL, UUID, uuid4, uuid5 # The part size clients should use. Every part except the last must be # exactly this size. Must be at least 5 MiB (the S3 minimum part size, @@ -91,6 +105,16 @@ class WrittenPart: storage_id: str +@dataclass(frozen=True) +class StoredObject: + """A committed object, as a download serves it: what to say about + the bytes, and the bytes themselves, in order.""" + content_type: str + etag: str + size: int + chunks: AsyncIterator[bytes] + + @dataclass(frozen=True) class StagedPart: """A part whose bytes are on disk under their final name but not @@ -150,6 +174,17 @@ def composite_etag(etags: Sequence[str]) -> str: return hashlib.md5(digests).hexdigest() + f"-{len(etags)}" +def _begin_upload_key(blob_id: str) -> UUID: + """The idempotency key for beginning one blob's upload. + + Derived from the blob ID rather than taken from the caller, + because the control plane both retries this inside a workflow and + re-runs it to validate that workflow's effects. "Begin the upload + for this blob" is one operation however many times it is asked + for, so the blob names it.""" + return uuid5(NAMESPACE_URL, f"reboot.std.blob.v1/begin-upload/{blob_id}") + + class FilesystemBlobStore: """Stores blob bytes as part files on the local filesystem, served over HTTP by the application (see `_http.py`). @@ -265,7 +300,27 @@ def part_path( f"part.{part_number:08d}.{storage_id}", ) - async def make_upload_directory( + async def begin_upload( + self, + context: ExternalContext, + blob_id: str, + content_type: str, + ) -> str: + """Establishes the session a blob's parts are written under and + returns it: the one already established, if there is one.""" + _, response = await StoredBlob.idempotently( + key=_begin_upload_key(blob_id), + ).BeginUpload( + context, + blob_id, + content_type=content_type, + ) + # After the session exists in state, so a directory is never + # left behind for a session nothing knows about. + await self._make_upload_directory(blob_id, response.upload_id) + return response.upload_id + + async def _make_upload_directory( self, blob_id: str, upload_id: str, @@ -380,30 +435,53 @@ async def stage_part( path=path, ) - async def discard_storage_id( + async def publish_part( self, - encoded_blob_id: str, + context: ExternalContext, + blob_id: str, upload_id: str, - part_number: int, - storage_id: str, - ) -> None: - """Drops one write of a part by name.""" - await _unlink_if_present( - self.part_path( - encoded_blob_id, upload_id, part_number, storage_id + staged: StagedPart, + ) -> bool: + """Makes a staged part's bytes part of the object, and says + whether it did. + + The bytes are on disk under a name of their own; whether the + object is made of them is `StoredBlob`'s to decide, and it + decides for every server that might be serving this blob. + Refused bytes are removed, which is safe because the file's + name belongs to this write alone: no manifest can point at it + unless this very claim succeeded. Anything that outlives an + interrupted request is reclaimed at completion, and with the + blob's directory on `delete`.""" + published = await StoredBlob.ref(blob_id).always().publish_part( + context, + upload_id=upload_id, + part=StoredPart( + number=staged.part.number, + size=staged.part.size, + etag=staged.part.etag, + storage_id=staged.part.storage_id, ), ) + if not published.published: + await _unlink_if_present(staged.path) + return False + if published.HasField("superseded_storage_id"): + # This part had been uploaded before. Nothing is made of + # the earlier bytes now, and the manifest that could still + # name them is refused at completion, so they are removed + # rather than left to accumulate a file per attempt. + await _unlink_if_present( + self.part_path( + _encode_blob_id(blob_id), + upload_id, + staged.part.number, + published.superseded_storage_id, + ), + ) + return True - async def discard_part(self, staged: StagedPart) -> None: - """Drops a part's bytes, for one the object turned out not to - be made of. - - Safe because the name belongs to this write alone: no manifest - can be pointing at it unless this write's own `PublishPart` - succeeded.""" - await _unlink_if_present(staged.path) - - async def reclaim( + async def _reclaim( self, encoded_blob_id: str, upload_id: str, @@ -443,7 +521,38 @@ async def reclaim( continue await _unlink_if_present(os.path.join(directory, name)) - async def read_part( + async def read( + self, + context: ExternalContext, + blob_id: str, + ) -> Optional[StoredObject]: + """The committed object stored for a blob, or `None` when there + is none: a blob whose upload never began, or is not finished, + has no bytes to serve. + + The bytes are the object's parts in part order, each the write + the manifest recorded and no later write of that part.""" + stored = await self._stored(context, blob_id) + if stored is None or not stored.committed: + return None + encoded = _encode_blob_id(blob_id) + parts = sorted(stored.parts, key=lambda part: part.number) + + async def chunks() -> AsyncIterator[bytes]: + for part in parts: + async for chunk in self._read_part( + encoded, stored.upload_id, part.number, part.storage_id + ): + yield chunk + + return StoredObject( + content_type=stored.content_type, + etag=stored.etag, + size=sum(part.size for part in parts), + chunks=chunks(), + ) + + async def _read_part( self, encoded_blob_id: str, upload_id: str, @@ -471,7 +580,170 @@ async def upload_directory_exists( os.path.join(self.blob_directory(encoded_blob_id), upload_id) ) - async def delete(self, blob_id: str) -> None: + async def _stored( + self, + context: ExternalContext, + blob_id: str, + ) -> Optional[filesystem_pb2.StoredBlob]: + """The metadata stored for a blob, or `None` when none is. + + A blob whose upload never began has no state at all, which the + framework reports by refusing the read rather than by + answering with an absent one.""" + try: + metadata = await StoredBlob.ref(blob_id).metadata(context) + except StoredBlob.MetadataAborted as aborted: + if isinstance( + aborted.error, + rbt.v1alpha1.errors_pb2.StateNotConstructed, + ): + return None + raise + return metadata.blob if metadata.HasField("blob") else None + + async def complete( + self, + context: ExternalContext, + blob_id: str, + upload_id: str, + content_type: str, + parts: Sequence[UploadedPart], + max_size: Optional[int] = None, + ) -> str: + """Finishes the object from the parts the client reports, + checking each against what was actually written, and returns + its ETag. Raises `BlobStoreError` for what can never succeed; + completing an already-completed blob returns its ETag.""" + stored = await self._stored(context, blob_id) + if stored is None: + raise BlobStoreError("no upload was ever begun for this blob") + if stored.committed: + # A retried completion. The object is finished and its + # ETag is what it was -- but reclaiming may not have run, + # or not finished, so it runs again from what was + # committed. + await self._reclaim( + _encode_blob_id(blob_id), + stored.upload_id, + [(part.number, part.storage_id) for part in stored.parts], + ) + return stored.etag + if stored.upload_id != upload_id: + # The parts that would be committed were written under a + # different session than the one being completed, so they + # are not the parts this verified. + raise BlobStoreError( + "the upload session being completed is not the one this " + "blob's parts were written under" + ) + + published = {part.number: part for part in stored.parts} + reported = {part.number: part for part in parts} + if len(reported) == 0: + raise BlobStoreError("no parts were reported") + + last_part_number = max(reported) + for number in sorted(reported): + part = published.get(number) + if part is None: + raise BlobStoreError(f"part {number} was never uploaded") + if part.etag != reported[number].etag.strip('"'): + raise BlobStoreError( + f"part {number} ETag mismatch: the uploaded bytes do " + "not match what was reported via `PartUploaded`" + ) + if part.size != reported[number].size: + raise BlobStoreError( + f"part {number} size mismatch: uploaded {part.size} " + f"bytes but {reported[number].size} were reported via " + "`PartUploaded`" + ) + if number != last_part_number and part.size != self._part_size: + # S3 rejects a short middle part with `EntityTooSmall`; + # reject it here too, so that an upload which cannot + # commit against the S3 store cannot commit against + # this one either. + raise BlobStoreError( + f"part {number} is {part.size} bytes, but every part " + f"except the last must be exactly {self._part_size} " + "bytes" + ) + + total_size = sum(published[number].size for number in reported) + # Checked against what the parts were found to hold, not + # against the sizes that were reported alongside them. + if max_size is not None and total_size > max_size: + raise BlobStoreError( + f"uploaded {total_size} bytes exceeds the maximum of " + f"{max_size}" + ) + + manifest = [ + StoredPart( + number=number, + size=published[number].size, + etag=published[number].etag, + storage_id=published[number].storage_id, + ) for number in sorted(reported) + ] + etag = composite_etag([part.etag for part in manifest]) + committed = await StoredBlob.ref(blob_id).always().commit( + context, + upload_id=upload_id, + content_type=content_type, + etag=etag, + parts=manifest, + ) + if not committed.committed: + raise BlobStoreError( + "a part was uploaded again while this upload was being " + "completed; report the parts and commit again" + ) + # The manifest is fixed, so anything else this session wrote + # -- a part uploaded and never reported, a version of a part + # that lost -- belongs to nothing and is safe to remove. Done + # after the commit, so a failure here leaves files behind + # rather than taking away bytes the object is made of; a retry + # reclaims them above. `commit` accepted this manifest, and + # refuses one whose parts have been superseded, so it is + # exactly what was recorded. + await self._reclaim( + _encode_blob_id(blob_id), + upload_id, + [(part.number, part.storage_id) for part in manifest], + ) + return etag + + async def delete( + self, + context: ExternalContext, + blob_id: str, + upload_ids: Sequence[str] = (), + ) -> None: + """Removes a blob's bytes and any unfinished upload of it. + Idempotent: deleting an absent blob succeeds. + + A part lives inside the blob's own directory, so removing the + directory removes any unfinished upload with it, whatever + `upload_ids` says. Forgotten before the bytes go, so that + nothing reads a manifest naming bytes that are already gone: + between the two a download would answer `200` and then run out + of file.""" + try: + await StoredBlob.ref(blob_id).always().forget(context) + except StoredBlob.ForgetAborted as aborted: + if isinstance( + aborted.error, + rbt.v1alpha1.errors_pb2.StateNotConstructed, + ): + # Nothing was ever stored for this blob, so there is + # nothing to forget and deleting it has succeeded. + pass + else: + raise + await self._remove_bytes(blob_id) + + async def _remove_bytes(self, blob_id: str) -> None: """Removes every byte this store holds for a blob.""" encoded = _encode_blob_id(blob_id) # Only a blob that is already gone is ignored: any other failure From 68e2ba43bca26e6875c1c93fbd20b8b29c248fb4 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:56:32 +0000 Subject: [PATCH 20/52] `reboot/std`: serve `BlobDataPlane` with one servicer for every store Before this change each data plane carried its own copy of the six `BlobDataPlane` RPCs: the filesystem one here and the S3 one in the Cloud were the same authorize-then-delegate code twice, and a third store would have made it three. `BlobDataPlaneServicer` now holds those RPCs once, and a data plane is a subclass answering the two questions that differ: which store, in `_blob_store()`, and who may call, in `_authorize()`. The store is anything with the `BlobStore` surface, a `Protocol` that names what the servicer needs of it -- S3's multipart upload as the control plane sees it. Its context-taking methods let a store keep its metadata in Reboot state, as the filesystem one does; a store whose metadata lives in an object store ignores the context. `FilesystemDataPlaneServicer` is left with its store and its own-application check. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY --- reboot/std/blob/v1/BUILD.bazel | 1 + reboot/std/blob/v1/_data_plane_servicer.py | 219 +++++++++++++++++++ reboot/std/blob/v1/_filesystem_data_plane.py | 140 +----------- 3 files changed, 230 insertions(+), 130 deletions(-) create mode 100644 reboot/std/blob/v1/_data_plane_servicer.py diff --git a/reboot/std/blob/v1/BUILD.bazel b/reboot/std/blob/v1/BUILD.bazel index da761d737..356350bfe 100644 --- a/reboot/std/blob/v1/BUILD.bazel +++ b/reboot/std/blob/v1/BUILD.bazel @@ -7,6 +7,7 @@ py_library( srcs = [ "_content_type.py", "_data_plane.py", + "_data_plane_servicer.py", "_filesystem_data_plane.py", "_http.py", "_store.py", diff --git a/reboot/std/blob/v1/_data_plane_servicer.py b/reboot/std/blob/v1/_data_plane_servicer.py new file mode 100644 index 000000000..5be772ac0 --- /dev/null +++ b/reboot/std/blob/v1/_data_plane_servicer.py @@ -0,0 +1,219 @@ +"""The `BlobDataPlane` gRPC servicer, for any store. + +Every data plane serves the same six RPCs the same way: refuse a +caller it does not serve, then hand the call to its store. What +differs from one data plane to the next is the store and who may +call, so a data plane is a subclass of `BlobDataPlaneServicer` that +answers those two questions, and a store is anything with the +`BlobStore` surface -- a directory of part files here, an object +store elsewhere. +""" + +import rbt.std.blob.v1.data_plane_pb2_grpc as data_plane_pb2_grpc +from rbt.std.blob.v1.data_plane_pb2 import ( + ConfigurationRequest, + ConfigurationResponse, + DataPlaneBeginUploadRequest, + DataPlaneBeginUploadResponse, + DataPlaneCompleteUploadRequest, + DataPlaneCompleteUploadResponse, + DataPlaneDeleteRequest, + DataPlaneDeleteResponse, + DataPlaneGetDownloadUrlRequest, + DataPlaneGetDownloadUrlResponse, + DataPlaneGetPartUploadInstructionsRequest, + DataPlaneGetPartUploadInstructionsResponse, + DataPlanePartUploadInstruction, +) +from reboot.aio.external import ExternalContext +from reboot.aio.interceptors import LegacyGrpcContext +from reboot.std.blob.v1._store import BlobStoreError, UploadedPart +from typing import Optional, Protocol, Sequence + + +class BlobStore(Protocol): + """What a `BlobDataPlaneServicer` needs of a store: S3's multipart + upload, seen from the control plane. + + Methods that may need to reach Reboot state for their metadata + take a `context` to do so with; a store whose metadata lives in an + object store ignores it.""" + + @property + def part_size(self) -> int: + """The part size clients must use; every part but the last is + exactly this size.""" + ... + + async def begin_upload( + self, + context: ExternalContext, + blob_id: str, + content_type: str, + ) -> str: + """Establishes the session a blob's parts are written under and + returns it, reusing an existing uncommitted one where it can.""" + ... + + def part_put_url( + self, + blob_id: str, + upload_id: str, + part_number: int, + ) -> str: + """A URL to `PUT` one part's bytes to.""" + ... + + async def complete( + self, + context: ExternalContext, + blob_id: str, + upload_id: str, + content_type: str, + parts: Sequence[UploadedPart], + max_size: Optional[int] = None, + ) -> str: + """Finishes the object from the parts the client reports, + checking each against what was actually stored, and returns + its ETag. Raises `BlobStoreError` for what can never succeed; + completing an already-completed blob returns its ETag.""" + ... + + def download_url( + self, + blob_id: str, + ttl_seconds: Optional[int] = None, + ) -> tuple[str, int]: + """A URL to `GET` the blob's bytes from, and how long it is + valid for.""" + ... + + async def delete( + self, + context: ExternalContext, + blob_id: str, + upload_ids: Sequence[str] = (), + ) -> None: + """Removes the blob's bytes and any unfinished upload of it. + Idempotent: deleting an absent blob succeeds.""" + ... + + +class BlobDataPlaneServicer(data_plane_pb2_grpc.BlobDataPlaneServicer): + """Serves `BlobDataPlane` from a `BlobStore`: each call is + authorized, then handed to the store. + + A subclass says which store, in `_blob_store()`, and who may call, + in `_authorize()`.""" + + async def _authorize(self, context: LegacyGrpcContext) -> None: + """Aborts the call unless its caller is one this data plane + serves.""" + raise NotImplementedError + + def _blob_store(self) -> BlobStore: + """The store this data plane serves from.""" + raise NotImplementedError + + def _context(self, grpc_context: LegacyGrpcContext) -> ExternalContext: + """The context the store reaches Reboot state with, on behalf + of a call `_authorize` has admitted.""" + return grpc_context.external_context(name="blob data plane") + + async def Configuration( + self, + request: ConfigurationRequest, + grpc_context: LegacyGrpcContext, + ) -> ConfigurationResponse: + await self._authorize(grpc_context) + return ConfigurationResponse(part_size=self._blob_store().part_size) + + async def BeginUpload( + self, + request: DataPlaneBeginUploadRequest, + grpc_context: LegacyGrpcContext, + ) -> DataPlaneBeginUploadResponse: + await self._authorize(grpc_context) + upload_id = await self._blob_store().begin_upload( + self._context(grpc_context), + request.blob_id, + request.content_type, + ) + return DataPlaneBeginUploadResponse(upload_id=upload_id) + + async def GetPartUploadInstructions( + self, + request: DataPlaneGetPartUploadInstructionsRequest, + grpc_context: LegacyGrpcContext, + ) -> DataPlaneGetPartUploadInstructionsResponse: + await self._authorize(grpc_context) + instructions = [ + DataPlanePartUploadInstruction( + part_number=part_number, + url=self._blob_store().part_put_url( + request.blob_id, + request.upload_id, + part_number, + ), + ) for part_number in request.part_numbers + ] + return DataPlaneGetPartUploadInstructionsResponse( + instructions=instructions + ) + + async def CompleteUpload( + self, + request: DataPlaneCompleteUploadRequest, + grpc_context: LegacyGrpcContext, + ) -> DataPlaneCompleteUploadResponse: + await self._authorize(grpc_context) + try: + etag = await self._blob_store().complete( + self._context(grpc_context), + request.blob_id, + request.upload_id, + request.content_type, + [ + UploadedPart( + number=part.number, etag=part.etag, size=part.size + ) for part in request.parts + ], + max_size=( + request.max_size if request.HasField("max_size") else None + ), + ) + return DataPlaneCompleteUploadResponse(etag=etag) + except BlobStoreError as error: + # A permanent failure: report it so the control plane can + # surface it and let the client re-upload. Transient + # failures raise other exceptions, which the control + # plane's workflow retries. + return DataPlaneCompleteUploadResponse(error=str(error)) + + async def GetDownloadUrl( + self, + request: DataPlaneGetDownloadUrlRequest, + grpc_context: LegacyGrpcContext, + ) -> DataPlaneGetDownloadUrlResponse: + await self._authorize(grpc_context) + url, ttl_seconds = self._blob_store().download_url( + request.blob_id, + request.ttl_seconds if request.HasField("ttl_seconds") else None, + ) + return DataPlaneGetDownloadUrlResponse( + url=url, + ttl_seconds=ttl_seconds, + ) + + async def Delete( + self, + request: DataPlaneDeleteRequest, + grpc_context: LegacyGrpcContext, + ) -> DataPlaneDeleteResponse: + await self._authorize(grpc_context) + await self._blob_store().delete( + self._context(grpc_context), + request.blob_id, + upload_ids=list(request.upload_ids), + ) + return DataPlaneDeleteResponse() diff --git a/reboot/std/blob/v1/_filesystem_data_plane.py b/reboot/std/blob/v1/_filesystem_data_plane.py index 4619950bf..037fc3248 100644 --- a/reboot/std/blob/v1/_filesystem_data_plane.py +++ b/reboot/std/blob/v1/_filesystem_data_plane.py @@ -21,50 +21,33 @@ Everything else is the store's: `FilesystemBlobStore` keeps the bytes and drives `StoredBlob`, the state machine that keeps the -metadata, so each call here is authorized and then handed over. -Nothing here holds state of its own, so any of a replica's servers -can serve any call. +metadata, and `BlobDataPlaneServicer` hands it every call once +authorized. Nothing here holds state of its own, so any of a +replica's servers can serve any call. """ import grpc -from rbt.std.blob.v1.data_plane_pb2 import ( - ConfigurationRequest, - ConfigurationResponse, - DataPlaneBeginUploadRequest, - DataPlaneBeginUploadResponse, - DataPlaneCompleteUploadRequest, - DataPlaneCompleteUploadResponse, - DataPlaneDeleteRequest, - DataPlaneDeleteResponse, - DataPlaneGetDownloadUrlRequest, - DataPlaneGetDownloadUrlResponse, - DataPlaneGetPartUploadInstructionsRequest, - DataPlaneGetPartUploadInstructionsResponse, - DataPlanePartUploadInstruction, -) -from rbt.std.blob.v1.data_plane_pb2_grpc import BlobDataPlaneServicer from reboot.aio.caller_id import CallerID -from reboot.aio.external import ExternalContext from reboot.aio.headers import CALLER_ID_HEADER from reboot.aio.interceptors import LegacyGrpcContext from reboot.aio.internals.contextvars import get_application_id -from reboot.std.blob.v1._store import ( - BlobStoreError, - FilesystemBlobStore, - UploadedPart, -) +from reboot.std.blob.v1._data_plane_servicer import BlobDataPlaneServicer +from reboot.std.blob.v1._store import FilesystemBlobStore class FilesystemDataPlaneServicer(BlobDataPlaneServicer): """Serves `BlobDataPlane` from the application whose blobs it - holds: each call is authorized, then handed to the store. + holds. The store is set by `BlobLibrary` once it knows where this application keeps them.""" _store: FilesystemBlobStore - async def _authorize_caller(self, context: LegacyGrpcContext) -> None: + def _blob_store(self) -> FilesystemBlobStore: + return self._store + + async def _authorize(self, context: LegacyGrpcContext) -> None: """Refuses anyone but this application's own code. This is the check `is_app_internal` makes, made by hand @@ -100,109 +83,6 @@ async def _authorize_caller(self, context: LegacyGrpcContext) -> None: ) raise RuntimeError("This is unreachable") - def _context(self, grpc_context: LegacyGrpcContext) -> ExternalContext: - """The context the store reaches `StoredBlob` with, on behalf - of a call `_authorize_caller` has admitted.""" - return grpc_context.external_context(name="blob data plane") - - async def Configuration( - self, - request: ConfigurationRequest, - grpc_context: LegacyGrpcContext, - ) -> ConfigurationResponse: - await self._authorize_caller(grpc_context) - return ConfigurationResponse(part_size=self._store.part_size) - - async def BeginUpload( - self, - request: DataPlaneBeginUploadRequest, - grpc_context: LegacyGrpcContext, - ) -> DataPlaneBeginUploadResponse: - await self._authorize_caller(grpc_context) - upload_id = await self._store.begin_upload( - self._context(grpc_context), - request.blob_id, - request.content_type, - ) - return DataPlaneBeginUploadResponse(upload_id=upload_id) - - async def GetPartUploadInstructions( - self, - request: DataPlaneGetPartUploadInstructionsRequest, - grpc_context: LegacyGrpcContext, - ) -> DataPlaneGetPartUploadInstructionsResponse: - await self._authorize_caller(grpc_context) - instructions = [ - DataPlanePartUploadInstruction( - part_number=part_number, - url=self._store.part_put_url( - request.blob_id, - request.upload_id, - part_number, - ), - ) for part_number in request.part_numbers - ] - return DataPlaneGetPartUploadInstructionsResponse( - instructions=instructions - ) - - async def CompleteUpload( - self, - request: DataPlaneCompleteUploadRequest, - grpc_context: LegacyGrpcContext, - ) -> DataPlaneCompleteUploadResponse: - await self._authorize_caller(grpc_context) - try: - etag = await self._store.complete( - self._context(grpc_context), - request.blob_id, - request.upload_id, - request.content_type, - [ - UploadedPart( - number=part.number, etag=part.etag, size=part.size - ) for part in request.parts - ], - max_size=( - request.max_size if request.HasField("max_size") else None - ), - ) - return DataPlaneCompleteUploadResponse(etag=etag) - except BlobStoreError as error: - # A permanent failure: report it so the control plane can - # surface it and let the client re-upload. Transient - # failures raise other exceptions, which the control - # plane's workflow retries. - return DataPlaneCompleteUploadResponse(error=str(error)) - - async def GetDownloadUrl( - self, - request: DataPlaneGetDownloadUrlRequest, - grpc_context: LegacyGrpcContext, - ) -> DataPlaneGetDownloadUrlResponse: - await self._authorize_caller(grpc_context) - url, ttl_seconds = self._store.download_url( - request.blob_id, - request.ttl_seconds if request.HasField("ttl_seconds") else None, - ) - return DataPlaneGetDownloadUrlResponse( - url=url, - ttl_seconds=ttl_seconds, - ) - - async def Delete( - self, - request: DataPlaneDeleteRequest, - grpc_context: LegacyGrpcContext, - ) -> DataPlaneDeleteResponse: - await self._authorize_caller(grpc_context) - await self._store.delete( - self._context(grpc_context), - request.blob_id, - upload_ids=list(request.upload_ids), - ) - return DataPlaneDeleteResponse() - def legacy_grpc_servicers() -> list[type]: return [FilesystemDataPlaneServicer] From cd9b87cbfaf1f34d14912ec6de9d557c63c784c0 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 19:44:11 +0000 Subject: [PATCH 21/52] `reboot/std/react`: retry a part's `PUT` the way its RPCs are retried An upload's control-plane calls go through the generated web client and get its retries, but before this change the bytes went out as a bare `fetch`, so one dropped connection or one 503 from a store failed the whole upload from the user's point of view, however many parts were already confirmed. A part's `PUT` is now attempted up to four times, with a doubling, jittered, capped delay in between: after a request that never got an answer, and after a store's 408, 429 or 5xx. A 403 is retried with a freshly minted URL, since on either store that is what an expired URL earns, and a part that starts late in a slow upload can outlive the minutes its URL was minted for. A 400, 404, 409 or 413 is refused for good, as is anything once the caller has aborted. What no attempt can fix is still a rejection of `upload()`, naming the part and the reason; the confirmed parts stay confirmed, so calling `upload()` again for the same blob resumes it. The hook's documentation now says so, and that a blob never committed is removed by the backend after a day, which is all the cleanup an abandoned upload needs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY --- reboot/std/react/blob/index.tsx | 152 +++++++++++++++++++++++++++----- 1 file changed, 132 insertions(+), 20 deletions(-) diff --git a/reboot/std/react/blob/index.tsx b/reboot/std/react/blob/index.tsx index 8e8aafdd8..c780bf936 100644 --- a/reboot/std/react/blob/index.tsx +++ b/reboot/std/react/blob/index.tsx @@ -9,7 +9,7 @@ // plane mints (the application's own data plane for the `filesystem` // store, presigned S3 URLs for the `s3` store — the uploader neither // knows nor cares which). Those `PUT`s are not Reboot RPCs, so they -// stay a plain `fetch`. +// are a plain `fetch`, with retries of their own below. import { useRebootClient } from "@reboot-dev/reboot-react"; import { Blob_Status } from "@reboot-dev/reboot-std-api/blob/v1/blob_pb.js"; @@ -28,6 +28,43 @@ export { useBlob }; // available bandwidth unused on any connection with real latency. const UPLOAD_CONCURRENCY = 4; +// How many times one part's `PUT` is attempted before the upload +// fails, and how the attempts are spaced: the delay doubles from the +// first one and is capped, so that a blip is ridden out in seconds +// while an outage is reported rather than waited out. Each delay is +// jittered, so that the parts of one window, which fail together, do +// not retry together. +const PUT_ATTEMPTS = 4; +const PUT_FIRST_RETRY_DELAY_MS = 500; +const PUT_MAX_RETRY_DELAY_MS = 5000; + +/** + * Resolves after `ms`, or rejects at once if `signal` aborts first. + */ +function delay(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + signal?.throwIfAborted(); + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, ms); + const onAbort = () => { + clearTimeout(timer); + reject(signal?.reason); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + +/** + * How one attempt to `PUT` a part ended: with the part's ETag, or with + * a failure that says whether another attempt is worth making, and + * whether it needs a freshly minted URL first. + */ +type PutAttempt = + | { ok: true; etag: string } + | { ok: false; retry: boolean; remint: boolean; reason: string }; + // How long `useBlobDownloadUrl` asks its URL to stay valid for. The // store caps what it grants; the granted value comes back on the // response. @@ -143,7 +180,11 @@ export class BlobUploader { /** * `PUT`s one part's bytes to an already-minted URL and reports it to - * the control plane. + * the control plane, retrying the `PUT` the way the control-plane + * calls are retried by their client. A failure that no attempt can + * fix, or that outlasts the attempts, is thrown; the part is then + * still pending, and a later `upload()` of the same blob picks it up + * again. */ private async putPartToUrl( partNumber: number, @@ -151,24 +192,36 @@ export class BlobUploader { bytes: globalThis.Blob | Uint8Array, options?: { signal?: AbortSignal } ): Promise { - const response = await fetch(url, { - method: "PUT", - body: bytes, - signal: options?.signal, - }); - if (!response.ok) { - throw new Error( - `Part ${partNumber} upload failed (${response.status}): ` + - `${await response.text()}` - ); - } - const etag = (response.headers.get("ETag") ?? "").replace(/"/g, ""); - if (etag === "") { - throw new Error( - `Part ${partNumber} upload returned no ETag; if this ` + - "application uses an S3-compatible store, its bucket CORS " + - "configuration must expose the `ETag` header" - ); + let etag: string; + for (let attempt = 1; ; attempt++) { + const outcome = await this.tryPutPart(url, bytes, options?.signal); + // Compared rather than negated: this package compiles without + // `strict`, and only an equality check narrows a discriminant + // then. + if (outcome.ok === false) { + if (!outcome.retry || attempt >= PUT_ATTEMPTS) { + throw new Error( + `Part ${partNumber} upload failed after ${attempt} ` + + `attempt${attempt === 1 ? "" : "s"}: ${outcome.reason}` + ); + } + if (outcome.remint) { + const { urls } = await this.instructions([partNumber], options); + const fresh = urls.get(partNumber); + if (fresh === undefined) { + throw new Error(`No upload URL for part ${partNumber}`); + } + url = fresh; + } + const backoff = Math.min( + PUT_MAX_RETRY_DELAY_MS, + PUT_FIRST_RETRY_DELAY_MS * 2 ** (attempt - 1) + ); + await delay(backoff * (0.5 + Math.random() / 2), options?.signal); + continue; + } + etag = outcome.etag; + break; } const size = bytes instanceof Uint8Array ? bytes.byteLength : bytes.size; await this.blob.partUploaded(this.context, { @@ -179,6 +232,59 @@ export class BlobUploader { this.confirmed.set(partNumber, size); } + /** + * One attempt to `PUT` a part. Retried: a request that never got an + * answer, and the answers a store gives while it is momentarily + * unable rather than unwilling (408, 429, 5xx). Retried with a fresh + * URL: 403, which on either store is what an expired URL earns, and + * an upload that started late in a slow session can outlive the + * minutes its URLs are minted for. Everything else is refused for + * good: the request itself is wrong (400), the session is gone + * (404), the blob is already committed (409), or the part is too + * large (413). + */ + private async tryPutPart( + url: string, + bytes: globalThis.Blob | Uint8Array, + signal?: AbortSignal + ): Promise { + let response: Response; + try { + response = await fetch(url, { method: "PUT", body: bytes, signal }); + } catch (error) { + // Aborting is the caller's doing, not the network's. + signal?.throwIfAborted(); + return { ok: false, retry: true, remint: false, reason: `${error}` }; + } + if (response.ok) { + const etag = (response.headers.get("ETag") ?? "").replace(/"/g, ""); + if (etag === "") { + return { + ok: false, + retry: false, + remint: false, + reason: + "the upload returned no ETag; if this application uses an " + + "S3-compatible store, its bucket CORS configuration must " + + "expose the `ETag` header", + }; + } + return { ok: true, etag }; + } + const reason = `${response.status}: ${await response.text()}`; + if (response.status === 403) { + return { ok: false, retry: true, remint: true, reason }; + } + if ( + response.status === 408 || + response.status === 429 || + response.status >= 500 + ) { + return { ok: false, retry: true, remint: false, reason }; + } + return { ok: false, retry: false, remint: false, reason }; + } + /** * Commits the upload and waits for the data plane to confirm, * returning the blob's ETag or the reason the commit failed. The @@ -297,6 +403,12 @@ export class BlobUploader { * const { upload } = useBlobUpload(); * ... * const { etag, error } = await upload(blobId, file); + * + * A resolved `error` is the data plane's verdict on the commit. A + * rejection is a part that could not be uploaded even after retries; + * the parts that did upload are kept, so calling `upload` again for + * the same blob resumes rather than restarts. A blob that is never + * committed is removed by the backend after a day. */ export function useBlobUpload(): { upload: ( From 272b6ca09fcbbb4f4e96da0311cbf705fae39d69 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:00:58 +0000 Subject: [PATCH 22/52] `tests/reboot/std/react`: unit-test the part `PUT`'s retries Before this change nothing in the repository exercised `reboot/std/react` on its own: the package was type-checked and driven through the chat-room example, so a retry policy could be wrong in ways no test would say, and a test of it would have had to mock the generated client and the browser both. The policy now lives in `reboot/std/react/blob/put.ts`, with the `fetch` it uses and the sleep between attempts handed in, and `index.tsx` calls `putPartWithRetries` with the real ones. The package's `exports` map keeps the module internal. A plain `js_test` (no Reboot backend, so no wheel) under `tests/reboot/std/react/blob` scripts `fetch` one answer per call and checks each branch of the policy: a first-time success, a 5xx followed by a dropped connection, a 403 retried on a freshly minted URL, giving up after the last attempt with the backoff bounded and capped, each status refused for good, a success without an `ETag`, and an abort mid-wait. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY --- reboot/std/react/blob/BUILD.bazel | 1 + reboot/std/react/blob/index.tsx | 136 ++-------------- reboot/std/react/blob/put.ts | 145 +++++++++++++++++ tests/reboot/std/react/blob/BUILD.bazel | 40 +++++ tests/reboot/std/react/blob/package.json | 3 + tests/reboot/std/react/blob/put_tests.ts | 196 +++++++++++++++++++++++ 6 files changed, 400 insertions(+), 121 deletions(-) create mode 100644 reboot/std/react/blob/put.ts create mode 100644 tests/reboot/std/react/blob/BUILD.bazel create mode 100644 tests/reboot/std/react/blob/package.json create mode 100644 tests/reboot/std/react/blob/put_tests.ts diff --git a/reboot/std/react/blob/BUILD.bazel b/reboot/std/react/blob/BUILD.bazel index 2858ee68f..434a9e6f9 100644 --- a/reboot/std/react/blob/BUILD.bazel +++ b/reboot/std/react/blob/BUILD.bazel @@ -5,6 +5,7 @@ ts_project( srcs = [ "index.tsx", "package.json", + "put.ts", ], declaration = True, tsconfig = { diff --git a/reboot/std/react/blob/index.tsx b/reboot/std/react/blob/index.tsx index c780bf936..48084edf5 100644 --- a/reboot/std/react/blob/index.tsx +++ b/reboot/std/react/blob/index.tsx @@ -9,7 +9,7 @@ // plane mints (the application's own data plane for the `filesystem` // store, presigned S3 URLs for the `s3` store — the uploader neither // knows nor cares which). Those `PUT`s are not Reboot RPCs, so they -// are a plain `fetch`, with retries of their own below. +// are a plain `fetch`, with retries of their own in `put.ts`. import { useRebootClient } from "@reboot-dev/reboot-react"; import { Blob_Status } from "@reboot-dev/reboot-std-api/blob/v1/blob_pb.js"; @@ -17,6 +17,7 @@ import { useBlob } from "@reboot-dev/reboot-std-api/blob/v1/blob_rbt_react.js"; import { Blob } from "@reboot-dev/reboot-std-api/blob/v1/blob_rbt_web.js"; import { WebContext } from "@reboot-dev/reboot-web"; import { useMemo } from "react"; +import { putPartWithRetries } from "./put.js"; // Re-exported so applications can reactively render blob metadata // (e.g. a progress bar for an attachment some *other* client is @@ -28,43 +29,6 @@ export { useBlob }; // available bandwidth unused on any connection with real latency. const UPLOAD_CONCURRENCY = 4; -// How many times one part's `PUT` is attempted before the upload -// fails, and how the attempts are spaced: the delay doubles from the -// first one and is capped, so that a blip is ridden out in seconds -// while an outage is reported rather than waited out. Each delay is -// jittered, so that the parts of one window, which fail together, do -// not retry together. -const PUT_ATTEMPTS = 4; -const PUT_FIRST_RETRY_DELAY_MS = 500; -const PUT_MAX_RETRY_DELAY_MS = 5000; - -/** - * Resolves after `ms`, or rejects at once if `signal` aborts first. - */ -function delay(ms: number, signal?: AbortSignal): Promise { - return new Promise((resolve, reject) => { - signal?.throwIfAborted(); - const timer = setTimeout(() => { - signal?.removeEventListener("abort", onAbort); - resolve(); - }, ms); - const onAbort = () => { - clearTimeout(timer); - reject(signal?.reason); - }; - signal?.addEventListener("abort", onAbort, { once: true }); - }); -} - -/** - * How one attempt to `PUT` a part ended: with the part's ETag, or with - * a failure that says whether another attempt is worth making, and - * whether it needs a freshly minted URL first. - */ -type PutAttempt = - | { ok: true; etag: string } - | { ok: false; retry: boolean; remint: boolean; reason: string }; - // How long `useBlobDownloadUrl` asks its URL to stay valid for. The // store caps what it grants; the granted value comes back on the // response. @@ -192,37 +156,20 @@ export class BlobUploader { bytes: globalThis.Blob | Uint8Array, options?: { signal?: AbortSignal } ): Promise { - let etag: string; - for (let attempt = 1; ; attempt++) { - const outcome = await this.tryPutPart(url, bytes, options?.signal); - // Compared rather than negated: this package compiles without - // `strict`, and only an equality check narrows a discriminant - // then. - if (outcome.ok === false) { - if (!outcome.retry || attempt >= PUT_ATTEMPTS) { - throw new Error( - `Part ${partNumber} upload failed after ${attempt} ` + - `attempt${attempt === 1 ? "" : "s"}: ${outcome.reason}` - ); - } - if (outcome.remint) { - const { urls } = await this.instructions([partNumber], options); - const fresh = urls.get(partNumber); - if (fresh === undefined) { - throw new Error(`No upload URL for part ${partNumber}`); - } - url = fresh; + const etag = await putPartWithRetries( + partNumber, + url, + bytes, + async () => { + const { urls } = await this.instructions([partNumber], options); + const fresh = urls.get(partNumber); + if (fresh === undefined) { + throw new Error(`No upload URL for part ${partNumber}`); } - const backoff = Math.min( - PUT_MAX_RETRY_DELAY_MS, - PUT_FIRST_RETRY_DELAY_MS * 2 ** (attempt - 1) - ); - await delay(backoff * (0.5 + Math.random() / 2), options?.signal); - continue; - } - etag = outcome.etag; - break; - } + return fresh; + }, + options + ); const size = bytes instanceof Uint8Array ? bytes.byteLength : bytes.size; await this.blob.partUploaded(this.context, { partNumber, @@ -232,59 +179,6 @@ export class BlobUploader { this.confirmed.set(partNumber, size); } - /** - * One attempt to `PUT` a part. Retried: a request that never got an - * answer, and the answers a store gives while it is momentarily - * unable rather than unwilling (408, 429, 5xx). Retried with a fresh - * URL: 403, which on either store is what an expired URL earns, and - * an upload that started late in a slow session can outlive the - * minutes its URLs are minted for. Everything else is refused for - * good: the request itself is wrong (400), the session is gone - * (404), the blob is already committed (409), or the part is too - * large (413). - */ - private async tryPutPart( - url: string, - bytes: globalThis.Blob | Uint8Array, - signal?: AbortSignal - ): Promise { - let response: Response; - try { - response = await fetch(url, { method: "PUT", body: bytes, signal }); - } catch (error) { - // Aborting is the caller's doing, not the network's. - signal?.throwIfAborted(); - return { ok: false, retry: true, remint: false, reason: `${error}` }; - } - if (response.ok) { - const etag = (response.headers.get("ETag") ?? "").replace(/"/g, ""); - if (etag === "") { - return { - ok: false, - retry: false, - remint: false, - reason: - "the upload returned no ETag; if this application uses an " + - "S3-compatible store, its bucket CORS configuration must " + - "expose the `ETag` header", - }; - } - return { ok: true, etag }; - } - const reason = `${response.status}: ${await response.text()}`; - if (response.status === 403) { - return { ok: false, retry: true, remint: true, reason }; - } - if ( - response.status === 408 || - response.status === 429 || - response.status >= 500 - ) { - return { ok: false, retry: true, remint: false, reason }; - } - return { ok: false, retry: false, remint: false, reason }; - } - /** * Commits the upload and waits for the data plane to confirm, * returning the blob's ETag or the reason the commit failed. The diff --git a/reboot/std/react/blob/put.ts b/reboot/std/react/blob/put.ts new file mode 100644 index 000000000..19c90d60b --- /dev/null +++ b/reboot/std/react/blob/put.ts @@ -0,0 +1,145 @@ +// How a part's bytes are `PUT` to the data plane: the retries a bare +// `fetch` does not have, so that an upload survives what the +// control-plane calls already survive through their client. Internal +// to the package; `index.tsx` is the only importer, and the package's +// `exports` map keeps it that way. + +// How many times one part's `PUT` is attempted before the upload +// fails, and how the attempts are spaced: the delay doubles from the +// first one and is capped, so that a blip is ridden out in seconds +// while an outage is reported rather than waited out. Each delay is +// jittered, so that the parts of one window, which fail together, do +// not retry together. +export const PUT_ATTEMPTS = 4; +export const PUT_FIRST_RETRY_DELAY_MS = 500; +export const PUT_MAX_RETRY_DELAY_MS = 5000; + +/** + * Resolves after `ms`, or rejects at once if `signal` aborts first. + */ +export function delay(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + signal?.throwIfAborted(); + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, ms); + const onAbort = () => { + clearTimeout(timer); + reject(signal?.reason); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + +/** + * How one attempt to `PUT` a part ended: with the part's ETag, or with + * a failure that says whether another attempt is worth making, and + * whether it needs a freshly minted URL first. + */ +export type PutAttempt = + | { ok: true; etag: string } + | { ok: false; retry: boolean; remint: boolean; reason: string }; + +/** + * What `putPartWithRetries` reaches the world through. Both default to + * the real thing; a test hands in its own. + */ +export interface PutPartDependencies { + fetch?: typeof globalThis.fetch; + sleep?: (ms: number, signal?: AbortSignal) => Promise; +} + +/** + * One attempt to `PUT` a part. Retried: a request that never got an + * answer, and the answers a store gives while it is momentarily unable + * rather than unwilling (408, 429, 5xx). Retried with a fresh URL: + * 403, which on either store is what an expired URL earns, and an + * upload that started late in a slow session can outlive the minutes + * its URLs are minted for. Everything else is refused for good: the + * request itself is wrong (400), the session is gone (404), the blob + * is already committed (409), or the part is too large (413). + */ +export async function tryPutPart( + url: string, + bytes: globalThis.Blob | Uint8Array, + signal?: AbortSignal, + fetchImpl: typeof globalThis.fetch = globalThis.fetch +): Promise { + let response: Response; + try { + response = await fetchImpl(url, { method: "PUT", body: bytes, signal }); + } catch (error) { + // Aborting is the caller's doing, not the network's. + signal?.throwIfAborted(); + return { ok: false, retry: true, remint: false, reason: `${error}` }; + } + if (response.ok) { + const etag = (response.headers.get("ETag") ?? "").replace(/"/g, ""); + if (etag === "") { + return { + ok: false, + retry: false, + remint: false, + reason: + "the upload returned no ETag; if this application uses an " + + "S3-compatible store, its bucket CORS configuration must " + + "expose the `ETag` header", + }; + } + return { ok: true, etag }; + } + const reason = `${response.status}: ${await response.text()}`; + if (response.status === 403) { + return { ok: false, retry: true, remint: true, reason }; + } + if ( + response.status === 408 || + response.status === 429 || + response.status >= 500 + ) { + return { ok: false, retry: true, remint: false, reason }; + } + return { ok: false, retry: false, remint: false, reason }; +} + +/** + * `PUT`s one part's bytes, retrying as `tryPutPart` advises and asking + * `remint` for a fresh URL when the old one has expired, and returns + * the part's ETag. Throws for a failure that no attempt can fix, or + * that outlasts the attempts. + */ +export async function putPartWithRetries( + partNumber: number, + url: string, + bytes: globalThis.Blob | Uint8Array, + remint: () => Promise, + options?: { signal?: AbortSignal } & PutPartDependencies +): Promise { + const fetchImpl = options?.fetch ?? globalThis.fetch; + const sleep = options?.sleep ?? delay; + for (let attempt = 1; ; attempt++) { + const outcome = await tryPutPart(url, bytes, options?.signal, fetchImpl); + // Compared rather than negated: this package compiles without + // `strict`, and only an equality check narrows a discriminant + // then. + if (outcome.ok === false) { + if (!outcome.retry || attempt >= PUT_ATTEMPTS) { + throw new Error( + `Part ${partNumber} upload failed after ${attempt} ` + + `attempt${attempt === 1 ? "" : "s"}: ${outcome.reason}` + ); + } + if (outcome.remint) { + url = await remint(); + } + const backoff = Math.min( + PUT_MAX_RETRY_DELAY_MS, + PUT_FIRST_RETRY_DELAY_MS * 2 ** (attempt - 1) + ); + await sleep(backoff * (0.5 + Math.random() / 2), options?.signal); + continue; + } + return outcome.etag; + } +} diff --git a/tests/reboot/std/react/blob/BUILD.bazel b/tests/reboot/std/react/blob/BUILD.bazel new file mode 100644 index 000000000..ba7099936 --- /dev/null +++ b/tests/reboot/std/react/blob/BUILD.bazel @@ -0,0 +1,40 @@ +load("@aspect_rules_js//js:defs.bzl", "js_test") +load("@aspect_rules_ts//ts:defs.bzl", "ts_project") + +ts_project( + name = "put_tests_ts", + srcs = [ + "put_tests.ts", + ":package.json", + ], + declaration = True, + tsconfig = { + "compilerOptions": { + "declaration": True, + "module": "nodenext", + "moduleResolution": "nodenext", + "target": "es2020", + }, + }, + visibility = ["//visibility:public"], + deps = [ + "//:node_modules/@types/node", + "//reboot/std/react/blob:blob_ts", + ], +) + +# A unit test of the part `PUT`'s retry policy, with `fetch` and the +# backoff sleep handed in; nothing here needs a Reboot backend, so +# this is a plain `js_test` rather than a `js_reboot_test`. +js_test( + name = "put_tests", + size = "small", + data = [ + ":put_tests_ts", + ], + entry_point = "put_tests.js", + node_options = [ + "--test", + ], + visibility = ["//visibility:public"], +) diff --git a/tests/reboot/std/react/blob/package.json b/tests/reboot/std/react/blob/package.json new file mode 100644 index 000000000..3dbc1ca59 --- /dev/null +++ b/tests/reboot/std/react/blob/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/tests/reboot/std/react/blob/put_tests.ts b/tests/reboot/std/react/blob/put_tests.ts new file mode 100644 index 000000000..1471ab6b6 --- /dev/null +++ b/tests/reboot/std/react/blob/put_tests.ts @@ -0,0 +1,196 @@ +import { strict as assert } from "node:assert"; +import test from "node:test"; +import { + PUT_ATTEMPTS, + PUT_FIRST_RETRY_DELAY_MS, + PUT_MAX_RETRY_DELAY_MS, + putPartWithRetries, +} from "../../../../../reboot/std/react/blob/put.js"; + +const BYTES = new Uint8Array([1, 2, 3]); + +/** + * A `fetch` that answers from a script, one entry per call: a status + * (with an `ETag` for a success), or an `Error` to reject with. It + * records what it was asked, so a test can see which URL each attempt + * went to. + */ +function scriptedFetch(script: (number | Error)[]): { + fetch: typeof globalThis.fetch; + urls: string[]; +} { + const urls: string[] = []; + const fetch = async (input: string | URL | Request): Promise => { + urls.push(input.toString()); + const next = script.shift(); + if (next === undefined) { + throw new Error("scripted fetch was called more often than scripted"); + } + if (next instanceof Error) { + throw next; + } + return new Response(`body for ${next}`, { + status: next, + headers: next === 200 ? { ETag: '"etag-of-the-part"' } : {}, + }); + }; + return { fetch, urls }; +} + +/** + * A `sleep` that never sleeps but remembers how long it was asked to, + * so a test can check the backoff. + */ +function recordingSleep(): { + sleep: (ms: number, signal?: AbortSignal) => Promise; + delays: number[]; +} { + const delays: number[] = []; + return { + sleep: async (ms: number) => { + delays.push(ms); + }, + delays, + }; +} + +function neverRemint(): Promise { + throw new Error("remint was not expected"); +} + +test("putPartWithRetries", async (t) => { + await t.test("returns the ETag of a first-time success", async () => { + const { fetch, urls } = scriptedFetch([200]); + const { sleep, delays } = recordingSleep(); + const etag = await putPartWithRetries( + 1, + "https://store/part", + BYTES, + neverRemint, + { fetch, sleep } + ); + assert.equal(etag, "etag-of-the-part"); + assert.deepEqual(urls, ["https://store/part"]); + assert.deepEqual(delays, []); + }); + + await t.test("retries a 5xx, then a dropped connection", async () => { + const { fetch, urls } = scriptedFetch([ + 503, + new TypeError("fetch failed"), + 200, + ]); + const { sleep, delays } = recordingSleep(); + const etag = await putPartWithRetries( + 1, + "https://store/part", + BYTES, + neverRemint, + { fetch, sleep } + ); + assert.equal(etag, "etag-of-the-part"); + assert.equal(urls.length, 3); + // One backoff per retry, doubling, with jitter of at most half. + assert.equal(delays.length, 2); + assert.ok(delays[0] >= PUT_FIRST_RETRY_DELAY_MS / 2); + assert.ok(delays[0] <= PUT_FIRST_RETRY_DELAY_MS); + assert.ok(delays[1] >= PUT_FIRST_RETRY_DELAY_MS); + assert.ok(delays[1] <= PUT_FIRST_RETRY_DELAY_MS * 2); + }); + + await t.test("retries a 403 with a freshly minted URL", async () => { + const { fetch, urls } = scriptedFetch([403, 200]); + const { sleep } = recordingSleep(); + let reminted = 0; + const etag = await putPartWithRetries( + 7, + "https://store/part?sig=expired", + BYTES, + async () => { + reminted += 1; + return "https://store/part?sig=fresh"; + }, + { fetch, sleep } + ); + assert.equal(etag, "etag-of-the-part"); + assert.equal(reminted, 1); + assert.deepEqual(urls, [ + "https://store/part?sig=expired", + "https://store/part?sig=fresh", + ]); + }); + + await t.test("gives up after the last attempt", async () => { + const { fetch, urls } = scriptedFetch( + Array.from({ length: PUT_ATTEMPTS }, () => 503) + ); + const { sleep, delays } = recordingSleep(); + await assert.rejects( + putPartWithRetries(3, "https://store/part", BYTES, neverRemint, { + fetch, + sleep, + }), + (error: Error) => + error.message.includes(`Part 3 upload failed after ${PUT_ATTEMPTS}`) && + error.message.includes("503: body for 503") + ); + assert.equal(urls.length, PUT_ATTEMPTS); + // The last attempt is not followed by a wait, and no wait exceeds + // the cap. + assert.equal(delays.length, PUT_ATTEMPTS - 1); + assert.ok(delays.every((ms) => ms <= PUT_MAX_RETRY_DELAY_MS)); + }); + + for (const status of [400, 404, 409, 413]) { + await t.test(`refuses a ${status} for good`, async () => { + const { fetch, urls } = scriptedFetch([status]); + const { sleep, delays } = recordingSleep(); + await assert.rejects( + putPartWithRetries(1, "https://store/part", BYTES, neverRemint, { + fetch, + sleep, + }), + (error: Error) => + error.message.includes("after 1 attempt:") && + error.message.includes(`${status}: body for ${status}`) + ); + assert.equal(urls.length, 1); + assert.deepEqual(delays, []); + }); + } + + await t.test("refuses a success without an ETag for good", async () => { + const fetch = async (): Promise => + new Response("stored", { status: 200 }); + const { sleep, delays } = recordingSleep(); + await assert.rejects( + putPartWithRetries(1, "https://store/part", BYTES, neverRemint, { + fetch, + sleep, + }), + (error: Error) => error.message.includes("returned no ETag") + ); + assert.deepEqual(delays, []); + }); + + await t.test("stops as soon as the caller aborts", async () => { + const controller = new AbortController(); + const { fetch, urls } = scriptedFetch([503, 200]); + // Abort while waiting to retry, the way a user leaving the page + // would; the `sleep` stands in for `delay`, which rejects with the + // abort reason. + const sleep = async (ms: number, signal?: AbortSignal) => { + controller.abort(new Error("user left")); + signal?.throwIfAborted(); + }; + await assert.rejects( + putPartWithRetries(1, "https://store/part", BYTES, neverRemint, { + fetch, + sleep, + signal: controller.signal, + }), + (error: Error) => error.message === "user left" + ); + assert.equal(urls.length, 1); + }); +}); From d60351988d3f8c0a4ddc6b2ea7a0d67c3d3a30e0 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:20:37 +0000 Subject: [PATCH 23/52] `rbt/std/blob`: say on `BlobDataPlane` who retries what Before this change the interface said every method had to be safe to call more than once, and left the reader to work out why: the retries live in the caller. A data plane retries nothing itself. The one failure a method declares is `CompleteUpload`'s `error`, for a completion that can never succeed; every other failure is an undeclared gRPC error, which the contract now says the caller retries until the call succeeds -- the control plane from its workflows, a client of the presigning methods through the control plane's client. One place for every retry keeps them understandable and lets a data plane stay stateless. The same contract is stated where a store implementer meets it, on `BlobStore`, and `CompleteUpload`'s `error` is documented as final for the blob: retrying the call would only repeat it, and so would re-uploading what was already uploaded, so the browser hook now says to upload again into a new blob. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY --- rbt/std/blob/v1/data_plane.proto | 39 +++++++++++++++------- reboot/std/blob/v1/_data_plane_servicer.py | 15 ++++++--- reboot/std/react/blob/index.tsx | 6 ++-- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/rbt/std/blob/v1/data_plane.proto b/rbt/std/blob/v1/data_plane.proto index 186467b56..cb83acae8 100644 --- a/rbt/std/blob/v1/data_plane.proto +++ b/rbt/std/blob/v1/data_plane.proto @@ -13,15 +13,27 @@ // framework options, so that it can be implemented by anything and // addressed by a bare URL (`REBOOT_BLOB_DATA_PLANE_URL`). // -// The control plane retries `BeginUpload`, `CompleteUpload`, and -// `Delete` inside workflows, so every method must be *safe* to call -// more than once for the same `blob_id`. `CompleteUpload` and `Delete` -// are naturally idempotent (a completed blob returns its ETag; -// deleting an absent blob succeeds). `BeginUpload` should reuse an -// existing uncommitted session where it can; a backend that can only -// mint fresh upload IDs is still acceptable — a lost response then -// orphans a session, which is reclaimed when the blob expires -// uncommitted or is deleted. +// Errors, and who retries them: a data plane retries nothing itself. +// The one failure a method declares is `CompleteUpload`'s `error`, +// for a completion that can never succeed. Every other failure -- a +// store that cannot be reached, a response that never arrived, a +// throttle, a 5xx -- is an undeclared gRPC error, and the contract is +// that the caller retries it until the call succeeds. The control +// plane does so from workflows, with backoff, for `BeginUpload`, +// `CompleteUpload` and `Delete`; a client of the presigning methods +// retries through the control plane's own client. Keeping every retry +// in the caller keeps it in one place, and lets a data plane stay +// stateless. +// +// That contract is what makes idempotency a requirement: every method +// must be *safe* to call more than once for the same `blob_id`, since +// a retry may repeat a call whose effect took but whose response was +// lost. `CompleteUpload` and `Delete` are naturally idempotent (a +// completed blob returns its ETag; deleting an absent blob succeeds). +// `BeginUpload` should reuse an existing uncommitted session where it +// can; a backend that can only mint fresh upload IDs is still +// acceptable — a lost response then orphans a session, which is +// reclaimed when the blob expires uncommitted or is deleted. syntax = "proto3"; @@ -127,9 +139,12 @@ message DataPlaneCompleteUploadResponse { // The committed object's composite ETag. string etag = 1; - // A permanent-failure reason: completion failed in a way the - // client can fix by re-uploading. The control plane reverts the - // blob to UPLOADING with this message. + // A permanent-failure reason: what the data plane holds can never + // complete as reported (a part's bytes or size differ from what + // was reported, a middle part is short, the total exceeds + // `max_size`, the session is gone). The control plane reverts the + // blob to UPLOADING with this message; retrying the call would + // only repeat it, so the caller does not. string error = 2; } } diff --git a/reboot/std/blob/v1/_data_plane_servicer.py b/reboot/std/blob/v1/_data_plane_servicer.py index 5be772ac0..a2a2787ed 100644 --- a/reboot/std/blob/v1/_data_plane_servicer.py +++ b/reboot/std/blob/v1/_data_plane_servicer.py @@ -37,7 +37,12 @@ class BlobStore(Protocol): Methods that may need to reach Reboot state for their metadata take a `context` to do so with; a store whose metadata lives in an - object store ignores it.""" + object store ignores it. + + A store retries nothing: it raises for what may pass on a later + attempt, which the servicer surfaces as a gRPC error for the + caller to retry (see `data_plane.proto`), and raises + `BlobStoreError` only for what never will.""" @property def part_size(self) -> int: @@ -184,10 +189,10 @@ async def CompleteUpload( ) return DataPlaneCompleteUploadResponse(etag=etag) except BlobStoreError as error: - # A permanent failure: report it so the control plane can - # surface it and let the client re-upload. Transient - # failures raise other exceptions, which the control - # plane's workflow retries. + # A permanent failure: reported in the response, since + # retrying the call would only repeat it. Transient failures + # raise other exceptions, which become the gRPC error the + # caller retries. return DataPlaneCompleteUploadResponse(error=str(error)) async def GetDownloadUrl( diff --git a/reboot/std/react/blob/index.tsx b/reboot/std/react/blob/index.tsx index 48084edf5..d8e0c5e86 100644 --- a/reboot/std/react/blob/index.tsx +++ b/reboot/std/react/blob/index.tsx @@ -298,8 +298,10 @@ export class BlobUploader { * ... * const { etag, error } = await upload(blobId, file); * - * A resolved `error` is the data plane's verdict on the commit. A - * rejection is a part that could not be uploaded even after retries; + * A resolved `error` is the data plane's verdict on the commit, and + * it is final for that blob: what was uploaded can never complete as + * reported, so upload again into a new blob. A rejection is a part + * that could not be uploaded even after retries; * the parts that did upload are kept, so calling `upload` again for * the same blob resumes rather than restarts. A blob that is never * committed is removed by the backend after a day. From ca484d59eb5e800cd2ca48bf1b56490d8d69d6cc Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 21:03:02 +0000 Subject: [PATCH 24/52] `reboot/std/blob`: call the same job by the same name on every layer Before this change a job changed name at each layer it passed through: `BlobDataPlane.CompleteUpload` called the store's `complete()`, `GetPartUploadInstructions` its `part_put_url()`, the byte routes signed with `signature_for_put()` and `signature_for_get()`, the browser fetched `instructions()` before a part upload, the metadata half of `Blob.Remove` was `StoredBlob.Forget`, and the workflow that `Blob.Commit` schedules was `Blob.CompleteUpload` -- the data plane's name for a different job. What `Blob.Create` schedules was `BeginUpload`, on every layer down to the `StoredBlob` constructor, though nothing is begun from the client's point of view: the blob is brought into existence, ready for parts, and the client's upload starts later with its first `PUT`. A reader following one job across the files learned a new name for it at every step. Every layer now uses one name per job. Creating a blob is `Create` everywhere: `Blob.Create` schedules `Blob.CreateWorkflow`, which calls `BlobDataPlane.Create` (S3's own name for it is `CreateMultipartUpload`), which delegates to the store's `create()`, which constructs `StoredBlob` through its `Create`; the filesystem store's own factory moves from `create` to `open` to free the name. Finishing an upload is a commit everywhere: `Blob.Commit` schedules `Blob.CommitWorkflow`, as `Remove` schedules `RemoveWorkflow`, which calls `BlobDataPlane.Commit`, which delegates to the store's `commit()`, which drives `StoredBlob.Commit`; the data plane had been the one layer to say "complete upload". The store's other methods are named after the RPC each one serves (`part_upload_url`, `download_url`, `delete`), so the shared servicer delegates every RPC to its namesake, and the byte routes sign a part upload and a download with `signature_for_part_upload()` and `signature_for_download()`. The browser's `partUploadInstructions()` is named after the RPC it calls. `StoredBlob.Forget` is `StoredBlob.Remove`, the metadata half of `Blob.Remove`; only the plain gRPC data plane can say `Delete`, since Reboot reserves that method name. The workflow half of a job on `Blob` is named after the writer that schedules it, with the `Workflow` suffix Cloud uses (`UpdateCard` and `UpdateCardWorkflow`). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY --- rbt/std/blob/v1/blob.proto | 44 ++++++------- rbt/std/blob/v1/data_plane.proto | 52 +++++++-------- rbt/std/blob/v1/filesystem.proto | 23 +++---- reboot/std/blob/v1/_data_plane_servicer.py | 40 ++++++------ reboot/std/blob/v1/_http.py | 4 +- reboot/std/blob/v1/_store.py | 57 ++++++++-------- reboot/std/blob/v1/_stored_blob.py | 32 ++++----- reboot/std/blob/v1/blob.py | 76 +++++++++++----------- reboot/std/react/blob/index.tsx | 27 ++++---- tests/reboot/std/blob/v1/blob_tests.py | 4 +- 10 files changed, 181 insertions(+), 178 deletions(-) diff --git a/rbt/std/blob/v1/blob.proto b/rbt/std/blob/v1/blob.proto index 66b6ddf78..0dadc1ea4 100644 --- a/rbt/std/blob/v1/blob.proto +++ b/rbt/std/blob/v1/blob.proto @@ -69,7 +69,7 @@ message Blob { // Parts may be uploaded. The initial status. UPLOADING = 0; - // `Commit` was called; the `CompleteUpload` workflow is + // `Commit` was called; the `CommitWorkflow` workflow is // finalizing the object in the data plane. COMMITTING = 1; @@ -77,7 +77,7 @@ message Blob { COMMITTED = 2; // `Remove` was called (or the upload expired); the - // `PerformRemove` workflow is removing the bytes from the data + // `RemoveWorkflow` workflow is removing the bytes from the data // plane. REMOVING = 3; @@ -115,7 +115,7 @@ message Blob { // `reboot.std.blob.v1.blob`. optional Downloaders downloaders = 10; - // Data-plane upload session ID, set by the `BeginUpload` workflow. + // Data-plane upload session ID, set by `CreateWorkflow`. // `GetPartUploadInstructions` reports `ready: false` until this is set. optional string upload_id = 6; @@ -130,7 +130,7 @@ message Blob { // different part boundaries can carry different ETags. optional string etag = 8; - // Why the most recent `CompleteUpload` attempt failed. Cleared on + // Why the most recent `CommitWorkflow` attempt failed. Cleared on // success; while set, `status` has reverted to `UPLOADING` so the // client can re-upload parts and `Commit` again. The message // describes the failure; it does not identify which parts, if any, @@ -188,9 +188,9 @@ message SetDownloadersResponse {} //////////////////////////////////////////////////////////////////////// -message BeginUploadRequest {} +message CreateWorkflowRequest {} -message BeginUploadResponse {} +message CreateWorkflowResponse {} //////////////////////////////////////////////////////////////////////// @@ -210,9 +210,9 @@ message PartUploadInstruction { } message GetPartUploadInstructionsResponse { - // False until the `BeginUpload` workflow has provisioned the - // data-plane upload session; read this method reactively and the - // instructions arrive as soon as it is true. + // False until `CreateWorkflow` has provisioned the data-plane + // upload session; read this method reactively and the instructions + // arrive as soon as it is true. bool ready = 1; // The part size uploads should use: every part except the last @@ -245,9 +245,9 @@ message CommitResponse {} //////////////////////////////////////////////////////////////////////// -message CompleteUploadRequest {} +message CommitWorkflowRequest {} -message CompleteUploadResponse {} +message CommitWorkflowResponse {} //////////////////////////////////////////////////////////////////////// @@ -306,9 +306,9 @@ message RemoveResponse {} //////////////////////////////////////////////////////////////////////// -message PerformRemoveRequest {} +message RemoveWorkflowRequest {} -message PerformRemoveResponse {} +message RemoveWorkflowResponse {} //////////////////////////////////////////////////////////////////////// @@ -319,9 +319,9 @@ message ExpireIfNotCommittedResponse {} //////////////////////////////////////////////////////////////////////// service BlobMethods { - // Creates the blob's metadata and schedules the `BeginUpload` - // workflow that provisions an upload session in the storage - // backend. Only application code may call this; it is the + // Creates the blob's metadata and schedules `CreateWorkflow`, which + // provisions an upload session in the storage backend. Only + // application code may call this; it is the // application's chance to enforce quota and size policy (directly, // or by setting `max_size`), to record which user may upload // (`uploader_id`), and to restrict who may download (`downloaders`). @@ -344,14 +344,14 @@ service BlobMethods { // Provisions the data-plane upload session and records its // `upload_id`. Scheduled by `Create`; not for direct use. - rpc BeginUpload(BeginUploadRequest) returns (BeginUploadResponse) { + rpc CreateWorkflow(CreateWorkflowRequest) returns (CreateWorkflowResponse) { option (rbt.v1alpha1.method) = { workflow: {}, }; } // Mints `PUT` URLs for the requested part numbers. Reports - // `ready: false` until `BeginUpload` has completed. + // `ready: false` until `CreateWorkflow` has completed. rpc GetPartUploadInstructions(GetPartUploadInstructionsRequest) returns (GetPartUploadInstructionsResponse) { option (rbt.v1alpha1.method) = { @@ -371,7 +371,7 @@ service BlobMethods { }; } - // Validates the reported parts and schedules the `CompleteUpload` + // Validates the reported parts and schedules the `CommitWorkflow` // workflow that finalizes the object in the data plane. // Observe the outcome reactively via `Info`: `status` becomes // `COMMITTED`, or reverts to `UPLOADING` with `commit_error` set. @@ -397,7 +397,7 @@ service BlobMethods { // Finalizes the object in the data plane (which validates the // reported part ETags). Scheduled by `Commit`; not for direct use. - rpc CompleteUpload(CompleteUploadRequest) returns (CompleteUploadResponse) { + rpc CommitWorkflow(CommitWorkflowRequest) returns (CommitWorkflowResponse) { option (rbt.v1alpha1.method) = { workflow: {}, }; @@ -422,7 +422,7 @@ service BlobMethods { }; } - // Marks the blob for deletion and schedules the `PerformRemove` + // Marks the blob for deletion and schedules the `RemoveWorkflow` // workflow that removes its bytes from the data plane. // (Named `Remove` because `Delete` is a reserved Reboot method // name.) @@ -434,7 +434,7 @@ service BlobMethods { // Removes the blob's bytes from the data plane. Scheduled by // `Remove` and `ExpireIfNotCommitted`; not for direct use. - rpc PerformRemove(PerformRemoveRequest) returns (PerformRemoveResponse) { + rpc RemoveWorkflow(RemoveWorkflowRequest) returns (RemoveWorkflowResponse) { option (rbt.v1alpha1.method) = { workflow: {}, }; diff --git a/rbt/std/blob/v1/data_plane.proto b/rbt/std/blob/v1/data_plane.proto index cb83acae8..1fc42cc3b 100644 --- a/rbt/std/blob/v1/data_plane.proto +++ b/rbt/std/blob/v1/data_plane.proto @@ -14,23 +14,22 @@ // addressed by a bare URL (`REBOOT_BLOB_DATA_PLANE_URL`). // // Errors, and who retries them: a data plane retries nothing itself. -// The one failure a method declares is `CompleteUpload`'s `error`, -// for a completion that can never succeed. Every other failure -- a -// store that cannot be reached, a response that never arrived, a -// throttle, a 5xx -- is an undeclared gRPC error, and the contract is -// that the caller retries it until the call succeeds. The control -// plane does so from workflows, with backoff, for `BeginUpload`, -// `CompleteUpload` and `Delete`; a client of the presigning methods -// retries through the control plane's own client. Keeping every retry -// in the caller keeps it in one place, and lets a data plane stay -// stateless. +// The one failure a method declares is `Commit`'s `error`, for a +// commit that can never succeed. Every other failure -- a store that +// cannot be reached, a response that never arrived, a throttle, a 5xx +// -- is an undeclared gRPC error, and the contract is that the caller +// retries it until the call succeeds. The control plane does so from +// workflows, with backoff, for `Create`, `Commit` and `Delete`; a +// client of the presigning methods retries through the control plane's +// own client. Keeping every retry in the caller keeps it in one place, +// and lets a data plane stay stateless. // // That contract is what makes idempotency a requirement: every method // must be *safe* to call more than once for the same `blob_id`, since // a retry may repeat a call whose effect took but whose response was -// lost. `CompleteUpload` and `Delete` are naturally idempotent (a -// completed blob returns its ETag; deleting an absent blob succeeds). -// `BeginUpload` should reuse an existing uncommitted session where it +// lost. `Commit` and `Delete` are naturally idempotent (a committed +// blob returns its ETag; deleting an absent blob succeeds). +// `Create` should reuse an existing uncommitted session where it // can; a backend that can only mint fresh upload IDs is still // acceptable — a lost response then orphans a session, which is // reclaimed when the blob expires uncommitted or is deleted. @@ -44,12 +43,12 @@ service BlobDataPlane { // plane: the client part size. Read once at application startup. rpc Configuration(ConfigurationRequest) returns (ConfigurationResponse); - // Provisions an upload session for a blob and returns its data-plane + // Creates the blob in the data plane: provisions the upload session + // its parts are written under and returns the session's data-plane // upload ID (e.g. an S3 multipart upload ID). Should reuse an // existing uncommitted session for the same blob where possible; a // fresh session is acceptable (see the idempotency note above). - rpc BeginUpload(DataPlaneBeginUploadRequest) - returns (DataPlaneBeginUploadResponse); + rpc Create(DataPlaneCreateRequest) returns (DataPlaneCreateResponse); // Mints URLs to which the client `PUT`s each requested part's bytes // directly. A URL is either absolute (e.g. a presigned S3 URL, @@ -64,10 +63,9 @@ service BlobDataPlane { // a permanent-failure `error` (ETag mismatch, over `max_size`, ...) // that the control plane surfaces so the client can re-upload. A // *transient* failure is a gRPC error instead, so the control-plane - // workflow retries. Idempotent: completing an already-completed blob - // returns its ETag. - rpc CompleteUpload(DataPlaneCompleteUploadRequest) - returns (DataPlaneCompleteUploadResponse); + // workflow retries. Idempotent: committing an already-committed + // blob returns its ETag. + rpc Commit(DataPlaneCommitRequest) returns (DataPlaneCommitResponse); // Mints a URL from which the committed object's bytes can be // downloaded: absolute (e.g. a presigned S3/CloudFront URL) or @@ -90,12 +88,12 @@ message ConfigurationResponse { //////////////////////////////////////////////////////////////////////// -message DataPlaneBeginUploadRequest { +message DataPlaneCreateRequest { string blob_id = 1; string content_type = 2; } -message DataPlaneBeginUploadResponse { +message DataPlaneCreateResponse { string upload_id = 1; } @@ -124,7 +122,7 @@ message DataPlaneUploadedPart { uint64 size = 3; } -message DataPlaneCompleteUploadRequest { +message DataPlaneCommitRequest { string blob_id = 1; string upload_id = 2; string content_type = 3; @@ -132,15 +130,15 @@ message DataPlaneCompleteUploadRequest { optional uint64 max_size = 5; } -message DataPlaneCompleteUploadResponse { - // Completion either finished the object or did not; there is no - // answer that is both. +message DataPlaneCommitResponse { + // A commit either finished the object or did not; there is no answer + // that is both. oneof outcome { // The committed object's composite ETag. string etag = 1; // A permanent-failure reason: what the data plane holds can never - // complete as reported (a part's bytes or size differ from what + // commit as reported (a part's bytes or size differ from what // was reported, a middle part is short, the total exceeds // `max_size`, the session is gone). The control plane reverts the // blob to UPLOADING with this message; retrying the call would diff --git a/rbt/std/blob/v1/filesystem.proto b/rbt/std/blob/v1/filesystem.proto index 3c849895b..c3894bb03 100644 --- a/rbt/std/blob/v1/filesystem.proto +++ b/rbt/std/blob/v1/filesystem.proto @@ -59,7 +59,7 @@ message StoredBlob { string content_type = 2; // The upload session the parts belong to. Absent before the first - // `BeginUpload`. + // `Create`. optional string upload_id = 3; // The object's ETag, derived from its parts' own. Absent until @@ -76,12 +76,12 @@ message StoredBlob { //////////////////////////////////////////////////////////////////////// -message StoredBlobBeginUploadRequest { +message StoredBlobCreateRequest { // MIME type to serve the bytes with. string content_type = 1; } -message StoredBlobBeginUploadResponse { +message StoredBlobCreateResponse { // The session to write parts under, whether freshly minted or the // one an earlier call already established. string upload_id = 1; @@ -146,20 +146,20 @@ message StoredBlobMetadataResponse { optional StoredBlob blob = 1; } -message StoredBlobForgetRequest {} +message StoredBlobRemoveRequest {} -message StoredBlobForgetResponse {} +message StoredBlobRemoveResponse {} //////////////////////////////////////////////////////////////////////// service StoredBlobMethods { - // Establishes the upload session parts are written under, and - // returns it. The control plane retries this inside a workflow, so + // Creates the object's metadata and the upload session its parts + // are written under, and returns the session. The control plane + // retries this inside a workflow, so // the caller asks for it once per workflow: a retry replays the // session that was already established rather than minting a // second one and orphaning the parts written under the first. - rpc BeginUpload(StoredBlobBeginUploadRequest) - returns (StoredBlobBeginUploadResponse) { + rpc Create(StoredBlobCreateRequest) returns (StoredBlobCreateResponse) { option (rbt.v1alpha1.method) = { writer: { constructor: {} }, }; @@ -194,8 +194,9 @@ service StoredBlobMethods { }; } - // Forgets the object, for a blob whose bytes are being deleted. - rpc Forget(StoredBlobForgetRequest) returns (StoredBlobForgetResponse) { + // Removes the object's metadata, for a blob whose bytes are being + // removed; the metadata half of `Blob.Remove`. + rpc Remove(StoredBlobRemoveRequest) returns (StoredBlobRemoveResponse) { option (rbt.v1alpha1.method) = { writer: {}, }; diff --git a/reboot/std/blob/v1/_data_plane_servicer.py b/reboot/std/blob/v1/_data_plane_servicer.py index a2a2787ed..72b6f5bc7 100644 --- a/reboot/std/blob/v1/_data_plane_servicer.py +++ b/reboot/std/blob/v1/_data_plane_servicer.py @@ -13,10 +13,10 @@ from rbt.std.blob.v1.data_plane_pb2 import ( ConfigurationRequest, ConfigurationResponse, - DataPlaneBeginUploadRequest, - DataPlaneBeginUploadResponse, - DataPlaneCompleteUploadRequest, - DataPlaneCompleteUploadResponse, + DataPlaneCommitRequest, + DataPlaneCommitResponse, + DataPlaneCreateRequest, + DataPlaneCreateResponse, DataPlaneDeleteRequest, DataPlaneDeleteResponse, DataPlaneGetDownloadUrlRequest, @@ -50,7 +50,7 @@ def part_size(self) -> int: exactly this size.""" ... - async def begin_upload( + async def create( self, context: ExternalContext, blob_id: str, @@ -60,7 +60,7 @@ async def begin_upload( returns it, reusing an existing uncommitted one where it can.""" ... - def part_put_url( + def part_upload_url( self, blob_id: str, upload_id: str, @@ -69,7 +69,7 @@ def part_put_url( """A URL to `PUT` one part's bytes to.""" ... - async def complete( + async def commit( self, context: ExternalContext, blob_id: str, @@ -81,7 +81,7 @@ async def complete( """Finishes the object from the parts the client reports, checking each against what was actually stored, and returns its ETag. Raises `BlobStoreError` for what can never succeed; - completing an already-completed blob returns its ETag.""" + committing an already-committed blob returns its ETag.""" ... def download_url( @@ -133,18 +133,18 @@ async def Configuration( await self._authorize(grpc_context) return ConfigurationResponse(part_size=self._blob_store().part_size) - async def BeginUpload( + async def Create( self, - request: DataPlaneBeginUploadRequest, + request: DataPlaneCreateRequest, grpc_context: LegacyGrpcContext, - ) -> DataPlaneBeginUploadResponse: + ) -> DataPlaneCreateResponse: await self._authorize(grpc_context) - upload_id = await self._blob_store().begin_upload( + upload_id = await self._blob_store().create( self._context(grpc_context), request.blob_id, request.content_type, ) - return DataPlaneBeginUploadResponse(upload_id=upload_id) + return DataPlaneCreateResponse(upload_id=upload_id) async def GetPartUploadInstructions( self, @@ -155,7 +155,7 @@ async def GetPartUploadInstructions( instructions = [ DataPlanePartUploadInstruction( part_number=part_number, - url=self._blob_store().part_put_url( + url=self._blob_store().part_upload_url( request.blob_id, request.upload_id, part_number, @@ -166,14 +166,14 @@ async def GetPartUploadInstructions( instructions=instructions ) - async def CompleteUpload( + async def Commit( self, - request: DataPlaneCompleteUploadRequest, + request: DataPlaneCommitRequest, grpc_context: LegacyGrpcContext, - ) -> DataPlaneCompleteUploadResponse: + ) -> DataPlaneCommitResponse: await self._authorize(grpc_context) try: - etag = await self._blob_store().complete( + etag = await self._blob_store().commit( self._context(grpc_context), request.blob_id, request.upload_id, @@ -187,13 +187,13 @@ async def CompleteUpload( request.max_size if request.HasField("max_size") else None ), ) - return DataPlaneCompleteUploadResponse(etag=etag) + return DataPlaneCommitResponse(etag=etag) except BlobStoreError as error: # A permanent failure: reported in the response, since # retrying the call would only repeat it. Transient failures # raise other exceptions, which become the gRPC error the # caller retries. - return DataPlaneCompleteUploadResponse(error=str(error)) + return DataPlaneCommitResponse(error=str(error)) async def GetDownloadUrl( self, diff --git a/reboot/std/blob/v1/_http.py b/reboot/std/blob/v1/_http.py index 8bf4ef7b0..3227db3f6 100644 --- a/reboot/std/blob/v1/_http.py +++ b/reboot/std/blob/v1/_http.py @@ -107,7 +107,7 @@ async def put_part(request: Request) -> Response: expiration = _unexpired_expiration(request) if expiration is None: return Response(status_code=403, content="URL expired") - expected = store.signature_for_put( + expected = store.signature_for_part_upload( blob, upload, part_number, expiration ) if not _signature_matches( @@ -168,7 +168,7 @@ async def get_blob(request: Request) -> Response: expiration = _unexpired_expiration(request) if expiration is None: return Response(status_code=403, content="URL expired") - expected = store.signature_for_get(blob, expiration) + expected = store.signature_for_download(blob, expiration) if not _signature_matches( expected, request.query_params.get("sig", "") ): diff --git a/reboot/std/blob/v1/_store.py b/reboot/std/blob/v1/_store.py index f100c4bab..c46fa7c74 100644 --- a/reboot/std/blob/v1/_store.py +++ b/reboot/std/blob/v1/_store.py @@ -13,7 +13,7 @@ ordered against each other. This store drives that state machine itself, so that it offers the -same surface an object store does (`begin_upload`, `complete`, +same surface an object store does (`create`, `commit`, `delete`, ...) and whoever serves it -- the `BlobDataPlane` servicer, the byte routes -- only authorizes and delegates. Its methods take the context they reach `StoredBlob` with; a store backed by an object @@ -77,11 +77,11 @@ class BlobStoreError(Exception): - """A permanent storage failure (e.g. a part missing at completion - time), reported to the control plane as a `CompleteUpload` `error` - so the client can re-upload. Transient failures (e.g. network - errors) are raised as their original exception types instead, - becoming gRPC errors that the control plane's workflow retries.""" + """A permanent storage failure (e.g. a part missing at commit + time), reported to the control plane as a `Commit` `error` so the + client can re-upload. Transient failures (e.g. network errors) are + raised as their original exception types instead, becoming gRPC + errors that the control plane's workflow retries.""" class PartTooLarge(Exception): @@ -174,15 +174,15 @@ def composite_etag(etags: Sequence[str]) -> str: return hashlib.md5(digests).hexdigest() + f"-{len(etags)}" -def _begin_upload_key(blob_id: str) -> UUID: - """The idempotency key for beginning one blob's upload. +def _create_key(blob_id: str) -> UUID: + """The idempotency key for creating one blob. Derived from the blob ID rather than taken from the caller, because the control plane both retries this inside a workflow and - re-runs it to validate that workflow's effects. "Begin the upload - for this blob" is one operation however many times it is asked - for, so the blob names it.""" - return uuid5(NAMESPACE_URL, f"reboot.std.blob.v1/begin-upload/{blob_id}") + re-runs it to validate that workflow's effects. "Create this blob" + is one operation however many times it is asked for, so the blob + names it.""" + return uuid5(NAMESPACE_URL, f"reboot.std.blob.v1/create/{blob_id}") class FilesystemBlobStore: @@ -213,7 +213,7 @@ def __init__( self._part_size = part_size @classmethod - async def create( + async def open( cls, directory: str, part_size: int = DEFAULT_PART_SIZE_BYTES, @@ -255,7 +255,7 @@ def _sign(self, *parts: str) -> str: return hmac.new(self._signing_key(), message, hashlib.sha256).hexdigest() - def signature_for_put( + def signature_for_part_upload( self, encoded_blob_id: str, upload_id: str, @@ -267,7 +267,7 @@ def signature_for_put( str(expiration) ) - def signature_for_get( + def signature_for_download( self, encoded_blob_id: str, expiration: int, @@ -300,17 +300,18 @@ def part_path( f"part.{part_number:08d}.{storage_id}", ) - async def begin_upload( + async def create( self, context: ExternalContext, blob_id: str, content_type: str, ) -> str: - """Establishes the session a blob's parts are written under and - returns it: the one already established, if there is one.""" + """Creates the blob: establishes the session its parts are + written under and returns it, the one already established if + there is one.""" _, response = await StoredBlob.idempotently( - key=_begin_upload_key(blob_id), - ).BeginUpload( + key=_create_key(blob_id), + ).Create( context, blob_id, content_type=content_type, @@ -338,7 +339,7 @@ async def _make_upload_directory( await _fsync_directory(self._directory) await _fsync_directory(self.blob_directory(encoded)) - def part_put_url( + def part_upload_url( self, blob_id: str, upload_id: str, @@ -346,7 +347,7 @@ def part_put_url( ) -> str: encoded = _encode_blob_id(blob_id) expiration = int(time.time()) + DEFAULT_URL_TTL_SECONDS - signature = self.signature_for_put( + signature = self.signature_for_part_upload( encoded, upload_id, part_number, expiration ) return ( @@ -367,7 +368,7 @@ def download_url( _MAX_URL_TTL_SECONDS, ) expiration = int(time.time()) + ttl - signature = self.signature_for_get(encoded, expiration) + signature = self.signature_for_download(encoded, expiration) url = (f"{BLOB_PATH}?blob={encoded}&exp={expiration}&sig={signature}") return url, ttl @@ -601,7 +602,7 @@ async def _stored( raise return metadata.blob if metadata.HasField("blob") else None - async def complete( + async def commit( self, context: ExternalContext, blob_id: str, @@ -613,7 +614,7 @@ async def complete( """Finishes the object from the parts the client reports, checking each against what was actually written, and returns its ETag. Raises `BlobStoreError` for what can never succeed; - completing an already-completed blob returns its ETag.""" + committing an already-committed blob returns its ETag.""" stored = await self._stored(context, blob_id) if stored is None: raise BlobStoreError("no upload was ever begun for this blob") @@ -730,8 +731,8 @@ async def delete( between the two a download would answer `200` and then run out of file.""" try: - await StoredBlob.ref(blob_id).always().forget(context) - except StoredBlob.ForgetAborted as aborted: + await StoredBlob.ref(blob_id).always().remove(context) + except StoredBlob.RemoveAborted as aborted: if isinstance( aborted.error, rbt.v1alpha1.errors_pb2.StateNotConstructed, @@ -747,7 +748,7 @@ async def _remove_bytes(self, blob_id: str) -> None: """Removes every byte this store holds for a blob.""" encoded = _encode_blob_id(blob_id) # Only a blob that is already gone is ignored: any other failure - # must reach the caller, or `PerformRemove` would report bytes + # must reach the caller, or `RemoveWorkflow` would report bytes # deleted that are still on disk. In a thread because # `aiofiles` has no `rmtree`. try: diff --git a/reboot/std/blob/v1/_stored_blob.py b/reboot/std/blob/v1/_stored_blob.py index a324b5a50..2f4aabcac 100644 --- a/reboot/std/blob/v1/_stored_blob.py +++ b/reboot/std/blob/v1/_stored_blob.py @@ -10,16 +10,16 @@ from rbt.std.blob.v1.filesystem_rbt import ( StoredBlob, - StoredBlobBeginUploadRequest, - StoredBlobBeginUploadResponse, StoredBlobCommitRequest, StoredBlobCommitResponse, - StoredBlobForgetRequest, - StoredBlobForgetResponse, + StoredBlobCreateRequest, + StoredBlobCreateResponse, StoredBlobMetadataRequest, StoredBlobMetadataResponse, StoredBlobPublishPartRequest, StoredBlobPublishPartResponse, + StoredBlobRemoveRequest, + StoredBlobRemoveResponse, ) from reboot.aio.auth.authorizers import allow_if, is_app_internal from reboot.aio.contexts import ReaderContext, WriterContext @@ -35,24 +35,24 @@ def authorizer(self) -> StoredBlob.Authorizer: # both are inside this application; a client's capability is # the signed URL it was given, not access to this state. return StoredBlob.Authorizer( - begin_upload=allow_if(any=[is_app_internal]), + create=allow_if(any=[is_app_internal]), publish_part=allow_if(any=[is_app_internal]), commit=allow_if(any=[is_app_internal]), metadata=allow_if(any=[is_app_internal]), - forget=allow_if(any=[is_app_internal]), + remove=allow_if(any=[is_app_internal]), ) - async def begin_upload( + async def create( self, context: WriterContext, - request: StoredBlobBeginUploadRequest, - ) -> StoredBlobBeginUploadResponse: + request: StoredBlobCreateRequest, + ) -> StoredBlobCreateResponse: self.state.committed = False self.state.content_type = request.content_type self.state.upload_id = uuid4().hex self.state.ClearField("etag") del self.state.parts[:] - return StoredBlobBeginUploadResponse(upload_id=self.state.upload_id) + return StoredBlobCreateResponse(upload_id=self.state.upload_id) async def publish_part( self, @@ -93,8 +93,8 @@ async def commit( request: StoredBlobCommitRequest, ) -> StoredBlobCommitResponse: if self.state.committed: - # `CompleteUpload` is retried by a workflow, so arriving at - # an object that is already finished is success, not a + # `CommitWorkflow` is a workflow, and retries, so arriving + # at an object that is already finished is success, not a # conflict. return StoredBlobCommitResponse(committed=True) @@ -128,17 +128,17 @@ async def metadata( ) -> StoredBlobMetadataResponse: return StoredBlobMetadataResponse(blob=self.state) - async def forget( + async def remove( self, context: WriterContext, - request: StoredBlobForgetRequest, - ) -> StoredBlobForgetResponse: + request: StoredBlobRemoveRequest, + ) -> StoredBlobRemoveResponse: self.state.committed = False self.state.content_type = "" self.state.ClearField("upload_id") self.state.ClearField("etag") del self.state.parts[:] - return StoredBlobForgetResponse() + return StoredBlobRemoveResponse() def servicers() -> list[type[StoredBlob.Servicer]]: diff --git a/reboot/std/blob/v1/blob.py b/reboot/std/blob/v1/blob.py index b62beb8df..b91b1d23a 100644 --- a/reboot/std/blob/v1/blob.py +++ b/reboot/std/blob/v1/blob.py @@ -37,16 +37,16 @@ from grpc.aio import AioRpcError from rbt.std.blob.v1.blob_rbt import ( AlreadyCommitted, - BeginUploadRequest, - BeginUploadResponse, Blob, BlobPart, CommitRequest, CommitResponse, - CompleteUploadRequest, - CompleteUploadResponse, + CommitWorkflowRequest, + CommitWorkflowResponse, CreateRequest, CreateResponse, + CreateWorkflowRequest, + CreateWorkflowResponse, ExpireIfNotCommittedRequest, ExpireIfNotCommittedResponse, GetDownloadUrlRequest, @@ -60,10 +60,10 @@ PartUploadedRequest, PartUploadedResponse, PartUploadInstruction, - PerformRemoveRequest, - PerformRemoveResponse, RemoveRequest, RemoveResponse, + RemoveWorkflowRequest, + RemoveWorkflowResponse, SetDownloadersRequest, SetDownloadersResponse, SizeMismatch, @@ -71,8 +71,8 @@ from rbt.std.blob.v1.data_plane_pb2 import ( ConfigurationRequest, ConfigurationResponse, - DataPlaneBeginUploadRequest, - DataPlaneCompleteUploadRequest, + DataPlaneCommitRequest, + DataPlaneCreateRequest, DataPlaneDeleteRequest, DataPlaneGetDownloadUrlRequest, DataPlaneGetPartUploadInstructionsRequest, @@ -203,9 +203,9 @@ def authorizer(self) -> Blob.Authorizer: return Blob.Authorizer( create=allow_if(any=[is_app_internal]), set_downloaders=allow_if(any=[is_app_internal]), - begin_upload=allow_if(any=[is_app_internal]), - complete_upload=allow_if(any=[is_app_internal]), - perform_remove=allow_if(any=[is_app_internal]), + create_workflow=allow_if(any=[is_app_internal]), + commit_workflow=allow_if(any=[is_app_internal]), + remove_workflow=allow_if(any=[is_app_internal]), expire_if_not_committed=allow_if(any=[is_app_internal]), # Either side may watch a blob: the uploader to follow # its own progress, a downloader to see when the bytes @@ -235,8 +235,8 @@ async def create( self.state.max_size = request.max_size # The data-plane side effect (provisioning the upload - # session) happens in the `BeginUpload` workflow, not here. - await self.ref().schedule().begin_upload(context) + # session) happens in `CreateWorkflow`. + await self.ref().schedule().create_workflow(context) # Expunge this blob if it is never committed. await self.ref().schedule( @@ -261,17 +261,17 @@ async def set_downloaders( return SetDownloadersResponse() @classmethod - async def begin_upload( + async def create_workflow( cls, context: WorkflowContext, - request: BeginUploadRequest, - ) -> BeginUploadResponse: + request: CreateWorkflowRequest, + ) -> CreateWorkflowResponse: state = await Blob.ref().read(context) async def provision() -> str: async with data_plane_stub(context) as data_plane: - response = await data_plane.BeginUpload( - DataPlaneBeginUploadRequest( + response = await data_plane.Create( + DataPlaneCreateRequest( blob_id=context.state_id, content_type=state.content_type, ) @@ -287,7 +287,7 @@ async def provision() -> str: # serializes each of them but holds nothing across the whole # method. Recording an upload session on a removed blob would # strand the data plane's directory forever, so drop the - # session instead. Deletion always wins, as in `CompleteUpload`. + # session instead. Deletion always wins, as in `CommitWorkflow`. removed = False async def record(state: Blob.State) -> None: @@ -311,7 +311,7 @@ async def record(state: Blob.State) -> None: ) ) - return BeginUploadResponse() + return CreateWorkflowResponse() async def get_part_upload_instructions( self, @@ -416,7 +416,7 @@ async def commit( request: CommitRequest, ) -> CommitResponse: if self.state.status == Blob.State.COMMITTING: - # Idempotent: the `CompleteUpload` workflow is already + # Idempotent: the `CommitWorkflow` workflow is already # scheduled. return CommitResponse() if self.state.status != Blob.State.UPLOADING: @@ -438,23 +438,23 @@ async def commit( # client watching `Info` doesn't observe the stale error while # this fresh attempt is in flight. self.state.ClearField("commit_error") - await self.ref().schedule().complete_upload(context) + await self.ref().schedule().commit_workflow(context) return CommitResponse() @classmethod - async def complete_upload( + async def commit_workflow( cls, context: WorkflowContext, - request: CompleteUploadRequest, - ) -> CompleteUploadResponse: + request: CommitWorkflowRequest, + ) -> CommitWorkflowResponse: state = await Blob.ref().read(context) # A concurrent `Remove` may have moved the blob out of # COMMITTING (deletion always wins); if so, don't finalize. if state.status != Blob.State.COMMITTING: - return CompleteUploadResponse() + return CommitWorkflowResponse() - complete_request = DataPlaneCompleteUploadRequest( + commit_request = DataPlaneCommitRequest( blob_id=context.state_id, upload_id=state.upload_id, content_type=state.content_type, @@ -466,7 +466,7 @@ async def complete_upload( ) ceiling = _size_ceiling(state) if ceiling is not None: - complete_request.max_size = ceiling + commit_request.max_size = ceiling async def attempt() -> tuple: # A response `error` is a *permanent* failure (e.g. an ETag @@ -474,13 +474,13 @@ async def attempt() -> tuple: # re-upload and re-commit. A gRPC error is transient and # propagates, so the workflow retries. async with data_plane_stub(context) as data_plane: - response = await data_plane.CompleteUpload(complete_request) + response = await data_plane.Commit(commit_request) if response.HasField("error"): return ("failed", response.error) return ("committed", response.etag) outcome, detail = await at_least_once_per_workflow( - "complete upload", context, attempt + "commit", context, attempt ) # Only transition if the blob is still COMMITTING: a @@ -517,7 +517,7 @@ async def cleanup() -> None: "cleanup orphaned bytes", context, cleanup ) - return CompleteUploadResponse() + return CommitWorkflowResponse() async def info( self, @@ -574,15 +574,15 @@ async def remove( ): return RemoveResponse() self.state.status = Blob.State.REMOVING - await self.ref().schedule().perform_remove(context) + await self.ref().schedule().remove_workflow(context) return RemoveResponse() @classmethod - async def perform_remove( + async def remove_workflow( cls, context: WorkflowContext, - request: PerformRemoveRequest, - ) -> PerformRemoveResponse: + request: RemoveWorkflowRequest, + ) -> RemoveWorkflowResponse: # An upload that never completed has parked bytes that # deleting the object does not reach, and the ID naming that @@ -606,7 +606,7 @@ async def record(state: Blob.State) -> None: del state.parts[:] await Blob.ref().write(context, record) - return PerformRemoveResponse() + return RemoveWorkflowResponse() async def expire_if_not_committed( self, @@ -615,7 +615,7 @@ async def expire_if_not_committed( ) -> ExpireIfNotCommittedResponse: if self.state.status == Blob.State.UPLOADING: self.state.status = Blob.State.REMOVING - await self.ref().schedule().perform_remove(context) + await self.ref().schedule().remove_workflow(context) elif self.state.status == Blob.State.COMMITTING: # A commit is in flight. If it fails it will revert to # UPLOADING and could then be abandoned, so re-arm the @@ -671,7 +671,7 @@ async def pre_run(self, application: Application) -> None: "data plane that serves its own URLs via " f"`{ENVVAR_BLOB_DATA_PLANE_URL}`." ) - store = await FilesystemBlobStore.create( + store = await FilesystemBlobStore.open( self._blobs_directory or blobs_directory() ) self._store = store diff --git a/reboot/std/react/blob/index.tsx b/reboot/std/react/blob/index.tsx index d8e0c5e86..6cb582b90 100644 --- a/reboot/std/react/blob/index.tsx +++ b/reboot/std/react/blob/index.tsx @@ -81,13 +81,13 @@ export class BlobUploader { * Fetches upload instructions for the given part numbers, waiting * for the blob's upload session to be provisioned. */ - async instructions( + async partUploadInstructions( partNumbers: number[], options?: { signal?: AbortSignal } ): Promise<{ partSize: number; urls: Map }> { - // `ready` is false until the `BeginUpload` workflow has - // provisioned the data-plane upload session, so watch until it - // flips rather than asking again on a timer. + // `ready` is false until `CreateWorkflow` has provisioned the + // data-plane upload session, so watch until it flips rather than + // asking again on a timer. options?.signal?.throwIfAborted(); const controller = new AbortController(); options?.signal?.addEventListener("abort", () => controller.abort(), { @@ -134,7 +134,7 @@ export class BlobUploader { bytes: globalThis.Blob | Uint8Array, options?: { signal?: AbortSignal } ): Promise { - const { urls } = await this.instructions([partNumber], options); + const { urls } = await this.partUploadInstructions([partNumber], options); const url = urls.get(partNumber); if (url === undefined) { throw new Error(`No upload URL for part ${partNumber}`); @@ -161,7 +161,10 @@ export class BlobUploader { url, bytes, async () => { - const { urls } = await this.instructions([partNumber], options); + const { urls } = await this.partUploadInstructions( + [partNumber], + options + ); const fresh = urls.get(partNumber); if (fresh === undefined) { throw new Error(`No upload URL for part ${partNumber}`); @@ -243,7 +246,7 @@ export class BlobUploader { info.parts.map((part) => [part.number, Number(part.size)]) ); - const { partSize } = await this.instructions([], options); + const { partSize } = await this.partUploadInstructions([], options); const totalBytes = data instanceof Uint8Array ? data.byteLength : data.size; const partCount = Math.max(1, Math.ceil(totalBytes / partSize)); @@ -259,13 +262,13 @@ export class BlobUploader { } } - // One `instructions` call per window rather than one per part, and - // no more URLs minted ahead of use than a window's worth: the URLs - // are short-lived, so fetching them all up front would see the - // later ones expire before their turn. + // One `partUploadInstructions` call per window rather than one per + // part, and no more URLs minted ahead of use than a window's worth: + // the URLs are short-lived, so fetching them all up front would see + // the later ones expire before their turn. for (let index = 0; index < pending.length; index += UPLOAD_CONCURRENCY) { const window = pending.slice(index, index + UPLOAD_CONCURRENCY); - const { urls } = await this.instructions(window, options); + const { urls } = await this.partUploadInstructions(window, options); await Promise.all( window.map(async (partNumber) => { const url = urls.get(partNumber); diff --git a/tests/reboot/std/blob/v1/blob_tests.py b/tests/reboot/std/blob/v1/blob_tests.py index f1b785c6f..ff1482a5a 100644 --- a/tests/reboot/std/blob/v1/blob_tests.py +++ b/tests/reboot/std/blob/v1/blob_tests.py @@ -47,8 +47,8 @@ async def asyncTearDown(self) -> None: await self.rbt.stop() async def _instructions(self, blob, part_numbers: list[int]): - """Fetches upload instructions, waiting for the `BeginUpload` - workflow to have provisioned the upload session.""" + """Fetches upload instructions, waiting for `CreateWorkflow` to + have provisioned the upload session.""" async for response in blob.reactively().get_part_upload_instructions( self.context, part_numbers=part_numbers, From 1164417c25a5a19343700998a3db2e0c01c707d4 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:18:28 +0000 Subject: [PATCH 25/52] fixup! `reboot/std`: move the filesystem data plane's bookkeeping into its store --- reboot/std/blob/v1/_store.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/reboot/std/blob/v1/_store.py b/reboot/std/blob/v1/_store.py index c46fa7c74..55f171c82 100644 --- a/reboot/std/blob/v1/_store.py +++ b/reboot/std/blob/v1/_store.py @@ -696,9 +696,15 @@ async def commit( parts=manifest, ) if not committed.committed: - raise BlobStoreError( + # A part was uploaded again between the manifest being read + # above and being committed, so what the manifest names is + # not the object any more. Not a `BlobStoreError`, since + # nothing about that is final: a retry of this call reads + # the manifest as it is now and verifies that one, and can + # succeed. + raise RuntimeError( "a part was uploaded again while this upload was being " - "completed; report the parts and commit again" + "committed" ) # The manifest is fixed, so anything else this session wrote # -- a part uploaded and never reported, a version of a part From d7c41f2e5cf9fb6e399fc7e2ae44b59afbd86fb6 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:19:51 +0000 Subject: [PATCH 26/52] amend! `tests/reboot/std/react`: unit-test the part `PUT`'s retries `tests/reboot/std/react`: unit-test the part `PUT`'s retries Before this change nothing in the repository exercised `reboot/std/react` on its own: the package was type-checked and driven through the chat-room example, so a retry policy could be wrong in ways no test would say, and a test of it would have had to mock the generated client and the browser both. The policy now lives in `reboot/std/react/blob/put.ts`, with the `fetch` it uses and the sleep between attempts handed in, and `index.tsx` calls `putPartWithRetries` with the real ones. The package's `exports` map keeps the module internal. A plain `js_test` (no Reboot backend, so no wheel) under `tests/reboot/std/react/blob` scripts `fetch` one answer per call and checks each branch of the policy: a first-time success, a 5xx followed by a dropped connection, a 408 followed by a 429, a 5xx and a 403 whose bodies never arrived, a 403 retried on a freshly minted URL, giving up after the last attempt with each wait double the one before, each status refused for good, a success without an `ETag`, and an abort mid-wait. Writing the tests found two things. Reading an error response's body sat outside the retry's `try`, so a connection dropped after a 503's headers rejected the upload instead of retrying it; the body is now read best-effort, since the status is the verdict and the body only explains it. And the cap on the backoff could never bind: four attempts wait at most 500, 1000 and 2000 milliseconds, so the cap and its constant are gone. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY --- reboot/std/react/blob/put.ts | 26 +++++--- tests/reboot/std/react/blob/put_tests.ts | 84 +++++++++++++++++++++--- 2 files changed, 92 insertions(+), 18 deletions(-) diff --git a/reboot/std/react/blob/put.ts b/reboot/std/react/blob/put.ts index 19c90d60b..87dbcd374 100644 --- a/reboot/std/react/blob/put.ts +++ b/reboot/std/react/blob/put.ts @@ -6,13 +6,12 @@ // How many times one part's `PUT` is attempted before the upload // fails, and how the attempts are spaced: the delay doubles from the -// first one and is capped, so that a blip is ridden out in seconds -// while an outage is reported rather than waited out. Each delay is -// jittered, so that the parts of one window, which fail together, do -// not retry together. +// first one, so that a blip is ridden out in a few seconds while an +// outage is reported rather than waited out. Each delay is jittered, +// so that the parts of one window, which fail together, do not retry +// together. export const PUT_ATTEMPTS = 4; export const PUT_FIRST_RETRY_DELAY_MS = 500; -export const PUT_MAX_RETRY_DELAY_MS = 5000; /** * Resolves after `ms`, or rejects at once if `signal` aborts first. @@ -89,7 +88,17 @@ export async function tryPutPart( } return { ok: true, etag }; } - const reason = `${response.status}: ${await response.text()}`; + // The status is the verdict; the body only explains it, so a body + // that cannot be read (a connection dropped after the headers) is + // not a reason to skip the retry the status calls for. + let body: string; + try { + body = await response.text(); + } catch (error) { + signal?.throwIfAborted(); + body = `(body could not be read: ${error})`; + } + const reason = `${response.status}: ${body}`; if (response.status === 403) { return { ok: false, retry: true, remint: true, reason }; } @@ -133,10 +142,7 @@ export async function putPartWithRetries( if (outcome.remint) { url = await remint(); } - const backoff = Math.min( - PUT_MAX_RETRY_DELAY_MS, - PUT_FIRST_RETRY_DELAY_MS * 2 ** (attempt - 1) - ); + const backoff = PUT_FIRST_RETRY_DELAY_MS * 2 ** (attempt - 1); await sleep(backoff * (0.5 + Math.random() / 2), options?.signal); continue; } diff --git a/tests/reboot/std/react/blob/put_tests.ts b/tests/reboot/std/react/blob/put_tests.ts index 1471ab6b6..f8c72d872 100644 --- a/tests/reboot/std/react/blob/put_tests.ts +++ b/tests/reboot/std/react/blob/put_tests.ts @@ -3,7 +3,6 @@ import test from "node:test"; import { PUT_ATTEMPTS, PUT_FIRST_RETRY_DELAY_MS, - PUT_MAX_RETRY_DELAY_MS, putPartWithRetries, } from "../../../../../reboot/std/react/blob/put.js"; @@ -11,11 +10,11 @@ const BYTES = new Uint8Array([1, 2, 3]); /** * A `fetch` that answers from a script, one entry per call: a status - * (with an `ETag` for a success), or an `Error` to reject with. It - * records what it was asked, so a test can see which URL each attempt - * went to. + * (with an `ETag` for a success), an `Error` to reject with, or a + * ready-made `Response`. It records what it was asked, so a test can + * see which URL each attempt went to. */ -function scriptedFetch(script: (number | Error)[]): { +function scriptedFetch(script: (number | Error | Response)[]): { fetch: typeof globalThis.fetch; urls: string[]; } { @@ -29,6 +28,9 @@ function scriptedFetch(script: (number | Error)[]): { if (next instanceof Error) { throw next; } + if (next instanceof Response) { + return next; + } return new Response(`body for ${next}`, { status: next, headers: next === 200 ? { ETag: '"etag-of-the-part"' } : {}, @@ -54,6 +56,21 @@ function recordingSleep(): { }; } +/** + * A response whose headers arrived but whose body never will: the + * connection dropped in between, so reading the body rejects. + */ +function headersOnly(status: number): Response { + return new Response( + new ReadableStream({ + start(controller) { + controller.error(new Error("connection reset")); + }, + }), + { status } + ); +} + function neverRemint(): Promise { throw new Error("remint was not expected"); } @@ -98,6 +115,53 @@ test("putPartWithRetries", async (t) => { assert.ok(delays[1] <= PUT_FIRST_RETRY_DELAY_MS * 2); }); + await t.test("retries a 408, then a 429", async () => { + const { fetch, urls } = scriptedFetch([408, 429, 200]); + const { sleep, delays } = recordingSleep(); + const etag = await putPartWithRetries( + 1, + "https://store/part", + BYTES, + neverRemint, + { fetch, sleep } + ); + assert.equal(etag, "etag-of-the-part"); + assert.equal(urls.length, 3); + assert.equal(delays.length, 2); + }); + + await t.test("retries a 503 whose body never arrived", async () => { + const { fetch, urls } = scriptedFetch([headersOnly(503), 200]); + const { sleep, delays } = recordingSleep(); + const etag = await putPartWithRetries( + 1, + "https://store/part", + BYTES, + neverRemint, + { fetch, sleep } + ); + assert.equal(etag, "etag-of-the-part"); + assert.equal(urls.length, 2); + assert.equal(delays.length, 1); + }); + + await t.test("re-mints on a 403 whose body never arrived", async () => { + const { fetch, urls } = scriptedFetch([headersOnly(403), 200]); + const { sleep } = recordingSleep(); + const etag = await putPartWithRetries( + 1, + "https://store/part?sig=expired", + BYTES, + async () => "https://store/part?sig=fresh", + { fetch, sleep } + ); + assert.equal(etag, "etag-of-the-part"); + assert.deepEqual(urls, [ + "https://store/part?sig=expired", + "https://store/part?sig=fresh", + ]); + }); + await t.test("retries a 403 with a freshly minted URL", async () => { const { fetch, urls } = scriptedFetch([403, 200]); const { sleep } = recordingSleep(); @@ -135,10 +199,14 @@ test("putPartWithRetries", async (t) => { error.message.includes("503: body for 503") ); assert.equal(urls.length, PUT_ATTEMPTS); - // The last attempt is not followed by a wait, and no wait exceeds - // the cap. + // The last attempt is not followed by a wait, and each wait is + // double the one before, jittered by at most half. assert.equal(delays.length, PUT_ATTEMPTS - 1); - assert.ok(delays.every((ms) => ms <= PUT_MAX_RETRY_DELAY_MS)); + delays.forEach((ms, index) => { + const backoff = PUT_FIRST_RETRY_DELAY_MS * 2 ** index; + assert.ok(ms >= backoff / 2, `wait ${index}: ${ms} < ${backoff / 2}`); + assert.ok(ms <= backoff, `wait ${index}: ${ms} > ${backoff}`); + }); }); for (const status of [400, 404, 409, 413]) { From d76f941451cca47a823662e81b506bfd8cc08149 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:20:13 +0000 Subject: [PATCH 27/52] amend! `reboot/std/react`: retry a part's `PUT` the way its RPCs are retried `reboot/std/react`: retry a part's `PUT` the way its RPCs are retried An upload's control-plane calls go through the generated web client and get its retries, but before this change the bytes went out as a bare `fetch`, so one dropped connection or one 503 from a store failed the whole upload from the user's point of view, however many parts were already confirmed. A part's `PUT` is now attempted up to four times, with a doubling, jittered delay in between: after a request that never got an answer, and after a store's 408, 429 or 5xx. A 403 is retried with a freshly minted URL, since on either store that is what an expired URL earns, and a part that starts late in a slow upload can outlive the minutes its URL was minted for. A 400, 404, 409 or 413 is refused for good, as is anything once the caller has aborted. What no attempt can fix is still a rejection of `upload()`, naming the part and the reason, and the rest of that part's window is stopped rather than left to run out its own retries against an upload already rejected; the confirmed parts stay confirmed, so calling `upload()` again for the same blob resumes it. The hook's documentation now says so, and that a blob never committed is removed by the backend after a day, which is all the cleanup an abandoned upload needs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY From 63fac1c9280092d2bac8d50bddd5f009e86dc884 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:20:30 +0000 Subject: [PATCH 28/52] fixup! `rbt/std/blob`: say on `BlobDataPlane` who retries what --- reboot/std/react/blob/index.tsx | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/reboot/std/react/blob/index.tsx b/reboot/std/react/blob/index.tsx index 6cb582b90..865c816e8 100644 --- a/reboot/std/react/blob/index.tsx +++ b/reboot/std/react/blob/index.tsx @@ -301,13 +301,16 @@ export class BlobUploader { * ... * const { etag, error } = await upload(blobId, file); * - * A resolved `error` is the data plane's verdict on the commit, and - * it is final for that blob: what was uploaded can never complete as - * reported, so upload again into a new blob. A rejection is a part - * that could not be uploaded even after retries; - * the parts that did upload are kept, so calling `upload` again for - * the same blob resumes rather than restarts. A blob that is never - * committed is removed by the backend after a day. + * A resolved `error` is the data plane's verdict on the commit: what + * was uploaded can never commit as reported. The blob is back to + * uploading, but `upload` cannot repair it, since it skips every + * part the blob already has and the verdict does not say which part + * is at fault; upload again into a new blob, or replace parts through + * `BlobUploader.putPart` and `commit` again. A rejection is a part + * that could not be uploaded even after retries; the parts that did + * upload are kept, so calling `upload` again for the same blob + * resumes rather than restarts. A blob that is never committed is + * removed by the backend after a day. */ export function useBlobUpload(): { upload: ( From 92320f165d00a2b592542b37a32570b0171e5776 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:20:52 +0000 Subject: [PATCH 29/52] amend! `rbt/std/blob`: say on `BlobDataPlane` who retries what `rbt/std/blob`: say on `BlobDataPlane` who retries what Before this change the interface said every method had to be safe to call more than once, and left the reader to work out why: the retries live in the caller. A data plane retries nothing itself. The one failure a method declares is `CompleteUpload`'s `error`, for a completion that can never succeed; every other failure is an undeclared gRPC error, which the contract now says the caller retries until the call succeeds -- the control plane from its workflows, a client of the presigning methods through the control plane's client. One place for every retry keeps them understandable and lets a data plane stay stateless. The same contract is stated where a store implementer meets it, on `BlobStore`, and `CompleteUpload`'s `error` is documented as final for that commit: retrying the call would only repeat it. The browser hook now says what that means for it: `upload()` again would skip every part the blob already has and repeat the verdict, so upload into a new blob, or replace parts through `BlobUploader` and commit again. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY From d360e1c787f462408a172800c0b6a855588befdd Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:21:12 +0000 Subject: [PATCH 30/52] fixup! `reboot/std/blob`: call the same job by the same name on every layer --- reboot/std/blob/v1/_store.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/reboot/std/blob/v1/_store.py b/reboot/std/blob/v1/_store.py index 55f171c82..6b85ba97d 100644 --- a/reboot/std/blob/v1/_store.py +++ b/reboot/std/blob/v1/_store.py @@ -452,7 +452,7 @@ async def publish_part( Refused bytes are removed, which is safe because the file's name belongs to this write alone: no manifest can point at it unless this very claim succeeded. Anything that outlives an - interrupted request is reclaimed at completion, and with the + interrupted request is reclaimed at commit, and with the blob's directory on `delete`.""" published = await StoredBlob.ref(blob_id).always().publish_part( context, @@ -470,7 +470,7 @@ async def publish_part( if published.HasField("superseded_storage_id"): # This part had been uploaded before. Nothing is made of # the earlier bytes now, and the manifest that could still - # name them is refused at completion, so they are removed + # name them is refused at commit, so they are removed # rather than left to accumulate a file per attempt. await _unlink_if_present( self.part_path( @@ -619,8 +619,8 @@ async def commit( if stored is None: raise BlobStoreError("no upload was ever begun for this blob") if stored.committed: - # A retried completion. The object is finished and its - # ETag is what it was -- but reclaiming may not have run, + # A retried commit. The object is finished and its ETag is + # what it was -- but reclaiming may not have run, # or not finished, so it runs again from what was # committed. await self._reclaim( @@ -631,10 +631,10 @@ async def commit( return stored.etag if stored.upload_id != upload_id: # The parts that would be committed were written under a - # different session than the one being completed, so they + # different session than the one being committed, so they # are not the parts this verified. raise BlobStoreError( - "the upload session being completed is not the one this " + "the upload session being committed is not the one this " "blob's parts were written under" ) @@ -732,8 +732,9 @@ async def delete( A part lives inside the blob's own directory, so removing the directory removes any unfinished upload with it, whatever - `upload_ids` says. Forgotten before the bytes go, so that - nothing reads a manifest naming bytes that are already gone: + `upload_ids` says. Removed from `StoredBlob` before the bytes + go, so that nothing reads a manifest naming bytes that are + already gone: between the two a download would answer `200` and then run out of file.""" try: @@ -744,7 +745,7 @@ async def delete( rbt.v1alpha1.errors_pb2.StateNotConstructed, ): # Nothing was ever stored for this blob, so there is - # nothing to forget and deleting it has succeeded. + # nothing to remove and deleting it has succeeded. pass else: raise From d173a22b18b49160078fac85894fa7798375bfcb Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:25:05 +0000 Subject: [PATCH 31/52] `reboot/std`: define the store contract's types beside the contract Before this change the contract a store implements was split across two modules: `BlobStore` lived beside the shared servicer, but the `UploadedPart` its `commit()` takes, the `BlobStoreError` it raises for a final failure and the `DEFAULT_PART_SIZE_BYTES` its `part_size` should answer lived in `_store.py`, the filesystem store. A store elsewhere could only raise the error the shared servicer recognizes by importing it from another store's private module, and the servicer, meant to serve any store, depended on that one. `BlobStoreError`, `UploadedPart` and `DEFAULT_PART_SIZE_BYTES` are now defined next to `BlobStore` in `_data_plane_servicer.py`, and the filesystem store imports them from there like any other store would; the servicer no longer imports the filesystem store at all. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY --- reboot/std/blob/v1/_data_plane_servicer.py | 23 ++++++++++++++++++- reboot/std/blob/v1/_store.py | 26 +++++----------------- tests/reboot/std/blob/v1/blob_tests.py | 2 +- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/reboot/std/blob/v1/_data_plane_servicer.py b/reboot/std/blob/v1/_data_plane_servicer.py index 72b6f5bc7..1b0767dec 100644 --- a/reboot/std/blob/v1/_data_plane_servicer.py +++ b/reboot/std/blob/v1/_data_plane_servicer.py @@ -10,6 +10,7 @@ """ import rbt.std.blob.v1.data_plane_pb2_grpc as data_plane_pb2_grpc +from dataclasses import dataclass from rbt.std.blob.v1.data_plane_pb2 import ( ConfigurationRequest, ConfigurationResponse, @@ -27,9 +28,29 @@ ) from reboot.aio.external import ExternalContext from reboot.aio.interceptors import LegacyGrpcContext -from reboot.std.blob.v1._store import BlobStoreError, UploadedPart from typing import Optional, Protocol, Sequence +# The part size a store should ask clients for. Every part except the +# last must be exactly this size. At least 5 MiB, the S3 minimum part +# size, so that an upload sized for one store can commit on any other. +DEFAULT_PART_SIZE_BYTES = 8 * 1024 * 1024 + + +class BlobStoreError(Exception): + """A permanent storage failure (e.g. a part missing at commit + time), reported to the control plane as a `Commit` `error` so the + client can re-upload. Transient failures (e.g. network errors) are + raised as their original exception types instead, becoming gRPC + errors that the control plane's workflow retries.""" + + +@dataclass(frozen=True) +class UploadedPart: + """One part of an upload, as reported by the client.""" + number: int + etag: str + size: int + class BlobStore(Protocol): """What a `BlobDataPlaneServicer` needs of a store: S3's multipart diff --git a/reboot/std/blob/v1/_store.py b/reboot/std/blob/v1/_store.py index 6b85ba97d..65d0baa31 100644 --- a/reboot/std/blob/v1/_store.py +++ b/reboot/std/blob/v1/_store.py @@ -41,14 +41,14 @@ from rbt.std.blob.v1.filesystem_rbt import StoredBlob, StoredPart from reboot.aio.external import ExternalContext from reboot.crypto import root_keys +from reboot.std.blob.v1._data_plane_servicer import ( + DEFAULT_PART_SIZE_BYTES, + BlobStoreError, + UploadedPart, +) from typing import AsyncIterator, Optional, Sequence from uuid import NAMESPACE_URL, UUID, uuid4, uuid5 -# The part size clients should use. Every part except the last must be -# exactly this size. Must be at least 5 MiB (the S3 minimum part size, -# mirrored here so that filesystem- and S3-backed data planes are -# interchangeable). -DEFAULT_PART_SIZE_BYTES = 8 * 1024 * 1024 # The maximum number of parts in one blob, following S3. MAX_PARTS = 10000 @@ -76,26 +76,10 @@ _STREAM_CHUNK_BYTES = 1024 * 1024 -class BlobStoreError(Exception): - """A permanent storage failure (e.g. a part missing at commit - time), reported to the control plane as a `Commit` `error` so the - client can re-upload. Transient failures (e.g. network errors) are - raised as their original exception types instead, becoming gRPC - errors that the control plane's workflow retries.""" - - class PartTooLarge(Exception): """A part's bytes exceeded the store's part size.""" -@dataclass(frozen=True) -class UploadedPart: - """One part of an upload, as reported by the client.""" - number: int - etag: str - size: int - - @dataclass(frozen=True) class WrittenPart: """One part of an upload, as this store found its bytes to be.""" diff --git a/tests/reboot/std/blob/v1/blob_tests.py b/tests/reboot/std/blob/v1/blob_tests.py index ff1482a5a..c7d46d709 100644 --- a/tests/reboot/std/blob/v1/blob_tests.py +++ b/tests/reboot/std/blob/v1/blob_tests.py @@ -11,7 +11,7 @@ ) from reboot.aio.applications import Application from reboot.aio.tests import Reboot -from reboot.std.blob.v1._store import DEFAULT_PART_SIZE_BYTES +from reboot.std.blob.v1._data_plane_servicer import DEFAULT_PART_SIZE_BYTES from reboot.std.blob.v1.blob import blob_library # How long the completion/`PUT` handshake waits before giving up, From d876bec702f1ac2ae8633576a45cd2b5bb658e75 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:25:06 +0000 Subject: [PATCH 32/52] fixup! `reboot/std/react`: retry a part's `PUT` the way its RPCs are retried --- reboot/std/react/blob/index.tsx | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/reboot/std/react/blob/index.tsx b/reboot/std/react/blob/index.tsx index 865c816e8..366c7c832 100644 --- a/reboot/std/react/blob/index.tsx +++ b/reboot/std/react/blob/index.tsx @@ -255,6 +255,18 @@ export class BlobUploader { uploadedBytes += size; } + // The parts of a window share one signal: the caller's, plus a + // stop as soon as one of them has failed for good, so that the + // others do not run out their retries and report parts to a blob + // whose upload has already been rejected. + const parts = new AbortController(); + options?.signal?.addEventListener( + "abort", + () => parts.abort(options?.signal?.reason), + { once: true } + ); + const partOptions = { signal: parts.signal }; + const pending: number[] = []; for (let partNumber = 1; partNumber <= partCount; partNumber++) { if (!this.confirmed.has(partNumber)) { @@ -280,7 +292,12 @@ export class BlobUploader { offset, Math.min(offset + partSize, totalBytes) ); - await this.putPartToUrl(partNumber, url, bytes, options); + try { + await this.putPartToUrl(partNumber, url, bytes, partOptions); + } catch (error) { + parts.abort(error); + throw error; + } uploadedBytes += bytes instanceof Uint8Array ? bytes.byteLength : bytes.size; options?.onProgress?.({ uploadedBytes, totalBytes }); From 2b5e19e4081916559c488b3a885329ac0f197864 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:43:34 +0000 Subject: [PATCH 33/52] fixup! `reboot/std`: move the filesystem data plane's bookkeeping into its store --- rbt/std/blob/v1/filesystem.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rbt/std/blob/v1/filesystem.proto b/rbt/std/blob/v1/filesystem.proto index c3894bb03..d88c04de9 100644 --- a/rbt/std/blob/v1/filesystem.proto +++ b/rbt/std/blob/v1/filesystem.proto @@ -132,8 +132,8 @@ message StoredBlobCommitRequest { message StoredBlobCommitResponse { // Whether the object was finished. False when a part was uploaded // again between this manifest being read and being committed: the - // bytes it names are gone, so the upload has to be reported and - // committed again. + // bytes it names are gone. Nothing about that is final; the caller + // reads the manifest as it is now and commits again. bool committed = 1; } From e9ba390115b517545d1273e362dafdeaae7d7453 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:43:35 +0000 Subject: [PATCH 34/52] fixup! `tests/reboot/std/react`: unit-test the part `PUT`'s retries --- reboot/std/react/blob/put.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reboot/std/react/blob/put.ts b/reboot/std/react/blob/put.ts index 87dbcd374..dd623c024 100644 --- a/reboot/std/react/blob/put.ts +++ b/reboot/std/react/blob/put.ts @@ -1,8 +1,8 @@ // How a part's bytes are `PUT` to the data plane: the retries a bare // `fetch` does not have, so that an upload survives what the // control-plane calls already survive through their client. Internal -// to the package; `index.tsx` is the only importer, and the package's -// `exports` map keeps it that way. +// to the package: the `exports` map leaves it out of the public +// surface. // How many times one part's `PUT` is attempted before the upload // fails, and how the attempts are spaced: the delay doubles from the From d88a934b58d07d2549ba75efc6dd0772cce21db3 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:45:04 +0000 Subject: [PATCH 35/52] `reboot/std/react`: answer an `upload()` of a blob no longer uploading Before this change `upload()` began by asking the blob for its part size through a reactive `GetPartUploadInstructions`, which a blob that is no longer uploading refuses with a declared error -- and the web client's reactive read retries a refusal forever, so `upload()` hung, until the caller's signal aborted, on any blob past `UPLOADING`. That is where the hook's promise that calling `upload()` again resumes an interrupted upload broke: an upload interrupted while waiting for its commit's verdict, or simply repeated after it succeeded, never came back. The same read stood behind the fresh URL a part asks for after a 403, so a part of a blob removed meanwhile waited forever instead of failing. `upload()` now reads the blob's status first and answers what it finds: a committed blob's ETag at once, the verdict of a commit already under way (`Commit` is idempotent while pending), and a rejection for a removed blob; only a blob still uploading goes on to its parts. `partUploadInstructions()` asks plainly before it watches, so a refusal surfaces as the declared error, and only a blob whose upload session is still being provisioned is watched for the moment it is ready. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY --- reboot/std/react/blob/index.tsx | 68 +++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/reboot/std/react/blob/index.tsx b/reboot/std/react/blob/index.tsx index 366c7c832..8510d5612 100644 --- a/reboot/std/react/blob/index.tsx +++ b/reboot/std/react/blob/index.tsx @@ -79,15 +79,27 @@ export class BlobUploader { /** * Fetches upload instructions for the given part numbers, waiting - * for the blob's upload session to be provisioned. + * for the blob's upload session to be provisioned. Rejects for a + * blob that is no longer uploading. */ async partUploadInstructions( partNumbers: number[], options?: { signal?: AbortSignal } ): Promise<{ partSize: number; urls: Map }> { - // `ready` is false until `CreateWorkflow` has provisioned the - // data-plane upload session, so watch until it flips rather than - // asking again on a timer. + // Asked plainly first: a blob that is no longer uploading refuses + // with a declared error, which a plain call surfaces and a + // reactive read would retry forever. Only a blob whose upload + // session is still being provisioned is then watched: `ready` is + // false until `CreateWorkflow` has provisioned the session, so + // watch until it flips rather than asking again on a timer. + const first = await this.blob.getPartUploadInstructions( + this.context, + { partNumbers }, + options + ); + if (first.ready) { + return this.instructionsFrom(first); + } options?.signal?.throwIfAborted(); const controller = new AbortController(); options?.signal?.addEventListener("abort", () => controller.abort(), { @@ -102,17 +114,9 @@ export class BlobUploader { { signal: controller.signal } ); for await (const response of responses) { - if (!response.ready) { - continue; + if (response.ready) { + return this.instructionsFrom(response); } - const urls = new Map(); - for (const instruction of response.instructions) { - urls.set( - instruction.partNumber, - new URL(instruction.url, this.options.url).toString() - ); - } - return { partSize: Number(response.partSize), urls }; } options?.signal?.throwIfAborted(); throw new Error( @@ -125,6 +129,24 @@ export class BlobUploader { } } + /** + * The part size and, per requested part, an absolute URL to `PUT` + * it to, from a ready response. + */ + private instructionsFrom(response: Blob.GetPartUploadInstructionsResponse): { + partSize: number; + urls: Map; + } { + const urls = new Map(); + for (const instruction of response.instructions) { + urls.set( + instruction.partNumber, + new URL(instruction.url, this.options.url).toString() + ); + } + return { partSize: Number(response.partSize), urls }; + } + /** * `PUT`s one part's bytes to the data plane and reports it to the * control plane. Idempotent per part number. @@ -242,6 +264,20 @@ export class BlobUploader { // Refresh what the control plane already has, so interrupted // uploads resume rather than restart. const info = await this.blob.info(this.context); + switch (info.status) { + case Blob_Status.COMMITTED: + // A previous attempt got all the way; there is nothing left + // to upload or to wait for. + return { etag: info.etag }; + case Blob_Status.COMMITTING: + // A previous attempt committed and was interrupted while + // waiting for the verdict; `Commit` is idempotent while it is + // pending, so wait for the verdict the way it would have. + return await this.commit(options); + case Blob_Status.REMOVING: + case Blob_Status.REMOVED: + throw new Error(`Blob ${this.options.blobId} has been removed`); + } this.confirmed = new Map( info.parts.map((part) => [part.number, Number(part.size)]) ); @@ -326,7 +362,9 @@ export class BlobUploader { * `BlobUploader.putPart` and `commit` again. A rejection is a part * that could not be uploaded even after retries; the parts that did * upload are kept, so calling `upload` again for the same blob - * resumes rather than restarts. A blob that is never committed is + * resumes rather than restarts: what is still missing is uploaded, + * a commit already under way is waited for, and a blob already + * committed is answered at once. A blob that is never committed is * removed by the backend after a day. */ export function useBlobUpload(): { From e715d2e47fc1ae1d763bb7a8bd2a01cbcd9c6af6 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:48:20 +0000 Subject: [PATCH 36/52] fixup! `reboot/std/react`: answer an `upload()` of a blob no longer uploading --- reboot/std/react/blob/index.tsx | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/reboot/std/react/blob/index.tsx b/reboot/std/react/blob/index.tsx index 8510d5612..294aedf15 100644 --- a/reboot/std/react/blob/index.tsx +++ b/reboot/std/react/blob/index.tsx @@ -214,13 +214,23 @@ export class BlobUploader { async commit(options?: { signal?: AbortSignal }): Promise { // `Commit` returns as soon as the blob is marked COMMITTING; the // data plane finalizes the object in a workflow, and the outcome - // lands back on the blob's state. Subscribe rather than re-read on - // a timer: `Info` is a reader, so the update is pushed. Committing - // first is safe because a reactive read always yields current - // state before any update, and `Commit` clears the error from a - // previous attempt as it marks the blob COMMITTING. + // lands back on the blob's state. Committing before watching is + // safe because a reactive read always yields current state before + // any update, and `Commit` clears the error from a previous + // attempt as it marks the blob COMMITTING. await this.blob.commit(this.context); + return await this.commitVerdict(options); + } + /** + * Waits for the verdict on a commit under way: the blob's ETag, or + * the reason the commit failed. + */ + private async commitVerdict(options?: { + signal?: AbortSignal; + }): Promise { + // Subscribe rather than re-read on a timer: `Info` is a reader, + // so the update is pushed. options?.signal?.throwIfAborted(); const controller = new AbortController(); options?.signal?.addEventListener("abort", () => controller.abort(), { @@ -271,9 +281,11 @@ export class BlobUploader { return { etag: info.etag }; case Blob_Status.COMMITTING: // A previous attempt committed and was interrupted while - // waiting for the verdict; `Commit` is idempotent while it is - // pending, so wait for the verdict the way it would have. - return await this.commit(options); + // waiting for the verdict. Only watch for it: the watch yields + // the current state first, so a verdict that has landed since + // is answered too, where another `Commit` would find a blob + // already committed, or restart one that has just failed. + return await this.commitVerdict(options); case Blob_Status.REMOVING: case Blob_Status.REMOVED: throw new Error(`Blob ${this.options.blobId} has been removed`); From 04f10575c0f61079637d0c7426f2405b52c58430 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:53:18 +0000 Subject: [PATCH 37/52] fixup! `reboot/std`: define the store contract's types beside the contract --- reboot/std/blob/v1/_store.py | 1 - 1 file changed, 1 deletion(-) diff --git a/reboot/std/blob/v1/_store.py b/reboot/std/blob/v1/_store.py index 65d0baa31..5d6e5e8df 100644 --- a/reboot/std/blob/v1/_store.py +++ b/reboot/std/blob/v1/_store.py @@ -49,7 +49,6 @@ from typing import AsyncIterator, Optional, Sequence from uuid import NAMESPACE_URL, UUID, uuid4, uuid5 - # The maximum number of parts in one blob, following S3. MAX_PARTS = 10000 From cc42e46a720aa1cff70b8fb6953b31ef5d5bc105 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:59:02 +0000 Subject: [PATCH 38/52] `reboot/std/blob`: let a client retry a data-plane blip behind a URL Before this change the two readers that hand a client something minted by the data plane -- `GetPartUploadInstructions` and `GetDownloadUrl` -- let a failure of that data-plane call escape as it was, and Reboot propagates an exception a method does not handle as `Unknown`, which a client treats as final. So a data plane that was unreachable for a moment failed an upload before its first part, or a part's fresh URL after a 403, although the contract on `BlobDataPlane` says the client of a presigning method retries through the control plane's own client: that client retries `Unavailable`, and never saw it. Both readers now turn a data-plane call's failure into `Unavailable`, which Reboot propagates as it is and the generated clients retry with backoff, so a blip behind a presigned URL is ridden out the way one behind a workflow is. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY --- reboot/std/blob/v1/blob.py | 39 +++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/reboot/std/blob/v1/blob.py b/reboot/std/blob/v1/blob.py index b91b1d23a..c610cd552 100644 --- a/reboot/std/blob/v1/blob.py +++ b/reboot/std/blob/v1/blob.py @@ -78,6 +78,7 @@ DataPlaneGetPartUploadInstructionsRequest, DataPlaneUploadedPart, ) +from reboot.aio.aborted import SystemAborted from reboot.aio.applications import Application, Library from reboot.aio.auth.authorizers import Authorizer, allow_if, is_app_internal from reboot.aio.backoff import Backoff @@ -191,6 +192,20 @@ def _downloader_or_open( return rbt.v1alpha1.errors_pb2.PermissionDenied() +def _data_plane_unavailable(error: AioRpcError) -> SystemAborted: + """A data-plane call's failure in the form a client retries. + + A reader that hands the data plane's answer straight to a client + -- a presigned URL -- has no workflow to retry the call from; the + data plane's contract leaves that retry to the client, and a + client retries `Unavailable` where `Unknown`, which any other + exception would propagate as, is final to it.""" + return SystemAborted( + rbt.v1alpha1.errors_pb2.Unavailable(), + message=f"data plane {error.code().name}: {error.details()}", + ) + + class BlobServicer(Blob.Servicer): # The part size the data plane reported, set by `BlobLibrary` @@ -348,14 +363,17 @@ async def get_part_upload_instructions( number for number in request.part_numbers if 1 <= number <= max_part_number ] - async with data_plane_stub(context) as data_plane: - response = await data_plane.GetPartUploadInstructions( - DataPlaneGetPartUploadInstructionsRequest( - blob_id=context.state_id, - upload_id=self.state.upload_id, - part_numbers=part_numbers, + try: + async with data_plane_stub(context) as data_plane: + response = await data_plane.GetPartUploadInstructions( + DataPlaneGetPartUploadInstructionsRequest( + blob_id=context.state_id, + upload_id=self.state.upload_id, + part_numbers=part_numbers, + ) ) - ) + except AioRpcError as error: + raise _data_plane_unavailable(error) from error instructions = [ PartUploadInstruction( part_number=instruction.part_number, @@ -556,8 +574,11 @@ async def get_download_url( ) if request.HasField("ttl_seconds"): download_request.ttl_seconds = request.ttl_seconds - async with data_plane_stub(context) as data_plane: - response = await data_plane.GetDownloadUrl(download_request) + try: + async with data_plane_stub(context) as data_plane: + response = await data_plane.GetDownloadUrl(download_request) + except AioRpcError as error: + raise _data_plane_unavailable(error) from error return GetDownloadUrlResponse( url=response.url, ttl_seconds=response.ttl_seconds, From 0265fc6dc386ab3e06693e9bfcbcef80c5c83551 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:59:15 +0000 Subject: [PATCH 39/52] `reboot/std/blob`: never leave a commit's verdict empty Before this change the browser knew a commit had failed by the `commit_error` on the blob being a non-empty string, but nothing made it one: the shared servicer forwarded a `BlobStoreError`'s text as it was, and the store contract never asked for any. A store that raised `BlobStoreError()` bare had the blob revert to uploading with an empty verdict, which the browser did not count as one, so an `upload()` waited for the verdict it had already been given, until its caller gave up. The servicer now reports a verdict with words in it however the store raised, and the browser reads the verdict off the status -- the blob reverting to uploading is the failure, as `blob.proto` says; the message only explains it -- so neither side depends on the other remembering to say something. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY --- reboot/std/blob/v1/_data_plane_servicer.py | 7 +++++-- reboot/std/react/blob/index.tsx | 8 ++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/reboot/std/blob/v1/_data_plane_servicer.py b/reboot/std/blob/v1/_data_plane_servicer.py index 1b0767dec..6d6d2b785 100644 --- a/reboot/std/blob/v1/_data_plane_servicer.py +++ b/reboot/std/blob/v1/_data_plane_servicer.py @@ -213,8 +213,11 @@ async def Commit( # A permanent failure: reported in the response, since # retrying the call would only repeat it. Transient failures # raise other exceptions, which become the gRPC error the - # caller retries. - return DataPlaneCommitResponse(error=str(error)) + # caller retries. The verdict is never empty, so that + # whatever reads it off the blob can tell it from none. + return DataPlaneCommitResponse( + error=str(error) or "the store refused the commit" + ) async def GetDownloadUrl( self, diff --git a/reboot/std/react/blob/index.tsx b/reboot/std/react/blob/index.tsx index 294aedf15..bd5d2d87e 100644 --- a/reboot/std/react/blob/index.tsx +++ b/reboot/std/react/blob/index.tsx @@ -244,8 +244,12 @@ export class BlobUploader { if (info.status === Blob_Status.COMMITTED) { return { etag: info.etag }; } - if (info.commitError !== undefined && info.commitError !== "") { - return { error: info.commitError }; + if (info.status === Blob_Status.UPLOADING) { + // The verdict is the status reverting; the message only + // explains it. + return { + error: info.commitError || "the data plane refused the commit", + }; } if ( info.status === Blob_Status.REMOVING || From a48ac7e677ae309e5371631e2bbb7e131d621aee Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:59:26 +0000 Subject: [PATCH 40/52] fixup! `reboot/std/blob`: call the same job by the same name on every layer --- rbt/std/blob/v1/filesystem.proto | 4 ++-- reboot/std/blob/v1/_store.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rbt/std/blob/v1/filesystem.proto b/rbt/std/blob/v1/filesystem.proto index d88c04de9..da82df002 100644 --- a/rbt/std/blob/v1/filesystem.proto +++ b/rbt/std/blob/v1/filesystem.proto @@ -186,8 +186,8 @@ service StoredBlobMethods { } // The metadata, reporting absence rather than raising, since a blob - // that was never begun and one that is merely uncommitted lead to - // different answers. + // that was never created and one that is merely uncommitted lead + // to different answers. rpc Metadata(StoredBlobMetadataRequest) returns (StoredBlobMetadataResponse) { option (rbt.v1alpha1.method) = { reader: {}, diff --git a/reboot/std/blob/v1/_store.py b/reboot/std/blob/v1/_store.py index 5d6e5e8df..8fd12599e 100644 --- a/reboot/std/blob/v1/_store.py +++ b/reboot/std/blob/v1/_store.py @@ -511,8 +511,8 @@ async def read( blob_id: str, ) -> Optional[StoredObject]: """The committed object stored for a blob, or `None` when there - is none: a blob whose upload never began, or is not finished, - has no bytes to serve. + is none: a blob that was never created, or whose upload is not + finished, has no bytes to serve. The bytes are the object's parts in part order, each the write the manifest recorded and no later write of that part.""" @@ -571,7 +571,7 @@ async def _stored( ) -> Optional[filesystem_pb2.StoredBlob]: """The metadata stored for a blob, or `None` when none is. - A blob whose upload never began has no state at all, which the + A blob that was never created has no state at all, which the framework reports by refusing the read rather than by answering with an absent one.""" try: @@ -600,7 +600,7 @@ async def commit( committing an already-committed blob returns its ETag.""" stored = await self._stored(context, blob_id) if stored is None: - raise BlobStoreError("no upload was ever begun for this blob") + raise BlobStoreError("this blob was never created") if stored.committed: # A retried commit. The object is finished and its ETag is # what it was -- but reclaiming may not have run, From 737b7c9ba1ca4c30d06ba255883344475a5b07db Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 23:15:37 +0000 Subject: [PATCH 41/52] fixup! `reboot/std/blob`: let a client retry a data-plane blip behind a URL --- rbt/std/blob/v1/data_plane.proto | 8 +++++--- reboot/std/blob/v1/blob.py | 27 +++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/rbt/std/blob/v1/data_plane.proto b/rbt/std/blob/v1/data_plane.proto index 1fc42cc3b..d7938a2de 100644 --- a/rbt/std/blob/v1/data_plane.proto +++ b/rbt/std/blob/v1/data_plane.proto @@ -19,10 +19,12 @@ // cannot be reached, a response that never arrived, a throttle, a 5xx // -- is an undeclared gRPC error, and the contract is that the caller // retries it until the call succeeds. The control plane does so from -// workflows, with backoff, for `Create`, `Commit` and `Delete`; a +// workflows, with backoff, for `Create`, `Commit` and `Delete`. A // client of the presigning methods retries through the control plane's -// own client. Keeping every retry in the caller keeps it in one place, -// and lets a data plane stay stateless. +// own client, which is handed only what can pass on a later attempt: +// those methods do nothing but sign, so failing to reach the data +// plane is retried and a refusal is final. Keeping every retry in the +// caller keeps it in one place, and lets a data plane stay stateless. // // That contract is what makes idempotency a requirement: every method // must be *safe* to call more than once for the same `blob_id`, since diff --git a/reboot/std/blob/v1/blob.py b/reboot/std/blob/v1/blob.py index c610cd552..b516ef9e1 100644 --- a/reboot/std/blob/v1/blob.py +++ b/reboot/std/blob/v1/blob.py @@ -29,6 +29,7 @@ anyone who knows the ID whenever either side is left open. """ +import grpc import log.log import rbt.v1alpha1.errors_pb2 import re @@ -192,14 +193,32 @@ def _downloader_or_open( return rbt.v1alpha1.errors_pb2.PermissionDenied() +# What can pass on a later attempt when a presigning call fails: such +# a call does nothing but reach the data plane and sign, so only +# failing to reach it is transient. A refusal, or a data plane that +# cannot sign, is final. +_TRANSIENT_DATA_PLANE_CODES = frozenset( + ( + grpc.StatusCode.UNAVAILABLE, + grpc.StatusCode.DEADLINE_EXCEEDED, + grpc.StatusCode.RESOURCE_EXHAUSTED, + grpc.StatusCode.CANCELLED, + ) +) + + +def _is_transient(error: AioRpcError) -> bool: + return error.code() in _TRANSIENT_DATA_PLANE_CODES + + def _data_plane_unavailable(error: AioRpcError) -> SystemAborted: - """A data-plane call's failure in the form a client retries. + """A transient data-plane failure in the form a client retries. A reader that hands the data plane's answer straight to a client -- a presigned URL -- has no workflow to retry the call from; the data plane's contract leaves that retry to the client, and a client retries `Unavailable` where `Unknown`, which any other - exception would propagate as, is final to it.""" + exception propagates as, is final to it.""" return SystemAborted( rbt.v1alpha1.errors_pb2.Unavailable(), message=f"data plane {error.code().name}: {error.details()}", @@ -373,6 +392,8 @@ async def get_part_upload_instructions( ) ) except AioRpcError as error: + if not _is_transient(error): + raise raise _data_plane_unavailable(error) from error instructions = [ PartUploadInstruction( @@ -578,6 +599,8 @@ async def get_download_url( async with data_plane_stub(context) as data_plane: response = await data_plane.GetDownloadUrl(download_request) except AioRpcError as error: + if not _is_transient(error): + raise raise _data_plane_unavailable(error) from error return GetDownloadUrlResponse( url=response.url, From b50668f51cd8974d73eee1c691cdf2884662d977 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 23:15:45 +0000 Subject: [PATCH 42/52] fixup! `reboot/std/react`: answer an `upload()` of a blob no longer uploading --- reboot/std/react/blob/index.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/reboot/std/react/blob/index.tsx b/reboot/std/react/blob/index.tsx index bd5d2d87e..2353634b8 100644 --- a/reboot/std/react/blob/index.tsx +++ b/reboot/std/react/blob/index.tsx @@ -255,7 +255,9 @@ export class BlobUploader { info.status === Blob_Status.REMOVING || info.status === Blob_Status.REMOVED ) { - return { error: "The blob was removed before it committed" }; + // Removal is not a verdict on the commit: there is no blob + // left to repair or to commit again. + throw new Error(`Blob ${this.options.blobId} has been removed`); } } options?.signal?.throwIfAborted(); From 781380e0d66caa28f1d29f826929d30800308b11 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 23:15:59 +0000 Subject: [PATCH 43/52] amend! `reboot/std/react`: answer an `upload()` of a blob no longer uploading `reboot/std/react`: answer an `upload()` of a blob no longer uploading Before this change `upload()` began by asking the blob for its part size through a reactive `GetPartUploadInstructions`, which a blob that is no longer uploading refuses with a declared error -- and the web client's reactive read retries a refusal forever, so `upload()` hung, until the caller's signal aborted, on any blob past `UPLOADING`. That is where the hook's promise that calling `upload()` again resumes an interrupted upload broke: an upload interrupted while waiting for its commit's verdict, or simply repeated after it succeeded, never came back. The same read stood behind the fresh URL a part asks for after a 403, so a part of a blob removed meanwhile waited forever instead of failing. `upload()` now reads the blob's status first and answers what it finds: a committed blob's ETag at once; the verdict of a commit already under way, watched for rather than asked for again, since a second `Commit` would find a blob committed meanwhile, or restart one that has just failed; and a rejection for a removed blob, which is what a removal during the wait for a verdict is too. Only a blob still uploading goes on to its parts. `partUploadInstructions()` asks plainly before it watches, so a refusal surfaces as the declared error, and only a blob whose upload session is still being provisioned is watched for the moment it is ready. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY From 6d786d7687c35a041a000d1d24493701374e54bb Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 23:15:59 +0000 Subject: [PATCH 44/52] amend! `reboot/std/blob`: let a client retry a data-plane blip behind a URL `reboot/std/blob`: let a client retry a data-plane blip behind a URL Before this change the two readers that hand a client something minted by the data plane -- `GetPartUploadInstructions` and `GetDownloadUrl` -- let a failure of that data-plane call escape as it was, and Reboot propagates an exception a method does not handle as `Unknown`, which a client treats as final. So a data plane that was unreachable for a moment failed an upload before its first part, or a part's fresh URL after a 403, although the contract on `BlobDataPlane` says the client of a presigning method retries through the control plane's own client: that client retries `Unavailable`, and never saw it. Both readers now turn a failure to reach the data plane into `Unavailable`, which Reboot propagates as it is and the generated clients retry with backoff, so a blip behind a presigned URL is ridden out the way one behind a workflow is. Only that kind of failure: a presigning call does nothing but reach the data plane and sign, so a refusal, or a data plane that cannot sign, stays the final error it was, rather than being retried until the caller gives up. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY From 46afe171a0140430f537a670791bd318aeb6cd54 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 23:33:13 +0000 Subject: [PATCH 45/52] fixup! `tests/reboot/std/react`: unit-test the part `PUT`'s retries --- reboot/std/react/blob/index.tsx | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/reboot/std/react/blob/index.tsx b/reboot/std/react/blob/index.tsx index 2353634b8..4c506e70f 100644 --- a/reboot/std/react/blob/index.tsx +++ b/reboot/std/react/blob/index.tsx @@ -166,11 +166,12 @@ export class BlobUploader { /** * `PUT`s one part's bytes to an already-minted URL and reports it to - * the control plane, retrying the `PUT` the way the control-plane - * calls are retried by their client. A failure that no attempt can - * fix, or that outlasts the attempts, is thrown; the part is then - * still pending, and a later `upload()` of the same blob picks it up - * again. + * the control plane. The `PUT` is retried a bounded number of times + * (the policy is in `put.ts`), unlike the control-plane calls, whose + * client retries them for as long as the caller waits. A failure + * that no attempt can fix, or that outlasts the attempts, is thrown; + * the part is then still pending, and a later `upload()` of the same + * blob picks it up again. */ private async putPartToUrl( partNumber: number, From dfc440fefd98396f4e8818b0017d71124c13b3be Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 23:33:13 +0000 Subject: [PATCH 46/52] amend! `reboot/std/react`: retry a part's `PUT` the way its RPCs are retried `reboot/std/react`: retry a part's `PUT` before failing the upload An upload's control-plane calls go through the generated web client and get its retries, but before this change the bytes went out as a bare `fetch`, so one dropped connection or one 503 from a store failed the whole upload from the user's point of view, however many parts were already confirmed. A part's `PUT` is now attempted up to four times, with a doubling, jittered delay in between: after a request that never got an answer, and after a store's 408, 429 or 5xx. A 403 is retried with a freshly minted URL, since on either store that is what an expired URL earns, and a part that starts late in a slow upload can outlive the minutes its URL was minted for. A 400, 404, 409 or 413 is refused for good, as is anything once the caller has aborted. What no attempt can fix is still a rejection of `upload()`, naming the part and the reason, and the rest of that part's window is stopped rather than left to run out its own retries against an upload already rejected; the confirmed parts stay confirmed, so calling `upload()` again for the same blob resumes it. The hook's documentation now says so, and that a blob never committed is removed by the backend after a day, which is all the cleanup an abandoned upload needs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY From f8f54643d8e9d55fcda2e21640bd4a61721954f7 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 23:48:45 +0000 Subject: [PATCH 47/52] fixup! `reboot/std/react`: answer an `upload()` of a blob no longer uploading --- reboot/std/react/blob/index.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/reboot/std/react/blob/index.tsx b/reboot/std/react/blob/index.tsx index 4c506e70f..d5835ae72 100644 --- a/reboot/std/react/blob/index.tsx +++ b/reboot/std/react/blob/index.tsx @@ -378,8 +378,11 @@ export class BlobUploader { * uploading, but `upload` cannot repair it, since it skips every * part the blob already has and the verdict does not say which part * is at fault; upload again into a new blob, or replace parts through - * `BlobUploader.putPart` and `commit` again. A rejection is a part - * that could not be uploaded even after retries; the parts that did + * `BlobUploader.putPart` and `commit` again. A rejection is either a + * part that could not be uploaded even after retries, or a refusal + * that another `upload` would only repeat: bytes that do not add up + * to the `size` the blob was created with, a blob removed meanwhile, + * or the caller's own abort. After the former, the parts that did * upload are kept, so calling `upload` again for the same blob * resumes rather than restarts: what is still missing is uploaded, * a commit already under way is waited for, and a blob already From e647133ff4e3878b61f543185b831517a1f97ea6 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 23:48:45 +0000 Subject: [PATCH 48/52] fixup! `tests/reboot/std/react`: unit-test the part `PUT`'s retries --- reboot/std/react/blob/put.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/reboot/std/react/blob/put.ts b/reboot/std/react/blob/put.ts index dd623c024..d26b01ab6 100644 --- a/reboot/std/react/blob/put.ts +++ b/reboot/std/react/blob/put.ts @@ -1,8 +1,9 @@ // How a part's bytes are `PUT` to the data plane: the retries a bare -// `fetch` does not have, so that an upload survives what the -// control-plane calls already survive through their client. Internal -// to the package: the `exports` map leaves it out of the public -// surface. +// `fetch` does not have, so that a blip does not fail an upload whose +// control-plane calls ride it out through their client. Bounded, +// unlike that client's retries: an outage is reported, not waited +// out. Internal to the package: the `exports` map leaves it out of +// the public surface. // How many times one part's `PUT` is attempted before the upload // fails, and how the attempts are spaced: the delay doubles from the From d383cfb5f8b905b6a0bccbb94566fa923bafcbcd Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Tue, 15 Sep 2026 23:48:45 +0000 Subject: [PATCH 49/52] amend! `reboot/std`: move the filesystem data plane's bookkeeping into its store `reboot/std`: move the filesystem data plane's bookkeeping into its store The S3 data plane's servicer authorizes a call and hands it to its store, because S3 keeps the metadata: the multipart session, the parts it holds, the object once completed. Before this change the filesystem data plane's servicer did that bookkeeping itself -- beginning a `StoredBlob` session, checking reported parts against written ones, committing a manifest, reclaiming what it did not name, forgetting a blob before removing its bytes -- and the byte routes drove `StoredBlob` directly as well, so three modules spoke the metadata protocol and the two servicers looked nothing alike. `FilesystemBlobStore` now drives `StoredBlob` itself and offers the surface `S3BlobStore` has: `begin_upload`, `part_put_url`, `complete`, `download_url` and `delete`, plus `publish_part` and `read` for the byte routes. Its methods take the context they reach `StoredBlob` with, which is the one honest difference from a store whose metadata lives in an object store. The servicer keeps the caller check and one delegating call per RPC, and `_http.py` no longer imports `StoredBlob`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01D8vHXG2KLHDUruv8642AXY From c23dabd0a5c5d7ee129265d229bef21bd7b5a8c8 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Wed, 16 Sep 2026 00:02:57 +0000 Subject: [PATCH 50/52] fixup! `tests/reboot/std/react`: unit-test the part `PUT`'s retries --- reboot/std/react/blob/put.ts | 7 +++++-- tests/reboot/std/react/blob/put_tests.ts | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/reboot/std/react/blob/put.ts b/reboot/std/react/blob/put.ts index d26b01ab6..714173624 100644 --- a/reboot/std/react/blob/put.ts +++ b/reboot/std/react/blob/put.ts @@ -58,7 +58,9 @@ export interface PutPartDependencies { * upload that started late in a slow session can outlive the minutes * its URLs are minted for. Everything else is refused for good: the * request itself is wrong (400), the session is gone (404), the blob - * is already committed (409), or the part is too large (413). + * is already committed (409), or the part is too large (413). The + * one 400 that is retried is S3's `RequestTimeout`, its answer to an + * upload whose socket stalled, which AWS documents as retryable. */ export async function tryPutPart( url: string, @@ -106,7 +108,8 @@ export async function tryPutPart( if ( response.status === 408 || response.status === 429 || - response.status >= 500 + response.status >= 500 || + (response.status === 400 && body.includes("RequestTimeout")) ) { return { ok: false, retry: true, remint: false, reason }; } diff --git a/tests/reboot/std/react/blob/put_tests.ts b/tests/reboot/std/react/blob/put_tests.ts index f8c72d872..23f3853fc 100644 --- a/tests/reboot/std/react/blob/put_tests.ts +++ b/tests/reboot/std/react/blob/put_tests.ts @@ -130,6 +130,29 @@ test("putPartWithRetries", async (t) => { assert.equal(delays.length, 2); }); + await t.test("retries S3's 400 RequestTimeout", async () => { + const { fetch, urls } = scriptedFetch([ + new Response( + "RequestTimeoutYour socket " + + "connection to the server was not read from or written to " + + "within the timeout period.", + { status: 400 } + ), + 200, + ]); + const { sleep, delays } = recordingSleep(); + const etag = await putPartWithRetries( + 1, + "https://store/part", + BYTES, + neverRemint, + { fetch, sleep } + ); + assert.equal(etag, "etag-of-the-part"); + assert.equal(urls.length, 2); + assert.equal(delays.length, 1); + }); + await t.test("retries a 503 whose body never arrived", async () => { const { fetch, urls } = scriptedFetch([headersOnly(503), 200]); const { sleep, delays } = recordingSleep(); From b26dba96bf2bfd393f51559d077fecd2ddac2569 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Wed, 16 Sep 2026 00:02:57 +0000 Subject: [PATCH 51/52] fixup! `reboot/std/react`: answer an `upload()` of a blob no longer uploading --- reboot/std/react/blob/index.tsx | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/reboot/std/react/blob/index.tsx b/reboot/std/react/blob/index.tsx index d5835ae72..ce82784d7 100644 --- a/reboot/std/react/blob/index.tsx +++ b/reboot/std/react/blob/index.tsx @@ -379,15 +379,16 @@ export class BlobUploader { * part the blob already has and the verdict does not say which part * is at fault; upload again into a new blob, or replace parts through * `BlobUploader.putPart` and `commit` again. A rejection is either a - * part that could not be uploaded even after retries, or a refusal - * that another `upload` would only repeat: bytes that do not add up - * to the `size` the blob was created with, a blob removed meanwhile, - * or the caller's own abort. After the former, the parts that did - * upload are kept, so calling `upload` again for the same blob - * resumes rather than restarts: what is still missing is uploaded, - * a commit already under way is waited for, and a blob already - * committed is answered at once. A blob that is never committed is - * removed by the backend after a day. + * refusal that another `upload` would only repeat -- bytes that do + * not add up to the `size` the blob was created with, or a blob + * removed meanwhile -- or an interruption: a part that could not be + * uploaded even after retries, or the caller's own abort. After an + * interruption the parts that did upload are kept, so calling + * `upload` again for the same blob resumes rather than restarts: + * what is still missing is uploaded, a commit already under way is + * waited for, and a blob already committed is answered at once. A + * blob that is never committed is removed by the backend after a + * day. */ export function useBlobUpload(): { upload: ( From 249d41120016c7c3be8af0cd1d743c4246e5e1d4 Mon Sep 17 00:00:00 2001 From: Robert-Jan Huijsman <22160949+rjhuijsman@users.noreply.github.com> Date: Wed, 16 Sep 2026 00:19:10 +0000 Subject: [PATCH 52/52] fixup! `reboot/std/react`: retry a part's `PUT` the way its RPCs are retried --- reboot/std/react/blob/index.tsx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/reboot/std/react/blob/index.tsx b/reboot/std/react/blob/index.tsx index ce82784d7..1183375ab 100644 --- a/reboot/std/react/blob/index.tsx +++ b/reboot/std/react/blob/index.tsx @@ -338,16 +338,18 @@ export class BlobUploader { const { urls } = await this.partUploadInstructions(window, options); await Promise.all( window.map(async (partNumber) => { - const url = urls.get(partNumber); - if (url === undefined) { - throw new Error(`No upload URL for part ${partNumber}`); - } const offset = (partNumber - 1) * partSize; const bytes = data.slice( offset, Math.min(offset + partSize, totalBytes) ); try { + // Inside the `try`, so that a part refused a URL stops its + // window-mates too. + const url = urls.get(partNumber); + if (url === undefined) { + throw new Error(`No upload URL for part ${partNumber}`); + } await this.putPartToUrl(partNumber, url, bytes, partOptions); } catch (error) { parts.abort(error);