From 142389c503dbd5acf9833e81b7b713740505f730 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 16 Oct 2026 03:44:24 +0530 Subject: [PATCH 1/3] chore: files changed crates/tinymold-remote/src/remote/test.rs Auto-committed-on: macbook --- crates/tinymold-remote/src/remote/test.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/tinymold-remote/src/remote/test.rs b/crates/tinymold-remote/src/remote/test.rs index be4c6b4..02c256a 100644 --- a/crates/tinymold-remote/src/remote/test.rs +++ b/crates/tinymold-remote/src/remote/test.rs @@ -361,6 +361,19 @@ fn an_upload_that_times_out_once_is_retried() { ); } +#[test] +fn a_connection_the_server_closed_is_retried() { + // nginx ends an HTTP/2 connection after keepalive_requests calls with a + // GOAWAY; tonic reports the calls in flight as internal h2 errors. The + // channel reconnects, so a retry goes through. + let status = |message: &str| Error::Status { + call: "Write", + status: Box::new(tonic::Status::internal(message.to_owned())), + }; + assert!(transient(&status("h2 protocol error: http2 error"))); + assert!(!transient(&status("the store is broken"))); +} + #[test] fn an_upload_that_keeps_failing_skips_only_the_entries_that_need_it() { let server = Server::start(); From e92e5806a0c363778c5f0c5522ac3f12fd1d2dba Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 16 Oct 2026 03:44:36 +0530 Subject: [PATCH 2/3] chore: files changed crates/tinymold-remote/src/remote/mod.rs Auto-committed-on: macbook --- crates/tinymold-remote/src/remote/mod.rs | 25 ++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/crates/tinymold-remote/src/remote/mod.rs b/crates/tinymold-remote/src/remote/mod.rs index 5134580..fd9347d 100644 --- a/crates/tinymold-remote/src/remote/mod.rs +++ b/crates/tinymold-remote/src/remote/mod.rs @@ -628,19 +628,24 @@ where } } -/// Whether `error` may not happen again: a call that timed out, or a server -/// that said it is unavailable, overloaded or aborted the call. A refusal, -/// such as a missing write credential, is not. +/// Whether `error` may not happen again: a call that timed out, a server +/// that said it is unavailable, overloaded or aborted the call, or a +/// connection it closed under the call (an HTTP/2 GOAWAY, which a proxy such +/// as nginx sends after its per-connection request limit; the channel +/// reconnects). A refusal, such as a missing write credential, is not. fn transient(error: &Error) -> bool { match error { Error::Timeout(_) => true, - Error::Status { status, .. } => matches!( - status.code(), - tonic::Code::Unavailable - | tonic::Code::DeadlineExceeded - | tonic::Code::ResourceExhausted - | tonic::Code::Aborted - ), + Error::Status { status, .. } => { + matches!( + status.code(), + tonic::Code::Unavailable + | tonic::Code::DeadlineExceeded + | tonic::Code::ResourceExhausted + | tonic::Code::Aborted + ) || (status.code() == tonic::Code::Internal + && status.message().starts_with("h2 protocol error")) + } _ => false, } } From 6f3fe5b7fe4efbbf795f1a8aa65860bb8404ea18 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 16 Oct 2026 03:45:31 +0530 Subject: [PATCH 3/3] fix(remote): retry calls cut off by an HTTP/2 GOAWAY Co-authored-by: Medulla --- docs/specs/remote.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/specs/remote.md b/docs/specs/remote.md index 46abb15..d115c2d 100644 --- a/docs/specs/remote.md +++ b/docs/specs/remote.md @@ -130,7 +130,10 @@ exit 1, on the first rejected write, such as missing write credentials. A call that times out, or that the server answers as `UNAVAILABLE`, `DEADLINE_EXCEEDED`, `RESOURCE_EXHAUSTED` or `ABORTED`, is retried after 1 s, -4 s and 15 s. A server sharing a saturated disk with the builds can stall a +4 s and 15 s. So is a call cut off by an HTTP/2 GOAWAY (tonic reports it as +an `INTERNAL` "h2 protocol error"): a TLS proxy in front of the server, such +as nginx, closes a connection after its per-connection request limit, and +the channel reconnects for the retry. A server sharing a saturated disk with the builds can stall a write for seconds, and before the retries one stall threw the whole push away: blobs went up, but no action did. A blob that still fails leaves out only the entries that name it; the report counts them as "not pushed after