Retry remote calls cut off by an HTTP/2 GOAWAY

crates/tinymold-remote/src/remote/mod.rs

@@ -628,19 +628,24 @@ where
     }
 }
 
-/// Whether `error` may not happen again: a call that timed out, or a server
-/// that said it is unavailable, overloaded or aborted the call. A refusal,
-/// such as a missing write credential, is not.
+/// Whether `error` may not happen again: a call that timed out, a server
+/// that said it is unavailable, overloaded or aborted the call, or a
+/// connection it closed under the call (an HTTP/2 GOAWAY, which a proxy such
+/// as nginx sends after its per-connection request limit; the channel
+/// reconnects). A refusal, such as a missing write credential, is not.
 fn transient(error: &Error) -> bool {
     match error {
         Error::Timeout(_) => true,
-        Error::Status { status, .. } => matches!(
-            status.code(),
-            tonic::Code::Unavailable
-                | tonic::Code::DeadlineExceeded
-                | tonic::Code::ResourceExhausted
-                | tonic::Code::Aborted
-        ),
+        Error::Status { status, .. } => {
+            matches!(
+                status.code(),
+                tonic::Code::Unavailable
+                    | tonic::Code::DeadlineExceeded
+                    | tonic::Code::ResourceExhausted
+                    | tonic::Code::Aborted
+            ) || (status.code() == tonic::Code::Internal
+                && status.message().starts_with("h2 protocol error"))
+        }
         _ => false,
     }
 }

crates/tinymold-remote/src/remote/test.rs

@@ -361,6 +361,19 @@ fn an_upload_that_times_out_once_is_retried() {
     );
 }
 
+#[test]
+fn a_connection_the_server_closed_is_retried() {
+    // nginx ends an HTTP/2 connection after keepalive_requests calls with a
+    // GOAWAY; tonic reports the calls in flight as internal h2 errors. The
+    // channel reconnects, so a retry goes through.
+    let status = |message: &str| Error::Status {
+        call: "Write",
+        status: Box::new(tonic::Status::internal(message.to_owned())),
+    };
+    assert!(transient(&status("h2 protocol error: http2 error")));
+    assert!(!transient(&status("the store is broken")));
+}
+
 #[test]
 fn an_upload_that_keeps_failing_skips_only_the_entries_that_need_it() {
     let server = Server::start();

docs/specs/remote.md

@@ -130,7 +130,10 @@ exit 1, on the first rejected write, such as missing write credentials.
 
 A call that times out, or that the server answers as `UNAVAILABLE`,
 `DEADLINE_EXCEEDED`, `RESOURCE_EXHAUSTED` or `ABORTED`, is retried after 1 s,
-4 s and 15 s. A server sharing a saturated disk with the builds can stall a
+4 s and 15 s. So is a call cut off by an HTTP/2 GOAWAY (tonic reports it as
+an `INTERNAL` "h2 protocol error"): a TLS proxy in front of the server, such
+as nginx, closes a connection after its per-connection request limit, and
+the channel reconnects for the retry. A server sharing a saturated disk with the builds can stall a
 write for seconds, and before the retries one stall threw the whole push
 away: blobs went up, but no action did. A blob that still fails leaves out
 only the entries that name it; the report counts them as "not pushed after