From fdf09dcdb71748959528255f5b8cbba1c2b47949 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 27 Oct 2026 16:05:58 -0700 Subject: [PATCH 1/3] disk_blob: bound, retry, and inspect HTTP blob requests HttpBlob sent each request through a hyper_util client with no connect timeout, no deadline for the response or its body, no TCP keepalive, and no retry. A reset connection fails the read at once, but when the network silently stops delivering a connection's packets, with no FIN or reset, the read never completes, and neither does the guest's: UEFI, GRUB, and Linux's storvsc all wait for the host to answer. microsoft/nvx#170 reports this in the VMM tests on Linux/KVM: UEFI guests intermittently stopped, in firmware, in GRUB, or after the OS had booted, until petri's ten-minute watchdog fired. In every stalled VM whose watchdog dump was retained, storvsp's in-flight table held a guest READ to the boot disk, whose image petri streams from Azure Blob Storage through HttpBlob, with ios_submitted past ios_completed, while the VMM itself stayed responsive. Near most of the stalls, other reads failed with "Connection reset by peer", and their guests went on; and VMs in separate processes that read the same blob stalled together, which points at the network path rather than at one VM. Every request is now bounded in time, retried when it fails transiently, and shown in inspect: - HttpBlobOptions, which the new HttpBlob::with_options takes and HttpBlob::new defaults, bounds each attempt: 10 s to connect, 20 s for the response headers, connecting and any TLS handshake included, and 20 s of silence while the body arrives, which limits stalls without cutting off a slow body that is still arriving. A 120 s deadline bounds each request as a whole, across its attempts and, for the HEAD request that opens the blob, its redirects. - Timeouts, lost connections, and statuses 408, 429, 500, 502, 503, and 504 are retried, for up to 4 attempts in all, after a backoff that starts at 250 ms and doubles up to 4 s. Each retry uses a new connection, and a timed-out attempt's connection is never returned to the pool. Other failures, such as any other error status, a server that ignores the range, or a short body, are returned at once. A request whose every attempt stalls fails after about 82 s with io::ErrorKind::TimedOut, which the guest sees as a failed read rather than one that never completes. - Connections send TCP keepalive probes after 30 s idle, 10 s apart, so that a pooled connection whose path dies while it is idle can be closed before a request is sent on it. On Linux, macOS, and most other Unix-like systems, it is closed after 3 unanswered probes, a minute after it went idle. hyper-util does not set the count on Windows, where the system's default of 10 probes applies, so there it is closed only after 130 s, when the pool, whose idle timeout is 90 s, would no longer reuse it anyway. Either way, a request sent on a dead connection is still bounded by the timeouts above and retried on a new connection. - Each request, body included, runs on the crate's tokio runtime, now with tokio's time feature, and is aborted if the read is dropped. Each retry, and each request that runs out of attempts or time, is logged as a rate-limited warning that names the request by its range rather than by its URL, which may carry a credential. - The blob's inspect node counts requests, retries, timeouts, and failures, and lists each request in progress under in_flight, with its range, attempt, state, elapsed time, and bytes received, so that a watchdog dump shows whether a stuck guest read is waiting on HTTP. - A read whose last byte would lie past u64::MAX is refused without a request, and one whose last byte is u64::MAX is requested without overflowing; read computed that offset unchecked. BlobDisk checks a guest's range against the disk's size first, so no guest read gets there. The test HTTP server can now stall before the response or mid-body, reset or close the connection, answer 503, send a slow body, keep its connections alive, and script its first requests, and it counts what it sees. The new tests cover each transient failure, alone and in a row; persistent stalls of a disk read and of the open; slow and endless bodies; a stalled connection, which is not reused; permanent failures, which are not retried; inspection of a stalled request; a dropped read, whose request is cancelled at once; and ranges that end at and past the largest offset. Without this change, 16 of the 18 new tests fail, 7 of them by hanging until the tests' 10 s limit; the other two check that a slow body is not taken for a stall and that permanent failures are not retried. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> (cherry picked from commit 2daddda84e1602cb6e59433ede01ca638d6e67a0) --- Cargo.lock | 5 + Guide/src/reference/backends/storage.md | 2 +- vm/devices/storage/disk_blob/Cargo.toml | 8 +- vm/devices/storage/disk_blob/src/blob/http.rs | 756 ++++++++++++++++-- .../disk_blob/src/tests/http_server.rs | 480 ++++++++--- vm/devices/storage/disk_blob/src/tests/mod.rs | 488 +++++++++++ 6 files changed, 1559 insertions(+), 180 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2c2dfb39b9..5a16463528c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1584,13 +1584,18 @@ dependencies = [ "hyper-tls", "hyper-util", "inspect", + "inspect_counters", "once_cell", "pal_async", + "parking_lot", "scsi_buffers", + "socket2", "storage_tests", "tempfile", + "test_with_tracing", "thiserror 2.0.16", "tokio", + "tracelimit", "vhd1_defs", "vm_resource", "zerocopy", diff --git a/Guide/src/reference/backends/storage.md b/Guide/src/reference/backends/storage.md index bd21d39438d..43b7b643337 100644 --- a/Guide/src/reference/backends/storage.md +++ b/Guide/src/reference/backends/storage.md @@ -18,7 +18,7 @@ blob, or a layered composition of multiple backends. | VhdmpDisk | `disk_vhdmp` | Windows vhdmp driver | Windows | Dynamic and differencing VHD/VHDX. | | VhdxDisk | [`vhdx`](vhdx.md) | VHDX file | Cross-platform | Pure-Rust VHDX parser. Dynamic, fixed, and differencing. | | FlatVmdkDisk | [`disk_vmdk`](vmdk.md) | VMDK extents | Cross-platform | Flat and read-only. Supports the `monolithicFlat` and `twoGbMaxExtentFlat` disk types. | -| BlobDisk | [`disk_blob`](https://openvmm.dev/rustdoc/linux/disk_blob/index.html) | HTTP / Azure Blob | Cross-platform | Read-only. HTTP range requests. | +| BlobDisk | [`disk_blob`](https://openvmm.dev/rustdoc/linux/disk_blob/index.html) | HTTP / Azure Blob | Cross-platform | Read-only. HTTP range requests, with timeouts and retries. | | BlockDeviceDisk | [`disk_blockdevice`](https://openvmm.dev/rustdoc/linux/disk_blockdevice/index.html) | Linux block device or file | Linux | io_uring, resize via uevent, PR passthrough. Default for raw files on Linux in both OpenHCL and OpenVMM. | | NvmeDisk | [`disk_nvme`](https://openvmm.dev/rustdoc/linux/disk_nvme/index.html) | Physical NVMe (VFIO) | Linux/Windows | User-mode NVMe driver. Resize via AEN. | | StripedDisk | [`disk_striped`](https://openvmm.dev/rustdoc/linux/disk_striped/index.html) | Multiple Disks | Cross-platform | Stripes data across underlying disks. | diff --git a/vm/devices/storage/disk_blob/Cargo.toml b/vm/devices/storage/disk_blob/Cargo.toml index a207a2365f2..3a23c9b0733 100644 --- a/vm/devices/storage/disk_blob/Cargo.toml +++ b/vm/devices/storage/disk_blob/Cargo.toml @@ -16,6 +16,8 @@ guestmem.workspace = true vm_resource.workspace = true inspect = { workspace = true, features = ["filepath"] } +inspect_counters.workspace = true +tracelimit.workspace = true anyhow.workspace = true async-trait.workspace = true @@ -26,17 +28,21 @@ hyper = { workspace = true, features = ["client", "http1", "http2"] } hyper-tls.workspace = true hyper-util = { workspace = true, features = ["client", "client-legacy", "http1", "http2"] } once_cell.workspace = true +parking_lot.workspace = true thiserror.workspace = true zerocopy.workspace = true # tokio use is allowed in this crate only. # FUTURE: replace this with our own executor -tokio = { version = "1", features = ["rt-multi-thread"] } +tokio = { version = "1", features = ["rt-multi-thread", "time"] } [dev-dependencies] disk_vhd1.workspace = true +inspect = { workspace = true, features = ["initiate"] } pal_async.workspace = true +socket2.workspace = true storage_tests.workspace = true tempfile.workspace = true +test_with_tracing.workspace = true [lints] workspace = true diff --git a/vm/devices/storage/disk_blob/src/blob/http.rs b/vm/devices/storage/disk_blob/src/blob/http.rs index 7d90be22503..f832dd2842a 100644 --- a/vm/devices/storage/disk_blob/src/blob/http.rs +++ b/vm/devices/storage/disk_blob/src/blob/http.rs @@ -14,42 +14,149 @@ use async_trait::async_trait; use http::uri::Scheme; use http_body_util::BodyExt; use http_body_util::Empty; +use hyper::Method; use hyper::Request; +use hyper::Response; use hyper::StatusCode; use hyper::Uri; +use hyper::body::Incoming; use hyper_tls::HttpsConnector; use hyper_util::client::legacy::Client; use hyper_util::client::legacy::connect::HttpConnector; use hyper_util::rt::TokioExecutor; use inspect::Inspect; +use inspect_counters::SharedCounter; use once_cell::sync::OnceCell; +use parking_lot::Mutex; +use std::collections::BTreeMap; +use std::fmt; use std::fmt::Debug; use std::io; +use std::sync::Arc; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; +use std::time::Duration; +use thiserror::Error; +use tokio::time::Instant; + +/// Timeouts and retries for the requests that an [`HttpBlob`] makes. +/// +/// A guest waits for its disk reads to complete, typically without a timeout +/// of its own, so a request that never completes hangs the guest. And a request +/// can stop making progress without failing: when the network silently stops +/// delivering a connection's packets, the client is left waiting for data that +/// will never come, with no reset or FIN to end the wait. So every request is +/// bounded in time: +/// +/// - Each attempt must receive the response headers within +/// [`response_timeout`](Self::response_timeout), and then more of the body at +/// least every [`body_idle_timeout`](Self::body_idle_timeout). These limit +/// silence rather than duration, so that a connection that has stopped is +/// detected without cutting off a slow response that is still arriving. +/// - An attempt that fails in a way that the next one might not -- by timing +/// out, by losing its connection, or with status 408, 429, 500, 502, 503, or +/// 504 -- is retried on a new connection, after a delay that starts at +/// [`initial_backoff`](Self::initial_backoff) and doubles with each retry, up +/// to [`max_backoff`](Self::max_backoff), for up to +/// [`max_attempts`](Self::max_attempts) attempts in all. Any other failure is +/// returned at once. +/// - [`operation_timeout`](Self::operation_timeout) bounds the request as a +/// whole, so that even a response that trickles in too slowly ever to finish +/// fails in bounded time. +/// +/// A request that fails returns an error, which the guest sees as a failed +/// read that it can handle, rather than as a read that never completes. +/// +/// The defaults suit a blob store such as Azure Blob Storage, where a range +/// request normally completes in tens of milliseconds. A connection that is +/// silent for 20 seconds, a thousand times that, is taken to be dead; a request +/// whose every attempt stalls fails after about 80 seconds; and no request +/// takes more than two minutes. That rides out a transient network failure, +/// but still fails a hopeless request well before a typical test watchdog, such +/// as petri's ten minutes, would intervene. +#[derive(Debug, Clone)] +pub struct HttpBlobOptions { + /// How long to wait for a TCP connection to be established. + pub connect_timeout: Duration, + /// How long an attempt waits for the response headers, including the time + /// to connect and to send the request. + pub response_timeout: Duration, + /// How long an attempt waits for more of the response body. + pub body_idle_timeout: Duration, + /// How long a request may take in all, across all its attempts. + pub operation_timeout: Duration, + /// How many attempts to make at a request, including the first. At least + /// one attempt is always made. + pub max_attempts: u32, + /// The delay before the first retry. + pub initial_backoff: Duration, + /// The longest delay before a retry. + pub max_backoff: Duration, +} + +impl Default for HttpBlobOptions { + fn default() -> Self { + Self { + connect_timeout: Duration::from_secs(10), + response_timeout: Duration::from_secs(20), + body_idle_timeout: Duration::from_secs(20), + operation_timeout: Duration::from_secs(120), + max_attempts: 4, + initial_backoff: Duration::from_millis(250), + max_backoff: Duration::from_secs(4), + } + } +} + +/// How long a connection may be idle before TCP keepalive probes check that +/// the server can still be reached. Without them, an idle pooled connection +/// whose network path has died is discovered only when a request is sent on it, +/// at the cost of a response timeout. With them, on Linux, macOS, and most +/// other Unix-like systems, it is closed after [`TCP_KEEPALIVE_RETRIES`] +/// unanswered probes, a minute after it went idle. On Windows, `hyper_util` +/// does not set the number of probes, so the system's default of 10 applies, +/// and the connection is closed only after 130 seconds, when the pool, whose +/// idle timeout defaults to 90 seconds, would no longer reuse it anyway. Either +/// way, a request sent on a dead connection is still bounded by the response, +/// body, and operation timeouts, and retried on a new connection. The probes +/// also keep the connection alive in middleboxes that drop idle flows. +const TCP_KEEPALIVE_TIME: Duration = Duration::from_secs(30); +/// The interval between TCP keepalive probes that go unanswered. +const TCP_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(10); +/// How many TCP keepalive probes go unanswered before the connection is closed, +/// except on Windows. See [`TCP_KEEPALIVE_TIME`]. +const TCP_KEEPALIVE_RETRIES: u32 = 3; + +type HttpClient = Client, Empty<&'static [u8]>>; /// A blob backed by an HTTP/HTTPS connection. +/// +/// Its requests are bounded in time, and retried, as [`HttpBlobOptions`] +/// describes. #[derive(Debug, Inspect)] pub struct HttpBlob { - #[inspect(skip)] - client: Client, Empty<&'static [u8]>>, + #[inspect(flatten)] + requester: Arc, #[inspect(debug)] version: http::Version, #[inspect(display)] uri: Uri, len: u64, - #[inspect(skip)] - tokio_handle: tokio::runtime::Handle, } static TOKIO_RUNTIME: OnceCell = OnceCell::new(); impl HttpBlob { - /// Connects to `url` and returns an object to access it as a blob. + /// Connects to `url` and returns an object to access it as a blob, with the + /// default [`HttpBlobOptions`]. pub async fn new(url: &str) -> anyhow::Result { - let mut uri: Uri = url.parse()?; + Self::with_options(url, HttpBlobOptions::default()).await + } - let connector = HttpsConnector::new(); - let builder = Client::builder(TokioExecutor::new()); - let client = builder.build(connector); + /// Connects to `url` and returns an object to access it as a blob, with the + /// timeouts and retries that `options` specifies. + pub async fn with_options(url: &str, options: HttpBlobOptions) -> anyhow::Result { + let uri: Uri = url.parse()?; let handle = TOKIO_RUNTIME .get_or_try_init(tokio::runtime::Runtime::new) @@ -57,28 +164,150 @@ impl HttpBlob { .handle() .clone(); + let requester = Arc::new(Requester::new(options, handle)); + let (uri, response) = requester.run(requester.clone().probe(uri)).await?; + + let len = response + .headers() + .get("Content-Length") + .context("missing blob length")? + .to_str() + .context("couldn't parse blob length")? + .parse() + .context("couldn't parse blob length")?; + + let version = response.version(); + + Ok(Self { + requester, + version, + uri, + len, + }) + } +} + +#[async_trait] +impl Blob for HttpBlob { + async fn read(&self, buf: &mut [u8], offset: u64) -> io::Result<()> { + // HTTP byte ranges are inclusive at both ends, so an empty one cannot + // be expressed, and computing its end offset would underflow. There is + // nothing to fetch in any case. + if buf.is_empty() { + return Ok(()); + } + // A request names the range by its first and last offsets, so the + // last must be representable. + if offset.checked_add(buf.len() as u64 - 1).is_none() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "read range ends past the largest representable offset", + )); + } + let requester = self.requester.clone(); + let uri = self.uri.clone(); + let len = buf.len(); + let data = self + .requester + .run(async move { requester.get(uri, offset, len).await }) + .await?; + buf.copy_from_slice(&data); + Ok(()) + } + + fn len(&self) -> u64 { + self.len + } +} + +/// Makes the requests for an [`HttpBlob`], with timeouts and retries. +#[derive(Debug, Inspect)] +struct Requester { + #[inspect(skip)] + options: HttpBlobOptions, + /// Makes first attempts, which may reuse idle connections. + #[inspect(skip)] + client: HttpClient, + /// Makes retries, each on a new connection: whatever broke the previous + /// attempt's connection may have broken the idle ones too. + #[inspect(skip)] + retry_client: HttpClient, + #[inspect(skip)] + tokio_handle: tokio::runtime::Handle, + /// Attempts made, retries included. + requests: SharedCounter, + /// Attempts made after a failed attempt. + retries: SharedCounter, + /// Attempts cut short by a timeout. + timeouts: SharedCounter, + /// Requests that failed, after any retries. + failures: SharedCounter, + /// The requests in progress. + in_flight: InFlightTable, + #[inspect(skip)] + next_id: AtomicU64, +} + +impl Requester { + fn new(options: HttpBlobOptions, tokio_handle: tokio::runtime::Handle) -> Self { + let mut http = HttpConnector::new(); + // The HTTPS connector hands HTTPS URIs to this one to connect. + http.enforce_http(false); + http.set_connect_timeout(Some(options.connect_timeout)); + http.set_keepalive(Some(TCP_KEEPALIVE_TIME)); + http.set_keepalive_interval(Some(TCP_KEEPALIVE_INTERVAL)); + // hyper_util ignores this on Windows. See TCP_KEEPALIVE_TIME. + http.set_keepalive_retries(Some(TCP_KEEPALIVE_RETRIES)); + let connector = HttpsConnector::new_with_connector(http); + Self { + client: Client::builder(TokioExecutor::new()).build(connector.clone()), + retry_client: Client::builder(TokioExecutor::new()) + .pool_max_idle_per_host(0) + .build(connector), + options, + tokio_handle, + requests: SharedCounter::new(), + retries: SharedCounter::new(), + timeouts: SharedCounter::new(), + failures: SharedCounter::new(), + in_flight: InFlightTable::default(), + next_id: AtomicU64::new(0), + } + } + + /// Runs `fut` on the tokio runtime, whose timers and connections it needs, + /// and returns its output. Dropping the returned future aborts `fut`. + async fn run(&self, fut: impl 'static + Send + Future) -> T { + let mut task = AbortOnDrop(self.tokio_handle.spawn(fut)); + match (&mut task.0).await { + Ok(output) => output, + // The task is only aborted when this future is dropped, so it must + // have panicked. + Err(err) => std::panic::resume_unwind(err.into_panic()), + } + } + + /// Follows `uri` through any redirects to the blob, and returns the blob's + /// URI and the response to a HEAD request for it. + async fn probe(self: Arc, mut uri: Uri) -> anyhow::Result<(Uri, Response)> { + // One deadline covers all the redirects. + let deadline = deadline_after(self.options.operation_timeout); + let response_timeout = self.options.response_timeout; let mut redirect_count = 0; - let response = loop { + loop { if redirect_count > 5 { anyhow::bail!("too many redirects"); } - let response = handle - .spawn( - client.request( - Request::builder() - .uri(&uri) - .method("HEAD") - .body(Empty::new()) - .unwrap(), - ), - ) + let response = self + .request(RequestKind::Head, deadline, |client, _| { + head_once(client, uri.clone(), response_timeout) + }) .await - .unwrap() .context("failed to query blob size")?; let next_uri: Uri = match response.status() { - StatusCode::OK => break response, + StatusCode::OK => return Ok((uri, response)), StatusCode::MOVED_PERMANENTLY | StatusCode::FOUND | StatusCode::TEMPORARY_REDIRECT @@ -101,81 +330,446 @@ impl HttpBlob { uri = next_uri; redirect_count += 1; - }; + } + } - let len = response - .headers() - .get("Content-Length") - .context("missing blob length")? - .to_str() - .context("couldn't parse blob length")? - .parse() - .context("couldn't parse blob length")?; + /// Reads `len` bytes at `offset` from the blob at `uri`. + async fn get(self: Arc, uri: Uri, offset: u64, len: usize) -> io::Result> { + let deadline = deadline_after(self.options.operation_timeout); + let response_timeout = self.options.response_timeout; + let body_idle_timeout = self.options.body_idle_timeout; + self.request( + RequestKind::Get { offset, len }, + deadline, + |client, progress| { + get_once( + client, + uri.clone(), + offset, + len, + response_timeout, + body_idle_timeout, + progress, + ) + }, + ) + .await + } - let version = response.version(); + /// Makes `request`, by calling `attempt` with the client to use and a + /// handle to report progress through, and retrying it as the options + /// say, until it succeeds, fails for good, or `deadline` passes. + async fn request( + self: &Arc, + request: RequestKind, + deadline: Instant, + mut attempt: F, + ) -> io::Result + where + F: FnMut(HttpClient, Progress) -> Fut, + Fut: Future>, + { + let tracker = self.track(request); + let mut backoff = self.options.initial_backoff.min(self.options.max_backoff); + let mut attempts = 0; + loop { + attempts += 1; + tracker.0.update(|entry| entry.start_attempt(attempts)); + self.requests.increment(); + let client = if attempts == 1 { + &self.client + } else { + &self.retry_client + }; + let (err, retry) = + match tokio::time::timeout_at(deadline, attempt(client.clone(), tracker.0.clone())) + .await + { + Ok(Ok(output)) => return Ok(output), + Ok(Err(Failure::Transient(err))) => (err, true), + Ok(Err(Failure::Timeout(err))) => { + self.timeouts.increment(); + (io::Error::new(io::ErrorKind::TimedOut, err), true) + } + Ok(Err(Failure::Permanent(err))) => { + self.failures.increment(); + return Err(err); + } + Err(_) => { + self.timeouts.increment(); + let err = RequestError::OperationTimeout { + request, + timeout: self.options.operation_timeout, + }; + (io::Error::new(io::ErrorKind::TimedOut, err), false) + } + }; - Ok(Self { - client, - version, - uri, - len, - tokio_handle: handle, + let next_attempt = Instant::now().checked_add(backoff); + if retry + && attempts < self.options.max_attempts + && next_attempt.is_some_and(|next| next < deadline) + { + tracelimit::warn_ratelimited!( + error = &err as &dyn std::error::Error, + %request, + attempt = attempts, + ?backoff, + "http blob request failed, retrying" + ); + self.retries.increment(); + tracker.0.update(|entry| entry.state = State::BackingOff); + tokio::time::sleep(backoff).await; + backoff = backoff.saturating_mul(2).min(self.options.max_backoff); + continue; + } + + self.failures.increment(); + tracelimit::warn_ratelimited!( + error = &err as &dyn std::error::Error, + %request, + attempts, + "http blob request failed" + ); + return Err(if retry && attempts > 1 { + io::Error::new( + err.kind(), + RequestError::Exhausted { + request, + attempts, + source: err, + }, + ) + } else { + err + }); + } + } + + /// Lists `request` in [`Requester::in_flight`] until the returned tracker + /// is dropped. + fn track(self: &Arc, request: RequestKind) -> Tracker { + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + self.in_flight.0.lock().insert( + id, + InFlight { + request, + attempt: 0, + state: State::AwaitingResponse, + started: Instant::now(), + received: 0, + }, + ); + Tracker(Progress { + requester: self.clone(), + id, }) } } -#[async_trait] -impl Blob for HttpBlob { - async fn read(&self, mut buf: &mut [u8], offset: u64) -> io::Result<()> { - // HTTP byte ranges are inclusive at both ends, so an empty one cannot - // be expressed, and computing the end offset below would underflow. - // There is nothing to fetch in any case. - if buf.is_empty() { - return Ok(()); +/// Returns the instant `timeout` from now, or a distant one if that cannot be +/// represented. +fn deadline_after(timeout: Duration) -> Instant { + let now = Instant::now(); + now.checked_add(timeout) + .unwrap_or_else(|| now + Duration::from_secs(60 * 60 * 24 * 365)) +} + +/// Makes one attempt at a HEAD request for `uri`. +async fn head_once( + client: HttpClient, + uri: Uri, + response_timeout: Duration, +) -> Result, Failure> { + let request = Request::builder() + .uri(uri) + .method(Method::HEAD) + .body(Empty::new()) + .unwrap(); + send(client, request, response_timeout).await +} + +/// Makes one attempt at reading `len` bytes at `offset` from the blob at `uri`. +async fn get_once( + client: HttpClient, + uri: Uri, + offset: u64, + len: usize, + response_timeout: Duration, + body_idle_timeout: Duration, + progress: Progress, +) -> Result, Failure> { + let request = Request::builder() + .uri(uri) + .header( + hyper::header::RANGE, + format!("bytes={}-{}", offset, offset + (len as u64 - 1)), + ) + .body(Empty::new()) + .unwrap(); + let response = send(client, request, response_timeout).await?; + + if !response.status().is_success() { + return Err(Failure::Permanent(io::Error::other(RequestError::Status( + response.status(), + )))); + } + + progress.update(|entry| entry.state = State::ReceivingBody); + let mut body = response.into_body(); + let mut data = Vec::with_capacity(len); + loop { + let frame = match tokio::time::timeout(body_idle_timeout, body.frame()).await { + Ok(Some(Ok(frame))) => frame, + Ok(None) => break, + // The connection failed part way through the body. + Ok(Some(Err(err))) => return Err(Failure::Transient(io::Error::other(err))), + Err(_) => { + return Err(Failure::Timeout(RequestError::BodyTimeout { + timeout: body_idle_timeout, + received: data.len(), + expected: len, + })); + } + }; + if let Some(chunk) = frame.data_ref() { + if chunk.len() > len - data.len() { + return Err(Failure::Permanent(io::Error::other( + "server did not respect range query", + ))); + } + data.extend_from_slice(chunk); + progress.update(|entry| entry.received = data.len()); } - let mut response = self - .tokio_handle - .spawn( - self.client.request( - Request::builder() - .uri(&self.uri) - .header( - hyper::header::RANGE, - format!("bytes={}-{}", offset, offset + buf.len() as u64 - 1,), - ) - .body(Empty::new()) - .unwrap(), - ), - ) - .await - .unwrap() - .map_err(io::Error::other)?; - - if !response.status().is_success() { - return Err(io::Error::other(response.status().to_string())); + } + + if data.len() < len { + // A well-formed response that is shorter than was asked for. Another + // attempt would get the same. + return Err(Failure::Permanent(io::ErrorKind::UnexpectedEof.into())); + } + + Ok(data) +} + +/// Sends `request`, and waits up to `response_timeout` for the response +/// headers. +async fn send( + client: HttpClient, + request: Request>, + response_timeout: Duration, +) -> Result, Failure> { + let response = match tokio::time::timeout(response_timeout, client.request(request)).await { + Ok(Ok(response)) => response, + // The connector gave up on connecting. + Ok(Err(err)) if is_timed_out(&err) => { + return Err(Failure::Timeout(RequestError::ConnectTimeout(err))); + } + // The request is well formed, so this is a failure to connect, or to + // keep the connection. + Ok(Err(err)) => return Err(Failure::Transient(io::Error::other(err))), + Err(_) => { + return Err(Failure::Timeout(RequestError::ResponseTimeout( + response_timeout, + ))); } + }; - while let Some(frame) = response.body_mut().frame().await { - let frame = frame.map_err(io::Error::other)?; - if let Some(data) = frame.data_ref() { - let len = data.len(); - if len > buf.len() { - return Err(io::Error::other("server did not respect range query")); - } - let (this, rest) = buf.split_at_mut(len); - this.copy_from_slice(data); - buf = rest; + let status = response.status(); + if matches!( + status, + StatusCode::REQUEST_TIMEOUT + | StatusCode::TOO_MANY_REQUESTS + | StatusCode::INTERNAL_SERVER_ERROR + | StatusCode::BAD_GATEWAY + | StatusCode::SERVICE_UNAVAILABLE + | StatusCode::GATEWAY_TIMEOUT + ) { + return Err(Failure::Transient(io::Error::other(RequestError::Status( + status, + )))); + } + + Ok(response) +} + +/// Returns whether `err`, or any error in its chain of sources, is a timeout, +/// as when the connector gives up on connecting. +fn is_timed_out(err: &(dyn std::error::Error + 'static)) -> bool { + let mut next = Some(err); + while let Some(err) = next { + if err + .downcast_ref::() + .is_some_and(|err| err.kind() == io::ErrorKind::TimedOut) + { + return true; + } + next = err.source(); + } + false +} + +/// Aborts a tokio task when dropped. +struct AbortOnDrop(tokio::task::JoinHandle); + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.abort(); + } +} + +/// Why an attempt failed, which decides whether to retry it. +enum Failure { + /// The attempt failed in a way that the next attempt might not. + Transient(io::Error), + /// The attempt timed out, which the next attempt might not. + Timeout(RequestError), + /// The attempt failed in a way that the next attempt would repeat. + Permanent(io::Error), +} + +#[derive(Debug, Error)] +enum RequestError { + #[error("connection timed out")] + ConnectTimeout(#[source] hyper_util::client::legacy::Error), + #[error("no response within {0:?}")] + ResponseTimeout(Duration), + #[error("no data for {timeout:?}, after {received} of {expected} bytes")] + BodyTimeout { + timeout: Duration, + received: usize, + expected: usize, + }, + #[error("{request} did not complete within {timeout:?}")] + OperationTimeout { + request: RequestKind, + timeout: Duration, + }, + #[error("{request} failed after {attempts} attempts")] + Exhausted { + request: RequestKind, + attempts: u32, + #[source] + source: io::Error, + }, + #[error("server returned {0}")] + Status(StatusCode), +} + +/// A request that an [`HttpBlob`] makes. +#[derive(Debug, Copy, Clone)] +enum RequestKind { + /// A HEAD request, to probe the blob's length. + Head, + /// A range request for `len` bytes at `offset`. `len` is not zero, and the + /// offset of the range's last byte, `offset + (len - 1)`, does not + /// overflow. + Get { offset: u64, len: usize }, +} + +impl fmt::Display for RequestKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + Self::Head => f.write_str("HEAD"), + Self::Get { offset, len } => { + write!(f, "GET bytes={}-{}", offset, offset + (len as u64 - 1)) } } + } +} - if !buf.is_empty() { - return Err(io::ErrorKind::UnexpectedEof.into()); +/// The requests in progress, by ID. +#[derive(Debug, Default)] +struct InFlightTable(Mutex>); + +impl Inspect for InFlightTable { + fn inspect(&self, req: inspect::Request<'_>) { + inspect::iter_by_key(self.0.lock().iter()).inspect(req); + } +} + +/// A request in progress, as inspection shows it. +#[derive(Debug, Inspect)] +struct InFlight { + /// The request, such as `GET bytes=0-4095`. + #[inspect(display)] + request: RequestKind, + /// The current attempt, counting from 1. + attempt: u32, + /// What the request is waiting for. + state: State, + /// How long ago the request started. + #[inspect(rename = "elapsed", with = "Instant::elapsed")] + started: Instant, + /// How much of the response body the current attempt has received. + received: usize, +} + +impl InFlight { + fn start_attempt(&mut self, attempt: u32) { + self.attempt = attempt; + self.state = State::AwaitingResponse; + self.received = 0; + } +} + +/// What a request in progress is waiting for. +#[derive(Debug, Copy, Clone, Inspect)] +enum State { + /// The response headers, connecting first if need be. + AwaitingResponse, + /// More of the response body. + ReceivingBody, + /// The delay before the next attempt. + BackingOff, +} + +/// A handle through which an attempt reports its progress to inspection. +#[derive(Clone)] +struct Progress { + requester: Arc, + id: u64, +} + +impl Progress { + fn update(&self, f: impl FnOnce(&mut InFlight)) { + if let Some(entry) = self.requester.in_flight.0.lock().get_mut(&self.id) { + f(entry); } + } +} - Ok(()) +/// Removes a request from [`Requester::in_flight`] when dropped. +struct Tracker(Progress); + +impl Drop for Tracker { + fn drop(&mut self) { + self.0.requester.in_flight.0.lock().remove(&self.0.id); } +} - fn len(&self) -> u64 { - self.len +#[cfg(test)] +mod tests { + use super::is_timed_out; + use std::io; + use test_with_tracing::test; + + /// An error caused by another, as the connector's errors are. + #[derive(Debug, thiserror::Error)] + #[error("wrapper")] + struct Wrapper(#[source] Box); + + #[test] + fn a_timeout_is_found_anywhere_in_the_chain() { + let timed_out = || io::Error::new(io::ErrorKind::TimedOut, "connect"); + assert!(is_timed_out(&timed_out())); + assert!(is_timed_out(&Wrapper(Box::new(Wrapper(Box::new( + timed_out() + )))))); + assert!(!is_timed_out(&io::Error::other("refused"))); + assert!(!is_timed_out(&Wrapper(Box::new(io::Error::new( + io::ErrorKind::ConnectionRefused, + "refused", + ))))); } } diff --git a/vm/devices/storage/disk_blob/src/tests/http_server.rs b/vm/devices/storage/disk_blob/src/tests/http_server.rs index 8b188d741b0..0ff3fdd2e57 100644 --- a/vm/devices/storage/disk_blob/src/tests/http_server.rs +++ b/vm/devices/storage/disk_blob/src/tests/http_server.rs @@ -9,17 +9,29 @@ //! importantly, the interesting cases are all responses a well-behaved server //! would never produce. See [`Behavior`]. //! +//! A [`Script`] can make the server misbehave for the first few requests only, +//! which models the transient failures that [`HttpBlob`] must recover from. The +//! server counts what it sees in [`Stats`], so that tests can wait for it to +//! reach a given point rather than sleep, and can check how many attempts a +//! read took. +//! //! [`HttpBlob`]: crate::blob::http::HttpBlob +use parking_lot::Condvar; +use parking_lot::Mutex; use std::io::Read; use std::io::Write; use std::net::TcpListener; use std::net::TcpStream; +use std::sync::Arc; +use std::time::Duration; +use std::time::Instant; -/// How the server should respond to range requests. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// How the server should respond to a request. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum Behavior { /// Answer with exactly the requested range. + #[default] Correct, /// Answer with one byte less than was asked for. `Blob::read` is required /// to report this as `UnexpectedEof` rather than as a partial success, @@ -32,140 +44,414 @@ pub enum Behavior { RangeNotSatisfiable, /// Send the headers, then close the connection part way through the body. CloseMidBody, + /// Read the request, then send nothing at all, holding the connection open + /// until the client closes it or the server is released. This is what a + /// client sees when the network silently stops delivering a connection's + /// packets: no data, but no FIN or reset to end the wait either. + StallBeforeResponse, + /// Send the headers and half the body, then stall as for + /// [`Behavior::StallBeforeResponse`]. + StallMidBody, + /// Reset the connection instead of answering. + ResetBeforeResponse, + /// Send the headers and half the body, then reset the connection. + ResetMidBody, + /// Answer `503 Service Unavailable`, as an overloaded or throttling server + /// does. + ServiceUnavailable, + /// Answer correctly, but send the body in `pieces` pieces, pausing for + /// `interval` before each: slow, but never silent for much longer than + /// `interval`. + SlowBody { pieces: usize, interval: Duration }, +} + +/// How the server responds to each request. +#[derive(Debug, Clone, Default)] +pub struct Script { + /// How to respond to the first HEAD requests, in order. Later ones are + /// answered correctly. A HEAD response has no body, so a behavior that + /// concerns the body acts as its counterpart that does not: a stall + /// stalls before the response, a reset resets before it, and the rest + /// answer correctly. + pub head: Vec, + /// How to respond to the first GET requests, in order. + pub get: Vec, + /// How to respond to the GET requests after those in `get`. + pub then: Behavior, + /// Serve further requests on a connection after a response, as real + /// servers do, rather than closing it after each one. + pub keep_alive: bool, +} + +/// What the server has seen so far. +#[derive(Debug, Clone, Copy, Default)] +pub struct Stats { + /// Connections accepted. + pub connections: usize, + /// HEAD requests received. + pub heads: usize, + /// GET requests received. + pub gets: usize, + /// Connections that the server has stalled. + pub stalled: usize, + /// Stalled connections that the client has since closed. + pub abandoned: usize, } /// A server bound to an ephemeral port on the loopback interface. pub struct TestHttpServer { port: u16, + shared: Arc, +} + +struct Shared { + content: Vec, + script: Script, + state: Mutex, + changed: Condvar, +} + +#[derive(Default)] +struct State { + stats: Stats, + released: bool, } impl TestHttpServer { - /// Starts a server publishing `content`. + /// Starts a server publishing `content`, which answers every range request + /// as `behavior` says. + pub fn new(content: Vec, behavior: Behavior) -> Self { + Self::with_script( + content, + Script { + then: behavior, + ..Default::default() + }, + ) + } + + /// Starts a server publishing `content`, which answers requests as `script` + /// says. /// - /// The serving thread runs until the process exits. Tests are short-lived + /// The listening thread runs until the process exits. Tests are short-lived /// and nextest gives each one its own process, so there is nothing to clean - /// up. - pub fn new(content: Vec, behavior: Behavior) -> Self { + /// up. Stalled connections are closed when the server is dropped. + pub fn with_script(content: Vec, script: Script) -> Self { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); + let shared = Arc::new(Shared { + content, + script, + state: Mutex::default(), + changed: Condvar::new(), + }); + let listener_shared = shared.clone(); std::thread::spawn(move || { for stream in listener.incoming() { let Ok(stream) = stream else { break }; - // A misbehaving response deliberately drops the connection, so - // errors here are expected and not worth reporting. - let _ = serve(stream, &content, behavior); + listener_shared.state.lock().stats.connections += 1; + listener_shared.changed.notify_all(); + let shared = listener_shared.clone(); + // A thread per connection, so that a stalled connection does + // not hold up the others. + std::thread::spawn(move || { + // A misbehaving response deliberately drops the connection, + // so errors here are expected and not worth reporting. + let _ = shared.serve(stream); + }); } }); - Self { port } + Self { port, shared } } /// The URL of the blob. pub fn url(&self) -> String { format!("http://127.0.0.1:{}/blob", self.port) } + + /// Returns what the server has seen so far. + pub fn stats(&self) -> Stats { + self.shared.state.lock().stats + } + + /// Waits up to `timeout` for `done` to return true for the server's + /// [`Stats`], and returns them. Panics, naming `what` it was waiting for, + /// if `done` is still false at the end. + pub fn wait_for( + &self, + what: &str, + timeout: Duration, + mut done: impl FnMut(&Stats) -> bool, + ) -> Stats { + let deadline = Instant::now() + timeout; + let mut state = self.shared.state.lock(); + while !done(&state.stats) { + if self + .shared + .changed + .wait_until(&mut state, deadline) + .timed_out() + && !done(&state.stats) + { + panic!("timed out waiting for {what}: {:?}", state.stats); + } + } + state.stats + } + + /// Ends every stall, now and in future, by closing the connection. + pub fn release(&self) { + self.shared.state.lock().released = true; + self.shared.changed.notify_all(); + } } -/// Reads one request and writes one response, then closes the connection. -fn serve(mut stream: TcpStream, content: &[u8], behavior: Behavior) -> std::io::Result<()> { - let request = read_headers(&mut stream)?; - let mut lines = request.lines(); - let method = lines - .next() - .and_then(|l| l.split(' ').next()) - .unwrap_or_default() - .to_owned(); - - // `HttpBlob::new` probes the length with a HEAD request. - if method.eq_ignore_ascii_case("HEAD") { - write!( - stream, - "HTTP/1.1 200 OK\r\n\ - Content-Length: {}\r\n\ - Accept-Ranges: bytes\r\n\ - Connection: close\r\n\r\n", - content.len() - )?; - return Ok(()); +impl Drop for TestHttpServer { + fn drop(&mut self) { + // A failing test may leave a client waiting on a stalled connection. + // Closing the connection ends the wait. + self.release(); } +} - // Header names are case-insensitive, and hyper does not necessarily send - // them in the case they were built with. - let range = lines.find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case("range") - .then(|| value.trim().strip_prefix("bytes=").map(str::to_owned)) - .flatten() - }); - let Some(range) = range else { - write!( - stream, - "HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n" - )?; - return Ok(()); - }; - let (start, end) = range.split_once('-').unwrap(); - let start: usize = start.parse().unwrap(); - // The range is inclusive at both ends. - let end: usize = end.parse().unwrap(); - - if behavior == Behavior::RangeNotSatisfiable || start >= content.len() { - write!( - stream, - "HTTP/1.1 416 Range Not Satisfiable\r\n\ - Content-Range: bytes */{}\r\n\ - Connection: close\r\n\r\n", - content.len() - )?; - return Ok(()); +impl Shared { + /// Serves requests on `stream` until it is closed. + fn serve(&self, mut stream: TcpStream) -> std::io::Result<()> { + // Send each write at once, so that the client sees the pieces and + // pauses of a response as the behavior describes them. + stream.set_nodelay(true)?; + loop { + let request = read_headers(&mut stream)?; + // An empty request means the client has closed the connection. + if request.is_empty() + || !self.respond(&mut stream, &request)? + || !self.script.keep_alive + { + return Ok(()); + } + } } - if behavior == Behavior::IgnoreRange { + /// Counts a request, and returns how to respond to it: as the next of the + /// `scripted` behaviors, if any remain, and otherwise as `then`. + fn count( + &self, + counter: impl FnOnce(&mut Stats) -> &mut usize, + scripted: &[Behavior], + then: Behavior, + ) -> Behavior { + let mut state = self.state.lock(); + let count = counter(&mut state.stats); + let behavior = scripted.get(*count).copied().unwrap_or(then); + *count += 1; + drop(state); + self.changed.notify_all(); + behavior + } + + /// Holds the connection open without sending anything, until the client + /// closes it or the server is released. + /// + /// Returns false, since the connection cannot carry another request. + fn stall(&self, stream: &mut TcpStream) -> bool { + self.state.lock().stats.stalled += 1; + self.changed.notify_all(); + // Poll, rather than block, so that a release is noticed promptly. + let _ = stream.set_read_timeout(Some(Duration::from_millis(10))); + let mut buf = [0; 512]; + while !self.state.lock().released { + match stream.read(&mut buf) { + // A stalled server ignores anything more that the client sends. + Ok(n) if n > 0 => {} + Err(err) + if matches!( + err.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) => {} + // The client has closed or reset the connection. + _ => { + self.state.lock().stats.abandoned += 1; + self.changed.notify_all(); + break; + } + } + } + false + } + + /// Answers one request. Returns whether the connection can carry another. + fn respond(&self, stream: &mut TcpStream, request: &str) -> std::io::Result { + let content = &self.content; + let mut lines = request.lines(); + let method = lines + .next() + .and_then(|l| l.split(' ').next()) + .unwrap_or_default() + .to_owned(); + let close = if self.script.keep_alive { + "" + } else { + "Connection: close\r\n" + }; + + // `HttpBlob::new` probes the length with a HEAD request. + if method.eq_ignore_ascii_case("HEAD") { + match self.count( + |stats| &mut stats.heads, + &self.script.head, + Behavior::Correct, + ) { + Behavior::StallBeforeResponse | Behavior::StallMidBody => { + return Ok(self.stall(stream)); + } + Behavior::ResetBeforeResponse | Behavior::ResetMidBody => return Ok(reset(stream)), + Behavior::ServiceUnavailable => return service_unavailable(stream, close), + _ => {} + } + write!( + stream, + "HTTP/1.1 200 OK\r\n\ + Content-Length: {}\r\n\ + Accept-Ranges: bytes\r\n\ + {close}\r\n", + content.len() + )?; + return Ok(true); + } + + let behavior = self.count(|stats| &mut stats.gets, &self.script.get, self.script.then); + + // Header names are case-insensitive, and hyper does not necessarily send + // them in the case they were built with. + let range = lines.find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("range") + .then(|| value.trim().strip_prefix("bytes=").map(str::to_owned)) + .flatten() + }); + let Some(range) = range else { + write!( + stream, + "HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n" + )?; + return Ok(false); + }; + let (start, end) = range.split_once('-').unwrap(); + let start: usize = start.parse().unwrap(); + // The range is inclusive at both ends. + let end: usize = end.parse().unwrap(); + + match behavior { + Behavior::StallBeforeResponse => return Ok(self.stall(stream)), + Behavior::ResetBeforeResponse => return Ok(reset(stream)), + Behavior::ServiceUnavailable => return service_unavailable(stream, close), + _ => {} + } + + if behavior == Behavior::RangeNotSatisfiable || start >= content.len() { + write!( + stream, + "HTTP/1.1 416 Range Not Satisfiable\r\n\ + Content-Range: bytes */{}\r\n\ + Content-Length: 0\r\n\ + {close}\r\n", + content.len() + )?; + return Ok(true); + } + + if behavior == Behavior::IgnoreRange { + write!( + stream, + "HTTP/1.1 200 OK\r\n\ + Content-Length: {}\r\n\ + {close}\r\n", + content.len() + )?; + stream.write_all(content)?; + return Ok(true); + } + + let end = end.min(content.len() - 1); + let full = &content[start..=end]; + + // `ShortBody` sends a well-formed response that is simply shorter than + // was asked for -- the Content-Length matches what is actually sent. + // That is precisely the case the `Blob` contract is about: the client + // sees a successful response, and must not mistake a partly filled + // buffer for success. A one-byte range cannot be shortened this way, + // since an empty body has no `Content-Range` that describes it, so send + // it in full. + let body = if behavior == Behavior::ShortBody && full.len() > 1 { + &full[..full.len() - 1] + } else { + full + }; + write!( stream, - "HTTP/1.1 200 OK\r\n\ + "HTTP/1.1 206 Partial Content\r\n\ + Content-Range: bytes {}-{}/{}\r\n\ Content-Length: {}\r\n\ - Connection: close\r\n\r\n", - content.len() + {close}\r\n", + start, + start + body.len() - 1, + content.len(), + body.len(), )?; - return stream.write_all(content); + + // Unlike `ShortBody`, the Content-Length above promises the full body, + // so sending less and then hanging up, or stalling, leaves the client + // with a truncated message rather than a well-formed short one. + let half = &body[..body.len() / 2]; + match behavior { + Behavior::CloseMidBody => { + stream.write_all(half)?; + Ok(false) + } + Behavior::StallMidBody => { + stream.write_all(half)?; + Ok(self.stall(stream)) + } + Behavior::ResetMidBody => { + stream.write_all(half)?; + Ok(reset(stream)) + } + Behavior::SlowBody { pieces, interval } => { + for piece in body.chunks(body.len().div_ceil(pieces.max(1))) { + std::thread::sleep(interval); + stream.write_all(piece)?; + } + Ok(true) + } + _ => { + stream.write_all(body)?; + Ok(true) + } + } } +} - let end = end.min(content.len() - 1); - let full = &content[start..=end]; - - // `ShortBody` sends a well-formed response that is simply shorter than was - // asked for -- the Content-Length matches what is actually sent. That is - // precisely the case the `Blob` contract is about: the client sees a - // successful response, and must not mistake a partly filled buffer for - // success. A one-byte range cannot be shortened this way, since an empty - // body has no `Content-Range` that describes it, so send it in full. - let body = if behavior == Behavior::ShortBody && full.len() > 1 { - &full[..full.len() - 1] - } else { - full - }; +/// Makes closing the connection send a reset rather than a FIN: with a zero +/// linger timeout, closing a socket discards any unsent data and aborts the +/// connection. +/// +/// Returns false, since the connection cannot carry another request. +fn reset(stream: &mut TcpStream) -> bool { + let _ = socket2::SockRef::from(&*stream).set_linger(Some(Duration::ZERO)); + false +} +/// Answers `503 Service Unavailable`. Returns whether the connection can carry +/// another request. +fn service_unavailable(stream: &mut TcpStream, close: &str) -> std::io::Result { write!( stream, - "HTTP/1.1 206 Partial Content\r\n\ - Content-Range: bytes {}-{}/{}\r\n\ - Content-Length: {}\r\n\ - Connection: close\r\n\r\n", - start, - start + body.len() - 1, - content.len(), - body.len(), + "HTTP/1.1 503 Service Unavailable\r\n\ + Content-Length: 0\r\n\ + {close}\r\n" )?; - - if behavior == Behavior::CloseMidBody { - // Unlike `ShortBody`, the Content-Length above promises the full body. - // Sending less and hanging up leaves the client with a truncated - // message rather than a well-formed short one. - stream.write_all(&body[..body.len() / 2])?; - return Ok(()); - } - - stream.write_all(body) + Ok(true) } /// Reads up to and including the blank line that terminates the headers. diff --git a/vm/devices/storage/disk_blob/src/tests/mod.rs b/vm/devices/storage/disk_blob/src/tests/mod.rs index 5794b4353c8..a1c220c5ac5 100644 --- a/vm/devices/storage/disk_blob/src/tests/mod.rs +++ b/vm/devices/storage/disk_blob/src/tests/mod.rs @@ -6,18 +6,29 @@ mod http_server; use crate::BlobDisk; +use crate::blob::Blob; use crate::blob::file::FileBlob; use crate::blob::http::HttpBlob; +use crate::blob::http::HttpBlobOptions; use disk_backend::Disk; use disk_backend::DiskError; use guestmem::GuestMemory; use http_server::Behavior; +use http_server::Script; use http_server::TestHttpServer; use pal_async::async_test; use scsi_buffers::OwnedRequestBuffers; use std::io::Read; use std::io::Seek; use std::io::SeekFrom; +use std::sync::Arc; +use std::sync::mpsc; +use std::task::Context; +use std::task::Waker; +use std::thread::JoinHandle; +use std::time::Duration; +use std::time::Instant; +use test_with_tracing::test; const SECTOR_SIZE: usize = 512; const DISK_SIZE: u64 = 1024 * 1024; @@ -150,3 +161,480 @@ async fn misbehaving_server_never_reports_success() { } } } + +/// How long a fault test waits for a request. One still pending by then is +/// taken to be hung for good, which is the failure these tests exist to catch. +const LIMIT: Duration = Duration::from_secs(10); + +/// Content in which every offset is distinguishable from its neighbors, so that +/// data from the wrong place does not pass for the right data. +fn content() -> Vec { + (0..DISK_SIZE).map(|i| (i % 251) as u8).collect() +} + +/// Timeouts short enough that a stall is detected in half a second, so that the +/// fault tests are quick, but still far longer than a loopback request takes. +fn fast_options() -> HttpBlobOptions { + HttpBlobOptions { + response_timeout: Duration::from_millis(500), + body_idle_timeout: Duration::from_millis(500), + operation_timeout: Duration::from_secs(5), + max_attempts: 3, + initial_backoff: Duration::from_millis(10), + max_backoff: Duration::from_millis(40), + ..Default::default() + } +} + +/// A future running on a thread of its own. See [`background`]. +struct Background { + recv: mpsc::Receiver, + thread: JoinHandle<()>, +} + +/// Runs `fut` on a thread of its own. +/// +/// The test's thread then only waits for the output, for at most [`LIMIT`], so +/// that a request that never completes fails the test rather than hanging the +/// test runner. +fn background(fut: impl 'static + Send + Future) -> Background { + let (send, recv) = mpsc::channel(); + let thread = std::thread::spawn(move || { + let output = pal_async::DefaultPool::run_with(async move |_| fut.await); + // The test may have stopped waiting. + let _ = send.send(output); + }); + Background { recv, thread } +} + +impl Background { + /// Waits up to [`LIMIT`] for the output, panicking if there is none. + fn wait(self, what: &str) -> T { + match self.recv.recv_timeout(LIMIT) { + Ok(output) => output, + Err(mpsc::RecvTimeoutError::Timeout) => { + panic!("{what} did not complete within {LIMIT:?}") + } + // The thread panicked. Report that panic rather than this one. + Err(mpsc::RecvTimeoutError::Disconnected) => { + std::panic::resume_unwind(self.thread.join().unwrap_err()) + } + } + } +} + +/// Runs `fut` to completion on a thread of its own, failing the test if that +/// takes longer than [`LIMIT`]. +fn bounded(what: &str, fut: impl 'static + Send + Future) -> T { + background(fut).wait(what) +} + +/// Opens the blob that `server` publishes and reads `len` bytes at `offset` +/// from it, within [`LIMIT`]. +fn read_blob( + server: &TestHttpServer, + options: HttpBlobOptions, + offset: u64, + len: usize, +) -> (HttpBlob, std::io::Result>) { + let url = server.url(); + bounded("read", async move { + let blob = HttpBlob::with_options(&url, options).await.unwrap(); + let mut buf = vec![0; len]; + let r = blob.read(&mut buf, offset).await.map(|()| buf); + (blob, r) + }) +} + +/// Returns the JSON for `path` in the blob's inspection output. +fn inspect_json(blob: &HttpBlob, path: &str) -> String { + inspect::inspect(path, blob).results().json().to_string() +} + +/// Reads through a server that fails the first GET requests as `failures` says, +/// and checks that the read recovers: one retry per failure, the right data, +/// and every stalled connection closed rather than leaked or reused. +fn check_recovers(failures: &[Behavior]) { + let content = content(); + let server = TestHttpServer::with_script( + content.clone(), + Script { + get: failures.to_vec(), + ..Default::default() + }, + ); + let (offset, len) = (12345, 64 * 1024); + let (_blob, r) = read_blob(&server, fast_options(), offset, len); + let data = r.unwrap_or_else(|err| panic!("{failures:?}: read failed: {err:?}")); + assert!( + data == content[offset as usize..][..len], + "{failures:?}: read returned the wrong data" + ); + assert_eq!( + server.stats().gets, + failures.len() + 1, + "{failures:?}: wrong number of attempts" + ); + server.wait_for("the stalled connections to be closed", LIMIT, |stats| { + stats.abandoned == stats.stalled + }); +} + +/// A server, or the network path to it, that goes silent before the response. +#[test] +fn stall_before_response_is_retried() { + check_recovers(&[Behavior::StallBeforeResponse]); +} + +/// A server, or the network path to it, that goes silent part way through the +/// body. +#[test] +fn stall_mid_body_is_retried() { + check_recovers(&[Behavior::StallMidBody]); +} + +#[test] +fn reset_before_response_is_retried() { + check_recovers(&[Behavior::ResetBeforeResponse]); +} + +#[test] +fn reset_mid_body_is_retried() { + check_recovers(&[Behavior::ResetMidBody]); +} + +#[test] +fn close_mid_body_is_retried() { + check_recovers(&[Behavior::CloseMidBody]); +} + +#[test] +fn service_unavailable_is_retried() { + check_recovers(&[Behavior::ServiceUnavailable]); +} + +/// Every attempt is bounded and retried alike, up to the attempt limit. +#[test] +fn consecutive_failures_are_retried() { + check_recovers(&[Behavior::ResetBeforeResponse, Behavior::StallMidBody]); +} + +/// A guest waits for its disk reads, typically without a timeout of its own, +/// so a read that never completes hangs the guest. When the server never +/// recovers, the read must fail instead, in bounded time. +#[test] +fn persistent_stall_fails_disk_read() { + let server = TestHttpServer::new(content(), Behavior::StallMidBody); + let url = server.url(); + let options = fast_options(); + let attempts = options.max_attempts as usize; + let r = bounded("disk read", async move { + let blob = HttpBlob::with_options(&url, options).await.unwrap(); + let disk = Disk::new(BlobDisk::new(blob)).unwrap(); + let mem = GuestMemory::allocate(SECTOR_SIZE * 8); + disk.read_vectored( + &OwnedRequestBuffers::linear(0, SECTOR_SIZE * 8, true).buffer(&mem), + 0, + ) + .await + }); + match r { + Err(DiskError::Io(err)) if err.kind() == std::io::ErrorKind::TimedOut => {} + r => panic!("expected a timeout, got {r:?}"), + } + assert_eq!(server.stats().gets, attempts); + server.wait_for("the stalled connections to be closed", LIMIT, |stats| { + stats.abandoned == attempts + }); +} + +/// A response that keeps arriving is not a stall, however long it takes. +#[test] +fn slow_body_is_not_a_stall() { + let content = content(); + let options = fast_options(); + // Each pause is well within the idle timeout, but the body as a whole takes + // longer than either timeout. + let interval = options.body_idle_timeout / 4; + let server = TestHttpServer::new( + content.clone(), + Behavior::SlowBody { + pieces: 8, + interval, + }, + ); + let start = Instant::now(); + let (_blob, r) = read_blob(&server, options.clone(), 0, 64 * 1024); + let elapsed = start.elapsed(); + assert!(r.unwrap() == content[..64 * 1024], "wrong data"); + assert_eq!(server.stats().gets, 1, "the slow response was retried"); + assert!( + elapsed > options.body_idle_timeout.max(options.response_timeout), + "the response was not slow: {elapsed:?}" + ); +} + +/// A response that keeps arriving, but too slowly ever to finish, is bounded by +/// the operation timeout instead. +#[test] +fn endless_body_is_bounded_by_operation_timeout() { + let options = HttpBlobOptions { + operation_timeout: Duration::from_millis(1500), + ..fast_options() + }; + // One byte at a time: the body would take minutes. + let server = TestHttpServer::new( + content(), + Behavior::SlowBody { + pieces: 4096, + interval: options.body_idle_timeout / 10, + }, + ); + let start = Instant::now(); + let (_blob, r) = read_blob(&server, options.clone(), 0, 4096); + let elapsed = start.elapsed(); + match r { + Err(err) if err.kind() == std::io::ErrorKind::TimedOut => {} + r => panic!("expected a timeout, got {:?}", r.map(|_| ())), + } + assert!( + elapsed >= options.operation_timeout, + "timed out early: {elapsed:?}" + ); + assert_eq!( + server.stats().gets, + 1, + "retried after the operation timeout" + ); +} + +/// Opening the blob probes it with a HEAD request, which must recover from a +/// stall too. +#[test] +fn stalled_head_is_retried() { + let server = TestHttpServer::with_script( + content(), + Script { + head: vec![Behavior::StallBeforeResponse], + ..Default::default() + }, + ); + let url = server.url(); + let blob = bounded("open", async move { + HttpBlob::with_options(&url, fast_options()).await.unwrap() + }); + assert_eq!(blob.len(), DISK_SIZE); + assert_eq!(server.stats().heads, 2); +} + +#[test] +fn persistent_head_stall_fails_open() { + let options = fast_options(); + let attempts = options.max_attempts as usize; + let server = TestHttpServer::with_script( + content(), + Script { + head: vec![Behavior::StallBeforeResponse; attempts + 1], + ..Default::default() + }, + ); + let url = server.url(); + let r = bounded("open", async move { + HttpBlob::with_options(&url, options).await.map(drop) + }); + assert!(r.is_err(), "opened a blob whose server never answered"); + assert_eq!(server.stats().heads, attempts); +} + +/// A connection whose request timed out must be closed, not returned to the +/// pool, where a later request would find it and stall in turn. +#[test] +fn stalled_connection_is_not_reused() { + let content = content(); + let server = TestHttpServer::with_script( + content.clone(), + Script { + get: vec![Behavior::Correct, Behavior::StallBeforeResponse], + keep_alive: true, + ..Default::default() + }, + ); + let url = server.url(); + let (blob, results) = bounded("reads", async move { + let blob = HttpBlob::with_options(&url, fast_options()).await.unwrap(); + let mut results = Vec::new(); + for i in 0..3 { + let mut buf = vec![0; 4096]; + results.push(blob.read(&mut buf, i * 4096).await.map(|()| buf)); + } + (blob, results) + }); + for (i, r) in results.into_iter().enumerate() { + let data = r.unwrap_or_else(|err| panic!("read {i} failed: {err:?}")); + assert!(data == content[i * 4096..][..4096], "read {i}: wrong data"); + } + // Only the stalled request was retried. + assert_eq!(inspect_json(&blob, "retries"), "1"); + assert_eq!(inspect_json(&blob, "timeouts"), "1"); + assert_eq!(server.stats().gets, 4); + server.wait_for("the stalled connection to be closed", LIMIT, |stats| { + stats.abandoned == 1 + }); +} + +/// Inspection shows what a request that is taking long is waiting for, so that +/// a hung guest's disk can be told apart from a slow one. +#[test] +fn inspect_shows_stalled_request() { + let content = content(); + let server = TestHttpServer::with_script( + content.clone(), + Script { + get: vec![Behavior::StallBeforeResponse], + ..Default::default() + }, + ); + let url = server.url(); + // Long enough that the stall does not time out while it is inspected. + // Releasing the server ends it instead. + let options = HttpBlobOptions { + response_timeout: LIMIT * 3, + operation_timeout: LIMIT * 3, + ..fast_options() + }; + let blob = Arc::new(bounded("open", async move { + HttpBlob::with_options(&url, options).await.unwrap() + })); + let read = background({ + let blob = blob.clone(); + async move { + let mut buf = vec![0; 4096]; + blob.read(&mut buf, 8192).await.map(|()| buf) + } + }); + server.wait_for("the request to stall", LIMIT, |stats| stats.stalled == 1); + let in_flight = inspect_json(&blob, "in_flight"); + for expected in [ + r#""request":"GET bytes=8192-12287""#, + r#""attempt":1"#, + r#""state":"awaiting_response""#, + ] { + assert!( + in_flight.contains(expected), + "{expected} not in {in_flight}" + ); + } + + // Ending the stall fails the attempt, and the retry succeeds. + server.release(); + let data = read.wait("read").unwrap(); + assert!(data == content[8192..][..4096], "wrong data"); + assert_eq!(inspect_json(&blob, "in_flight"), "{}"); + assert_eq!(inspect_json(&blob, "retries"), "1"); + assert_eq!(inspect_json(&blob, "timeouts"), "0"); +} + +/// Dropping a read cancels its request at once: the connection is closed, the +/// request leaves `in_flight`, and it is not retried. Otherwise a read that its +/// caller has given up on would hold its connection, and might be retried, +/// until the request's own timeouts ended it, two minutes later by default. +#[test] +fn dropped_read_is_cancelled() { + let server = TestHttpServer::new(content(), Behavior::StallBeforeResponse); + let url = server.url(); + // Long enough that nothing times out, and so nothing is retried, before + // the read is dropped and its cancellation checked. + let options = HttpBlobOptions { + response_timeout: LIMIT * 3, + operation_timeout: LIMIT * 3, + ..fast_options() + }; + let blob = bounded("open", async move { + HttpBlob::with_options(&url, options).await.unwrap() + }); + + // The first poll starts the request, which runs on the blob's tokio + // runtime. + let mut buf = vec![0; 4096]; + let mut read = blob.read(&mut buf, 8192); + let poll = read.as_mut().poll(&mut Context::from_waker(Waker::noop())); + assert!(poll.is_pending(), "the read did not wait for the server"); + server.wait_for("the request to stall", LIMIT, |stats| stats.stalled == 1); + let in_flight = inspect_json(&blob, "in_flight"); + assert!( + in_flight.contains(r#""request":"GET bytes=8192-12287""#), + "the stalled read is not in flight: {in_flight}" + ); + + drop(read); + + // Well within the request's own timeouts, so that only the read's + // cancellation can close its connection and end it in time. + let bound = LIMIT / 2; + server.wait_for( + "the dropped read's connection to be closed", + bound, + |stats| stats.abandoned == 1, + ); + let deadline = Instant::now() + bound; + while inspect_json(&blob, "in_flight") != "{}" { + assert!( + Instant::now() < deadline, + "the dropped read is still in flight" + ); + std::thread::sleep(Duration::from_millis(10)); + } + // With the request gone, nothing is left to retry it, so these counts are + // final: one connection and one request each for the open and the read. + let stats = server.stats(); + assert_eq!(stats.gets, 1, "the dropped read was retried"); + assert_eq!( + stats.connections, 2, + "the dropped read opened another connection" + ); +} + +/// A failure that a new attempt would only repeat is returned at once. +#[test] +fn permanent_errors_are_not_retried() { + for behavior in [ + Behavior::ShortBody, + Behavior::IgnoreRange, + Behavior::RangeNotSatisfiable, + ] { + let server = TestHttpServer::new(content(), behavior); + let (_blob, r) = read_blob(&server, fast_options(), 0, 4096); + assert!(r.is_err(), "{behavior:?}: reported success"); + assert_eq!(server.stats().gets, 1, "{behavior:?}: retried"); + } +} + +/// A range whose last byte would lie past the largest offset that a request +/// can name is refused, without a request, rather than overflowing. +#[test] +fn range_past_the_largest_offset_is_refused() { + let server = TestHttpServer::new(content(), Behavior::Correct); + let (_blob, r) = read_blob(&server, fast_options(), u64::MAX - 100, 4096); + match r { + Err(err) if err.kind() == std::io::ErrorKind::InvalidInput => {} + r => panic!("expected InvalidInput, got {:?}", r.map(|_| ())), + } + assert_eq!(server.stats().gets, 0); +} + +/// A range that ends at the largest offset that a request can name is +/// requested like any other, rather than overflowing as its last offset is +/// computed. The server fails every attempt with 503, so that the read also +/// names the range in its warnings and its error. +#[test] +fn range_ending_at_the_largest_offset_is_requested() { + let server = TestHttpServer::new(content(), Behavior::ServiceUnavailable); + let options = fast_options(); + let attempts = options.max_attempts as usize; + let offset = u64::MAX - 4095; + let (_blob, r) = read_blob(&server, options, offset, 4096); + let err = r.map(|_| ()).expect_err("reported success"); + let range = format!("GET bytes={offset}-{}", u64::MAX); + assert!(err.to_string().contains(&range), "{range} not in {err}"); + assert_eq!(server.stats().gets, attempts); +} From 3317be8bbba1ff89e95a8b4d8a2229dcc0759783 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 6 Nov 2026 18:57:03 -0800 Subject: [PATCH 2/3] disk_blob: test that a stalled TLS handshake fails the open HttpBlob's tests serve the blob over plain HTTP, but blob URLs are normally HTTPS, as are those of the images that petri streams from Azure Blob Storage. There, each new connection begins with a TLS handshake, which the connector carries out before any request is sent, so a server, or the network path to it, that goes silent once the TCP handshake is done stalls the client inside the connector. HttpBlob bounds that stall only because its response timeout wraps all of client.request(), connecting included. A change that bounded only the wait for the response would leave the handshake unbounded, and no test would notice. stalled_tls_handshake_fails_open opens an https:// URL whose server, a plain TcpListener, accepts each connection and reads the ClientHello but never answers it. The open must fail with a timeout after its attempts, each on a connection of its own that began a TLS handshake and that is closed when the attempt gives up. Without the previous commit, the open never completes, and the test fails at the tests' 10 s limit; with it, the open fails in about 1.6 s. For microsoft/nvx#170, like the previous commit. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vm/devices/storage/disk_blob/src/tests/mod.rs | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/vm/devices/storage/disk_blob/src/tests/mod.rs b/vm/devices/storage/disk_blob/src/tests/mod.rs index a1c220c5ac5..0c3d3722723 100644 --- a/vm/devices/storage/disk_blob/src/tests/mod.rs +++ b/vm/devices/storage/disk_blob/src/tests/mod.rs @@ -446,6 +446,101 @@ fn persistent_head_stall_fails_open() { assert_eq!(server.stats().heads, attempts); } +/// What a [`silent_server`] sees. +#[derive(Debug)] +enum SilentEvent { + /// It accepted a connection. + Accepted, + /// The client closed a connection, having first sent `first`, if it sent + /// anything at all. + Closed { first: Option }, +} + +/// Starts a server on the loopback interface that accepts connections but +/// never sends a byte on them, as a server, or the network path to it, that +/// goes silent once the TCP handshake is done. Returns its port, and a channel +/// on which it reports each connection that it accepts and each that the +/// client closes. +/// +/// Like [`TestHttpServer`], it runs until the process exits. +fn silent_server() -> (u16, mpsc::Receiver) { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let (send, recv) = mpsc::channel(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { break }; + if send.send(SilentEvent::Accepted).is_err() { + // The test has finished. + break; + } + let send = send.clone(); + std::thread::spawn(move || { + let mut first = None; + let mut buf = [0; 4096]; + // Take in whatever the client sends, and answer none of it. + loop { + match stream.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(_) => { + first.get_or_insert(buf[0]); + } + } + } + let _ = send.send(SilentEvent::Closed { first }); + }); + } + }); + (port, recv) +} + +/// The TLS record type of handshake messages, with which the ClientHello that +/// opens a TLS connection begins. +const TLS_HANDSHAKE_RECORD: u8 = 0x16; + +/// Blobs are normally served over HTTPS, so each new connection begins with a +/// TLS handshake, inside the connector and before any request is sent. When +/// the server, or the network path to it, goes silent before answering the +/// ClientHello, that stall must be bounded like the others: the open fails +/// with a timeout after its attempts, each on a connection of its own that is +/// closed when the attempt gives up, rather than waiting for good. +#[test] +fn stalled_tls_handshake_fails_open() { + let (port, events) = silent_server(); + let options = fast_options(); + let attempts = options.max_attempts as usize; + let url = format!("https://127.0.0.1:{port}/blob"); + let r = bounded("open", async move { + HttpBlob::with_options(&url, options).await.map(drop) + }); + let err = r.expect_err("opened a blob whose server never answered the TLS handshake"); + assert!( + err.downcast_ref::() + .is_some_and(|err| err.kind() == std::io::ErrorKind::TimedOut), + "expected a timeout, got: {err:#}" + ); + + let mut accepted = 0; + let mut closed = 0; + while closed < attempts { + match events.recv_timeout(LIMIT) { + Ok(SilentEvent::Accepted) => accepted += 1, + Ok(SilentEvent::Closed { first }) => { + assert_eq!( + first, + Some(TLS_HANDSHAKE_RECORD), + "a connection did not begin a TLS handshake" + ); + closed += 1; + } + Err(_) => panic!("only {closed} of {attempts} stalled connections were closed"), + } + } + // The server reports each connection's acceptance before its close, so + // every connection that has been closed has been counted. + assert_eq!(accepted, attempts, "wrong number of connections"); +} + /// A connection whose request timed out must be closed, not returned to the /// pool, where a later request would find it and stall in turn. #[test] From c73a71d5a0982500430b20eb72c3733a21a35df0 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 6 Nov 2026 20:09:12 -0800 Subject: [PATCH 3/3] disk_blob: keep the HTTP blob fault tests reliable under load The fault tests timed out every request after 500 ms, both the ones that the test server stalls and the ones that it answers. On a heavily loaded machine, a request that the server answers can take longer than that: in 50 runs of cargo test -p disk_blob on Windows, with every core busy with other builds, 5 failed. In three of them, requests that should have succeeded timed out, adding attempts that the tests did not expect or failing the read; in one, four tests could not open the blob, because their HEAD requests timed out on every attempt; and in one, a persistent stall test timed out an attempt before the server had stalled it, so the server never saw that connection stall. Each test now shortens only the timeouts that it tests, based on a STALL_TIMEOUT of 2 s, and gives every other one LIMIT, 10 s, as long as the test waits for a request. Tests in which nothing stalls can no longer time out a request, and a test that expects a request to succeed alongside a stall has four times the margin it had. A stall in the body is timed out only once the headers have arrived, so every attempt gets as far as the stall. The tests that stall every attempt make two attempts rather than three, so that the longest tests take about 4 s, well within LIMIT and within the 10 s that nextest's ci profile allows a test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vm/devices/storage/disk_blob/src/tests/mod.rs | 122 ++++++++++++------ 1 file changed, 82 insertions(+), 40 deletions(-) diff --git a/vm/devices/storage/disk_blob/src/tests/mod.rs b/vm/devices/storage/disk_blob/src/tests/mod.rs index 0c3d3722723..1a5f05c731c 100644 --- a/vm/devices/storage/disk_blob/src/tests/mod.rs +++ b/vm/devices/storage/disk_blob/src/tests/mod.rs @@ -166,19 +166,28 @@ async fn misbehaving_server_never_reports_success() { /// taken to be hung for good, which is the failure these tests exist to catch. const LIMIT: Duration = Duration::from_secs(10); +/// How long a fault test's options let a stall last before they time it out. +/// +/// A request that the test server answers can take more than a second on a +/// heavily loaded machine, so a test that expects such a request to succeed +/// under the same timeout needs this margin. No test waits out more than two +/// stalls, so that each stays well within [`LIMIT`]. +const STALL_TIMEOUT: Duration = Duration::from_secs(2); + /// Content in which every offset is distinguishable from its neighbors, so that /// data from the wrong place does not pass for the right data. fn content() -> Vec { (0..DISK_SIZE).map(|i| (i % 251) as u8).collect() } -/// Timeouts short enough that a stall is detected in half a second, so that the -/// fault tests are quick, but still far longer than a loopback request takes. -fn fast_options() -> HttpBlobOptions { +/// Options under which no request times out before the test stops waiting for +/// it, for the tests in which nothing stalls, so that a slow machine does not +/// turn a request that should succeed into a retry. +fn patient_options() -> HttpBlobOptions { HttpBlobOptions { - response_timeout: Duration::from_millis(500), - body_idle_timeout: Duration::from_millis(500), - operation_timeout: Duration::from_secs(5), + response_timeout: LIMIT, + body_idle_timeout: LIMIT, + operation_timeout: LIMIT, max_attempts: 3, initial_backoff: Duration::from_millis(10), max_backoff: Duration::from_millis(40), @@ -186,6 +195,24 @@ fn fast_options() -> HttpBlobOptions { } } +/// Options that time out a stall before the response after [`STALL_TIMEOUT`]. +fn response_stall_options() -> HttpBlobOptions { + HttpBlobOptions { + response_timeout: STALL_TIMEOUT, + ..patient_options() + } +} + +/// Options that time out a stall in the body after [`STALL_TIMEOUT`], but wait +/// for the response headers as long as the test does, so that every attempt +/// gets as far as the stall. +fn body_stall_options() -> HttpBlobOptions { + HttpBlobOptions { + body_idle_timeout: STALL_TIMEOUT, + ..patient_options() + } +} + /// A future running on a thread of its own. See [`background`]. struct Background { recv: mpsc::Receiver, @@ -252,9 +279,10 @@ fn inspect_json(blob: &HttpBlob, path: &str) -> String { } /// Reads through a server that fails the first GET requests as `failures` says, -/// and checks that the read recovers: one retry per failure, the right data, -/// and every stalled connection closed rather than leaked or reused. -fn check_recovers(failures: &[Behavior]) { +/// with `options`, and checks that the read recovers: one retry per failure, +/// the right data, and every stalled connection closed rather than leaked or +/// reused. +fn check_recovers(options: HttpBlobOptions, failures: &[Behavior]) { let content = content(); let server = TestHttpServer::with_script( content.clone(), @@ -264,7 +292,7 @@ fn check_recovers(failures: &[Behavior]) { }, ); let (offset, len) = (12345, 64 * 1024); - let (_blob, r) = read_blob(&server, fast_options(), offset, len); + let (_blob, r) = read_blob(&server, options, offset, len); let data = r.unwrap_or_else(|err| panic!("{failures:?}: read failed: {err:?}")); assert!( data == content[offset as usize..][..len], @@ -283,40 +311,43 @@ fn check_recovers(failures: &[Behavior]) { /// A server, or the network path to it, that goes silent before the response. #[test] fn stall_before_response_is_retried() { - check_recovers(&[Behavior::StallBeforeResponse]); + check_recovers(response_stall_options(), &[Behavior::StallBeforeResponse]); } /// A server, or the network path to it, that goes silent part way through the /// body. #[test] fn stall_mid_body_is_retried() { - check_recovers(&[Behavior::StallMidBody]); + check_recovers(body_stall_options(), &[Behavior::StallMidBody]); } #[test] fn reset_before_response_is_retried() { - check_recovers(&[Behavior::ResetBeforeResponse]); + check_recovers(patient_options(), &[Behavior::ResetBeforeResponse]); } #[test] fn reset_mid_body_is_retried() { - check_recovers(&[Behavior::ResetMidBody]); + check_recovers(patient_options(), &[Behavior::ResetMidBody]); } #[test] fn close_mid_body_is_retried() { - check_recovers(&[Behavior::CloseMidBody]); + check_recovers(patient_options(), &[Behavior::CloseMidBody]); } #[test] fn service_unavailable_is_retried() { - check_recovers(&[Behavior::ServiceUnavailable]); + check_recovers(patient_options(), &[Behavior::ServiceUnavailable]); } /// Every attempt is bounded and retried alike, up to the attempt limit. #[test] fn consecutive_failures_are_retried() { - check_recovers(&[Behavior::ResetBeforeResponse, Behavior::StallMidBody]); + check_recovers( + body_stall_options(), + &[Behavior::ResetBeforeResponse, Behavior::StallMidBody], + ); } /// A guest waits for its disk reads, typically without a timeout of its own, @@ -326,7 +357,11 @@ fn consecutive_failures_are_retried() { fn persistent_stall_fails_disk_read() { let server = TestHttpServer::new(content(), Behavior::StallMidBody); let url = server.url(); - let options = fast_options(); + // Two stalled attempts are enough to show that the retry is bounded too. + let options = HttpBlobOptions { + max_attempts: 2, + ..body_stall_options() + }; let attempts = options.max_attempts as usize; let r = bounded("disk read", async move { let blob = HttpBlob::with_options(&url, options).await.unwrap(); @@ -352,7 +387,10 @@ fn persistent_stall_fails_disk_read() { #[test] fn slow_body_is_not_a_stall() { let content = content(); - let options = fast_options(); + let options = HttpBlobOptions { + response_timeout: STALL_TIMEOUT, + ..body_stall_options() + }; // Each pause is well within the idle timeout, but the body as a whole takes // longer than either timeout. let interval = options.body_idle_timeout / 4; @@ -379,8 +417,8 @@ fn slow_body_is_not_a_stall() { #[test] fn endless_body_is_bounded_by_operation_timeout() { let options = HttpBlobOptions { - operation_timeout: Duration::from_millis(1500), - ..fast_options() + operation_timeout: STALL_TIMEOUT * 2, + ..body_stall_options() }; // One byte at a time: the body would take minutes. let server = TestHttpServer::new( @@ -421,7 +459,9 @@ fn stalled_head_is_retried() { ); let url = server.url(); let blob = bounded("open", async move { - HttpBlob::with_options(&url, fast_options()).await.unwrap() + HttpBlob::with_options(&url, response_stall_options()) + .await + .unwrap() }); assert_eq!(blob.len(), DISK_SIZE); assert_eq!(server.stats().heads, 2); @@ -429,7 +469,11 @@ fn stalled_head_is_retried() { #[test] fn persistent_head_stall_fails_open() { - let options = fast_options(); + // Two stalled attempts are enough to show that the retry is bounded too. + let options = HttpBlobOptions { + max_attempts: 2, + ..response_stall_options() + }; let attempts = options.max_attempts as usize; let server = TestHttpServer::with_script( content(), @@ -507,7 +551,11 @@ const TLS_HANDSHAKE_RECORD: u8 = 0x16; #[test] fn stalled_tls_handshake_fails_open() { let (port, events) = silent_server(); - let options = fast_options(); + // Two stalled attempts are enough to show that the retry is bounded too. + let options = HttpBlobOptions { + max_attempts: 2, + ..response_stall_options() + }; let attempts = options.max_attempts as usize; let url = format!("https://127.0.0.1:{port}/blob"); let r = bounded("open", async move { @@ -556,7 +604,9 @@ fn stalled_connection_is_not_reused() { ); let url = server.url(); let (blob, results) = bounded("reads", async move { - let blob = HttpBlob::with_options(&url, fast_options()).await.unwrap(); + let blob = HttpBlob::with_options(&url, response_stall_options()) + .await + .unwrap(); let mut results = Vec::new(); for i in 0..3 { let mut buf = vec![0; 4096]; @@ -590,13 +640,9 @@ fn inspect_shows_stalled_request() { }, ); let url = server.url(); - // Long enough that the stall does not time out while it is inspected. + // Patient, so that the stall does not time out while it is inspected. // Releasing the server ends it instead. - let options = HttpBlobOptions { - response_timeout: LIMIT * 3, - operation_timeout: LIMIT * 3, - ..fast_options() - }; + let options = patient_options(); let blob = Arc::new(bounded("open", async move { HttpBlob::with_options(&url, options).await.unwrap() })); @@ -637,13 +683,9 @@ fn inspect_shows_stalled_request() { fn dropped_read_is_cancelled() { let server = TestHttpServer::new(content(), Behavior::StallBeforeResponse); let url = server.url(); - // Long enough that nothing times out, and so nothing is retried, before + // Patient, so that nothing times out, and so nothing is retried, before // the read is dropped and its cancellation checked. - let options = HttpBlobOptions { - response_timeout: LIMIT * 3, - operation_timeout: LIMIT * 3, - ..fast_options() - }; + let options = patient_options(); let blob = bounded("open", async move { HttpBlob::with_options(&url, options).await.unwrap() }); @@ -698,7 +740,7 @@ fn permanent_errors_are_not_retried() { Behavior::RangeNotSatisfiable, ] { let server = TestHttpServer::new(content(), behavior); - let (_blob, r) = read_blob(&server, fast_options(), 0, 4096); + let (_blob, r) = read_blob(&server, patient_options(), 0, 4096); assert!(r.is_err(), "{behavior:?}: reported success"); assert_eq!(server.stats().gets, 1, "{behavior:?}: retried"); } @@ -709,7 +751,7 @@ fn permanent_errors_are_not_retried() { #[test] fn range_past_the_largest_offset_is_refused() { let server = TestHttpServer::new(content(), Behavior::Correct); - let (_blob, r) = read_blob(&server, fast_options(), u64::MAX - 100, 4096); + let (_blob, r) = read_blob(&server, patient_options(), u64::MAX - 100, 4096); match r { Err(err) if err.kind() == std::io::ErrorKind::InvalidInput => {} r => panic!("expected InvalidInput, got {:?}", r.map(|_| ())), @@ -724,7 +766,7 @@ fn range_past_the_largest_offset_is_refused() { #[test] fn range_ending_at_the_largest_offset_is_requested() { let server = TestHttpServer::new(content(), Behavior::ServiceUnavailable); - let options = fast_options(); + let options = patient_options(); let attempts = options.max_attempts as usize; let offset = u64::MAX - 4095; let (_blob, r) = read_blob(&server, options, offset, 4096);