diff --git a/Cargo.lock b/Cargo.lock index b2c2dfb39b9..5a16463528c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1584,13 +1584,18 @@ dependencies = [ "hyper-tls", "hyper-util", "inspect", + "inspect_counters", "once_cell", "pal_async", + "parking_lot", "scsi_buffers", + "socket2", "storage_tests", "tempfile", + "test_with_tracing", "thiserror 2.0.16", "tokio", + "tracelimit", "vhd1_defs", "vm_resource", "zerocopy", diff --git a/Guide/src/reference/backends/storage.md b/Guide/src/reference/backends/storage.md index bd21d39438d..43b7b643337 100644 --- a/Guide/src/reference/backends/storage.md +++ b/Guide/src/reference/backends/storage.md @@ -18,7 +18,7 @@ blob, or a layered composition of multiple backends. | VhdmpDisk | `disk_vhdmp` | Windows vhdmp driver | Windows | Dynamic and differencing VHD/VHDX. | | VhdxDisk | [`vhdx`](vhdx.md) | VHDX file | Cross-platform | Pure-Rust VHDX parser. Dynamic, fixed, and differencing. | | FlatVmdkDisk | [`disk_vmdk`](vmdk.md) | VMDK extents | Cross-platform | Flat and read-only. Supports the `monolithicFlat` and `twoGbMaxExtentFlat` disk types. | -| BlobDisk | [`disk_blob`](https://openvmm.dev/rustdoc/linux/disk_blob/index.html) | HTTP / Azure Blob | Cross-platform | Read-only. HTTP range requests. | +| BlobDisk | [`disk_blob`](https://openvmm.dev/rustdoc/linux/disk_blob/index.html) | HTTP / Azure Blob | Cross-platform | Read-only. HTTP range requests, with timeouts and retries. | | BlockDeviceDisk | [`disk_blockdevice`](https://openvmm.dev/rustdoc/linux/disk_blockdevice/index.html) | Linux block device or file | Linux | io_uring, resize via uevent, PR passthrough. Default for raw files on Linux in both OpenHCL and OpenVMM. | | NvmeDisk | [`disk_nvme`](https://openvmm.dev/rustdoc/linux/disk_nvme/index.html) | Physical NVMe (VFIO) | Linux/Windows | User-mode NVMe driver. Resize via AEN. | | StripedDisk | [`disk_striped`](https://openvmm.dev/rustdoc/linux/disk_striped/index.html) | Multiple Disks | Cross-platform | Stripes data across underlying disks. | diff --git a/vm/devices/storage/disk_blob/Cargo.toml b/vm/devices/storage/disk_blob/Cargo.toml index a207a2365f2..3a23c9b0733 100644 --- a/vm/devices/storage/disk_blob/Cargo.toml +++ b/vm/devices/storage/disk_blob/Cargo.toml @@ -16,6 +16,8 @@ guestmem.workspace = true vm_resource.workspace = true inspect = { workspace = true, features = ["filepath"] } +inspect_counters.workspace = true +tracelimit.workspace = true anyhow.workspace = true async-trait.workspace = true @@ -26,17 +28,21 @@ hyper = { workspace = true, features = ["client", "http1", "http2"] } hyper-tls.workspace = true hyper-util = { workspace = true, features = ["client", "client-legacy", "http1", "http2"] } once_cell.workspace = true +parking_lot.workspace = true thiserror.workspace = true zerocopy.workspace = true # tokio use is allowed in this crate only. # FUTURE: replace this with our own executor -tokio = { version = "1", features = ["rt-multi-thread"] } +tokio = { version = "1", features = ["rt-multi-thread", "time"] } [dev-dependencies] disk_vhd1.workspace = true +inspect = { workspace = true, features = ["initiate"] } pal_async.workspace = true +socket2.workspace = true storage_tests.workspace = true tempfile.workspace = true +test_with_tracing.workspace = true [lints] workspace = true diff --git a/vm/devices/storage/disk_blob/src/blob/http.rs b/vm/devices/storage/disk_blob/src/blob/http.rs index 7d90be22503..f832dd2842a 100644 --- a/vm/devices/storage/disk_blob/src/blob/http.rs +++ b/vm/devices/storage/disk_blob/src/blob/http.rs @@ -14,42 +14,149 @@ use async_trait::async_trait; use http::uri::Scheme; use http_body_util::BodyExt; use http_body_util::Empty; +use hyper::Method; use hyper::Request; +use hyper::Response; use hyper::StatusCode; use hyper::Uri; +use hyper::body::Incoming; use hyper_tls::HttpsConnector; use hyper_util::client::legacy::Client; use hyper_util::client::legacy::connect::HttpConnector; use hyper_util::rt::TokioExecutor; use inspect::Inspect; +use inspect_counters::SharedCounter; use once_cell::sync::OnceCell; +use parking_lot::Mutex; +use std::collections::BTreeMap; +use std::fmt; use std::fmt::Debug; use std::io; +use std::sync::Arc; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; +use std::time::Duration; +use thiserror::Error; +use tokio::time::Instant; + +/// Timeouts and retries for the requests that an [`HttpBlob`] makes. +/// +/// A guest waits for its disk reads to complete, typically without a timeout +/// of its own, so a request that never completes hangs the guest. And a request +/// can stop making progress without failing: when the network silently stops +/// delivering a connection's packets, the client is left waiting for data that +/// will never come, with no reset or FIN to end the wait. So every request is +/// bounded in time: +/// +/// - Each attempt must receive the response headers within +/// [`response_timeout`](Self::response_timeout), and then more of the body at +/// least every [`body_idle_timeout`](Self::body_idle_timeout). These limit +/// silence rather than duration, so that a connection that has stopped is +/// detected without cutting off a slow response that is still arriving. +/// - An attempt that fails in a way that the next one might not -- by timing +/// out, by losing its connection, or with status 408, 429, 500, 502, 503, or +/// 504 -- is retried on a new connection, after a delay that starts at +/// [`initial_backoff`](Self::initial_backoff) and doubles with each retry, up +/// to [`max_backoff`](Self::max_backoff), for up to +/// [`max_attempts`](Self::max_attempts) attempts in all. Any other failure is +/// returned at once. +/// - [`operation_timeout`](Self::operation_timeout) bounds the request as a +/// whole, so that even a response that trickles in too slowly ever to finish +/// fails in bounded time. +/// +/// A request that fails returns an error, which the guest sees as a failed +/// read that it can handle, rather than as a read that never completes. +/// +/// The defaults suit a blob store such as Azure Blob Storage, where a range +/// request normally completes in tens of milliseconds. A connection that is +/// silent for 20 seconds, a thousand times that, is taken to be dead; a request +/// whose every attempt stalls fails after about 80 seconds; and no request +/// takes more than two minutes. That rides out a transient network failure, +/// but still fails a hopeless request well before a typical test watchdog, such +/// as petri's ten minutes, would intervene. +#[derive(Debug, Clone)] +pub struct HttpBlobOptions { + /// How long to wait for a TCP connection to be established. + pub connect_timeout: Duration, + /// How long an attempt waits for the response headers, including the time + /// to connect and to send the request. + pub response_timeout: Duration, + /// How long an attempt waits for more of the response body. + pub body_idle_timeout: Duration, + /// How long a request may take in all, across all its attempts. + pub operation_timeout: Duration, + /// How many attempts to make at a request, including the first. At least + /// one attempt is always made. + pub max_attempts: u32, + /// The delay before the first retry. + pub initial_backoff: Duration, + /// The longest delay before a retry. + pub max_backoff: Duration, +} + +impl Default for HttpBlobOptions { + fn default() -> Self { + Self { + connect_timeout: Duration::from_secs(10), + response_timeout: Duration::from_secs(20), + body_idle_timeout: Duration::from_secs(20), + operation_timeout: Duration::from_secs(120), + max_attempts: 4, + initial_backoff: Duration::from_millis(250), + max_backoff: Duration::from_secs(4), + } + } +} + +/// How long a connection may be idle before TCP keepalive probes check that +/// the server can still be reached. Without them, an idle pooled connection +/// whose network path has died is discovered only when a request is sent on it, +/// at the cost of a response timeout. With them, on Linux, macOS, and most +/// other Unix-like systems, it is closed after [`TCP_KEEPALIVE_RETRIES`] +/// unanswered probes, a minute after it went idle. On Windows, `hyper_util` +/// does not set the number of probes, so the system's default of 10 applies, +/// and the connection is closed only after 130 seconds, when the pool, whose +/// idle timeout defaults to 90 seconds, would no longer reuse it anyway. Either +/// way, a request sent on a dead connection is still bounded by the response, +/// body, and operation timeouts, and retried on a new connection. The probes +/// also keep the connection alive in middleboxes that drop idle flows. +const TCP_KEEPALIVE_TIME: Duration = Duration::from_secs(30); +/// The interval between TCP keepalive probes that go unanswered. +const TCP_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(10); +/// How many TCP keepalive probes go unanswered before the connection is closed, +/// except on Windows. See [`TCP_KEEPALIVE_TIME`]. +const TCP_KEEPALIVE_RETRIES: u32 = 3; + +type HttpClient = Client, Empty<&'static [u8]>>; /// A blob backed by an HTTP/HTTPS connection. +/// +/// Its requests are bounded in time, and retried, as [`HttpBlobOptions`] +/// describes. #[derive(Debug, Inspect)] pub struct HttpBlob { - #[inspect(skip)] - client: Client, Empty<&'static [u8]>>, + #[inspect(flatten)] + requester: Arc, #[inspect(debug)] version: http::Version, #[inspect(display)] uri: Uri, len: u64, - #[inspect(skip)] - tokio_handle: tokio::runtime::Handle, } static TOKIO_RUNTIME: OnceCell = OnceCell::new(); impl HttpBlob { - /// Connects to `url` and returns an object to access it as a blob. + /// Connects to `url` and returns an object to access it as a blob, with the + /// default [`HttpBlobOptions`]. pub async fn new(url: &str) -> anyhow::Result { - let mut uri: Uri = url.parse()?; + Self::with_options(url, HttpBlobOptions::default()).await + } - let connector = HttpsConnector::new(); - let builder = Client::builder(TokioExecutor::new()); - let client = builder.build(connector); + /// Connects to `url` and returns an object to access it as a blob, with the + /// timeouts and retries that `options` specifies. + pub async fn with_options(url: &str, options: HttpBlobOptions) -> anyhow::Result { + let uri: Uri = url.parse()?; let handle = TOKIO_RUNTIME .get_or_try_init(tokio::runtime::Runtime::new) @@ -57,28 +164,150 @@ impl HttpBlob { .handle() .clone(); + let requester = Arc::new(Requester::new(options, handle)); + let (uri, response) = requester.run(requester.clone().probe(uri)).await?; + + let len = response + .headers() + .get("Content-Length") + .context("missing blob length")? + .to_str() + .context("couldn't parse blob length")? + .parse() + .context("couldn't parse blob length")?; + + let version = response.version(); + + Ok(Self { + requester, + version, + uri, + len, + }) + } +} + +#[async_trait] +impl Blob for HttpBlob { + async fn read(&self, buf: &mut [u8], offset: u64) -> io::Result<()> { + // HTTP byte ranges are inclusive at both ends, so an empty one cannot + // be expressed, and computing its end offset would underflow. There is + // nothing to fetch in any case. + if buf.is_empty() { + return Ok(()); + } + // A request names the range by its first and last offsets, so the + // last must be representable. + if offset.checked_add(buf.len() as u64 - 1).is_none() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "read range ends past the largest representable offset", + )); + } + let requester = self.requester.clone(); + let uri = self.uri.clone(); + let len = buf.len(); + let data = self + .requester + .run(async move { requester.get(uri, offset, len).await }) + .await?; + buf.copy_from_slice(&data); + Ok(()) + } + + fn len(&self) -> u64 { + self.len + } +} + +/// Makes the requests for an [`HttpBlob`], with timeouts and retries. +#[derive(Debug, Inspect)] +struct Requester { + #[inspect(skip)] + options: HttpBlobOptions, + /// Makes first attempts, which may reuse idle connections. + #[inspect(skip)] + client: HttpClient, + /// Makes retries, each on a new connection: whatever broke the previous + /// attempt's connection may have broken the idle ones too. + #[inspect(skip)] + retry_client: HttpClient, + #[inspect(skip)] + tokio_handle: tokio::runtime::Handle, + /// Attempts made, retries included. + requests: SharedCounter, + /// Attempts made after a failed attempt. + retries: SharedCounter, + /// Attempts cut short by a timeout. + timeouts: SharedCounter, + /// Requests that failed, after any retries. + failures: SharedCounter, + /// The requests in progress. + in_flight: InFlightTable, + #[inspect(skip)] + next_id: AtomicU64, +} + +impl Requester { + fn new(options: HttpBlobOptions, tokio_handle: tokio::runtime::Handle) -> Self { + let mut http = HttpConnector::new(); + // The HTTPS connector hands HTTPS URIs to this one to connect. + http.enforce_http(false); + http.set_connect_timeout(Some(options.connect_timeout)); + http.set_keepalive(Some(TCP_KEEPALIVE_TIME)); + http.set_keepalive_interval(Some(TCP_KEEPALIVE_INTERVAL)); + // hyper_util ignores this on Windows. See TCP_KEEPALIVE_TIME. + http.set_keepalive_retries(Some(TCP_KEEPALIVE_RETRIES)); + let connector = HttpsConnector::new_with_connector(http); + Self { + client: Client::builder(TokioExecutor::new()).build(connector.clone()), + retry_client: Client::builder(TokioExecutor::new()) + .pool_max_idle_per_host(0) + .build(connector), + options, + tokio_handle, + requests: SharedCounter::new(), + retries: SharedCounter::new(), + timeouts: SharedCounter::new(), + failures: SharedCounter::new(), + in_flight: InFlightTable::default(), + next_id: AtomicU64::new(0), + } + } + + /// Runs `fut` on the tokio runtime, whose timers and connections it needs, + /// and returns its output. Dropping the returned future aborts `fut`. + async fn run(&self, fut: impl 'static + Send + Future) -> T { + let mut task = AbortOnDrop(self.tokio_handle.spawn(fut)); + match (&mut task.0).await { + Ok(output) => output, + // The task is only aborted when this future is dropped, so it must + // have panicked. + Err(err) => std::panic::resume_unwind(err.into_panic()), + } + } + + /// Follows `uri` through any redirects to the blob, and returns the blob's + /// URI and the response to a HEAD request for it. + async fn probe(self: Arc, mut uri: Uri) -> anyhow::Result<(Uri, Response)> { + // One deadline covers all the redirects. + let deadline = deadline_after(self.options.operation_timeout); + let response_timeout = self.options.response_timeout; let mut redirect_count = 0; - let response = loop { + loop { if redirect_count > 5 { anyhow::bail!("too many redirects"); } - let response = handle - .spawn( - client.request( - Request::builder() - .uri(&uri) - .method("HEAD") - .body(Empty::new()) - .unwrap(), - ), - ) + let response = self + .request(RequestKind::Head, deadline, |client, _| { + head_once(client, uri.clone(), response_timeout) + }) .await - .unwrap() .context("failed to query blob size")?; let next_uri: Uri = match response.status() { - StatusCode::OK => break response, + StatusCode::OK => return Ok((uri, response)), StatusCode::MOVED_PERMANENTLY | StatusCode::FOUND | StatusCode::TEMPORARY_REDIRECT @@ -101,81 +330,446 @@ impl HttpBlob { uri = next_uri; redirect_count += 1; - }; + } + } - let len = response - .headers() - .get("Content-Length") - .context("missing blob length")? - .to_str() - .context("couldn't parse blob length")? - .parse() - .context("couldn't parse blob length")?; + /// Reads `len` bytes at `offset` from the blob at `uri`. + async fn get(self: Arc, uri: Uri, offset: u64, len: usize) -> io::Result> { + let deadline = deadline_after(self.options.operation_timeout); + let response_timeout = self.options.response_timeout; + let body_idle_timeout = self.options.body_idle_timeout; + self.request( + RequestKind::Get { offset, len }, + deadline, + |client, progress| { + get_once( + client, + uri.clone(), + offset, + len, + response_timeout, + body_idle_timeout, + progress, + ) + }, + ) + .await + } - let version = response.version(); + /// Makes `request`, by calling `attempt` with the client to use and a + /// handle to report progress through, and retrying it as the options + /// say, until it succeeds, fails for good, or `deadline` passes. + async fn request( + self: &Arc, + request: RequestKind, + deadline: Instant, + mut attempt: F, + ) -> io::Result + where + F: FnMut(HttpClient, Progress) -> Fut, + Fut: Future>, + { + let tracker = self.track(request); + let mut backoff = self.options.initial_backoff.min(self.options.max_backoff); + let mut attempts = 0; + loop { + attempts += 1; + tracker.0.update(|entry| entry.start_attempt(attempts)); + self.requests.increment(); + let client = if attempts == 1 { + &self.client + } else { + &self.retry_client + }; + let (err, retry) = + match tokio::time::timeout_at(deadline, attempt(client.clone(), tracker.0.clone())) + .await + { + Ok(Ok(output)) => return Ok(output), + Ok(Err(Failure::Transient(err))) => (err, true), + Ok(Err(Failure::Timeout(err))) => { + self.timeouts.increment(); + (io::Error::new(io::ErrorKind::TimedOut, err), true) + } + Ok(Err(Failure::Permanent(err))) => { + self.failures.increment(); + return Err(err); + } + Err(_) => { + self.timeouts.increment(); + let err = RequestError::OperationTimeout { + request, + timeout: self.options.operation_timeout, + }; + (io::Error::new(io::ErrorKind::TimedOut, err), false) + } + }; - Ok(Self { - client, - version, - uri, - len, - tokio_handle: handle, + let next_attempt = Instant::now().checked_add(backoff); + if retry + && attempts < self.options.max_attempts + && next_attempt.is_some_and(|next| next < deadline) + { + tracelimit::warn_ratelimited!( + error = &err as &dyn std::error::Error, + %request, + attempt = attempts, + ?backoff, + "http blob request failed, retrying" + ); + self.retries.increment(); + tracker.0.update(|entry| entry.state = State::BackingOff); + tokio::time::sleep(backoff).await; + backoff = backoff.saturating_mul(2).min(self.options.max_backoff); + continue; + } + + self.failures.increment(); + tracelimit::warn_ratelimited!( + error = &err as &dyn std::error::Error, + %request, + attempts, + "http blob request failed" + ); + return Err(if retry && attempts > 1 { + io::Error::new( + err.kind(), + RequestError::Exhausted { + request, + attempts, + source: err, + }, + ) + } else { + err + }); + } + } + + /// Lists `request` in [`Requester::in_flight`] until the returned tracker + /// is dropped. + fn track(self: &Arc, request: RequestKind) -> Tracker { + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + self.in_flight.0.lock().insert( + id, + InFlight { + request, + attempt: 0, + state: State::AwaitingResponse, + started: Instant::now(), + received: 0, + }, + ); + Tracker(Progress { + requester: self.clone(), + id, }) } } -#[async_trait] -impl Blob for HttpBlob { - async fn read(&self, mut buf: &mut [u8], offset: u64) -> io::Result<()> { - // HTTP byte ranges are inclusive at both ends, so an empty one cannot - // be expressed, and computing the end offset below would underflow. - // There is nothing to fetch in any case. - if buf.is_empty() { - return Ok(()); +/// Returns the instant `timeout` from now, or a distant one if that cannot be +/// represented. +fn deadline_after(timeout: Duration) -> Instant { + let now = Instant::now(); + now.checked_add(timeout) + .unwrap_or_else(|| now + Duration::from_secs(60 * 60 * 24 * 365)) +} + +/// Makes one attempt at a HEAD request for `uri`. +async fn head_once( + client: HttpClient, + uri: Uri, + response_timeout: Duration, +) -> Result, Failure> { + let request = Request::builder() + .uri(uri) + .method(Method::HEAD) + .body(Empty::new()) + .unwrap(); + send(client, request, response_timeout).await +} + +/// Makes one attempt at reading `len` bytes at `offset` from the blob at `uri`. +async fn get_once( + client: HttpClient, + uri: Uri, + offset: u64, + len: usize, + response_timeout: Duration, + body_idle_timeout: Duration, + progress: Progress, +) -> Result, Failure> { + let request = Request::builder() + .uri(uri) + .header( + hyper::header::RANGE, + format!("bytes={}-{}", offset, offset + (len as u64 - 1)), + ) + .body(Empty::new()) + .unwrap(); + let response = send(client, request, response_timeout).await?; + + if !response.status().is_success() { + return Err(Failure::Permanent(io::Error::other(RequestError::Status( + response.status(), + )))); + } + + progress.update(|entry| entry.state = State::ReceivingBody); + let mut body = response.into_body(); + let mut data = Vec::with_capacity(len); + loop { + let frame = match tokio::time::timeout(body_idle_timeout, body.frame()).await { + Ok(Some(Ok(frame))) => frame, + Ok(None) => break, + // The connection failed part way through the body. + Ok(Some(Err(err))) => return Err(Failure::Transient(io::Error::other(err))), + Err(_) => { + return Err(Failure::Timeout(RequestError::BodyTimeout { + timeout: body_idle_timeout, + received: data.len(), + expected: len, + })); + } + }; + if let Some(chunk) = frame.data_ref() { + if chunk.len() > len - data.len() { + return Err(Failure::Permanent(io::Error::other( + "server did not respect range query", + ))); + } + data.extend_from_slice(chunk); + progress.update(|entry| entry.received = data.len()); } - let mut response = self - .tokio_handle - .spawn( - self.client.request( - Request::builder() - .uri(&self.uri) - .header( - hyper::header::RANGE, - format!("bytes={}-{}", offset, offset + buf.len() as u64 - 1,), - ) - .body(Empty::new()) - .unwrap(), - ), - ) - .await - .unwrap() - .map_err(io::Error::other)?; - - if !response.status().is_success() { - return Err(io::Error::other(response.status().to_string())); + } + + if data.len() < len { + // A well-formed response that is shorter than was asked for. Another + // attempt would get the same. + return Err(Failure::Permanent(io::ErrorKind::UnexpectedEof.into())); + } + + Ok(data) +} + +/// Sends `request`, and waits up to `response_timeout` for the response +/// headers. +async fn send( + client: HttpClient, + request: Request>, + response_timeout: Duration, +) -> Result, Failure> { + let response = match tokio::time::timeout(response_timeout, client.request(request)).await { + Ok(Ok(response)) => response, + // The connector gave up on connecting. + Ok(Err(err)) if is_timed_out(&err) => { + return Err(Failure::Timeout(RequestError::ConnectTimeout(err))); + } + // The request is well formed, so this is a failure to connect, or to + // keep the connection. + Ok(Err(err)) => return Err(Failure::Transient(io::Error::other(err))), + Err(_) => { + return Err(Failure::Timeout(RequestError::ResponseTimeout( + response_timeout, + ))); } + }; - while let Some(frame) = response.body_mut().frame().await { - let frame = frame.map_err(io::Error::other)?; - if let Some(data) = frame.data_ref() { - let len = data.len(); - if len > buf.len() { - return Err(io::Error::other("server did not respect range query")); - } - let (this, rest) = buf.split_at_mut(len); - this.copy_from_slice(data); - buf = rest; + let status = response.status(); + if matches!( + status, + StatusCode::REQUEST_TIMEOUT + | StatusCode::TOO_MANY_REQUESTS + | StatusCode::INTERNAL_SERVER_ERROR + | StatusCode::BAD_GATEWAY + | StatusCode::SERVICE_UNAVAILABLE + | StatusCode::GATEWAY_TIMEOUT + ) { + return Err(Failure::Transient(io::Error::other(RequestError::Status( + status, + )))); + } + + Ok(response) +} + +/// Returns whether `err`, or any error in its chain of sources, is a timeout, +/// as when the connector gives up on connecting. +fn is_timed_out(err: &(dyn std::error::Error + 'static)) -> bool { + let mut next = Some(err); + while let Some(err) = next { + if err + .downcast_ref::() + .is_some_and(|err| err.kind() == io::ErrorKind::TimedOut) + { + return true; + } + next = err.source(); + } + false +} + +/// Aborts a tokio task when dropped. +struct AbortOnDrop(tokio::task::JoinHandle); + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.abort(); + } +} + +/// Why an attempt failed, which decides whether to retry it. +enum Failure { + /// The attempt failed in a way that the next attempt might not. + Transient(io::Error), + /// The attempt timed out, which the next attempt might not. + Timeout(RequestError), + /// The attempt failed in a way that the next attempt would repeat. + Permanent(io::Error), +} + +#[derive(Debug, Error)] +enum RequestError { + #[error("connection timed out")] + ConnectTimeout(#[source] hyper_util::client::legacy::Error), + #[error("no response within {0:?}")] + ResponseTimeout(Duration), + #[error("no data for {timeout:?}, after {received} of {expected} bytes")] + BodyTimeout { + timeout: Duration, + received: usize, + expected: usize, + }, + #[error("{request} did not complete within {timeout:?}")] + OperationTimeout { + request: RequestKind, + timeout: Duration, + }, + #[error("{request} failed after {attempts} attempts")] + Exhausted { + request: RequestKind, + attempts: u32, + #[source] + source: io::Error, + }, + #[error("server returned {0}")] + Status(StatusCode), +} + +/// A request that an [`HttpBlob`] makes. +#[derive(Debug, Copy, Clone)] +enum RequestKind { + /// A HEAD request, to probe the blob's length. + Head, + /// A range request for `len` bytes at `offset`. `len` is not zero, and the + /// offset of the range's last byte, `offset + (len - 1)`, does not + /// overflow. + Get { offset: u64, len: usize }, +} + +impl fmt::Display for RequestKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + Self::Head => f.write_str("HEAD"), + Self::Get { offset, len } => { + write!(f, "GET bytes={}-{}", offset, offset + (len as u64 - 1)) } } + } +} - if !buf.is_empty() { - return Err(io::ErrorKind::UnexpectedEof.into()); +/// The requests in progress, by ID. +#[derive(Debug, Default)] +struct InFlightTable(Mutex>); + +impl Inspect for InFlightTable { + fn inspect(&self, req: inspect::Request<'_>) { + inspect::iter_by_key(self.0.lock().iter()).inspect(req); + } +} + +/// A request in progress, as inspection shows it. +#[derive(Debug, Inspect)] +struct InFlight { + /// The request, such as `GET bytes=0-4095`. + #[inspect(display)] + request: RequestKind, + /// The current attempt, counting from 1. + attempt: u32, + /// What the request is waiting for. + state: State, + /// How long ago the request started. + #[inspect(rename = "elapsed", with = "Instant::elapsed")] + started: Instant, + /// How much of the response body the current attempt has received. + received: usize, +} + +impl InFlight { + fn start_attempt(&mut self, attempt: u32) { + self.attempt = attempt; + self.state = State::AwaitingResponse; + self.received = 0; + } +} + +/// What a request in progress is waiting for. +#[derive(Debug, Copy, Clone, Inspect)] +enum State { + /// The response headers, connecting first if need be. + AwaitingResponse, + /// More of the response body. + ReceivingBody, + /// The delay before the next attempt. + BackingOff, +} + +/// A handle through which an attempt reports its progress to inspection. +#[derive(Clone)] +struct Progress { + requester: Arc, + id: u64, +} + +impl Progress { + fn update(&self, f: impl FnOnce(&mut InFlight)) { + if let Some(entry) = self.requester.in_flight.0.lock().get_mut(&self.id) { + f(entry); } + } +} - Ok(()) +/// Removes a request from [`Requester::in_flight`] when dropped. +struct Tracker(Progress); + +impl Drop for Tracker { + fn drop(&mut self) { + self.0.requester.in_flight.0.lock().remove(&self.0.id); } +} - fn len(&self) -> u64 { - self.len +#[cfg(test)] +mod tests { + use super::is_timed_out; + use std::io; + use test_with_tracing::test; + + /// An error caused by another, as the connector's errors are. + #[derive(Debug, thiserror::Error)] + #[error("wrapper")] + struct Wrapper(#[source] Box); + + #[test] + fn a_timeout_is_found_anywhere_in_the_chain() { + let timed_out = || io::Error::new(io::ErrorKind::TimedOut, "connect"); + assert!(is_timed_out(&timed_out())); + assert!(is_timed_out(&Wrapper(Box::new(Wrapper(Box::new( + timed_out() + )))))); + assert!(!is_timed_out(&io::Error::other("refused"))); + assert!(!is_timed_out(&Wrapper(Box::new(io::Error::new( + io::ErrorKind::ConnectionRefused, + "refused", + ))))); } } diff --git a/vm/devices/storage/disk_blob/src/tests/http_server.rs b/vm/devices/storage/disk_blob/src/tests/http_server.rs index 8b188d741b0..0ff3fdd2e57 100644 --- a/vm/devices/storage/disk_blob/src/tests/http_server.rs +++ b/vm/devices/storage/disk_blob/src/tests/http_server.rs @@ -9,17 +9,29 @@ //! importantly, the interesting cases are all responses a well-behaved server //! would never produce. See [`Behavior`]. //! +//! A [`Script`] can make the server misbehave for the first few requests only, +//! which models the transient failures that [`HttpBlob`] must recover from. The +//! server counts what it sees in [`Stats`], so that tests can wait for it to +//! reach a given point rather than sleep, and can check how many attempts a +//! read took. +//! //! [`HttpBlob`]: crate::blob::http::HttpBlob +use parking_lot::Condvar; +use parking_lot::Mutex; use std::io::Read; use std::io::Write; use std::net::TcpListener; use std::net::TcpStream; +use std::sync::Arc; +use std::time::Duration; +use std::time::Instant; -/// How the server should respond to range requests. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// How the server should respond to a request. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum Behavior { /// Answer with exactly the requested range. + #[default] Correct, /// Answer with one byte less than was asked for. `Blob::read` is required /// to report this as `UnexpectedEof` rather than as a partial success, @@ -32,140 +44,414 @@ pub enum Behavior { RangeNotSatisfiable, /// Send the headers, then close the connection part way through the body. CloseMidBody, + /// Read the request, then send nothing at all, holding the connection open + /// until the client closes it or the server is released. This is what a + /// client sees when the network silently stops delivering a connection's + /// packets: no data, but no FIN or reset to end the wait either. + StallBeforeResponse, + /// Send the headers and half the body, then stall as for + /// [`Behavior::StallBeforeResponse`]. + StallMidBody, + /// Reset the connection instead of answering. + ResetBeforeResponse, + /// Send the headers and half the body, then reset the connection. + ResetMidBody, + /// Answer `503 Service Unavailable`, as an overloaded or throttling server + /// does. + ServiceUnavailable, + /// Answer correctly, but send the body in `pieces` pieces, pausing for + /// `interval` before each: slow, but never silent for much longer than + /// `interval`. + SlowBody { pieces: usize, interval: Duration }, +} + +/// How the server responds to each request. +#[derive(Debug, Clone, Default)] +pub struct Script { + /// How to respond to the first HEAD requests, in order. Later ones are + /// answered correctly. A HEAD response has no body, so a behavior that + /// concerns the body acts as its counterpart that does not: a stall + /// stalls before the response, a reset resets before it, and the rest + /// answer correctly. + pub head: Vec, + /// How to respond to the first GET requests, in order. + pub get: Vec, + /// How to respond to the GET requests after those in `get`. + pub then: Behavior, + /// Serve further requests on a connection after a response, as real + /// servers do, rather than closing it after each one. + pub keep_alive: bool, +} + +/// What the server has seen so far. +#[derive(Debug, Clone, Copy, Default)] +pub struct Stats { + /// Connections accepted. + pub connections: usize, + /// HEAD requests received. + pub heads: usize, + /// GET requests received. + pub gets: usize, + /// Connections that the server has stalled. + pub stalled: usize, + /// Stalled connections that the client has since closed. + pub abandoned: usize, } /// A server bound to an ephemeral port on the loopback interface. pub struct TestHttpServer { port: u16, + shared: Arc, +} + +struct Shared { + content: Vec, + script: Script, + state: Mutex, + changed: Condvar, +} + +#[derive(Default)] +struct State { + stats: Stats, + released: bool, } impl TestHttpServer { - /// Starts a server publishing `content`. + /// Starts a server publishing `content`, which answers every range request + /// as `behavior` says. + pub fn new(content: Vec, behavior: Behavior) -> Self { + Self::with_script( + content, + Script { + then: behavior, + ..Default::default() + }, + ) + } + + /// Starts a server publishing `content`, which answers requests as `script` + /// says. /// - /// The serving thread runs until the process exits. Tests are short-lived + /// The listening thread runs until the process exits. Tests are short-lived /// and nextest gives each one its own process, so there is nothing to clean - /// up. - pub fn new(content: Vec, behavior: Behavior) -> Self { + /// up. Stalled connections are closed when the server is dropped. + pub fn with_script(content: Vec, script: Script) -> Self { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); + let shared = Arc::new(Shared { + content, + script, + state: Mutex::default(), + changed: Condvar::new(), + }); + let listener_shared = shared.clone(); std::thread::spawn(move || { for stream in listener.incoming() { let Ok(stream) = stream else { break }; - // A misbehaving response deliberately drops the connection, so - // errors here are expected and not worth reporting. - let _ = serve(stream, &content, behavior); + listener_shared.state.lock().stats.connections += 1; + listener_shared.changed.notify_all(); + let shared = listener_shared.clone(); + // A thread per connection, so that a stalled connection does + // not hold up the others. + std::thread::spawn(move || { + // A misbehaving response deliberately drops the connection, + // so errors here are expected and not worth reporting. + let _ = shared.serve(stream); + }); } }); - Self { port } + Self { port, shared } } /// The URL of the blob. pub fn url(&self) -> String { format!("http://127.0.0.1:{}/blob", self.port) } + + /// Returns what the server has seen so far. + pub fn stats(&self) -> Stats { + self.shared.state.lock().stats + } + + /// Waits up to `timeout` for `done` to return true for the server's + /// [`Stats`], and returns them. Panics, naming `what` it was waiting for, + /// if `done` is still false at the end. + pub fn wait_for( + &self, + what: &str, + timeout: Duration, + mut done: impl FnMut(&Stats) -> bool, + ) -> Stats { + let deadline = Instant::now() + timeout; + let mut state = self.shared.state.lock(); + while !done(&state.stats) { + if self + .shared + .changed + .wait_until(&mut state, deadline) + .timed_out() + && !done(&state.stats) + { + panic!("timed out waiting for {what}: {:?}", state.stats); + } + } + state.stats + } + + /// Ends every stall, now and in future, by closing the connection. + pub fn release(&self) { + self.shared.state.lock().released = true; + self.shared.changed.notify_all(); + } } -/// Reads one request and writes one response, then closes the connection. -fn serve(mut stream: TcpStream, content: &[u8], behavior: Behavior) -> std::io::Result<()> { - let request = read_headers(&mut stream)?; - let mut lines = request.lines(); - let method = lines - .next() - .and_then(|l| l.split(' ').next()) - .unwrap_or_default() - .to_owned(); - - // `HttpBlob::new` probes the length with a HEAD request. - if method.eq_ignore_ascii_case("HEAD") { - write!( - stream, - "HTTP/1.1 200 OK\r\n\ - Content-Length: {}\r\n\ - Accept-Ranges: bytes\r\n\ - Connection: close\r\n\r\n", - content.len() - )?; - return Ok(()); +impl Drop for TestHttpServer { + fn drop(&mut self) { + // A failing test may leave a client waiting on a stalled connection. + // Closing the connection ends the wait. + self.release(); } +} - // Header names are case-insensitive, and hyper does not necessarily send - // them in the case they were built with. - let range = lines.find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case("range") - .then(|| value.trim().strip_prefix("bytes=").map(str::to_owned)) - .flatten() - }); - let Some(range) = range else { - write!( - stream, - "HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n" - )?; - return Ok(()); - }; - let (start, end) = range.split_once('-').unwrap(); - let start: usize = start.parse().unwrap(); - // The range is inclusive at both ends. - let end: usize = end.parse().unwrap(); - - if behavior == Behavior::RangeNotSatisfiable || start >= content.len() { - write!( - stream, - "HTTP/1.1 416 Range Not Satisfiable\r\n\ - Content-Range: bytes */{}\r\n\ - Connection: close\r\n\r\n", - content.len() - )?; - return Ok(()); +impl Shared { + /// Serves requests on `stream` until it is closed. + fn serve(&self, mut stream: TcpStream) -> std::io::Result<()> { + // Send each write at once, so that the client sees the pieces and + // pauses of a response as the behavior describes them. + stream.set_nodelay(true)?; + loop { + let request = read_headers(&mut stream)?; + // An empty request means the client has closed the connection. + if request.is_empty() + || !self.respond(&mut stream, &request)? + || !self.script.keep_alive + { + return Ok(()); + } + } } - if behavior == Behavior::IgnoreRange { + /// Counts a request, and returns how to respond to it: as the next of the + /// `scripted` behaviors, if any remain, and otherwise as `then`. + fn count( + &self, + counter: impl FnOnce(&mut Stats) -> &mut usize, + scripted: &[Behavior], + then: Behavior, + ) -> Behavior { + let mut state = self.state.lock(); + let count = counter(&mut state.stats); + let behavior = scripted.get(*count).copied().unwrap_or(then); + *count += 1; + drop(state); + self.changed.notify_all(); + behavior + } + + /// Holds the connection open without sending anything, until the client + /// closes it or the server is released. + /// + /// Returns false, since the connection cannot carry another request. + fn stall(&self, stream: &mut TcpStream) -> bool { + self.state.lock().stats.stalled += 1; + self.changed.notify_all(); + // Poll, rather than block, so that a release is noticed promptly. + let _ = stream.set_read_timeout(Some(Duration::from_millis(10))); + let mut buf = [0; 512]; + while !self.state.lock().released { + match stream.read(&mut buf) { + // A stalled server ignores anything more that the client sends. + Ok(n) if n > 0 => {} + Err(err) + if matches!( + err.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) => {} + // The client has closed or reset the connection. + _ => { + self.state.lock().stats.abandoned += 1; + self.changed.notify_all(); + break; + } + } + } + false + } + + /// Answers one request. Returns whether the connection can carry another. + fn respond(&self, stream: &mut TcpStream, request: &str) -> std::io::Result { + let content = &self.content; + let mut lines = request.lines(); + let method = lines + .next() + .and_then(|l| l.split(' ').next()) + .unwrap_or_default() + .to_owned(); + let close = if self.script.keep_alive { + "" + } else { + "Connection: close\r\n" + }; + + // `HttpBlob::new` probes the length with a HEAD request. + if method.eq_ignore_ascii_case("HEAD") { + match self.count( + |stats| &mut stats.heads, + &self.script.head, + Behavior::Correct, + ) { + Behavior::StallBeforeResponse | Behavior::StallMidBody => { + return Ok(self.stall(stream)); + } + Behavior::ResetBeforeResponse | Behavior::ResetMidBody => return Ok(reset(stream)), + Behavior::ServiceUnavailable => return service_unavailable(stream, close), + _ => {} + } + write!( + stream, + "HTTP/1.1 200 OK\r\n\ + Content-Length: {}\r\n\ + Accept-Ranges: bytes\r\n\ + {close}\r\n", + content.len() + )?; + return Ok(true); + } + + let behavior = self.count(|stats| &mut stats.gets, &self.script.get, self.script.then); + + // Header names are case-insensitive, and hyper does not necessarily send + // them in the case they were built with. + let range = lines.find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("range") + .then(|| value.trim().strip_prefix("bytes=").map(str::to_owned)) + .flatten() + }); + let Some(range) = range else { + write!( + stream, + "HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n" + )?; + return Ok(false); + }; + let (start, end) = range.split_once('-').unwrap(); + let start: usize = start.parse().unwrap(); + // The range is inclusive at both ends. + let end: usize = end.parse().unwrap(); + + match behavior { + Behavior::StallBeforeResponse => return Ok(self.stall(stream)), + Behavior::ResetBeforeResponse => return Ok(reset(stream)), + Behavior::ServiceUnavailable => return service_unavailable(stream, close), + _ => {} + } + + if behavior == Behavior::RangeNotSatisfiable || start >= content.len() { + write!( + stream, + "HTTP/1.1 416 Range Not Satisfiable\r\n\ + Content-Range: bytes */{}\r\n\ + Content-Length: 0\r\n\ + {close}\r\n", + content.len() + )?; + return Ok(true); + } + + if behavior == Behavior::IgnoreRange { + write!( + stream, + "HTTP/1.1 200 OK\r\n\ + Content-Length: {}\r\n\ + {close}\r\n", + content.len() + )?; + stream.write_all(content)?; + return Ok(true); + } + + let end = end.min(content.len() - 1); + let full = &content[start..=end]; + + // `ShortBody` sends a well-formed response that is simply shorter than + // was asked for -- the Content-Length matches what is actually sent. + // That is precisely the case the `Blob` contract is about: the client + // sees a successful response, and must not mistake a partly filled + // buffer for success. A one-byte range cannot be shortened this way, + // since an empty body has no `Content-Range` that describes it, so send + // it in full. + let body = if behavior == Behavior::ShortBody && full.len() > 1 { + &full[..full.len() - 1] + } else { + full + }; + write!( stream, - "HTTP/1.1 200 OK\r\n\ + "HTTP/1.1 206 Partial Content\r\n\ + Content-Range: bytes {}-{}/{}\r\n\ Content-Length: {}\r\n\ - Connection: close\r\n\r\n", - content.len() + {close}\r\n", + start, + start + body.len() - 1, + content.len(), + body.len(), )?; - return stream.write_all(content); + + // Unlike `ShortBody`, the Content-Length above promises the full body, + // so sending less and then hanging up, or stalling, leaves the client + // with a truncated message rather than a well-formed short one. + let half = &body[..body.len() / 2]; + match behavior { + Behavior::CloseMidBody => { + stream.write_all(half)?; + Ok(false) + } + Behavior::StallMidBody => { + stream.write_all(half)?; + Ok(self.stall(stream)) + } + Behavior::ResetMidBody => { + stream.write_all(half)?; + Ok(reset(stream)) + } + Behavior::SlowBody { pieces, interval } => { + for piece in body.chunks(body.len().div_ceil(pieces.max(1))) { + std::thread::sleep(interval); + stream.write_all(piece)?; + } + Ok(true) + } + _ => { + stream.write_all(body)?; + Ok(true) + } + } } +} - let end = end.min(content.len() - 1); - let full = &content[start..=end]; - - // `ShortBody` sends a well-formed response that is simply shorter than was - // asked for -- the Content-Length matches what is actually sent. That is - // precisely the case the `Blob` contract is about: the client sees a - // successful response, and must not mistake a partly filled buffer for - // success. A one-byte range cannot be shortened this way, since an empty - // body has no `Content-Range` that describes it, so send it in full. - let body = if behavior == Behavior::ShortBody && full.len() > 1 { - &full[..full.len() - 1] - } else { - full - }; +/// Makes closing the connection send a reset rather than a FIN: with a zero +/// linger timeout, closing a socket discards any unsent data and aborts the +/// connection. +/// +/// Returns false, since the connection cannot carry another request. +fn reset(stream: &mut TcpStream) -> bool { + let _ = socket2::SockRef::from(&*stream).set_linger(Some(Duration::ZERO)); + false +} +/// Answers `503 Service Unavailable`. Returns whether the connection can carry +/// another request. +fn service_unavailable(stream: &mut TcpStream, close: &str) -> std::io::Result { write!( stream, - "HTTP/1.1 206 Partial Content\r\n\ - Content-Range: bytes {}-{}/{}\r\n\ - Content-Length: {}\r\n\ - Connection: close\r\n\r\n", - start, - start + body.len() - 1, - content.len(), - body.len(), + "HTTP/1.1 503 Service Unavailable\r\n\ + Content-Length: 0\r\n\ + {close}\r\n" )?; - - if behavior == Behavior::CloseMidBody { - // Unlike `ShortBody`, the Content-Length above promises the full body. - // Sending less and hanging up leaves the client with a truncated - // message rather than a well-formed short one. - stream.write_all(&body[..body.len() / 2])?; - return Ok(()); - } - - stream.write_all(body) + Ok(true) } /// Reads up to and including the blank line that terminates the headers. diff --git a/vm/devices/storage/disk_blob/src/tests/mod.rs b/vm/devices/storage/disk_blob/src/tests/mod.rs index 5794b4353c8..1a5f05c731c 100644 --- a/vm/devices/storage/disk_blob/src/tests/mod.rs +++ b/vm/devices/storage/disk_blob/src/tests/mod.rs @@ -6,18 +6,29 @@ mod http_server; use crate::BlobDisk; +use crate::blob::Blob; use crate::blob::file::FileBlob; use crate::blob::http::HttpBlob; +use crate::blob::http::HttpBlobOptions; use disk_backend::Disk; use disk_backend::DiskError; use guestmem::GuestMemory; use http_server::Behavior; +use http_server::Script; use http_server::TestHttpServer; use pal_async::async_test; use scsi_buffers::OwnedRequestBuffers; use std::io::Read; use std::io::Seek; use std::io::SeekFrom; +use std::sync::Arc; +use std::sync::mpsc; +use std::task::Context; +use std::task::Waker; +use std::thread::JoinHandle; +use std::time::Duration; +use std::time::Instant; +use test_with_tracing::test; const SECTOR_SIZE: usize = 512; const DISK_SIZE: u64 = 1024 * 1024; @@ -150,3 +161,617 @@ async fn misbehaving_server_never_reports_success() { } } } + +/// How long a fault test waits for a request. One still pending by then is +/// taken to be hung for good, which is the failure these tests exist to catch. +const LIMIT: Duration = Duration::from_secs(10); + +/// How long a fault test's options let a stall last before they time it out. +/// +/// A request that the test server answers can take more than a second on a +/// heavily loaded machine, so a test that expects such a request to succeed +/// under the same timeout needs this margin. No test waits out more than two +/// stalls, so that each stays well within [`LIMIT`]. +const STALL_TIMEOUT: Duration = Duration::from_secs(2); + +/// Content in which every offset is distinguishable from its neighbors, so that +/// data from the wrong place does not pass for the right data. +fn content() -> Vec { + (0..DISK_SIZE).map(|i| (i % 251) as u8).collect() +} + +/// Options under which no request times out before the test stops waiting for +/// it, for the tests in which nothing stalls, so that a slow machine does not +/// turn a request that should succeed into a retry. +fn patient_options() -> HttpBlobOptions { + HttpBlobOptions { + response_timeout: LIMIT, + body_idle_timeout: LIMIT, + operation_timeout: LIMIT, + max_attempts: 3, + initial_backoff: Duration::from_millis(10), + max_backoff: Duration::from_millis(40), + ..Default::default() + } +} + +/// Options that time out a stall before the response after [`STALL_TIMEOUT`]. +fn response_stall_options() -> HttpBlobOptions { + HttpBlobOptions { + response_timeout: STALL_TIMEOUT, + ..patient_options() + } +} + +/// Options that time out a stall in the body after [`STALL_TIMEOUT`], but wait +/// for the response headers as long as the test does, so that every attempt +/// gets as far as the stall. +fn body_stall_options() -> HttpBlobOptions { + HttpBlobOptions { + body_idle_timeout: STALL_TIMEOUT, + ..patient_options() + } +} + +/// A future running on a thread of its own. See [`background`]. +struct Background { + recv: mpsc::Receiver, + thread: JoinHandle<()>, +} + +/// Runs `fut` on a thread of its own. +/// +/// The test's thread then only waits for the output, for at most [`LIMIT`], so +/// that a request that never completes fails the test rather than hanging the +/// test runner. +fn background(fut: impl 'static + Send + Future) -> Background { + let (send, recv) = mpsc::channel(); + let thread = std::thread::spawn(move || { + let output = pal_async::DefaultPool::run_with(async move |_| fut.await); + // The test may have stopped waiting. + let _ = send.send(output); + }); + Background { recv, thread } +} + +impl Background { + /// Waits up to [`LIMIT`] for the output, panicking if there is none. + fn wait(self, what: &str) -> T { + match self.recv.recv_timeout(LIMIT) { + Ok(output) => output, + Err(mpsc::RecvTimeoutError::Timeout) => { + panic!("{what} did not complete within {LIMIT:?}") + } + // The thread panicked. Report that panic rather than this one. + Err(mpsc::RecvTimeoutError::Disconnected) => { + std::panic::resume_unwind(self.thread.join().unwrap_err()) + } + } + } +} + +/// Runs `fut` to completion on a thread of its own, failing the test if that +/// takes longer than [`LIMIT`]. +fn bounded(what: &str, fut: impl 'static + Send + Future) -> T { + background(fut).wait(what) +} + +/// Opens the blob that `server` publishes and reads `len` bytes at `offset` +/// from it, within [`LIMIT`]. +fn read_blob( + server: &TestHttpServer, + options: HttpBlobOptions, + offset: u64, + len: usize, +) -> (HttpBlob, std::io::Result>) { + let url = server.url(); + bounded("read", async move { + let blob = HttpBlob::with_options(&url, options).await.unwrap(); + let mut buf = vec![0; len]; + let r = blob.read(&mut buf, offset).await.map(|()| buf); + (blob, r) + }) +} + +/// Returns the JSON for `path` in the blob's inspection output. +fn inspect_json(blob: &HttpBlob, path: &str) -> String { + inspect::inspect(path, blob).results().json().to_string() +} + +/// Reads through a server that fails the first GET requests as `failures` says, +/// with `options`, and checks that the read recovers: one retry per failure, +/// the right data, and every stalled connection closed rather than leaked or +/// reused. +fn check_recovers(options: HttpBlobOptions, failures: &[Behavior]) { + let content = content(); + let server = TestHttpServer::with_script( + content.clone(), + Script { + get: failures.to_vec(), + ..Default::default() + }, + ); + let (offset, len) = (12345, 64 * 1024); + let (_blob, r) = read_blob(&server, options, offset, len); + let data = r.unwrap_or_else(|err| panic!("{failures:?}: read failed: {err:?}")); + assert!( + data == content[offset as usize..][..len], + "{failures:?}: read returned the wrong data" + ); + assert_eq!( + server.stats().gets, + failures.len() + 1, + "{failures:?}: wrong number of attempts" + ); + server.wait_for("the stalled connections to be closed", LIMIT, |stats| { + stats.abandoned == stats.stalled + }); +} + +/// A server, or the network path to it, that goes silent before the response. +#[test] +fn stall_before_response_is_retried() { + check_recovers(response_stall_options(), &[Behavior::StallBeforeResponse]); +} + +/// A server, or the network path to it, that goes silent part way through the +/// body. +#[test] +fn stall_mid_body_is_retried() { + check_recovers(body_stall_options(), &[Behavior::StallMidBody]); +} + +#[test] +fn reset_before_response_is_retried() { + check_recovers(patient_options(), &[Behavior::ResetBeforeResponse]); +} + +#[test] +fn reset_mid_body_is_retried() { + check_recovers(patient_options(), &[Behavior::ResetMidBody]); +} + +#[test] +fn close_mid_body_is_retried() { + check_recovers(patient_options(), &[Behavior::CloseMidBody]); +} + +#[test] +fn service_unavailable_is_retried() { + check_recovers(patient_options(), &[Behavior::ServiceUnavailable]); +} + +/// Every attempt is bounded and retried alike, up to the attempt limit. +#[test] +fn consecutive_failures_are_retried() { + check_recovers( + body_stall_options(), + &[Behavior::ResetBeforeResponse, Behavior::StallMidBody], + ); +} + +/// A guest waits for its disk reads, typically without a timeout of its own, +/// so a read that never completes hangs the guest. When the server never +/// recovers, the read must fail instead, in bounded time. +#[test] +fn persistent_stall_fails_disk_read() { + let server = TestHttpServer::new(content(), Behavior::StallMidBody); + let url = server.url(); + // Two stalled attempts are enough to show that the retry is bounded too. + let options = HttpBlobOptions { + max_attempts: 2, + ..body_stall_options() + }; + let attempts = options.max_attempts as usize; + let r = bounded("disk read", async move { + let blob = HttpBlob::with_options(&url, options).await.unwrap(); + let disk = Disk::new(BlobDisk::new(blob)).unwrap(); + let mem = GuestMemory::allocate(SECTOR_SIZE * 8); + disk.read_vectored( + &OwnedRequestBuffers::linear(0, SECTOR_SIZE * 8, true).buffer(&mem), + 0, + ) + .await + }); + match r { + Err(DiskError::Io(err)) if err.kind() == std::io::ErrorKind::TimedOut => {} + r => panic!("expected a timeout, got {r:?}"), + } + assert_eq!(server.stats().gets, attempts); + server.wait_for("the stalled connections to be closed", LIMIT, |stats| { + stats.abandoned == attempts + }); +} + +/// A response that keeps arriving is not a stall, however long it takes. +#[test] +fn slow_body_is_not_a_stall() { + let content = content(); + let options = HttpBlobOptions { + response_timeout: STALL_TIMEOUT, + ..body_stall_options() + }; + // Each pause is well within the idle timeout, but the body as a whole takes + // longer than either timeout. + let interval = options.body_idle_timeout / 4; + let server = TestHttpServer::new( + content.clone(), + Behavior::SlowBody { + pieces: 8, + interval, + }, + ); + let start = Instant::now(); + let (_blob, r) = read_blob(&server, options.clone(), 0, 64 * 1024); + let elapsed = start.elapsed(); + assert!(r.unwrap() == content[..64 * 1024], "wrong data"); + assert_eq!(server.stats().gets, 1, "the slow response was retried"); + assert!( + elapsed > options.body_idle_timeout.max(options.response_timeout), + "the response was not slow: {elapsed:?}" + ); +} + +/// A response that keeps arriving, but too slowly ever to finish, is bounded by +/// the operation timeout instead. +#[test] +fn endless_body_is_bounded_by_operation_timeout() { + let options = HttpBlobOptions { + operation_timeout: STALL_TIMEOUT * 2, + ..body_stall_options() + }; + // One byte at a time: the body would take minutes. + let server = TestHttpServer::new( + content(), + Behavior::SlowBody { + pieces: 4096, + interval: options.body_idle_timeout / 10, + }, + ); + let start = Instant::now(); + let (_blob, r) = read_blob(&server, options.clone(), 0, 4096); + let elapsed = start.elapsed(); + match r { + Err(err) if err.kind() == std::io::ErrorKind::TimedOut => {} + r => panic!("expected a timeout, got {:?}", r.map(|_| ())), + } + assert!( + elapsed >= options.operation_timeout, + "timed out early: {elapsed:?}" + ); + assert_eq!( + server.stats().gets, + 1, + "retried after the operation timeout" + ); +} + +/// Opening the blob probes it with a HEAD request, which must recover from a +/// stall too. +#[test] +fn stalled_head_is_retried() { + let server = TestHttpServer::with_script( + content(), + Script { + head: vec![Behavior::StallBeforeResponse], + ..Default::default() + }, + ); + let url = server.url(); + let blob = bounded("open", async move { + HttpBlob::with_options(&url, response_stall_options()) + .await + .unwrap() + }); + assert_eq!(blob.len(), DISK_SIZE); + assert_eq!(server.stats().heads, 2); +} + +#[test] +fn persistent_head_stall_fails_open() { + // Two stalled attempts are enough to show that the retry is bounded too. + let options = HttpBlobOptions { + max_attempts: 2, + ..response_stall_options() + }; + let attempts = options.max_attempts as usize; + let server = TestHttpServer::with_script( + content(), + Script { + head: vec![Behavior::StallBeforeResponse; attempts + 1], + ..Default::default() + }, + ); + let url = server.url(); + let r = bounded("open", async move { + HttpBlob::with_options(&url, options).await.map(drop) + }); + assert!(r.is_err(), "opened a blob whose server never answered"); + assert_eq!(server.stats().heads, attempts); +} + +/// What a [`silent_server`] sees. +#[derive(Debug)] +enum SilentEvent { + /// It accepted a connection. + Accepted, + /// The client closed a connection, having first sent `first`, if it sent + /// anything at all. + Closed { first: Option }, +} + +/// Starts a server on the loopback interface that accepts connections but +/// never sends a byte on them, as a server, or the network path to it, that +/// goes silent once the TCP handshake is done. Returns its port, and a channel +/// on which it reports each connection that it accepts and each that the +/// client closes. +/// +/// Like [`TestHttpServer`], it runs until the process exits. +fn silent_server() -> (u16, mpsc::Receiver) { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let (send, recv) = mpsc::channel(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { break }; + if send.send(SilentEvent::Accepted).is_err() { + // The test has finished. + break; + } + let send = send.clone(); + std::thread::spawn(move || { + let mut first = None; + let mut buf = [0; 4096]; + // Take in whatever the client sends, and answer none of it. + loop { + match stream.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(_) => { + first.get_or_insert(buf[0]); + } + } + } + let _ = send.send(SilentEvent::Closed { first }); + }); + } + }); + (port, recv) +} + +/// The TLS record type of handshake messages, with which the ClientHello that +/// opens a TLS connection begins. +const TLS_HANDSHAKE_RECORD: u8 = 0x16; + +/// Blobs are normally served over HTTPS, so each new connection begins with a +/// TLS handshake, inside the connector and before any request is sent. When +/// the server, or the network path to it, goes silent before answering the +/// ClientHello, that stall must be bounded like the others: the open fails +/// with a timeout after its attempts, each on a connection of its own that is +/// closed when the attempt gives up, rather than waiting for good. +#[test] +fn stalled_tls_handshake_fails_open() { + let (port, events) = silent_server(); + // Two stalled attempts are enough to show that the retry is bounded too. + let options = HttpBlobOptions { + max_attempts: 2, + ..response_stall_options() + }; + let attempts = options.max_attempts as usize; + let url = format!("https://127.0.0.1:{port}/blob"); + let r = bounded("open", async move { + HttpBlob::with_options(&url, options).await.map(drop) + }); + let err = r.expect_err("opened a blob whose server never answered the TLS handshake"); + assert!( + err.downcast_ref::() + .is_some_and(|err| err.kind() == std::io::ErrorKind::TimedOut), + "expected a timeout, got: {err:#}" + ); + + let mut accepted = 0; + let mut closed = 0; + while closed < attempts { + match events.recv_timeout(LIMIT) { + Ok(SilentEvent::Accepted) => accepted += 1, + Ok(SilentEvent::Closed { first }) => { + assert_eq!( + first, + Some(TLS_HANDSHAKE_RECORD), + "a connection did not begin a TLS handshake" + ); + closed += 1; + } + Err(_) => panic!("only {closed} of {attempts} stalled connections were closed"), + } + } + // The server reports each connection's acceptance before its close, so + // every connection that has been closed has been counted. + assert_eq!(accepted, attempts, "wrong number of connections"); +} + +/// A connection whose request timed out must be closed, not returned to the +/// pool, where a later request would find it and stall in turn. +#[test] +fn stalled_connection_is_not_reused() { + let content = content(); + let server = TestHttpServer::with_script( + content.clone(), + Script { + get: vec![Behavior::Correct, Behavior::StallBeforeResponse], + keep_alive: true, + ..Default::default() + }, + ); + let url = server.url(); + let (blob, results) = bounded("reads", async move { + let blob = HttpBlob::with_options(&url, response_stall_options()) + .await + .unwrap(); + let mut results = Vec::new(); + for i in 0..3 { + let mut buf = vec![0; 4096]; + results.push(blob.read(&mut buf, i * 4096).await.map(|()| buf)); + } + (blob, results) + }); + for (i, r) in results.into_iter().enumerate() { + let data = r.unwrap_or_else(|err| panic!("read {i} failed: {err:?}")); + assert!(data == content[i * 4096..][..4096], "read {i}: wrong data"); + } + // Only the stalled request was retried. + assert_eq!(inspect_json(&blob, "retries"), "1"); + assert_eq!(inspect_json(&blob, "timeouts"), "1"); + assert_eq!(server.stats().gets, 4); + server.wait_for("the stalled connection to be closed", LIMIT, |stats| { + stats.abandoned == 1 + }); +} + +/// Inspection shows what a request that is taking long is waiting for, so that +/// a hung guest's disk can be told apart from a slow one. +#[test] +fn inspect_shows_stalled_request() { + let content = content(); + let server = TestHttpServer::with_script( + content.clone(), + Script { + get: vec![Behavior::StallBeforeResponse], + ..Default::default() + }, + ); + let url = server.url(); + // Patient, so that the stall does not time out while it is inspected. + // Releasing the server ends it instead. + let options = patient_options(); + let blob = Arc::new(bounded("open", async move { + HttpBlob::with_options(&url, options).await.unwrap() + })); + let read = background({ + let blob = blob.clone(); + async move { + let mut buf = vec![0; 4096]; + blob.read(&mut buf, 8192).await.map(|()| buf) + } + }); + server.wait_for("the request to stall", LIMIT, |stats| stats.stalled == 1); + let in_flight = inspect_json(&blob, "in_flight"); + for expected in [ + r#""request":"GET bytes=8192-12287""#, + r#""attempt":1"#, + r#""state":"awaiting_response""#, + ] { + assert!( + in_flight.contains(expected), + "{expected} not in {in_flight}" + ); + } + + // Ending the stall fails the attempt, and the retry succeeds. + server.release(); + let data = read.wait("read").unwrap(); + assert!(data == content[8192..][..4096], "wrong data"); + assert_eq!(inspect_json(&blob, "in_flight"), "{}"); + assert_eq!(inspect_json(&blob, "retries"), "1"); + assert_eq!(inspect_json(&blob, "timeouts"), "0"); +} + +/// Dropping a read cancels its request at once: the connection is closed, the +/// request leaves `in_flight`, and it is not retried. Otherwise a read that its +/// caller has given up on would hold its connection, and might be retried, +/// until the request's own timeouts ended it, two minutes later by default. +#[test] +fn dropped_read_is_cancelled() { + let server = TestHttpServer::new(content(), Behavior::StallBeforeResponse); + let url = server.url(); + // Patient, so that nothing times out, and so nothing is retried, before + // the read is dropped and its cancellation checked. + let options = patient_options(); + let blob = bounded("open", async move { + HttpBlob::with_options(&url, options).await.unwrap() + }); + + // The first poll starts the request, which runs on the blob's tokio + // runtime. + let mut buf = vec![0; 4096]; + let mut read = blob.read(&mut buf, 8192); + let poll = read.as_mut().poll(&mut Context::from_waker(Waker::noop())); + assert!(poll.is_pending(), "the read did not wait for the server"); + server.wait_for("the request to stall", LIMIT, |stats| stats.stalled == 1); + let in_flight = inspect_json(&blob, "in_flight"); + assert!( + in_flight.contains(r#""request":"GET bytes=8192-12287""#), + "the stalled read is not in flight: {in_flight}" + ); + + drop(read); + + // Well within the request's own timeouts, so that only the read's + // cancellation can close its connection and end it in time. + let bound = LIMIT / 2; + server.wait_for( + "the dropped read's connection to be closed", + bound, + |stats| stats.abandoned == 1, + ); + let deadline = Instant::now() + bound; + while inspect_json(&blob, "in_flight") != "{}" { + assert!( + Instant::now() < deadline, + "the dropped read is still in flight" + ); + std::thread::sleep(Duration::from_millis(10)); + } + // With the request gone, nothing is left to retry it, so these counts are + // final: one connection and one request each for the open and the read. + let stats = server.stats(); + assert_eq!(stats.gets, 1, "the dropped read was retried"); + assert_eq!( + stats.connections, 2, + "the dropped read opened another connection" + ); +} + +/// A failure that a new attempt would only repeat is returned at once. +#[test] +fn permanent_errors_are_not_retried() { + for behavior in [ + Behavior::ShortBody, + Behavior::IgnoreRange, + Behavior::RangeNotSatisfiable, + ] { + let server = TestHttpServer::new(content(), behavior); + let (_blob, r) = read_blob(&server, patient_options(), 0, 4096); + assert!(r.is_err(), "{behavior:?}: reported success"); + assert_eq!(server.stats().gets, 1, "{behavior:?}: retried"); + } +} + +/// A range whose last byte would lie past the largest offset that a request +/// can name is refused, without a request, rather than overflowing. +#[test] +fn range_past_the_largest_offset_is_refused() { + let server = TestHttpServer::new(content(), Behavior::Correct); + let (_blob, r) = read_blob(&server, patient_options(), u64::MAX - 100, 4096); + match r { + Err(err) if err.kind() == std::io::ErrorKind::InvalidInput => {} + r => panic!("expected InvalidInput, got {:?}", r.map(|_| ())), + } + assert_eq!(server.stats().gets, 0); +} + +/// A range that ends at the largest offset that a request can name is +/// requested like any other, rather than overflowing as its last offset is +/// computed. The server fails every attempt with 503, so that the read also +/// names the range in its warnings and its error. +#[test] +fn range_ending_at_the_largest_offset_is_requested() { + let server = TestHttpServer::new(content(), Behavior::ServiceUnavailable); + let options = patient_options(); + let attempts = options.max_attempts as usize; + let offset = u64::MAX - 4095; + let (_blob, r) = read_blob(&server, options, offset, 4096); + let err = r.map(|_| ()).expect_err("reported success"); + let range = format!("GET bytes={offset}-{}", u64::MAX); + assert!(err.to_string().contains(&range), "{range} not in {err}"); + assert_eq!(server.stats().gets, attempts); +}