diff --git a/Guide/src/reference/openvmm/management/cli.md b/Guide/src/reference/openvmm/management/cli.md index aa0ae46c22d..f8c214630dd 100644 --- a/Guide/src/reference/openvmm/management/cli.md +++ b/Guide/src/reference/openvmm/management/cli.md @@ -237,8 +237,10 @@ describes the source definitions. with `--restore-network-policy`. Omitting the option preserves the existing guest-gateway-to-host-loopback mapping, from the IPv4 gateway to host IPv4 loopback and from the IPv6 - gateway to host IPv6 loopback, without publishing guest ports. `deny` blocks - general gateway and host-local destinations and rejects every + gateway to host IPv6 loopback, without publishing guest ports. The guest + then reaches every host-loopback listener that its egress policy permits, + which is why microVM consoles take no TCP endpoint; see `--virtio-console`. + `deny` blocks general gateway and host-local destinations and rejects every `--host-loopback-forward`. `--network-proxy ` preserves one exact proxy endpoint when @@ -1038,17 +1040,16 @@ Serial devices can be configured to appear as different devices inside the guest A microVM accepts these explicit attachment policies: - * `listen=PATH` or `listen=tcp:IP:PORT`: save the canonical endpoint and - recreate the optional listener on restore. Unix sockets must be beside the - snapshot directory. Windows pipes use the - `//./pipe/openvmm-microvm-` namespace. TCP ports must be nonzero. - TCP addresses must be loopback addresses. A restore caller can resupply - `--virtio-console listen=` with `--restore-snapshot` to - replace the saved listener identity while preserving the attachment - contract. Omitting the option recreates the saved listener endpoint. - * `connect=PATH` or `connect=tcp:IP:PORT`: require a client connection before - vCPUs start. Cold boot and restore use a five-second timeout. The - restore command must explicitly resupply the matching client attachment. + * `listen=PATH`: save the canonical endpoint and recreate the optional + listener on restore. Unix sockets must be beside the snapshot directory. + Windows pipes use the `//./pipe/openvmm-microvm-` namespace. A + restore caller can resupply `--virtio-console listen=` with + `--restore-snapshot` to replace the saved listener identity while + preserving the attachment contract. Omitting the option recreates the + saved listener endpoint. + * `connect=PATH`: require a client connection before vCPUs start. Cold boot + and restore use a five-second timeout. The restore command must explicitly + resupply the matching client attachment. * `console`: require the restore caller to supply the same inherited terminal attachment. Portb recovery output moves to stderr while the terminal is attached to `hvc1`. A guest-requested exit writes the boot console output @@ -1056,6 +1057,14 @@ Serial devices can be configured to appear as different devices inside the guest * `none`: keep the device present and explicitly discard guest TX while disconnected. + A microVM refuses TCP endpoints, `listen=tcp:IP:PORT` and + `connect=tcp:IP:PORT`, and a snapshot whose boot console has one. Unless + `--host-loopback deny` is given, a guest's gateway maps the guest's TCP flows + onto host loopback, so any guest on the host whose egress policy permits the + port could connect to a loopback TCP listener, its own console's or another + VM's, or to the listener that a client endpoint connects to. The boot console + does not authenticate its client, so such a guest would drive the console. + A generic byte stream guarantees no replay up to OpenVMM's backend write boundary; it cannot prove that the remote application consumed bytes without its own acknowledgment protocol. @@ -1138,8 +1147,8 @@ The `BACKEND` argument is the same for all serial devices: up to listen on the given path. Serial input and output is relayed to this pipe/socket. * `listen=tcp:IP:PORT`: As with `listen=PATH`, but listen for TCP - connections on the given IP address and port. A microVM requires a - loopback IP such as `127.0.0.1` or `::1`. + connections on the given IP address and port. A microVM's consoles + refuse it and `connect=tcp:IP:PORT`; see `--virtio-console`. * `connect=PATH`: Connect to an existing named pipe or Unix socket. * `connect=tcp:IP:PORT`: Connect to an existing TCP listener. diff --git a/openvmm/openvmm_entry/src/cli_args/microvm.rs b/openvmm/openvmm_entry/src/cli_args/microvm.rs index 57ee6d6a840..df556e109c6 100644 --- a/openvmm/openvmm_entry/src/cli_args/microvm.rs +++ b/openvmm/openvmm_entry/src/cli_args/microvm.rs @@ -1027,17 +1027,23 @@ impl Options { "microVM requires virtio-console on its fixed MMIO transport" ); if let Some(console) = &self.virtio_console { + anyhow::ensure!( + !matches!( + console, + SerialConfigCli::Tcp(_) | SerialConfigCli::ConnectTcp(_) + ), + "{}", + crate::microvm::MICROVM_CONSOLE_TCP_REFUSED + ); anyhow::ensure!( matches!( console, SerialConfigCli::Pipe(_) - | SerialConfigCli::Tcp(_) | SerialConfigCli::ConnectPipe(_) - | SerialConfigCli::ConnectTcp(_) | SerialConfigCli::Console | SerialConfigCli::None ), - "microVM virtio-console requires listen=..., connect=..., console, or none" + "microVM virtio-console requires listen=PATH, connect=PATH, console, or none" ); } anyhow::ensure!( @@ -2440,15 +2446,39 @@ mod tests { .unwrap(); valid_mshv.validate_microvm_options().unwrap(); } + let console_endpoint = if cfg!(windows) { + "listen=//./pipe/openvmm-microvm-console-test" + } else { + "listen=console.sock" + }; let valid_console = Options::try_parse_from([ "openvmm", "--machine", "microvm", "--virtio-console", - "listen=tcp:127.0.0.1:5555", + console_endpoint, ]) .unwrap(); valid_console.validate_microvm_options().unwrap(); + // Guests reach loopback TCP through their gateways, so the boot + // console takes no TCP endpoint. + for tcp in ["listen=tcp:127.0.0.1:5555", "connect=tcp:127.0.0.1:5555"] { + let error = Options::try_parse_from([ + "openvmm", + "--machine", + "microvm", + "--virtio-console", + tcp, + ]) + .unwrap() + .validate_microvm_options() + .unwrap_err(); + assert_eq!( + error.to_string(), + crate::microvm::MICROVM_CONSOLE_TCP_REFUSED, + "{tcp}" + ); + } let valid_control_console = Options::try_parse_from([ "openvmm", "--machine", @@ -2570,7 +2600,7 @@ mod tests { "--machine", "microvm", "--virtio-console", - "listen=tcp:127.0.0.1:5555", + console_endpoint, "--virtio-console-pcie-port", "port0", ], diff --git a/openvmm/openvmm_entry/src/microvm/config.rs b/openvmm/openvmm_entry/src/microvm/config.rs index c3969b9c5cf..d759623c89c 100644 --- a/openvmm/openvmm_entry/src/microvm/config.rs +++ b/openvmm/openvmm_entry/src/microvm/config.rs @@ -306,7 +306,6 @@ impl<'a> MicrovmConfigBuilder<'a> { self.resources.console_socket_cleanup = microvm_console_socket_cleanup(path)?; Some(backend) } - SerialConfigCli::Tcp(address) => Some(serial_io::bind_tcp_serial(&address)?), SerialConfigCli::ConnectPipe(path) => Some( serial_io::connect::connect_serial_with_timeout( &path, @@ -321,14 +320,6 @@ impl<'a> MicrovmConfigBuilder<'a> { ) })?, ), - SerialConfigCli::ConnectTcp(address) => { - Some(serial_io::connect::connect_tcp_serial( - &address, - Duration::from_millis( - openvmm_defs::microvm::MICROVM_CONSOLE_RECONNECT_TIMEOUT_MS, - ), - )?) - } SerialConfigCli::Console => { let (backend, _, relay) = setup_host_console( "virtio-console", diff --git a/openvmm/openvmm_entry/src/microvm/console.rs b/openvmm/openvmm_entry/src/microvm/console.rs index 317b57b6bf8..5f60188ca5e 100644 --- a/openvmm/openvmm_entry/src/microvm/console.rs +++ b/openvmm/openvmm_entry/src/microvm/console.rs @@ -26,6 +26,14 @@ pub(crate) const MICROVM_CONTROL_CONSOLE_STABLE_ID: &str = "console:microvm-cont pub(crate) const MICROVM_CONSOLE_ATTACHMENT_KIND: &str = "virtio-console"; pub(crate) const MICROVM_CONTROL_CONSOLE_ATTACHMENT_KIND: &str = "virtio-control-console"; +/// Why a microVM console refuses a TCP endpoint. Unless host loopback is +/// denied, a guest's gateway maps the guest's TCP flows onto host loopback, +/// so any guest on the host whose egress policy permits the port could +/// connect to a loopback TCP listener, its own console's or another VM's, or +/// to the listener that a client endpoint connects to. The boot console does +/// not authenticate its client, so that guest would drive the console. +pub(crate) const MICROVM_CONSOLE_TCP_REFUSED: &str = "microVM consoles do not accept TCP endpoints, which guests can reach through the host-loopback mapping of their gateways; use listen=PATH or connect=PATH with a Unix socket or named pipe"; + pub(super) fn microvm_console_attachments_share_endpoint( left: &openvmm_helpers::snapshot::microvm::SnapshotAttachment, right: &openvmm_helpers::snapshot::microvm::SnapshotAttachment, @@ -251,47 +259,8 @@ fn microvm_console_attachment_from_cli_with_identity( }, ) } - SerialConfigCli::Tcp(address) | SerialConfigCli::ConnectTcp(address) => { - anyhow::ensure!( - address.port() != 0, - "microVM console TCP endpoint requires a nonzero stable port" - ); - anyhow::ensure!( - address.ip().is_loopback(), - "microVM console TCP endpoints must use a loopback address" - ); - let is_client = matches!(config, SerialConfigCli::ConnectTcp(_)); - ( - if is_client { - SerialConfigCli::ConnectTcp(*address) - } else { - SerialConfigCli::Tcp(*address) - }, - VirtioConsoleBackendKind::Tcp, - "tcp", - address.to_string(), - if is_client { - VirtioConsoleAttachmentMode::Connect - } else { - VirtioConsoleAttachmentMode::Listen - }, - if is_client { - VirtioConsoleReconnectPolicy::ReconnectClient - } else { - VirtioConsoleReconnectPolicy::RecreateListener - }, - if is_client { - "reconnect-client" - } else { - "recreate-listener" - }, - is_client, - if is_client { - openvmm_defs::microvm::MICROVM_CONSOLE_RECONNECT_TIMEOUT_MS - } else { - 0 - }, - ) + SerialConfigCli::Tcp(_) | SerialConfigCli::ConnectTcp(_) => { + anyhow::bail!("{MICROVM_CONSOLE_TCP_REFUSED}") } SerialConfigCli::Console => ( SerialConfigCli::Console, @@ -316,7 +285,7 @@ fn microvm_console_attachment_from_cli_with_identity( 0, ), _ => anyhow::bail!( - "microVM virtio-console requires listen=..., connect=..., console, or none" + "microVM virtio-console requires listen=PATH, connect=PATH, console, or none" ), }; anyhow::ensure!( @@ -457,7 +426,6 @@ pub(crate) fn validate_microvm_console_attachment_namespace( "microVM console pipe is outside //./pipe/openvmm-microvm-" ); } - "tcp" => {} "provider" => {} "disconnected" => {} kind => anyhow::bail!("unsupported microVM console identity kind '{kind}'"), @@ -482,6 +450,12 @@ fn microvm_console_attachment_from_snapshot_with_identity( && attachment.length == 0, "snapshot has an invalid microVM console attachment" ); + // Older captures could record a TCP endpoint, which a restore would + // expose again. + anyhow::ensure!( + attachment.identity_kind != "tcp", + "snapshot microVM console has a TCP endpoint, which guests can reach through the host-loopback mapping of their gateways; capture the snapshot again with a Unix socket or named pipe" + ); let identity = std::str::from_utf8(&attachment.identity) .context("snapshot microVM console identity is not valid UTF-8")?; if attachment.reconnect_policy == "reconnect-client" { @@ -529,19 +503,9 @@ fn microvm_console_attachment_from_snapshot_with_identity( { SerialConfigCli::Pipe(PathBuf::from(identity)) } - ("recreate-listener", "tcp") => SerialConfigCli::Tcp( - identity - .parse() - .context("snapshot microVM console TCP identity is invalid")?, - ), ("reconnect-client", "unix-socket" | "named-pipe") if !control_console => { SerialConfigCli::ConnectPipe(PathBuf::from(identity)) } - ("reconnect-client", "tcp") => SerialConfigCli::ConnectTcp( - identity - .parse() - .context("snapshot microVM console TCP identity is invalid")?, - ), ("require-inherited-attachment", "provider") => requested .cloned() .context("snapshot requires --virtio-console console as a replacement attachment")?, @@ -823,7 +787,7 @@ mod tests { #[test] fn client_attachment_is_required_and_bounded() { - let requested = SerialConfigCli::ConnectTcp("127.0.0.1:5555".parse().unwrap()); + let (_directory, requested) = client_endpoint(); let (_, resource, snapshot) = microvm_console_attachment_from_cli(&requested).unwrap(); assert!(snapshot.required); assert_eq!(snapshot.reconnect_policy, "reconnect-client"); @@ -839,10 +803,30 @@ mod tests { assert!(microvm_console_attachment_from_snapshot(&snapshot, None).is_err()); let (restored, _, restored_snapshot) = microvm_console_attachment_from_snapshot(&snapshot, Some(&requested)).unwrap(); - assert!(matches!(restored, SerialConfigCli::ConnectTcp(_))); + assert!(matches!(restored, SerialConfigCli::ConnectPipe(_))); assert_eq!(restored_snapshot, snapshot); } + /// Returns a client endpoint of this host's kind, and the directory that + /// holds a Unix socket's path. + fn client_endpoint() -> (Option, SerialConfigCli) { + #[cfg(windows)] + { + ( + None, + SerialConfigCli::ConnectPipe(PathBuf::from( + "//./pipe/openvmm-microvm-client-console", + )), + ) + } + #[cfg(not(windows))] + { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("client.sock"); + (Some(directory), SerialConfigCli::ConnectPipe(path)) + } + } + #[cfg(unix)] #[test] fn listener_attachment_accepts_fresh_restore_identity_after_source_cleanup() { @@ -1006,19 +990,67 @@ mod tests { } #[test] - fn tcp_attachment_rejects_non_loopback_addresses() { - assert!( - microvm_console_attachment_from_cli(&SerialConfigCli::Tcp( - "0.0.0.0:5555".parse().unwrap() - )) - .is_err() - ); - assert!( - microvm_console_attachment_from_cli(&SerialConfigCli::ConnectTcp( - "192.0.2.1:5555".parse().unwrap() - )) - .is_err() - ); + fn tcp_attachments_are_refused() { + // A guest reaches host loopback through its gateway, so no address, + // loopback included, makes a TCP listener or its client safe. + for address in [ + "127.0.0.1:5555", + "[::1]:5555", + "0.0.0.0:5555", + "192.0.2.1:5555", + ] { + let address = address.parse().unwrap(); + for config in [ + SerialConfigCli::Tcp(address), + SerialConfigCli::ConnectTcp(address), + ] { + let error = microvm_console_attachment_from_cli(&config).unwrap_err(); + assert_eq!(error.to_string(), MICROVM_CONSOLE_TCP_REFUSED, "{config:?}"); + } + } + } + + #[test] + fn tcp_snapshot_attachments_are_refused_at_restore() { + let listener = openvmm_helpers::snapshot::microvm::SnapshotAttachment { + stable_id: MICROVM_CONSOLE_STABLE_ID.to_owned(), + kind: MICROVM_CONSOLE_ATTACHMENT_KIND.to_owned(), + required: false, + reconnect_policy: "recreate-listener".to_owned(), + identity_kind: "tcp".to_owned(), + identity: b"127.0.0.1:5555".to_vec(), + length: 0, + reconnect_timeout_ms: 0, + }; + let client = openvmm_helpers::snapshot::microvm::SnapshotAttachment { + required: true, + reconnect_policy: "reconnect-client".to_owned(), + reconnect_timeout_ms: openvmm_defs::microvm::MICROVM_CONSOLE_RECONNECT_TIMEOUT_MS, + ..listener.clone() + }; + let address = "127.0.0.1:5555".parse().unwrap(); + // Neither the saved endpoint nor a replacement of any kind restores. + for attachment in [&listener, &client] { + for requested in [ + None, + Some(SerialConfigCli::Tcp(address)), + Some(SerialConfigCli::ConnectTcp(address)), + Some(SerialConfigCli::None), + ] { + let error = + microvm_console_attachment_from_snapshot(attachment, requested.as_ref()) + .unwrap_err(); + assert!( + error.to_string().contains("TCP endpoint"), + "{}: {error}", + attachment.reconnect_policy + ); + } + assert!( + validate_microvm_console_attachment_namespace(attachment, Path::new("snapshot")) + .is_err() + ); + } } #[test] @@ -1033,13 +1065,8 @@ mod tests { .is_ok() ); assert!( - microvm_console_attachment_from_snapshot( - &snapshot, - Some(&SerialConfigCli::ConnectTcp( - "127.0.0.1:5555".parse().unwrap() - )) - ) - .is_err() + microvm_console_attachment_from_snapshot(&snapshot, Some(&SerialConfigCli::None)) + .is_err() ); } diff --git a/openvmm/openvmm_entry/src/microvm/mod.rs b/openvmm/openvmm_entry/src/microvm/mod.rs index a113ca6355a..fb905a4508f 100644 --- a/openvmm/openvmm_entry/src/microvm/mod.rs +++ b/openvmm/openvmm_entry/src/microvm/mod.rs @@ -18,6 +18,7 @@ pub(crate) use config::MicrovmConfigBuilder; pub(crate) use console::MICROVM_CONSOLE_ATTACHMENT_KIND; #[cfg(test)] pub(crate) use console::MICROVM_CONSOLE_STABLE_ID; +pub(crate) use console::MICROVM_CONSOLE_TCP_REFUSED; #[cfg(any(feature = "ttrpc", feature = "grpc"))] pub(crate) use console::MICROVM_CONTROL_CONSOLE_ATTACHMENT_KIND; #[cfg(any(feature = "ttrpc", feature = "grpc"))] diff --git a/openvmm/openvmm_entry/src/ttrpc/microvm.rs b/openvmm/openvmm_entry/src/ttrpc/microvm.rs index 6f42ac4b265..a48b5b9385d 100644 --- a/openvmm/openvmm_entry/src/ttrpc/microvm.rs +++ b/openvmm/openvmm_entry/src/ttrpc/microvm.rs @@ -951,10 +951,6 @@ impl CreateVm { crate::microvm::microvm_console_socket_cleanup(path)?; (backend, VirtioConsoleDisconnectPolicy::Retain) } - SerialConfigCli::Tcp(address) => ( - crate::serial_io::bind_tcp_serial(&address)?, - VirtioConsoleDisconnectPolicy::Retain, - ), SerialConfigCli::ConnectPipe(path) => ( crate::serial_io::connect::connect_serial_with_timeout( &path, @@ -970,15 +966,6 @@ impl CreateVm { })?, VirtioConsoleDisconnectPolicy::Retain, ), - SerialConfigCli::ConnectTcp(address) => ( - crate::serial_io::connect::connect_tcp_serial( - &address, - Duration::from_millis( - openvmm_defs::microvm::MICROVM_CONSOLE_RECONNECT_TIMEOUT_MS, - ), - )?, - VirtioConsoleDisconnectPolicy::Retain, - ), SerialConfigCli::None => ( DisconnectedSerialBackendHandle.into_resource(), VirtioConsoleDisconnectPolicy::Discard, diff --git a/openvmm/openvmm_helpers/src/snapshot/microvm.rs b/openvmm/openvmm_helpers/src/snapshot/microvm.rs index 7a593c8824e..a563b8a7774 100644 --- a/openvmm/openvmm_helpers/src/snapshot/microvm.rs +++ b/openvmm/openvmm_helpers/src/snapshot/microvm.rs @@ -1308,13 +1308,15 @@ pub fn microvm_machine_contract( }) .transpose()?; if let Some(attachment) = console_attachment { + // A boot console never has a TCP endpoint, which a guest could reach + // through the host-loopback mapping of its gateway. let policy_is_valid = match attachment.reconnect_policy.as_str() { "recreate-listener" => { !attachment.required && attachment.reconnect_timeout_ms == 0 && matches!( attachment.identity_kind.as_str(), - "unix-socket" | "named-pipe" | "tcp" + "unix-socket" | "named-pipe" ) } "reconnect-client" => { @@ -1323,7 +1325,7 @@ pub fn microvm_machine_contract( == openvmm_defs::microvm::MICROVM_CONSOLE_RECONNECT_TIMEOUT_MS && matches!( attachment.identity_kind.as_str(), - "unix-socket" | "named-pipe" | "tcp" + "unix-socket" | "named-pipe" ) } "require-inherited-attachment" => { @@ -2535,8 +2537,8 @@ mod tests { kind: "virtio-console".to_owned(), required: false, reconnect_policy: "recreate-listener".to_owned(), - identity_kind: "tcp".to_owned(), - identity: b"127.0.0.1:5555".to_vec(), + identity_kind: "unix-socket".to_owned(), + identity: b"/run/nvx/console.sock".to_vec(), length: 0, reconnect_timeout_ms: 0, } @@ -2667,6 +2669,12 @@ mod tests { } fn generated_console_contract() -> SnapshotMachineContract { + console_contract(microvm_console_attachment()).unwrap() + } + + /// Returns the contract of a snapshot whose boot console has the + /// attachment `console`. + fn console_contract(console: SnapshotAttachment) -> anyhow::Result { microvm_machine_contract( "whp", MICROVM_BOOT_LAYOUT_VERSION, @@ -2674,7 +2682,7 @@ mod tests { .to_owned(), None, None, - Some(microvm_console_attachment()), + Some(console), None, Vec::new(), 1, @@ -2698,7 +2706,6 @@ mod tests { crate::snapshot::time::tests::test_time_contract(), crate::snapshot::time::tests::test_cpu_profile(), ) - .unwrap() } fn generated_control_console_contract_with_attachment( @@ -3312,6 +3319,31 @@ mod tests { assert_eq!(contract.attachments, [microvm_console_attachment()]); } + #[test] + fn generated_microvm_console_contract_refuses_tcp_endpoints() { + // A guest reaches loopback TCP through its gateway's host-loopback + // mapping, so neither a listener nor a client console is on TCP. + let listener = SnapshotAttachment { + identity_kind: "tcp".to_owned(), + identity: b"127.0.0.1:5555".to_vec(), + ..microvm_console_attachment() + }; + let client = SnapshotAttachment { + required: true, + reconnect_policy: "reconnect-client".to_owned(), + reconnect_timeout_ms: openvmm_defs::microvm::MICROVM_CONSOLE_RECONNECT_TIMEOUT_MS, + ..listener.clone() + }; + for attachment in [listener, client] { + let policy = attachment.reconnect_policy.clone(); + let error = console_contract(attachment).unwrap_err(); + assert!( + error.to_string().contains("unsupported reconnect policy"), + "{policy}: {error}" + ); + } + } + #[test] fn generated_microvm_control_console_contract_has_distinct_fixed_abi() { let contract = generated_control_console_contract(); @@ -3795,7 +3827,7 @@ mod tests { manifest.vp_count = 1; manifest.machine_contract = Some(contract.clone()); let mut expected = contract; - expected.attachments[0].identity = b"127.0.0.1:6666".to_vec(); + expected.attachments[0].identity = b"/run/nvx/other-console.sock".to_vec(); let error = validate_microvm_machine_contract(&manifest, &expected).unwrap_err(); assert!(error.to_string().contains("attachment inventory")); } diff --git a/vm/devices/virtio/virtio_resources/src/console/attachment.rs b/vm/devices/virtio/virtio_resources/src/console/attachment.rs index a21452734e1..1462164c7d9 100644 --- a/vm/devices/virtio/virtio_resources/src/console/attachment.rs +++ b/vm/devices/virtio/virtio_resources/src/console/attachment.rs @@ -17,11 +17,13 @@ pub enum VirtioConsoleDisconnectPolicy { Retain, } +/// The kind of a console's host endpoint. A TCP endpoint is not among them: +/// a guest could reach a loopback TCP listener through the host-loopback +/// mapping of its gateway. #[derive(Copy, Clone, Debug, Eq, PartialEq, MeshPayload)] pub enum VirtioConsoleBackendKind { UnixSocket, NamedPipe, - Tcp, Inherited, Disconnected, }