[nexus] add an ExternalHttpClient for sending requests outside the rack

clients/ddm-admin-client/src/lib.rs

@@ -11,10 +11,10 @@ pub use ddm_admin_client::types;
 
 use ddm_admin_client::Client as InnerClient;
 use either::Either;
+use omicron_common::address::BOOTSTRAP_PREFIX;
+use omicron_common::address::BOOTSTRAP_SLED_SUBNET_PREFIX_LENGTH;
 use omicron_common::address::DDMD_PORT;
 use oxnet::Ipv6Net;
-use sled_hardware_types::underlay::BOOTSTRAP_MASK;
-use sled_hardware_types::underlay::BOOTSTRAP_PREFIX;
 use sled_hardware_types::underlay::BootstrapInterface;
 use slog::Logger;
 use slog_error_chain::SlogInlineError;
@@ -114,7 +114,8 @@ impl Client {
         Ok(prefixes.into_values().flat_map(|prefixes| {
             prefixes.into_iter().flat_map(|prefix| {
                 let mut segments = prefix.destination.addr().segments();
-                if prefix.destination.width() == BOOTSTRAP_MASK
+                if prefix.destination.width()
+                    == BOOTSTRAP_SLED_SUBNET_PREFIX_LENGTH
                     && segments[0] == BOOTSTRAP_PREFIX
                 {
                     Either::Left(interfaces.iter().map(move |interface| {

common/src/address.rs

@@ -19,6 +19,11 @@ use std::{
     sync::LazyLock,
 };
 
+/// The routing prefix length of the bootstrap network /40.
+///
+/// This is distinct from [`BOOTSTRAP_SLED_SUBNET_PREFIX_LENGTH`], which is the
+/// prefix length of the /64 subnet for an individual sled's bootstrap
+/// addresses.
 pub const BOOTSTRAP_SUBNET_PREFIX_LENGTH: u8 = 40;
 pub const AZ_PREFIX_LENGTH: u8 = 48;
 pub const RACK_PREFIX_LENGTH: u8 = 56;
@@ -449,6 +454,33 @@ pub const CP_SERVICES_RESERVED_ADDRESSES: u16 = 0xFFFF;
 /// with each sled in the Reconfigurator blueprint.
 pub const SLED_RESERVED_ADDRESSES: u16 = 2;
 
+/// Initial octets of IPv6 for bootstrap addresses.
+pub const BOOTSTRAP_PREFIX: u16 = 0xfdb0;
+
+/// IPv6 subnet for the *entire* bootstrap network.
+///
+/// This is a /16 subnet with the prefix [`BOOTSTRAP_PREFIX`]. Individual sled
+/// bootstrap networks are /64 subnets within this range, with the rest of the
+/// prefix constructed from the sled's MAC address. This constant is intended to
+/// be used for checking whether an address is a bootstrap address on *any*
+/// sled's bootstrap network.
+pub const BOOTSTRAP_NETWORK_SUBNET: Ipv6Net = Ipv6Net::new_unchecked(
+    Ipv6Addr::new(BOOTSTRAP_PREFIX, 0, 0, 0, 0, 0, 0, 0),
+    16,
+);
+
+/// IPv6 prefix length for bootstrap network sled subnets.
+///
+/// This is the length of the prefix for the bootstrap network subnet *of an
+/// individual sled*. The prefix begins with [`BOOTSTRAP_PREFIX`], and the rest
+/// of the /64 prefix is constructed from the sled's MAC address.
+///
+/// This is distinct from the prefix length of [`BOOTSTRAP_NETWORK_SUBNET`],
+/// which is the /16 subnet that contains *all* sled bootstrap networks, and
+/// from [`BOOTSTRAP_SUBNET_PREFIX_LENGTH`], which is the prefix length of the
+/// announced bootstrap route.
+pub const BOOTSTRAP_SLED_SUBNET_PREFIX_LENGTH: u8 = 64;
+
 /// Wraps an [`Ipv6Net`] with a compile-time prefix length.
 #[derive(
     Debug,
@@ -529,6 +561,25 @@ impl<'de, const N: u8> Deserialize<'de> for Ipv6Subnet<N> {
     }
 }
 
+/// A rack's underlay subnet, along with the AZ-wide underlay subnet
+/// containing it.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub struct UnderlaySubnets {
+    pub rack_subnet: Ipv6Subnet<RACK_PREFIX_LENGTH>,
+    pub az_subnet: Ipv6Subnet<AZ_PREFIX_LENGTH>,
+}
+
+impl UnderlaySubnets {
+    /// Constructs a new `UnderlaySubnets` from the rack subnet, widening it to
+    /// determine the AZ subnet.
+    pub fn new(rack_subnet: Ipv6Subnet<RACK_PREFIX_LENGTH>) -> Self {
+        Self {
+            rack_subnet,
+            az_subnet: Ipv6Subnet::new(rack_subnet.net().addr()),
+        }
+    }
+}
+
 /// Represents a subnet which may be used for contacting DNS services.
 #[derive(
     Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord,

nexus-config/src/nexus_config.rs

@@ -276,6 +276,33 @@ struct UnvalidatedTunables {
     load_timeout: Option<std::time::Duration>,
 }
 
+/// Whether HTTP clients for external services permit requests to loopback
+/// addresses.
+///
+/// This is an enum rather than a `bool`, so that if you want to turn on the
+/// test-only config, you have to type the string "yes_for_test_purposes_only"
+/// in the config file so you know what you're doing.
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    Default,
+    Deserialize,
+    Eq,
+    JsonSchema,
+    PartialEq,
+    Serialize,
+)]
+#[serde(rename_all = "snake_case")]
+pub enum TreatLoopbackAsExternal {
+    /// Loopback addresses are considered "external". This must only be used
+    /// in test environments.
+    YesForTestPurposesOnly,
+    /// Loopback addresses are rejected (the default).
+    #[default]
+    No,
+}
+
 /// Configuration for HTTP clients to external services.
 #[derive(
     Clone, Debug, Default, Deserialize, PartialEq, Serialize, JsonSchema,
@@ -285,6 +312,15 @@ pub struct ExternalHttpClientConfig {
     /// specified interface name.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub interface: Option<String>,
+    /// Whether external HTTP clients are permitted to make requests to
+    /// loopback addresses (and names in the special-use "localhost." zone).
+    ///
+    /// External HTTP clients normally refuse to make requests to any address
+    /// that isn't external to the rack, including loopback addresses. Test
+    /// environments, however, run their "external" servers on localhost, so
+    /// the test suite needs a way to turn that off.
+    #[serde(default)]
+    pub treat_loopback_as_external: TreatLoopbackAsExternal,
 }
 
 /// Tunable configuration parameters, intended for use in test environments or
@@ -1266,6 +1302,7 @@ mod test {
             external_dns_servers = [ "1.1.1.1", "9.9.9.9" ]
             [deployment.external_http_clients]
             interface = "opte0"
+            treat_loopback_as_external = "yes_for_test_purposes_only"
             [deployment.dropshot_external]
             bind_address = "10.1.2.3:4567"
             default_request_body_max_bytes = 1024
@@ -1414,6 +1451,7 @@ mod test {
                     ],
                     external_http_clients: ExternalHttpClientConfig {
                         interface: Some("opte0".to_string()),
+                        treat_loopback_as_external: TreatLoopbackAsExternal::YesForTestPurposesOnly,
                     },
                 },
                 pkg: PackageConfig {

nexus/Cargo.toml

@@ -131,6 +131,7 @@ tokio-util = { workspace = true, features = ["codec", "rt"] }
 tough.workspace = true
 trust-quorum-types.workspace = true
 tufaceous-artifact.workspace = true
+url.workspace = true
 usdt.workspace = true
 uuid.workspace = true
 
@@ -211,6 +212,7 @@ sp-sim.workspace = true
 strum.workspace = true
 subprocess.workspace = true
 term.workspace = true
+transient-dns-server.workspace = true
 tufaceous.workspace = true
 tufaceous-artifact.workspace = true
 tufaceous-lib.workspace = true

nexus/examples/config-second.toml

@@ -49,6 +49,13 @@ techport_external_server_port = 0
 # These are the DNS servers it should use.
 external_dns_servers = ["1.1.1.1", "9.9.9.9"]
 
+# Simulated/development deployments run "external" servers (e.g. webhook
+# receivers) on localhost, so external HTTP clients must be permitted to
+# connect to loopback addresses. This must never be enabled in production
+# configs.
+[deployment.external_http_clients]
+treat_loopback_as_external = "yes_for_test_purposes_only"
+
 [deployment.dropshot_external]
 # IP Address and TCP port on which to listen for the external API
 # This config file uses 12222 to avoid colliding with the usual 12220 that's

nexus/examples/config.toml

@@ -37,6 +37,13 @@ rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"
 # These are the DNS servers it should use.
 external_dns_servers = ["1.1.1.1", "9.9.9.9"]
 
+# Simulated/development deployments run "external" servers (e.g. webhook
+# receivers) on localhost, so external HTTP clients must be permitted to
+# connect to loopback addresses. This must never be enabled in production
+# configs.
+[deployment.external_http_clients]
+treat_loopback_as_external = "yes_for_test_purposes_only"
+
 [deployment.dropshot_external]
 # IP Address and TCP port on which to listen for the external API
 bind_address = "127.0.0.1:12220"

nexus/src/app/background/init.rs

@@ -147,6 +147,7 @@ use super::tasks::vpc_routes;
 use super::tasks::webhook_deliverator;
 use crate::Nexus;
 use crate::app::background::tasks::populate_switch_ports;
+use crate::app::external_client::ExternalHttpClient;
 use crate::app::oximeter::PRODUCER_LEASE_DURATION;
 use crate::app::quiesce::NexusQuiesceHandle;
 use crate::app::saga::StartSaga;
@@ -1357,11 +1358,11 @@ pub struct BackgroundTasksData {
     pub tuf_artifact_replication_rx: mpsc::Receiver<ArtifactsWithPlan>,
     /// Channel for exposing the latest loaded blueprint
     pub blueprint_load_tx: watch::Sender<Option<LoadedTargetBlueprint>>,
-    /// `reqwest::Client` for webhook delivery requests.
+    /// [`ExternalHttpClient`] for webhook delivery requests.
     ///
     /// This is shared with the external API as it's also used when sending
     /// webhook liveness probe requests from the API.
-    pub webhook_delivery_client: reqwest::Client,
+    pub webhook_delivery_client: ExternalHttpClient,
     /// Channel for configuring pending MGS updates
     pub mgs_updates_tx: watch::Sender<PendingMgsUpdates>,
     /// handle for controlling Nexus quiesce

nexus/src/app/background/tasks/webhook_deliverator.rs

@@ -31,6 +31,7 @@
 //! [`app::webhook`]: crate::app::webhook
 
 use crate::app::background::BackgroundTask;
+use crate::app::external_client::ExternalHttpClient;
 use crate::app::webhook::ReceiverClient;
 use futures::future::BoxFuture;
 use nexus_db_queries::context::OpContext;
@@ -102,7 +103,7 @@ use std::sync::Arc;
 pub struct WebhookDeliverator {
     datastore: Arc<DataStore>,
     nexus_id: OmicronZoneUuid,
-    client: reqwest::Client,
+    client: ExternalHttpClient,
     cfg: DeliveryConfig,
 }
 
@@ -135,7 +136,7 @@ impl WebhookDeliverator {
         datastore: Arc<DataStore>,
         cfg: DeliveryConfig,
         nexus_id: OmicronZoneUuid,
-        client: reqwest::Client,
+        client: ExternalHttpClient,
     ) -> Self {
         Self { datastore, nexus_id, cfg, client }
     }