From 3c5f34987354ab1e2f82338b47237e2ff082304c Mon Sep 17 00:00:00 2001 From: MasterPtato Date: Thu, 27 Aug 2026 15:38:19 -0700 Subject: [PATCH] feat(outbound): gate serverless outbound requests behind an SSRF policy --- Cargo.lock | 19 + Cargo.toml | 4 + docs/content/docs/debugging.mdx | 2 +- docs/content/docs/troubleshooting.mdx | 8 + engine/artifacts/config-schema.json | 68 ++++ engine/packages/config/src/config/mod.rs | 11 + engine/packages/config/src/config/outbound.rs | 60 +++ engine/packages/outbound-guard/Cargo.toml | 20 + engine/packages/outbound-guard/src/client.rs | 82 ++++ engine/packages/outbound-guard/src/lib.rs | 14 + engine/packages/outbound-guard/src/policy.rs | 375 ++++++++++++++++++ .../packages/outbound-guard/tests/policy.rs | 240 +++++++++++ engine/packages/pegboard-outbound/Cargo.toml | 3 + engine/packages/pegboard-outbound/src/lib.rs | 56 ++- engine/packages/pegboard/Cargo.toml | 1 + .../pegboard/src/ops/runner_config/upsert.rs | 18 +- .../src/ops/serverless_metadata/fetch.rs | 37 +- .../pegboard/src/workflows/serverless/conn.rs | 58 ++- engine/packages/pools/Cargo.toml | 1 + engine/packages/pools/src/pools.rs | 3 +- engine/packages/pools/src/reqwest.rs | 63 ++- engine/packages/types/src/actor/error.rs | 3 + .../src/app/runner-pool-error-popover.tsx | 15 + .../components/actors/actor-status-label.tsx | 18 + frontend/src/queries/types.ts | 1 + .../dev-host/rivet-engine/config.jsonc | 3 + .../dc-a/rivet-engine/0/config.jsonc | 3 + .../dc-a/rivet-engine/1/config.jsonc | 3 + .../dc-a/rivet-engine/2/config.jsonc | 3 + .../dc-b/rivet-engine/0/config.jsonc | 3 + .../dc-b/rivet-engine/1/config.jsonc | 3 + .../dc-b/rivet-engine/2/config.jsonc | 3 + .../dc-c/rivet-engine/0/config.jsonc | 3 + .../dc-c/rivet-engine/1/config.jsonc | 3 + .../dc-c/rivet-engine/2/config.jsonc | 3 + .../dev-multidc-multinode/docker-compose.yml | 9 - .../dc-a/rivet-engine/config.jsonc | 3 + .../dc-b/rivet-engine/config.jsonc | 3 + .../dc-c/rivet-engine/config.jsonc | 3 + .../dev-multinode/rivet-engine/0/config.jsonc | 3 + .../dev-multinode/rivet-engine/1/config.jsonc | 3 + .../dev-multinode/rivet-engine/2/config.jsonc | 3 + .../compose/dev/rivet-engine/config.jsonc | 3 + self-host/compose/template/src/main.ts | 2 +- .../src/services/edge/rivet-engine.ts | 5 + 45 files changed, 1210 insertions(+), 37 deletions(-) create mode 100644 engine/packages/config/src/config/outbound.rs create mode 100644 engine/packages/outbound-guard/Cargo.toml create mode 100644 engine/packages/outbound-guard/src/client.rs create mode 100644 engine/packages/outbound-guard/src/lib.rs create mode 100644 engine/packages/outbound-guard/src/policy.rs create mode 100644 engine/packages/outbound-guard/tests/policy.rs diff --git a/Cargo.lock b/Cargo.lock index a2ade3bdd1..293c5c4da5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3951,6 +3951,7 @@ dependencies = [ "rivet-envoy-protocol", "rivet-error", "rivet-metrics", + "rivet-outbound-guard", "rivet-pools", "rivet-runner-protocol", "rivet-runtime", @@ -4112,12 +4113,15 @@ dependencies = [ "rivet-config", "rivet-envoy-protocol", "rivet-metrics", + "rivet-outbound-guard", + "rivet-pools", "rivet-runtime", "rivet-types", "tokio", "tracing", "universaldb", "universalpubsub", + "url", "vbare", ] @@ -5693,6 +5697,20 @@ dependencies = [ "tracing", ] +[[package]] +name = "rivet-outbound-guard" +version = "2.3.7" +dependencies = [ + "anyhow", + "ipnet", + "reqwest 0.12.22", + "rivet-config", + "thiserror 1.0.69", + "tokio", + "tracing", + "url", +] + [[package]] name = "rivet-perf" version = "2.3.7" @@ -5719,6 +5737,7 @@ dependencies = [ "rivet-async-nats", "rivet-config", "rivet-metrics", + "rivet-outbound-guard", "rivet-util", "rustls", "serde", diff --git a/Cargo.toml b/Cargo.toml index a9048c159f..7953efdc0b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ members = [ "engine/packages/metrics", "engine/packages/metrics-server", "engine/packages/namespace", + "engine/packages/outbound-guard", "engine/packages/pegboard", "engine/packages/pegboard-envoy", "engine/packages/pegboard-gateway", @@ -500,6 +501,9 @@ members = [ [workspace.dependencies.namespace] path = "engine/packages/namespace" + [workspace.dependencies.rivet-outbound-guard] + path = "engine/packages/outbound-guard" + [workspace.dependencies.pegboard] path = "engine/packages/pegboard" diff --git a/docs/content/docs/debugging.mdx b/docs/content/docs/debugging.mdx index 93cfe96e1b..432ec7deaf 100644 --- a/docs/content/docs/debugging.mdx +++ b/docs/content/docs/debugging.mdx @@ -199,7 +199,7 @@ Returns the configured provider settings per datacenter and the latest pool erro } ``` -`runner_pool_error` mirrors actor scheduling errors such as `serverless_http_error`, `serverless_connection_error`, and `serverless_stream_ended_early`. +`runner_pool_error` mirrors actor scheduling errors such as `serverless_http_error`, `serverless_connection_error`, `serverless_destination_blocked`, and `serverless_stream_ended_early`. ### Check Serverless Provider Health diff --git a/docs/content/docs/troubleshooting.mdx b/docs/content/docs/troubleshooting.mdx index f06c30fe14..75ba34704d 100644 --- a/docs/content/docs/troubleshooting.mdx +++ b/docs/content/docs/troubleshooting.mdx @@ -97,6 +97,14 @@ Rivet was unable to connect to your serverless endpoint. Check that: - Your server is publicly reachable from the internet. - There are no DNS or firewall issues blocking the connection. +### `serverless_destination_blocked` + +The configured serverless URL points at a destination Rivet is not allowed to reach. Rivet dials serverless endpoints from inside its own network, so by default it refuses any destination that is not publicly routable: private ranges, link-local and cloud metadata addresses, carrier-grade NAT, and other reserved ranges. Check that: + +- Your endpoint URL is publicly reachable, and is not an internal hostname or address. +- The URL scheme is `http` or `https`, and the URL does not embed credentials. +- If you are self-hosting and intentionally point runners at an address on your own network, set `outbound.allow_private_networks` to `true` in your engine config, or list the specific range in `outbound.allow_cidrs`. + ### `serverless_stream_ended_early` The connection to your serverless endpoint was terminated before the actor finished. This usually means your serverless function hit its execution time limit. Ensure that your Rivet provider's request lifespan is configured to match the max duration of your serverless platform. diff --git a/engine/artifacts/config-schema.json b/engine/artifacts/config-schema.json index 9147b7a7d1..23533f876c 100644 --- a/engine/artifacts/config-schema.json +++ b/engine/artifacts/config-schema.json @@ -106,6 +106,17 @@ } ] }, + "outbound": { + "default": null, + "anyOf": [ + { + "$ref": "#/definitions/Outbound" + }, + { + "type": "null" + } + ] + }, "pegboard": { "default": null, "anyOf": [ @@ -803,6 +814,63 @@ }, "additionalProperties": false }, + "Outbound": { + "description": "Policy for outbound HTTP requests to user-configured destinations, such as serverless runner URLs.\n\nThese requests originate from inside the trusted engine network, so without restrictions a caller who can configure a runner can reach internal-only services. The defaults deny every non-globally-routable destination except loopback.", + "type": "object", + "properties": { + "allow_cidrs": { + "description": "Additional CIDRs that are always permitted, evaluated after `deny_cidrs`.\n\nUse this to reach a specific internal service without opening up the whole private range.", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + }, + "allow_insecure_scheme": { + "description": "Allow plaintext `http://` destinations. When disabled only `https://` is permitted.", + "type": [ + "boolean", + "null" + ] + }, + "allow_loopback": { + "description": "Allow destinations that resolve to loopback addresses (127.0.0.0/8, ::1).\n\nEnabled by default so local development against `http://localhost:...` works without configuration.", + "type": [ + "boolean", + "null" + ] + }, + "allow_private_networks": { + "description": "Allow destinations that resolve to private, link-local, shared (CGNAT), or otherwise non-globally-routable addresses.\n\nSelf-hosted deployments that point runners at addresses on their own network, such as a Docker Compose service name, need this enabled.", + "type": [ + "boolean", + "null" + ] + }, + "deny_cidrs": { + "description": "Additional CIDRs that are always denied. Takes precedence over every allow rule.", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + }, + "max_redirects": { + "description": "Maximum number of redirects to follow. Every hop is re-checked against this policy.", + "type": [ + "integer", + "null" + ], + "format": "uint", + "minimum": 0.0 + } + }, + "additionalProperties": false + }, "Pegboard": { "type": "object", "properties": { diff --git a/engine/packages/config/src/config/mod.rs b/engine/packages/config/src/config/mod.rs index 5ee03c6d2a..babed64296 100644 --- a/engine/packages/config/src/config/mod.rs +++ b/engine/packages/config/src/config/mod.rs @@ -11,6 +11,7 @@ pub mod db; pub mod guard; pub mod logs; pub mod metrics; +pub mod outbound; pub mod pegboard; pub mod pubsub; pub mod pyroscope; @@ -27,6 +28,7 @@ pub use db::Database; pub use guard::*; pub use logs::*; pub use metrics::*; +pub use outbound::*; pub use pegboard::*; pub use pubsub::PubSub; pub use pyroscope::*; @@ -110,6 +112,9 @@ pub struct Root { #[serde(default)] pub pyroscope: Option, + + #[serde(default)] + pub outbound: Option, } impl Default for Root { @@ -130,6 +135,7 @@ impl Default for Root { sqlite: None, metrics: Default::default(), pyroscope: None, + outbound: None, } } } @@ -145,6 +151,11 @@ impl Root { self.api_peer.as_ref().unwrap_or(&DEFAULT) } + pub fn outbound(&self) -> &Outbound { + static DEFAULT: LazyLock = LazyLock::new(Outbound::default); + self.outbound.as_ref().unwrap_or(&DEFAULT) + } + pub fn pegboard(&self) -> &Pegboard { static DEFAULT: LazyLock = LazyLock::new(Pegboard::default); self.pegboard.as_ref().unwrap_or(&DEFAULT) diff --git a/engine/packages/config/src/config/outbound.rs b/engine/packages/config/src/config/outbound.rs new file mode 100644 index 0000000000..d0aa65b35f --- /dev/null +++ b/engine/packages/config/src/config/outbound.rs @@ -0,0 +1,60 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Policy for outbound HTTP requests to user-configured destinations, such as serverless runner +/// URLs. +/// +/// These requests originate from inside the trusted engine network, so without restrictions a +/// caller who can configure a runner can reach internal-only services. The defaults deny every +/// non-globally-routable destination except loopback. +#[derive(Debug, Serialize, Deserialize, Clone, Default, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Outbound { + /// Allow destinations that resolve to loopback addresses (127.0.0.0/8, ::1). + /// + /// Enabled by default so local development against `http://localhost:...` works without + /// configuration. + pub allow_loopback: Option, + /// Allow destinations that resolve to private, link-local, shared (CGNAT), or otherwise + /// non-globally-routable addresses. + /// + /// Self-hosted deployments that point runners at addresses on their own network, such as a + /// Docker Compose service name, need this enabled. + pub allow_private_networks: Option, + /// Allow plaintext `http://` destinations. When disabled only `https://` is permitted. + pub allow_insecure_scheme: Option, + /// Additional CIDRs that are always permitted, evaluated after `deny_cidrs`. + /// + /// Use this to reach a specific internal service without opening up the whole private range. + pub allow_cidrs: Option>, + /// Additional CIDRs that are always denied. Takes precedence over every allow rule. + pub deny_cidrs: Option>, + /// Maximum number of redirects to follow. Every hop is re-checked against this policy. + pub max_redirects: Option, +} + +impl Outbound { + pub fn allow_loopback(&self) -> bool { + self.allow_loopback.unwrap_or(true) + } + + pub fn allow_private_networks(&self) -> bool { + self.allow_private_networks.unwrap_or(false) + } + + pub fn allow_insecure_scheme(&self) -> bool { + self.allow_insecure_scheme.unwrap_or(true) + } + + pub fn allow_cidrs(&self) -> &[String] { + self.allow_cidrs.as_deref().unwrap_or(&[]) + } + + pub fn deny_cidrs(&self) -> &[String] { + self.deny_cidrs.as_deref().unwrap_or(&[]) + } + + pub fn max_redirects(&self) -> usize { + self.max_redirects.unwrap_or(4) + } +} diff --git a/engine/packages/outbound-guard/Cargo.toml b/engine/packages/outbound-guard/Cargo.toml new file mode 100644 index 0000000000..a47d61ea49 --- /dev/null +++ b/engine/packages/outbound-guard/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "rivet-outbound-guard" +publish = false +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true + +[dependencies] +anyhow.workspace = true +ipnet.workspace = true +reqwest.workspace = true +rivet-config.workspace = true +thiserror.workspace = true +tokio.workspace = true +tracing.workspace = true +url.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/engine/packages/outbound-guard/src/client.rs b/engine/packages/outbound-guard/src/client.rs new file mode 100644 index 0000000000..8dc44cedf8 --- /dev/null +++ b/engine/packages/outbound-guard/src/client.rs @@ -0,0 +1,82 @@ +use std::net::SocketAddr; +use std::sync::Arc; + +use reqwest::dns::{Addrs, Name, Resolve, Resolving}; + +use crate::policy::{BlockReason, Policy}; + +/// A DNS resolver that drops every address the policy disallows. +/// +/// Placing the check here rather than before the request closes the DNS rebinding window: reqwest +/// connects to exactly the addresses this returns, and it runs for every redirect hop as well as +/// the initial request. +#[derive(Debug)] +pub struct GuardedResolver { + policy: Arc, +} + +impl GuardedResolver { + pub fn new(policy: Arc) -> Self { + GuardedResolver { policy } + } +} + +impl Resolve for GuardedResolver { + fn resolve(&self, name: Name) -> Resolving { + let policy = self.policy.clone(); + let host = name.as_str().to_string(); + + Box::pin(async move { + // The port is discarded by the connector, which substitutes the real one. + let resolved = tokio::net::lookup_host((host.as_str(), 0)) + .await + .map_err(|err| { + tracing::debug!(%host, ?err, "failed to resolve outbound host"); + Box::new(BlockReason::ResolutionFailed { host: host.clone() }) + as Box + })? + .map(|addr| addr.ip()) + .collect::>(); + + let addrs = policy + .filter_addrs(&host, resolved)? + .into_iter() + .map(|ip| SocketAddr::new(ip, 0)) + .collect::>(); + + Ok(Box::new(addrs.into_iter()) as Addrs) + }) + } +} + +/// Build the redirect policy a guarded client must be configured with. +/// +/// Every hop is re-checked, so a destination cannot bounce the engine somewhere it was not +/// allowed to reach directly. +pub fn redirect_policy(policy: Arc) -> reqwest::redirect::Policy { + let max_redirects = policy.max_redirects(); + + reqwest::redirect::Policy::custom(move |attempt| { + if attempt.previous().len() >= max_redirects { + return attempt.error(BlockReason::TooManyRedirects { max: max_redirects }); + } + + match policy.check_url(attempt.url()) { + Ok(()) => attempt.follow(), + Err(reason) => { + tracing::debug!(url = %attempt.url(), %reason, "blocked outbound redirect"); + attempt.error(reason) + } + } + }) +} + +/// Recover the [`BlockReason`] that caused a request to fail, if the policy is what stopped it. +/// +/// The reason is buried in the source chain of a `reqwest::Error`, so callers that want to tell +/// "the destination is not allowed" apart from "the destination is down" have to walk it. +pub fn block_reason(err: &anyhow::Error) -> Option { + err.chain() + .find_map(|err| err.downcast_ref::()) + .cloned() +} diff --git a/engine/packages/outbound-guard/src/lib.rs b/engine/packages/outbound-guard/src/lib.rs new file mode 100644 index 0000000000..4076d4299f --- /dev/null +++ b/engine/packages/outbound-guard/src/lib.rs @@ -0,0 +1,14 @@ +//! Destination policy for outbound HTTP requests to user-configured URLs. +//! +//! Serverless runner URLs are supplied by whoever can write a runner config, and the engine dials +//! them from inside its own trusted network. This crate is the trust boundary for those requests: +//! it decides which destinations are reachable, and supplies the resolver and redirect policy that +//! enforce that decision at connect time. +//! +//! Clients are built in `rivet_pools::reqwest`, which owns every `reqwest::Client` in the process. + +mod client; +mod policy; + +pub use client::{GuardedResolver, block_reason, redirect_policy}; +pub use policy::{AddressClass, BlockReason, Policy}; diff --git a/engine/packages/outbound-guard/src/policy.rs b/engine/packages/outbound-guard/src/policy.rs new file mode 100644 index 0000000000..a0d255c9b7 --- /dev/null +++ b/engine/packages/outbound-guard/src/policy.rs @@ -0,0 +1,375 @@ +use std::fmt; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + +use anyhow::{Context, Result}; +use ipnet::IpNet; +use url::Url; + +/// Why a destination was rejected. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum BlockReason { + #[error("url is not a valid absolute url")] + InvalidUrl, + #[error("scheme {scheme:?} is not allowed, expected http or https")] + UnsupportedScheme { scheme: String }, + #[error("plaintext http is not allowed, use https")] + InsecureScheme, + #[error("url must not contain embedded credentials")] + EmbeddedCredentials, + #[error("url has no host")] + MissingHost, + #[error("address {addr} is not an allowed destination ({class})")] + BlockedAddress { addr: IpAddr, class: AddressClass }, + #[error("address {addr} is explicitly denied")] + DeniedAddress { addr: IpAddr }, + #[error("host {host:?} resolved to no allowed addresses")] + NoAllowedAddresses { host: String }, + #[error("failed to resolve host {host:?}")] + ResolutionFailed { host: String }, + #[error("exceeded the maximum of {max} redirects")] + TooManyRedirects { max: usize }, +} + +/// The reason an address is not globally routable. +/// +/// Every class except [`AddressClass::Loopback`] is governed by +/// `outbound.allow_private_networks`. An address that matches no class is globally +/// routable and always allowed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AddressClass { + /// `0.0.0.0` or `::`, which the kernel routes to a local interface. + Unspecified, + /// `127.0.0.0/8` or `::1`. + Loopback, + /// `169.254.0.0/16` or `fe80::/10`. Covers the cloud metadata endpoint. + LinkLocal, + /// `10/8`, `172.16/12`, `192.168/16`. + Private, + /// `255.255.255.255`. + Broadcast, + /// `224.0.0.0/4` or `ff00::/8`. + Multicast, + /// `192.0.2.0/24`, `198.51.100.0/24`, `203.0.113.0/24`, or `2001:db8::/32`. + Documentation, + /// `100.64.0.0/10`, the carrier-grade NAT range. + Shared, + /// `192.0.0.0/24`, reserved for IETF protocol assignments. + ProtocolAssignments, + /// `198.18.0.0/15`, reserved for network benchmarking. + Benchmarking, + /// `240.0.0.0/4`. + Reserved, + /// `fc00::/7`, the IPv6 equivalent of the private ranges. + UniqueLocal, + /// `100::/64`, which is discarded rather than routed. + Discard, + /// An IPv6 address carrying an IPv4 destination that is itself globally routable. + /// + /// These are held to the same rule as the private classes because they are an easy way to + /// smuggle a destination past a filter that only understands one address family. + Ipv4Embedded, +} + +impl fmt::Display for AddressClass { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let name = match self { + AddressClass::Unspecified => "unspecified", + AddressClass::Loopback => "loopback", + AddressClass::LinkLocal => "link-local", + AddressClass::Private => "private", + AddressClass::Broadcast => "broadcast", + AddressClass::Multicast => "multicast", + AddressClass::Documentation => "documentation", + AddressClass::Shared => "shared", + AddressClass::ProtocolAssignments => "protocol-assignments", + AddressClass::Benchmarking => "benchmarking", + AddressClass::Reserved => "reserved", + AddressClass::UniqueLocal => "unique-local", + AddressClass::Discard => "discard", + AddressClass::Ipv4Embedded => "ipv4-embedded", + }; + + f.write_str(name) + } +} + +impl AddressClass { + /// Classify an address, or `None` if it is globally routable. + pub fn of(addr: IpAddr) -> Option { + match addr { + IpAddr::V4(v4) => AddressClass::of_ipv4(v4), + IpAddr::V6(v6) => AddressClass::of_ipv6(v6), + } + } + + fn of_ipv4(addr: Ipv4Addr) -> Option { + let [a, b, c, _] = addr.octets(); + + if addr.is_unspecified() { + return Some(AddressClass::Unspecified); + } + if addr.is_loopback() { + return Some(AddressClass::Loopback); + } + if addr.is_link_local() { + return Some(AddressClass::LinkLocal); + } + if addr.is_private() { + return Some(AddressClass::Private); + } + if addr.is_broadcast() { + return Some(AddressClass::Broadcast); + } + if addr.is_multicast() { + return Some(AddressClass::Multicast); + } + if addr.is_documentation() { + return Some(AddressClass::Documentation); + } + if a == 100 && (64..128).contains(&b) { + return Some(AddressClass::Shared); + } + if a == 192 && b == 0 && c == 0 { + return Some(AddressClass::ProtocolAssignments); + } + if a == 198 && (b == 18 || b == 19) { + return Some(AddressClass::Benchmarking); + } + if a >= 240 { + return Some(AddressClass::Reserved); + } + + None + } + + fn of_ipv6(addr: Ipv6Addr) -> Option { + // An IPv4 address wearing an IPv6 costume routes to the embedded IPv4 destination, so + // classify it as that address rather than trusting the outer form. + if let Some(v4) = unwrap_embedded_ipv4(addr) { + return AddressClass::of_ipv4(v4).or(Some(AddressClass::Ipv4Embedded)); + } + + let segments = addr.segments(); + + if addr.is_unspecified() { + return Some(AddressClass::Unspecified); + } + if addr.is_loopback() { + return Some(AddressClass::Loopback); + } + if addr.is_multicast() { + return Some(AddressClass::Multicast); + } + if segments[0] & 0xfe00 == 0xfc00 { + return Some(AddressClass::UniqueLocal); + } + if segments[0] & 0xffc0 == 0xfe80 { + return Some(AddressClass::LinkLocal); + } + if segments[0] == 0x2001 && segments[1] == 0x0db8 { + return Some(AddressClass::Documentation); + } + if segments[0] == 0x0100 && segments[1..4] == [0, 0, 0] { + return Some(AddressClass::Discard); + } + + None + } +} + +/// Extract the IPv4 destination an IPv6 address actually routes to, if any. +/// +/// Covers IPv4-mapped (`::ffff:0:0/96`), IPv4-compatible (`::/96`), and the well-known NAT64 +/// prefix (`64:ff9b::/96`). +fn unwrap_embedded_ipv4(addr: Ipv6Addr) -> Option { + if let Some(v4) = addr.to_ipv4_mapped() { + return Some(v4); + } + + let segments = addr.segments(); + let tail = Ipv4Addr::new( + (segments[6] >> 8) as u8, + (segments[6] & 0xff) as u8, + (segments[7] >> 8) as u8, + (segments[7] & 0xff) as u8, + ); + + if segments[0] == 0x0064 && segments[1] == 0xff9b && segments[2..6] == [0, 0, 0, 0] { + return Some(tail); + } + + // IPv4-compatible addresses are deprecated but still routed. `::` and `::1` have their own + // classes, so skip anything in the lowest /104. + if segments[0..6] == [0, 0, 0, 0, 0, 0] && segments[6] != 0 { + return Some(tail); + } + + None +} + +/// Which destinations the engine may reach on behalf of a user-supplied URL. +#[derive(Debug, Clone)] +pub struct Policy { + allow_loopback: bool, + allow_private_networks: bool, + allow_insecure_scheme: bool, + allow_cidrs: Vec, + deny_cidrs: Vec, + max_redirects: usize, +} + +impl Policy { + pub fn from_config(config: &rivet_config::Config) -> Result { + let outbound = config.outbound(); + + Ok(Policy { + allow_loopback: outbound.allow_loopback(), + allow_private_networks: outbound.allow_private_networks(), + allow_insecure_scheme: outbound.allow_insecure_scheme(), + allow_cidrs: parse_cidrs(outbound.allow_cidrs(), "outbound.allow_cidrs")?, + deny_cidrs: parse_cidrs(outbound.deny_cidrs(), "outbound.deny_cidrs")?, + max_redirects: outbound.max_redirects(), + }) + } + + pub fn max_redirects(&self) -> usize { + self.max_redirects + } + + /// Check everything about a destination that can be known without resolving DNS. + /// + /// This is the gate that runs when a runner config is written, so a bad URL is rejected before + /// it is ever stored. It is also re-run on every redirect hop. + pub fn check_url(&self, url: &Url) -> Result<(), BlockReason> { + match url.scheme() { + "https" => {} + "http" => { + if !self.allow_insecure_scheme { + return Err(BlockReason::InsecureScheme); + } + } + scheme => { + return Err(BlockReason::UnsupportedScheme { + scheme: scheme.to_string(), + }); + } + } + + // Credentials in the URL would be replayed to whatever the destination redirects to. + if !url.username().is_empty() || url.password().is_some() { + return Err(BlockReason::EmbeddedCredentials); + } + + let Some(host) = url.host() else { + return Err(BlockReason::MissingHost); + }; + + // A hostname is checked once it resolves, at connect time. An address literal skips the + // resolver entirely, so it has to be checked here. + match host { + url::Host::Ipv4(addr) => self.check_addr(IpAddr::V4(addr)), + url::Host::Ipv6(addr) => self.check_addr(IpAddr::V6(addr)), + url::Host::Domain(_) => Ok(()), + } + } + + /// Check a single resolved address. + pub fn check_addr(&self, addr: IpAddr) -> Result<(), BlockReason> { + // An IPv6 address that carries an IPv4 destination has to match CIDR rules written for + // either form, so both are tested. + let mut forms = vec![addr]; + if let IpAddr::V6(v6) = addr { + if let Some(v4) = unwrap_embedded_ipv4(v6) { + forms.push(IpAddr::V4(v4)); + } + } + + // An explicit deny always wins, including over the allow list. + if forms + .iter() + .any(|form| self.deny_cidrs.iter().any(|net| net.contains(form))) + { + return Err(BlockReason::DeniedAddress { addr }); + } + + if forms + .iter() + .any(|form| self.allow_cidrs.iter().any(|net| net.contains(form))) + { + return Ok(()); + } + + let Some(class) = AddressClass::of(addr) else { + return Ok(()); + }; + + let allowed = match class { + AddressClass::Loopback => self.allow_loopback, + AddressClass::Unspecified + | AddressClass::LinkLocal + | AddressClass::Private + | AddressClass::Broadcast + | AddressClass::Multicast + | AddressClass::Documentation + | AddressClass::Shared + | AddressClass::ProtocolAssignments + | AddressClass::Benchmarking + | AddressClass::Reserved + | AddressClass::UniqueLocal + | AddressClass::Discard + | AddressClass::Ipv4Embedded => self.allow_private_networks, + }; + + if allowed { + Ok(()) + } else { + Err(BlockReason::BlockedAddress { addr, class }) + } + } + + /// Filter a resolver's answer down to the addresses this policy permits. + /// + /// Dropping individual addresses rather than rejecting the whole answer keeps a dual-stack + /// host reachable when only one of its families is allowed, and still guarantees the + /// connection can only land on an address that passed. + pub fn filter_addrs( + &self, + host: &str, + addrs: impl IntoIterator, + ) -> Result, BlockReason> { + let mut allowed = Vec::new(); + + for addr in addrs { + match self.check_addr(addr) { + Ok(()) => allowed.push(addr), + Err(reason) => { + tracing::debug!(%host, %addr, %reason, "dropped disallowed resolved address"); + } + } + } + + if allowed.is_empty() { + Err(BlockReason::NoAllowedAddresses { + host: host.to_string(), + }) + } else { + Ok(allowed) + } + } +} + +fn parse_cidrs(raw: &[String], label: &str) -> Result> { + raw.iter() + .map(|entry| { + let entry = entry.trim(); + // Accept a bare address as a single-host CIDR so operators do not have to write /32. + if let Ok(addr) = entry.parse::() { + return Ok(IpNet::from(addr)); + } + + entry + .parse::() + .with_context(|| format!("invalid cidr in {label}: {entry:?}")) + }) + .collect() +} diff --git a/engine/packages/outbound-guard/tests/policy.rs b/engine/packages/outbound-guard/tests/policy.rs new file mode 100644 index 0000000000..dd4b19b368 --- /dev/null +++ b/engine/packages/outbound-guard/tests/policy.rs @@ -0,0 +1,240 @@ +use std::net::IpAddr; + +use rivet_config::config::{Outbound, Root}; +use rivet_outbound_guard::{AddressClass, BlockReason, Policy}; +use url::Url; + +fn policy(outbound: Outbound) -> Policy { + let config = rivet_config::Config::from_root(Root { + outbound: Some(outbound), + ..Default::default() + }); + + Policy::from_config(&config).expect("policy should build") +} + +fn default_policy() -> Policy { + policy(Outbound::default()) +} + +fn check(policy: &Policy, url: &str) -> Result<(), BlockReason> { + policy.check_url(&Url::parse(url).expect("test url should parse")) +} + +fn addr(raw: &str) -> IpAddr { + raw.parse().expect("test address should parse") +} + +#[test] +fn allows_public_destinations() { + let policy = default_policy(); + + check(&policy, "https://runner.example.com/start").expect("public host should be allowed"); + check(&policy, "https://8.8.8.8/start").expect("public literal should be allowed"); + policy + .check_addr(addr("2606:4700::1")) + .expect("public v6 should be allowed"); +} + +#[test] +fn allows_loopback_by_default() { + let policy = default_policy(); + + check(&policy, "http://127.0.0.1:6420/start").expect("loopback v4 should be allowed"); + check(&policy, "http://[::1]:6420/start").expect("loopback v6 should be allowed"); +} + +#[test] +fn denies_loopback_when_disabled() { + let policy = policy(Outbound { + allow_loopback: Some(false), + ..Default::default() + }); + + assert_eq!( + check(&policy, "http://127.0.0.1:6420/start"), + Err(BlockReason::BlockedAddress { + addr: addr("127.0.0.1"), + class: AddressClass::Loopback, + }), + ); +} + +#[test] +fn denies_private_ranges_by_default() { + let policy = default_policy(); + + for (raw, class) in [ + ("10.0.0.5", AddressClass::Private), + ("172.16.4.1", AddressClass::Private), + ("192.168.1.1", AddressClass::Private), + ("169.254.169.254", AddressClass::LinkLocal), + ("100.64.0.1", AddressClass::Shared), + ("198.18.0.1", AddressClass::Benchmarking), + ("0.0.0.0", AddressClass::Unspecified), + ("240.0.0.1", AddressClass::Reserved), + ("fd00::1", AddressClass::UniqueLocal), + ("fe80::1", AddressClass::LinkLocal), + ] { + assert_eq!( + policy.check_addr(addr(raw)), + Err(BlockReason::BlockedAddress { + addr: addr(raw), + class, + }), + "{raw} should be blocked", + ); + } +} + +#[test] +fn denies_ipv6_wrapped_ipv4_metadata_endpoint() { + let policy = default_policy(); + + // The same destination reached through three different IPv6 encodings. + for raw in ["::ffff:169.254.169.254", "64:ff9b::169.254.169.254"] { + assert_eq!( + policy.check_addr(addr(raw)), + Err(BlockReason::BlockedAddress { + addr: addr(raw), + class: AddressClass::LinkLocal, + }), + "{raw} should be blocked", + ); + } +} + +#[test] +fn allows_private_ranges_when_enabled() { + let policy = policy(Outbound { + allow_private_networks: Some(true), + ..Default::default() + }); + + policy + .check_addr(addr("10.0.0.5")) + .expect("private should be allowed when enabled"); + policy + .check_addr(addr("169.254.169.254")) + .expect("link-local should be allowed when enabled"); +} + +#[test] +fn allow_cidrs_open_a_single_destination() { + let policy = policy(Outbound { + allow_cidrs: Some(vec!["10.1.2.0/24".to_string(), "192.168.5.9".to_string()]), + ..Default::default() + }); + + policy + .check_addr(addr("10.1.2.7")) + .expect("allow-listed cidr should be allowed"); + policy + .check_addr(addr("192.168.5.9")) + .expect("bare address should be read as a single-host cidr"); + assert!( + policy.check_addr(addr("10.1.3.7")).is_err(), + "address outside the allow-listed cidr should stay blocked", + ); +} + +#[test] +fn deny_cidrs_win_over_allow_rules() { + let policy = policy(Outbound { + allow_private_networks: Some(true), + allow_cidrs: Some(vec!["169.254.0.0/16".to_string()]), + deny_cidrs: Some(vec!["169.254.169.254".to_string()]), + ..Default::default() + }); + + policy + .check_addr(addr("169.254.1.1")) + .expect("rest of the range should still be reachable"); + assert_eq!( + policy.check_addr(addr("169.254.169.254")), + Err(BlockReason::DeniedAddress { + addr: addr("169.254.169.254"), + }), + ); +} + +#[test] +fn deny_cidrs_catch_the_ipv6_wrapped_form() { + let policy = policy(Outbound { + allow_private_networks: Some(true), + deny_cidrs: Some(vec!["169.254.169.254".to_string()]), + ..Default::default() + }); + + assert_eq!( + policy.check_addr(addr("::ffff:169.254.169.254")), + Err(BlockReason::DeniedAddress { + addr: addr("::ffff:169.254.169.254"), + }), + ); +} + +#[test] +fn rejects_non_http_schemes() { + let policy = default_policy(); + + assert_eq!( + check(&policy, "file:///etc/passwd"), + Err(BlockReason::UnsupportedScheme { + scheme: "file".to_string(), + }), + ); + assert_eq!( + check(&policy, "gopher://example.com/"), + Err(BlockReason::UnsupportedScheme { + scheme: "gopher".to_string(), + }), + ); +} + +#[test] +fn rejects_plaintext_http_when_disabled() { + let policy = policy(Outbound { + allow_insecure_scheme: Some(false), + ..Default::default() + }); + + assert_eq!( + check(&policy, "http://runner.example.com/"), + Err(BlockReason::InsecureScheme), + ); + check(&policy, "https://runner.example.com/").expect("https should still be allowed"); +} + +#[test] +fn rejects_embedded_credentials() { + let policy = default_policy(); + + assert_eq!( + check(&policy, "https://user:pass@runner.example.com/"), + Err(BlockReason::EmbeddedCredentials), + ); +} + +#[test] +fn filter_addrs_keeps_only_allowed_answers() { + let policy = default_policy(); + + let allowed = policy + .filter_addrs( + "rebind.example.com", + [addr("10.0.0.1"), addr("93.184.216.34")], + ) + .expect("a dual answer with one public address should still connect"); + assert_eq!(allowed, vec![addr("93.184.216.34")]); + + assert_eq!( + policy.filter_addrs( + "rebind.example.com", + [addr("10.0.0.1"), addr("192.168.0.1")] + ), + Err(BlockReason::NoAllowedAddresses { + host: "rebind.example.com".to_string(), + }), + ); +} diff --git a/engine/packages/pegboard-outbound/Cargo.toml b/engine/packages/pegboard-outbound/Cargo.toml index e48c0ba430..6095f44b01 100644 --- a/engine/packages/pegboard-outbound/Cargo.toml +++ b/engine/packages/pegboard-outbound/Cargo.toml @@ -18,10 +18,13 @@ reqwest.workspace = true rivet-config.workspace = true rivet-envoy-protocol.workspace = true rivet-metrics.workspace = true +rivet-outbound-guard.workspace = true +rivet-pools.workspace = true rivet-runtime.workspace = true rivet-types.workspace = true tokio.workspace = true tracing.workspace = true universaldb.workspace = true universalpubsub.workspace = true +url.workspace = true vbare.workspace = true diff --git a/engine/packages/pegboard-outbound/src/lib.rs b/engine/packages/pegboard-outbound/src/lib.rs index 00697ea7b0..4a1341fd7c 100644 --- a/engine/packages/pegboard-outbound/src/lib.rs +++ b/engine/packages/pegboard-outbound/src/lib.rs @@ -327,6 +327,7 @@ fn error_label(error: &RunnerPoolError) -> &'static str { match error { RunnerPoolError::ServerlessHttpError { .. } => "http_error", RunnerPoolError::ServerlessConnectionError { .. } => "connection_error", + RunnerPoolError::ServerlessDestinationBlocked { .. } => "destination_blocked", RunnerPoolError::ServerlessStreamEndedEarly => "stream_ended_early", RunnerPoolError::ServerlessInvalidSsePayload { .. } => "invalid_payload", RunnerPoolError::Downgrade => "downgrade", @@ -345,6 +346,7 @@ fn status_label(error: &RunnerPoolError) -> &'static str { _ => "other", }, RunnerPoolError::ServerlessConnectionError { .. } + | RunnerPoolError::ServerlessDestinationBlocked { .. } | RunnerPoolError::ServerlessStreamEndedEarly | RunnerPoolError::ServerlessInvalidSsePayload { .. } | RunnerPoolError::Downgrade @@ -362,6 +364,7 @@ fn error_result_label(error: &RunnerPoolError) -> &'static str { _ => "error_http_other", }, RunnerPoolError::ServerlessConnectionError { .. } => "error_connection", + RunnerPoolError::ServerlessDestinationBlocked { .. } => "error_destination_blocked", RunnerPoolError::ServerlessStreamEndedEarly => "error_stream_ended", RunnerPoolError::ServerlessInvalidSsePayload { .. } => "error_invalid_payload", RunnerPoolError::Downgrade => "error_downgrade", @@ -441,7 +444,37 @@ async fn serverless_outbound_req( let endpoint_url = format!("{}/start", url.trim_end_matches('/')); - let client = rivet_pools::reqwest::client_no_timeout().await?; + // The client only sees this URL as a host to connect to: an address literal never reaches its + // resolver, and its redirect policy only runs on later hops. So the scheme, credentials, and a + // literal destination have to be checked here, which also re-gates configs stored before this + // policy existed. + let policy = rivet_pools::reqwest::outbound_policy(ctx.config()).await?; + let block_reason = match url::Url::parse(&endpoint_url) { + Ok(parsed_url) => policy.check_url(&parsed_url).err(), + Err(_) => Some(rivet_outbound_guard::BlockReason::InvalidUrl), + }; + if let Some(reason) = block_reason { + tracing::warn!( + ?namespace_id, + %pool_name, + %reason, + "serverless url is not an allowed destination, dropping outbound req" + ); + + report_error( + ctx, + namespace_id, + pool_name, + RunnerPoolError::ServerlessDestinationBlocked { + reason: reason.to_string(), + }, + ) + .await; + + return Ok(()); + } + + let client = rivet_pools::reqwest::guarded_client_no_timeout(ctx.config()).await?; let req = client .post(endpoint_url.clone()) .body(payload) @@ -550,13 +583,20 @@ async fn serverless_outbound_req( Err(err) => { let wrapped_err = anyhow::Error::from(err); - let error = RunnerPoolError::ServerlessConnectionError { - // Print entire error chain - message: wrapped_err - .chain() - .map(|err| err.to_string()) - .collect::>() - .join("\n"), + // A hostname that only resolves to disallowed addresses is rejected by the + // resolver, which the pre-flight check above cannot see. + let error = match rivet_outbound_guard::block_reason(&wrapped_err) { + Some(reason) => RunnerPoolError::ServerlessDestinationBlocked { + reason: reason.to_string(), + }, + None => RunnerPoolError::ServerlessConnectionError { + // Print entire error chain + message: wrapped_err + .chain() + .map(|err| err.to_string()) + .collect::>() + .join("\n"), + }, }; report_error(ctx, namespace_id, &pool_name, error.clone()).await; observe_req_duration( diff --git a/engine/packages/pegboard/Cargo.toml b/engine/packages/pegboard/Cargo.toml index 6e9701246f..d4b1654f08 100644 --- a/engine/packages/pegboard/Cargo.toml +++ b/engine/packages/pegboard/Cargo.toml @@ -30,6 +30,7 @@ rivet-data.workspace = true rivet-envoy-protocol.workspace = true rivet-error.workspace = true rivet-metrics.workspace = true +rivet-outbound-guard.workspace = true rivet-pools.workspace = true rivet-runner-protocol.workspace = true rivet-runtime.workspace = true diff --git a/engine/packages/pegboard/src/ops/runner_config/upsert.rs b/engine/packages/pegboard/src/ops/runner_config/upsert.rs index e5d2c3c4b3..ad3865612e 100644 --- a/engine/packages/pegboard/src/ops/runner_config/upsert.rs +++ b/engine/packages/pegboard/src/ops/runner_config/upsert.rs @@ -34,9 +34,23 @@ pub async fn pegboard_runner_config_upsert(ctx: &OperationCtx, input: &Input) -> slots_per_runner, .. } => { - if let Err(err) = url::Url::parse(url) { + let parsed_url = match url::Url::parse(url) { + Ok(parsed_url) => parsed_url, + Err(err) => { + return Err(errors::RunnerConfig::Invalid { + reason: format!("invalid serverless url: {err}"), + } + .build()); + } + }; + + // Reject destinations the engine is not allowed to reach before the config is stored. + // Requests are checked again when they connect, which catches configs written before + // this gate existed and hosts whose DNS answer changes afterwards. + let policy = rivet_pools::reqwest::outbound_policy(ctx.config()).await?; + if let Err(reason) = policy.check_url(&parsed_url) { return Err(errors::RunnerConfig::Invalid { - reason: format!("invalid serverless url: {err}"), + reason: format!("invalid serverless url: {reason}"), } .build()); } diff --git a/engine/packages/pegboard/src/ops/serverless_metadata/fetch.rs b/engine/packages/pegboard/src/ops/serverless_metadata/fetch.rs index 2d25ba3c70..da4dab6cd8 100644 --- a/engine/packages/pegboard/src/ops/serverless_metadata/fetch.rs +++ b/engine/packages/pegboard/src/ops/serverless_metadata/fetch.rs @@ -21,6 +21,7 @@ pub struct Input { #[derive(Clone, Debug, PartialEq, Eq)] pub enum ServerlessMetadataError { InvalidRequest {}, + DestinationBlocked { reason: String }, RequestFailed {}, RequestTimedOut {}, NonSuccessStatus { status_code: u16, body: String }, @@ -59,6 +60,14 @@ impl From for ServerlessMetadataErrorEnvelope { details: None, metadata: serde_json::json!({ "kind": "invalid_request" }), }, + ServerlessMetadataError::DestinationBlocked { reason } => Self { + message: "serverless endpoint is not an allowed destination".to_string(), + details: Some(reason.clone()), + metadata: serde_json::json!({ + "kind": "destination_blocked", + "reason": reason, + }), + }, ServerlessMetadataError::RequestFailed {} => Self { message: "failed to reach serverless endpoint".to_string(), details: None, @@ -158,8 +167,19 @@ pub async fn pegboard_serverless_metadata_fetch( let metadata_url = format!("{}/metadata", trimmed_url.trim_end_matches('/')); - if reqwest::Url::parse(&metadata_url).is_err() { + let Ok(parsed_url) = reqwest::Url::parse(&metadata_url) else { return Ok(Err(ServerlessMetadataError::InvalidRequest {})); + }; + + // The client only sees this URL as a host to connect to: an address literal never reaches its + // resolver, and its redirect policy only runs on later hops. So the scheme, credentials, and a + // literal destination have to be checked here. This op also backs the health check endpoint, + // where an unchecked URL is a probe for whatever the engine can reach. + let policy = rivet_pools::reqwest::outbound_policy(ctx.config()).await?; + if let Err(reason) = policy.check_url(&parsed_url) { + return Ok(Err(ServerlessMetadataError::DestinationBlocked { + reason: reason.to_string(), + })); } let mut header_map = ReqwestHeaderMap::new(); @@ -177,7 +197,7 @@ pub async fn pegboard_serverless_metadata_fetch( header_map.insert(header_name, header_value); } - let client = match rivet_pools::reqwest::client().await { + let client = match rivet_pools::reqwest::guarded_client(ctx.config()).await { Ok(c) => c, Err(_) => return Ok(Err(ServerlessMetadataError::RequestFailed {})), }; @@ -193,7 +213,18 @@ pub async fn pegboard_serverless_metadata_fetch( { Ok(r) => r, Err(err) => { - return Ok(Err(if err.is_timeout() { + let is_timeout = err.is_timeout(); + let err = anyhow::Error::from(err); + + // A hostname that only resolves to disallowed addresses is rejected by the resolver, + // which the pre-flight check above cannot see. + if let Some(reason) = rivet_outbound_guard::block_reason(&err) { + return Ok(Err(ServerlessMetadataError::DestinationBlocked { + reason: reason.to_string(), + })); + } + + return Ok(Err(if is_timeout { ServerlessMetadataError::RequestTimedOut {} } else { ServerlessMetadataError::RequestFailed {} diff --git a/engine/packages/pegboard/src/workflows/serverless/conn.rs b/engine/packages/pegboard/src/workflows/serverless/conn.rs index 0064fab984..d621ba4976 100644 --- a/engine/packages/pegboard/src/workflows/serverless/conn.rs +++ b/engine/packages/pegboard/src/workflows/serverless/conn.rs @@ -299,9 +299,22 @@ async fn outbound_req_inner( let endpoint_url = format!("{}/start", url.trim_end_matches('/')); + // The client only sees this URL as a host to connect to: an address literal never reaches its + // resolver, and its redirect policy only runs on later hops. So the scheme, credentials, and a + // literal destination have to be checked here, which also re-gates configs stored before this + // policy existed. + let policy = rivet_pools::reqwest::outbound_policy(ctx.config()).await?; + let block_reason = match url::Url::parse(&endpoint_url) { + Ok(parsed_url) => policy.check_url(&parsed_url).err(), + Err(_) => Some(rivet_outbound_guard::BlockReason::InvalidUrl), + }; + if let Some(reason) = block_reason { + return Ok(blocked_destination(ctx, input, reason).await); + } + tracing::debug!(%endpoint_url, "sending outbound req"); - let client = rivet_pools::reqwest::client_no_timeout().await?; + let client = rivet_pools::reqwest::guarded_client_no_timeout(ctx.config()).await?; let req = client.get(endpoint_url).headers(headers); let conn_started = Instant::now(); @@ -411,11 +424,11 @@ async fn outbound_req_inner( _ => { let wrapped_err = anyhow::Error::from(err); - report_error( - ctx, - input.namespace_id, - &input.runner_name, - RunnerPoolError::ServerlessConnectionError { + let error = match rivet_outbound_guard::block_reason(&wrapped_err) { + Some(reason) => RunnerPoolError::ServerlessDestinationBlocked { + reason: reason.to_string(), + }, + None => RunnerPoolError::ServerlessConnectionError { // Print entire error chain message: wrapped_err .chain() @@ -423,8 +436,9 @@ async fn outbound_req_inner( .collect::>() .join("\n"), }, - ) - .await; + }; + + report_error(ctx, input.namespace_id, &input.runner_name, error).await; return Err(wrapped_err); } @@ -484,6 +498,34 @@ async fn outbound_req_inner( Ok(OutboundReqOutput::Draining { drain_sent: true }) } +/// Report a destination the policy refuses to dial and stop the connection loop. +/// +/// Retrying cannot help: the config has to change before this URL becomes reachable. +async fn blocked_destination( + ctx: &ActivityCtx, + input: &OutboundReqInput, + reason: rivet_outbound_guard::BlockReason, +) -> OutboundReqOutput { + tracing::warn!( + namespace_id = %input.namespace_id, + runner_name = %input.runner_name, + %reason, + "serverless url is not an allowed destination, ending outbound req" + ); + + report_error( + ctx, + input.namespace_id, + &input.runner_name, + RunnerPoolError::ServerlessDestinationBlocked { + reason: reason.to_string(), + }, + ) + .await; + + OutboundReqOutput::Draining { drain_sent: false } +} + /// Reads from the adjacent serverless runner wf which is keeping track of signals while this workflow runs /// outbound requests. #[tracing::instrument(skip_all)] diff --git a/engine/packages/pools/Cargo.toml b/engine/packages/pools/Cargo.toml index 2dcf4390b7..00fe13a97d 100644 --- a/engine/packages/pools/Cargo.toml +++ b/engine/packages/pools/Cargo.toml @@ -17,6 +17,7 @@ hyper-util.workspace = true lazy_static.workspace = true reqwest.workspace = true rivet-config.workspace = true +rivet-outbound-guard.workspace = true rivet-metrics.workspace = true rivet-util.workspace = true rustls.workspace = true diff --git a/engine/packages/pools/src/pools.rs b/engine/packages/pools/src/pools.rs index e2a10d1ac9..0c3acf8c91 100644 --- a/engine/packages/pools/src/pools.rs +++ b/engine/packages/pools/src/pools.rs @@ -46,7 +46,8 @@ impl Pools { // Initialize here to avoid cold starts elsewhere crate::reqwest::client().await?; - crate::reqwest::client_no_timeout().await?; + crate::reqwest::guarded_client(pool.config()).await?; + crate::reqwest::guarded_client_no_timeout(pool.config()).await?; Ok(pool) } diff --git a/engine/packages/pools/src/reqwest.rs b/engine/packages/pools/src/reqwest.rs index 779c08e219..24179f2bca 100644 --- a/engine/packages/pools/src/reqwest.rs +++ b/engine/packages/pools/src/reqwest.rs @@ -1,10 +1,21 @@ +use std::sync::Arc; + +use anyhow::{Context, Result}; use reqwest::Client; +use rivet_outbound_guard::{GuardedResolver, Policy}; use tokio::sync::OnceCell; static CLIENT: OnceCell = OnceCell::const_new(); -static CLIENT_NO_TIMEOUT: OnceCell = OnceCell::const_new(); +static GUARDED_CLIENT: OnceCell = OnceCell::const_new(); +static GUARDED_CLIENT_NO_TIMEOUT: OnceCell = OnceCell::const_new(); +static OUTBOUND_POLICY: OnceCell> = OnceCell::const_new(); static CLIENT_USER_AGENT: &str = concat!("RivetEngine/", env!("CARGO_PKG_VERSION")); +/// Client for trusted destinations inside the engine network, such as peer datacenters and epoxy +/// replicas. +/// +/// Never use this for a URL that came from user configuration. Those go through +/// [`guarded_client`], which restricts what the request can reach. pub async fn client() -> Result { CLIENT .get_or_try_init(|| async { @@ -17,9 +28,53 @@ pub async fn client() -> Result { .cloned() } -pub async fn client_no_timeout() -> Result { - CLIENT_NO_TIMEOUT - .get_or_try_init(|| async { Client::builder().user_agent(CLIENT_USER_AGENT).build() }) +/// Client for destinations that come from user configuration, such as serverless runner URLs. +/// +/// The `outbound` security policy is enforced at DNS resolution time and on every redirect, so +/// these requests cannot be steered at services only reachable from inside the engine network. +pub async fn guarded_client(config: &rivet_config::Config) -> Result { + GUARDED_CLIENT + .get_or_try_init(|| async { + build_guarded_client(config, Some(std::time::Duration::from_secs(30))).await + }) + .await + .cloned() +} + +/// Same as [`guarded_client`] but without a request timeout, for long-lived streaming requests +/// such as the serverless SSE connection. +pub async fn guarded_client_no_timeout(config: &rivet_config::Config) -> Result { + GUARDED_CLIENT_NO_TIMEOUT + .get_or_try_init(|| async { build_guarded_client(config, None).await }) + .await + .cloned() +} + +async fn build_guarded_client( + config: &rivet_config::Config, + timeout: Option, +) -> Result { + let policy = outbound_policy(config).await?; + + let mut builder = Client::builder() + .user_agent(CLIENT_USER_AGENT) + .dns_resolver(Arc::new(GuardedResolver::new(policy.clone()))) + .redirect(rivet_outbound_guard::redirect_policy(policy)); + + if let Some(timeout) = timeout { + builder = builder.timeout(timeout); + } + + builder.build().context("failed building guarded client") +} + +/// The destination policy applied to every request to a user-configured URL. +/// +/// Callers use this to reject a URL at the point it is submitted, before it is ever stored. The +/// guarded clients apply the same policy again when they connect. +pub async fn outbound_policy(config: &rivet_config::Config) -> Result> { + OUTBOUND_POLICY + .get_or_try_init(|| async { Policy::from_config(config).map(Arc::new) }) .await .cloned() } diff --git a/engine/packages/types/src/actor/error.rs b/engine/packages/types/src/actor/error.rs index c8a5e94095..158fddc036 100644 --- a/engine/packages/types/src/actor/error.rs +++ b/engine/packages/types/src/actor/error.rs @@ -15,6 +15,9 @@ pub enum RunnerPoolError { /// Serverless: SSE connection or network error ServerlessConnectionError { message: String }, + /// Serverless: the configured URL is not a destination the engine is allowed to reach + ServerlessDestinationBlocked { reason: String }, + /// Serverless: Runner sent invalid payload ServerlessInvalidSsePayload { message: String, diff --git a/frontend/src/app/runner-pool-error-popover.tsx b/frontend/src/app/runner-pool-error-popover.tsx index 268a9f118f..18a1f1b371 100644 --- a/frontend/src/app/runner-pool-error-popover.tsx +++ b/frontend/src/app/runner-pool-error-popover.tsx @@ -43,6 +43,7 @@ interface ClassifiedError { kind: | "serverless_http" | "serverless_connection" + | "serverless_destination_blocked" | "serverless_invalid_sse" | "serverless_stream_ended_early" | "downgrade" @@ -116,6 +117,18 @@ function classifyRunnerError(error: RivetActorError): ClassifiedError { fingerprint: `conn:${e.serverless_connection_error.message.slice(0, 64)}`, }), ) + .with( + P.shape({ + serverless_destination_blocked: P.shape({ reason: P.string }), + }), + (e) => ({ + severity: "error", + kind: "serverless_destination_blocked", + title: "Serverless URL is not an allowed destination", + body: e.serverless_destination_blocked.reason, + fingerprint: `blocked:${e.serverless_destination_blocked.reason.slice(0, 64)}`, + }), + ) .with( P.shape({ serverless_invalid_sse_payload: P.shape({ message: P.string }), @@ -502,6 +515,8 @@ function describeKind(kind: ClassifiedError["kind"]): string { return "Runner pool was downgraded to an unsupported version. Revert to a higher version."; case "serverless_stream_ended_early": return "Connection terminated before the runner stopped. Check the request lifespan limits on your serverless provider."; + case "serverless_destination_blocked": + return "The configured serverless URL points at a destination Rivet is not allowed to reach. Use a publicly routable URL, or allow the address range in the engine's outbound configuration."; case "internal": return "An internal error occurred in the runner pool."; default: diff --git a/frontend/src/components/actors/actor-status-label.tsx b/frontend/src/components/actors/actor-status-label.tsx index 7ce47442c2..5d08591928 100644 --- a/frontend/src/components/actors/actor-status-label.tsx +++ b/frontend/src/components/actors/actor-status-label.tsx @@ -103,6 +103,7 @@ export function ActorError({ error }: { error: object | string }) { .or(P.shape({ serverless_http_error: P.any })) .or(P.string) .or(P.shape({ serverless_connection_error: P.any })) + .or(P.shape({ serverless_destination_blocked: P.any })) .or(P.shape({ serverless_invalid_sse_payload: P.any })), }), (err) => , @@ -221,6 +222,23 @@ export function RunnerPoolError({ error }: { error: RivetActorError }) { ); }, ) + .with( + P.shape({ + serverless_destination_blocked: P.shape({ reason: P.string }), + }), + (errObj) => { + const reason = errObj.serverless_destination_blocked?.reason; + return ( + <> +

+ Serverless endpoint URL is not an allowed + destination +

+ {reason ? : null} + + ); + }, + ) .with( P.shape({ serverless_invalid_sse_payload: P.shape({ message: P.string }), diff --git a/frontend/src/queries/types.ts b/frontend/src/queries/types.ts index fdffce099f..f69860e2b3 100644 --- a/frontend/src/queries/types.ts +++ b/frontend/src/queries/types.ts @@ -35,4 +35,5 @@ export type RivetActorError = | { runner_id: string } | { serverless_http_error: unknown } | { serverless_connection_error: unknown } + | { serverless_destination_blocked: unknown } | { serverless_invalid_sse_payload: unknown }; diff --git a/self-host/compose/dev-host/rivet-engine/config.jsonc b/self-host/compose/dev-host/rivet-engine/config.jsonc index ee79af406a..1cd7e292d1 100644 --- a/self-host/compose/dev-host/rivet-engine/config.jsonc +++ b/self-host/compose/dev-host/rivet-engine/config.jsonc @@ -21,6 +21,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@127.0.0.1:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/0/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/0/config.jsonc index 29ca3ee404..058a1b0a44 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/0/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/0/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-a:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/1/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/1/config.jsonc index 29ca3ee404..058a1b0a44 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/1/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/1/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-a:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/2/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/2/config.jsonc index 29ca3ee404..058a1b0a44 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/2/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/2/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-a:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/0/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/0/config.jsonc index 0fe743310a..e8136b2656 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/0/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/0/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-b:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/1/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/1/config.jsonc index 0fe743310a..e8136b2656 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/1/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/1/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-b:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/2/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/2/config.jsonc index 0fe743310a..e8136b2656 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/2/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/2/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-b:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/0/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/0/config.jsonc index 5a639b2fd4..be419e4ab6 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/0/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/0/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-c:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/1/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/1/config.jsonc index 5a639b2fd4..be419e4ab6 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/1/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/1/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-c:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/2/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/2/config.jsonc index 5a639b2fd4..be419e4ab6 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/2/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/2/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-c:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/docker-compose.yml b/self-host/compose/dev-multidc-multinode/docker-compose.yml index 81b39d39eb..a971a9716d 100644 --- a/self-host/compose/dev-multidc-multinode/docker-compose.yml +++ b/self-host/compose/dev-multidc-multinode/docker-compose.yml @@ -208,7 +208,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-a:4317 stop_grace_period: 0s depends_on: @@ -254,7 +253,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-a:4317 stop_grace_period: 0s depends_on: @@ -298,7 +296,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-a:4317 stop_grace_period: 0s depends_on: @@ -519,7 +516,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-b:4317 stop_grace_period: 0s depends_on: @@ -563,7 +559,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-b:4317 stop_grace_period: 0s depends_on: @@ -607,7 +602,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-b:4317 stop_grace_period: 0s depends_on: @@ -826,7 +820,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-c:4317 stop_grace_period: 0s depends_on: @@ -870,7 +863,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-c:4317 stop_grace_period: 0s depends_on: @@ -914,7 +906,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-c:4317 stop_grace_period: 0s depends_on: diff --git a/self-host/compose/dev-multidc/datacenters/dc-a/rivet-engine/config.jsonc b/self-host/compose/dev-multidc/datacenters/dc-a/rivet-engine/config.jsonc index ac3c4bc3c6..eaef4154fa 100644 --- a/self-host/compose/dev-multidc/datacenters/dc-a/rivet-engine/config.jsonc +++ b/self-host/compose/dev-multidc/datacenters/dc-a/rivet-engine/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-a:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc/datacenters/dc-b/rivet-engine/config.jsonc b/self-host/compose/dev-multidc/datacenters/dc-b/rivet-engine/config.jsonc index 8e5c6aaa45..c42e0a0ffb 100644 --- a/self-host/compose/dev-multidc/datacenters/dc-b/rivet-engine/config.jsonc +++ b/self-host/compose/dev-multidc/datacenters/dc-b/rivet-engine/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-b:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc/datacenters/dc-c/rivet-engine/config.jsonc b/self-host/compose/dev-multidc/datacenters/dc-c/rivet-engine/config.jsonc index 6ba3be6d65..6a6238fec7 100644 --- a/self-host/compose/dev-multidc/datacenters/dc-c/rivet-engine/config.jsonc +++ b/self-host/compose/dev-multidc/datacenters/dc-c/rivet-engine/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-c:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multinode/rivet-engine/0/config.jsonc b/self-host/compose/dev-multinode/rivet-engine/0/config.jsonc index 13e9996d6e..b57d5c4e01 100644 --- a/self-host/compose/dev-multinode/rivet-engine/0/config.jsonc +++ b/self-host/compose/dev-multinode/rivet-engine/0/config.jsonc @@ -21,6 +21,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multinode/rivet-engine/1/config.jsonc b/self-host/compose/dev-multinode/rivet-engine/1/config.jsonc index 13e9996d6e..b57d5c4e01 100644 --- a/self-host/compose/dev-multinode/rivet-engine/1/config.jsonc +++ b/self-host/compose/dev-multinode/rivet-engine/1/config.jsonc @@ -21,6 +21,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multinode/rivet-engine/2/config.jsonc b/self-host/compose/dev-multinode/rivet-engine/2/config.jsonc index 13e9996d6e..b57d5c4e01 100644 --- a/self-host/compose/dev-multinode/rivet-engine/2/config.jsonc +++ b/self-host/compose/dev-multinode/rivet-engine/2/config.jsonc @@ -21,6 +21,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres:5432/rivet_engine" }, diff --git a/self-host/compose/dev/rivet-engine/config.jsonc b/self-host/compose/dev/rivet-engine/config.jsonc index 4c6312b6d3..cad7a1e600 100644 --- a/self-host/compose/dev/rivet-engine/config.jsonc +++ b/self-host/compose/dev/rivet-engine/config.jsonc @@ -21,6 +21,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres:5432/rivet_engine" }, diff --git a/self-host/compose/template/src/main.ts b/self-host/compose/template/src/main.ts index 82a08e1f73..4a984d1f70 100644 --- a/self-host/compose/template/src/main.ts +++ b/self-host/compose/template/src/main.ts @@ -18,7 +18,7 @@ import { generateDatacenterVectorClient } from "./services/edge/vector-client"; import { generateDatacenterVectorServer } from "./services/edge/vector-server"; function generateTemplate(templateName: string, config: TemplateConfig) { - const outputDir = path.join(__dirname, "../../../", templateName); + const outputDir = path.join(__dirname, "../../", templateName); // Remove existing directory if it exists if (fs.existsSync(outputDir)) { diff --git a/self-host/compose/template/src/services/edge/rivet-engine.ts b/self-host/compose/template/src/services/edge/rivet-engine.ts index 1c5b8247da..554d7b8454 100644 --- a/self-host/compose/template/src/services/edge/rivet-engine.ts +++ b/self-host/compose/template/src/services/edge/rivet-engine.ts @@ -45,6 +45,11 @@ export function generateDatacenterRivetEngine( host: "0.0.0.0", }, topology, + // Serverless runner URLs in a compose deployment point at other containers on this + // network, which the engine refuses to dial by default. + outbound: { + allow_private_networks: true, + }, postgres: { url: `postgresql://postgres:postgres@${context.getServiceHost("postgres", datacenter.name)}:5432/rivet_engine`, },