From 4304282457da375671e82f898d9d4678c3fd071e Mon Sep 17 00:00:00 2001 From: Mathias Staab Date: Mon, 20 Jul 2026 18:57:27 +0200 Subject: [PATCH] federation: reuse established tunnel instead of re-dialing peer:9191 The federation send path only looked up dest:9191 in children_ss and dialed a new outbound DTLS connection on a miss. When our egress NAT rewrites the source port, the peer keys our tunnel under the rewritten port, so its own send path misses and it dials back into us. That dial is redundant (a healthy tunnel between the two coturns already exists) and it is exactly the handshake that fails: our conntrack entry for the dial occupies the local:9191->peer:9191 tuple, so the kernel's clash resolution rewrites the inbound ClientHello fragments to a fresh source port each, and the handshake can never complete (one-way media, paired 10s handshake timeouts on the receiving side). Maintain a peers-by-ip index (peer IP, port zeroed -> established federation socket), registered when any federation handshake completes and unregistered in delete_socket_from_map so entries can never dangle. On a send-path miss of dest:9191, reuse any established tunnel to that peer IP instead of dialing; DTLS application data flows both ways regardless of client/server role (the server-side PONG already proves this path). Only dial when no established tunnel exists at all. One side deploying this heals a pairing: the redundant dial never happens, and the clash precondition is never created on the peer's node. Same thread model as children_ss (federation listener thread). --- src/apps/relay/dtls_listener.c | 5 +- src/apps/relay/federation.c | 97 +++++++++++++++++++++++++- src/apps/relay/federation.h | 2 + src/apps/relay/ns_ioalib_engine_impl.c | 7 ++ src/apps/relay/ns_ioalib_impl.h | 1 + 5 files changed, 110 insertions(+), 2 deletions(-) diff --git a/src/apps/relay/dtls_listener.c b/src/apps/relay/dtls_listener.c index ba2bf1d1f0..eb4c03dc8b 100644 --- a/src/apps/relay/dtls_listener.c +++ b/src/apps/relay/dtls_listener.c @@ -359,12 +359,15 @@ static int handle_udp_packet(dtls_listener_relay_server_type *server, struct mes //TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "%s: federation_listener: DTLS connection complete!\n", __FUNCTION__); IOA_EVENT_DEL(s->federation_handshake_tmr); // Stop federation handshake timer if running // We are connected now - start the heartbeat timer - s->federation_heartbeat_pings_outstanding = 0; + s->federation_heartbeat_pings_outstanding = 0; if(SSL_is_server(s->ssl)) { federation_start_server_heartbeat_timer(s); } else { federation_start_client_heartbeat_timer(s); } + // Index this established tunnel by peer IP so the send path can + // reuse it instead of dialing a redundant connection to peer:9191. + federation_register_established_peer(s); } send_ssl_backlog_buffers(s); // Send any data packets that have been queued waiting for handshake to finish diff --git a/src/apps/relay/federation.c b/src/apps/relay/federation.c index 55fc653252..4ce8071650 100644 --- a/src/apps/relay/federation.c +++ b/src/apps/relay/federation.c @@ -521,6 +521,87 @@ void federation_init(ioa_engine_handle e) { federation_load_certificates(); } +// --------------------------------------------------------------------------- +// Established-peer index: peer IP (port zeroed) -> established federation socket. +// +// The send path historically only looked up dest:9191 in children_ss and dialed +// a brand-new outbound connection on a miss. When the peer sits behind a NAT +// that rewrites our source port, the peer keys our tunnel under the rewritten +// port, its own send path misses on dest:9191 and it dials back into us - a +// redundant handshake that is also the one that fails behind port-rewriting +// NATs (the dial's own NAT/conntrack entry occupies the local:9191->peer:9191 +// tuple, so the peer's inbound ClientHello gets rewritten to fresh source +// ports and can never complete). This index lets the send path find ANY +// established tunnel to the destination peer (typically the accepted, +// peer-initiated one) and reuse it instead of dialing. +// +// Thread model: identical to children_ss. All accesses happen on the federation +// listener thread - registration from handle_udp_packet, lookup from +// federation_send_data_imp (marshaled there via the bufferevent pair), and +// unregistration from close paths of federation child sockets, whose timers and +// reads all run on the listener engine. Sockets of other threads never carry +// the federation_established_registered flag, so their close path never touches +// the map. +static ur_addr_map *federation_peers_by_ip = NULL; + +static void federation_peer_ip_key(const ioa_addr *addr, ioa_addr *key) { + addr_cpy(key, addr); + addr_set_port(key, 0); +} + +void federation_register_established_peer(ioa_socket_handle s) { + if (!s || !s->e) { + return; + } + if (!federation_peers_by_ip) { + federation_peers_by_ip = (ur_addr_map *)allocate_super_memory_engine(s->e, sizeof(ur_addr_map)); + ur_addr_map_init(federation_peers_by_ip); + } + ioa_addr key; + federation_peer_ip_key(get_remote_addr_from_ioa_socket(s), &key); + // Last-wins: a newer established tunnel to the same peer replaces the entry. + ur_addr_map_put(federation_peers_by_ip, &key, (ur_addr_map_value_type)s); + s->federation_established_registered = 1; +} + +void federation_unregister_established_peer(ioa_socket_handle s) { + if (!s || !s->federation_established_registered) { + return; + } + s->federation_established_registered = 0; + if (!federation_peers_by_ip) { + return; + } + ioa_addr key; + federation_peer_ip_key(get_remote_addr_from_ioa_socket(s), &key); + ur_addr_map_value_type mvt = 0; + // Only remove if the index still points at THIS socket - a newer established + // connection to the same peer may have replaced it. + if ((ur_addr_map_get(federation_peers_by_ip, &key, &mvt) > 0) && ((ioa_socket_handle)mvt == s)) { + ur_addr_map_del(federation_peers_by_ip, &key, NULL); + } +} + +static ioa_socket_handle federation_find_established_by_peer_ip(const ioa_addr *dest_addr) { + if (!federation_peers_by_ip) { + return NULL; + } + ioa_addr key; + federation_peer_ip_key(dest_addr, &key); + ur_addr_map_value_type mvt = 0; + if (!(ur_addr_map_get(federation_peers_by_ip, &key, &mvt) > 0) || !mvt) { + return NULL; + } + ioa_socket_handle s = (ioa_socket_handle)mvt; + // Unregistration on close guarantees entries are never dangling, but the + // socket may be on its way out or mid-renegotiation; never hand those back. + if (ioa_socket_tobeclosed(s) || !s->ssl || !SSL_is_init_finished(s->ssl)) { + return NULL; + } + return s; +} +// --------------------------------------------------------------------------- + int federation_send_data_imp(dtls_listener_relay_server_type* server, ioa_addr* dest_addr, ioa_network_buffer_handle nbh, int ttl, int tos, int* skip) { int fd = server->udp_listen_s->fd; @@ -540,7 +621,21 @@ int federation_send_data_imp(dtls_listener_relay_server_type* server, ioa_addr* chs = (ioa_socket_handle) mvt; } - if(chs == NULL) { + if(chs == NULL) { + // No outbound connection keyed dest:9191. Before dialing, reuse any + // established tunnel to this peer - typically the accepted connection the + // peer dialed to us (its source port may have been rewritten by NAT, so + // it is keyed under a different tuple). DTLS application data flows both + // ways regardless of client/server role, and dialing here is both + // redundant and the handshake that fails behind port-rewriting NATs. + chs = federation_find_established_by_peer_ip(dest_addr); + if(chs) { + addr_debug_print(eve(server->verbose), get_remote_addr_from_ioa_socket(chs), + "Federation: reusing established tunnel instead of dialing, peer"); + } + } + + if(chs == NULL) { // No connection yet, DTLS connect to dest_addr SSL* ssl = SSL_new(turn_params.federation_dtls_client_ctx); SSL_set_options(ssl, SSL_OP_COOKIE_EXCHANGE); diff --git a/src/apps/relay/federation.h b/src/apps/relay/federation.h index f0af5f80d4..a8e2233dea 100644 --- a/src/apps/relay/federation.h +++ b/src/apps/relay/federation.h @@ -26,6 +26,8 @@ void federation_whitelist_add(char* hostname, char* issuer); void federation_start_handshake_timer(ioa_socket_handle s); void federation_start_client_heartbeat_timer(ioa_socket_handle s); void federation_start_server_heartbeat_timer(ioa_socket_handle s); +void federation_register_established_peer(ioa_socket_handle s); +void federation_unregister_established_peer(ioa_socket_handle s); /////////////////////////////////////////// diff --git a/src/apps/relay/ns_ioalib_engine_impl.c b/src/apps/relay/ns_ioalib_engine_impl.c index 1ab25a1917..4f5ee3e6cf 100644 --- a/src/apps/relay/ns_ioalib_engine_impl.c +++ b/src/apps/relay/ns_ioalib_engine_impl.c @@ -1283,7 +1283,14 @@ void add_socket_to_map(ioa_socket_handle s, ur_addr_map *amap) { } } +/* federation.c - remove socket from the established-peer index (no-op unless + * the socket was registered there; see federation_register_established_peer) */ +void federation_unregister_established_peer(ioa_socket_handle s); + void delete_socket_from_map(ioa_socket_handle s) { + if (s) { + federation_unregister_established_peer(s); + } if (s && s->sockets_container) { ur_addr_map_del(s->sockets_container, &(s->remote_addr), NULL); diff --git a/src/apps/relay/ns_ioalib_impl.h b/src/apps/relay/ns_ioalib_impl.h index 4cb09a7750..d95ad1ee60 100644 --- a/src/apps/relay/ns_ioalib_impl.h +++ b/src/apps/relay/ns_ioalib_impl.h @@ -187,6 +187,7 @@ struct _ioa_socket { ioa_timer_handle federation_handshake_tmr; ioa_timer_handle federation_heartbeat_tmr; int federation_heartbeat_pings_outstanding; + int federation_established_registered; /* socket is in the federation peers-by-ip index */ uint32_t ssl_renegs; int in_write; int bound;