Fix race condition in http reverse proxy for websockets

When a WebSocket connection is established through the lws http reverse proxy, the
LWS_CALLBACK_ESTABLISHED callback was being called immediately. Under load, this
could lead to a race condition where the user code would write to the socket before
the connection was fully established on the other side of the proxy, causing the
connection to be dropped.

This patch fixes the issue by deferring the LWS_CALLBACK_ESTABLISHED
callback for proxied WebSocket connections. An lws_sul is added in order to
ensure the caller completed setting up the proxy connection before we try
to do anything with it.

Author: google-labs-jules[bot]

lib/core-net/dummy-callback.c

@@ -125,7 +125,7 @@ lws_callback_ws_proxy(struct lws *wsi, enum lws_callback_reasons reason,
 		 * If the parent has started to close, don't try to
 		 * upgrade it, just let it go.
 		 */
-		if (lwsi_state(wsi->parent) >= LRS_RETURNED_CLOSE)
+		if ((lwsi_state(wsi->parent) & 0xff) >= LRS_RETURNED_CLOSE)
 			return -1;
 
 		if (lws_process_ws_upgrade2(wsi->parent))

lib/core-net/private-lib-core-net.h

@@ -669,6 +669,9 @@ struct lws {
 	lws_sorted_usec_list_t		sul_timeout;
 	lws_sorted_usec_list_t		sul_hrtimer;
 	lws_sorted_usec_list_t		sul_validity;
+#if defined(LWS_WITH_HTTP_PROXY)
+	lws_sorted_usec_list_t		sul_ws_proxy_est;
+#endif
 	lws_sorted_usec_list_t		sul_connect_timeout;
 #if defined(WIN32)
 	lws_sorted_usec_list_t		win32_sul_connect_async_check;

lib/core-net/wsi-timeout.c

@@ -30,6 +30,9 @@ __lws_wsi_remove_from_sul(struct lws *wsi)
 	lws_sul_cancel(&wsi->sul_timeout);
 	lws_sul_cancel(&wsi->sul_hrtimer);
 	lws_sul_cancel(&wsi->sul_validity);
+#if defined(LWS_WITH_HTTP_PROXY)
+	lws_sul_cancel(&wsi->sul_ws_proxy_est);
+#endif
 #if defined(LWS_WITH_SYS_FAULT_INJECTION)
 	lws_sul_cancel(&wsi->sul_fault_timedclose);
 #endif

lib/roles/ws/ops-ws.c

@@ -24,6 +24,27 @@
 
 #include <private-lib-core.h>
 
+#if defined(LWS_WITH_HTTP_PROXY)
+static void
+lws_ws_proxy_est_cb(lws_sorted_usec_list_t *sul)
+{
+	struct lws *wsi = lws_container_of(sul, struct lws, sul_ws_proxy_est);
+
+	lwsi_set_state(wsi, LRS_ESTABLISHED);
+
+	if (wsi->a.protocol->callback)
+		if (wsi->a.protocol->callback(wsi, LWS_CALLBACK_ESTABLISHED,
+					    wsi->user_space,
+#ifdef LWS_WITH_TLS
+					    wsi->tls.ssl,
+#else
+					    NULL,
+#endif
+					    wsi->h2_stream_carries_ws))
+			lws_wsi_close(wsi, LWS_TO_KILL_ASYNC);
+}
+#endif
+
 #define LWS_CPYAPP(ptr, str) { strcpy(ptr, str); ptr += strlen(str); }
 
 /*
@@ -840,7 +861,10 @@ lws_server_init_wsi_for_ws(struct lws *wsi)
 {
 	int n;
 
-	lwsi_set_state(wsi, LRS_ESTABLISHED);
+#if defined(LWS_WITH_HTTP_PROXY)
+	if (!wsi->proxied_ws_parent) 
+		lwsi_set_state(wsi, LRS_ESTABLISHED);
+#endif
 
 	/*
 	 * create the frame buffer for this connection according to the
@@ -861,7 +885,15 @@ lws_server_init_wsi_for_ws(struct lws *wsi)
 
 	/* notify user code that we're ready to roll */
 
-	if (wsi->a.protocol->callback)
+	if (wsi->a.protocol->callback) {
+#if defined(LWS_WITH_HTTP_PROXY)
+		if (wsi->proxied_ws_parent) {
+			lws_sul_schedule(wsi->a.context, wsi->tsi,
+					 &wsi->sul_ws_proxy_est,
+					 lws_ws_proxy_est_cb, 5000);
+			goto validity;
+		}
+#endif
 		if (wsi->a.protocol->callback(wsi, LWS_CALLBACK_ESTABLISHED,
 					    wsi->user_space,
 #ifdef LWS_WITH_TLS
@@ -871,6 +903,10 @@ lws_server_init_wsi_for_ws(struct lws *wsi)
 #endif
 					    wsi->h2_stream_carries_ws))
 			return 1;
+	}
+#if defined(LWS_WITH_HTTP_PROXY)
+validity:
+#endif
 
 	lws_validity_confirmed(wsi);
 	lwsl_debug("ws established\n");