diff --git a/docs/wsmc-observability.md b/docs/wsmc-observability.md new file mode 100644 index 000000000..c84ab1c2c --- /dev/null +++ b/docs/wsmc-observability.md @@ -0,0 +1,25 @@ +# WSMC observability + +Set `OTEL_METRICS_ENABLED=true` and configure Gate's normal OTLP or Prometheus exporter. The WSMC +listener publishes low-cardinality transport metrics. Client IP addresses and literal virtual hosts +are intentionally not used as labels. + +| Metric | Purpose | +| --- | --- | +| `gate.wsmc.active_sessions` | Current accepted sessions by mode | +| `gate.wsmc.requests` | Accepted and rejected handshakes by result and mode | +| `gate.wsmc.bytes` | Stream bytes by direction and mode | +| `gate.wsmc.frames` | Binary messages emitted by Gate | +| `gate.wsmc.write.duration` | Time spent writing one coalesced batch | +| `gate.wsmc.batch.bytes` | Coalesced stream batch size | +| `gate.wsmc.pending.bytes` | Pending bytes after a completed write | +| `gate.wsmc.target_frame.bytes` | Adaptive outgoing message payload target | +| `gate.wsmc.handshake.duration` | HTTP upgrade duration by result and mode | +| `gate.wsmc.session.duration` | Accepted session lifetime by mode | +| `gate.wsmc.events` | Idle, backpressure, close-flush and write failures | + +The compatibility test follows `rikka0w0/wsmc` commit +`ba665afdf3bd3d1feb90431dd0d53cd077b98f61`: RFC 6455 V13, binary messages, a custom HTTP Host, +and a default maximum frame payload of 65536 bytes. WSMC treats message payloads as a continuous +Minecraft TCP byte stream, so Gate may safely coalesce and re-slice messages without adding an +application-level acknowledgement or retransmission protocol. diff --git a/pkg/edition/java/lite/websocket.go b/pkg/edition/java/lite/websocket.go index c360180c5..fc39298fb 100644 --- a/pkg/edition/java/lite/websocket.go +++ b/pkg/edition/java/lite/websocket.go @@ -87,13 +87,16 @@ func ServeWebSocket(ctx context.Context, opts WebSocketOptions) error { } func (s *webSocketServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + requestStarted := time.Now() cfg := s.opts.Config() wsCfg := cfg.WebSocket if !wsCfg.Enabled { + wsmcMetrics.request("disabled", "unknown", time.Since(requestStarted)) http.Error(w, "WebSocket transport disabled", http.StatusServiceUnavailable) return } if path := wsCfg.EffectivePath(); path != "" && r.URL.Path != path { + wsmcMetrics.request("invalid_path", "unknown", time.Since(requestStarted)) http.NotFound(w, r) return } @@ -101,12 +104,14 @@ func (s *webSocketServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { virtualHost := webSocketHost(r.Host) matchedHost, route, groups := FindRouteWithGroups(virtualHost, cfg.Routes...) if route == nil || !route.WebSocket.Enabled { + wsmcMetrics.request("no_route", "unknown", time.Since(requestStarted)) http.Error(w, "No WebSocket route", http.StatusNotFound) return } remoteAddr, clientIP := s.clientAddr(r, wsCfg) if !s.acquireIP(clientIP, wsCfg.EffectiveMaxConnectionsPerIP()) { + wsmcMetrics.request("quota_rejected", string(route.WebSocket.EffectiveMode()), time.Since(requestStarted)) http.Error(w, "Too many WebSocket connections", http.StatusTooManyRequests) return } @@ -118,12 +123,19 @@ func (s *webSocketServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { } wsConn, err := websocket.Accept(w, r, &websocket.AcceptOptions{CompressionMode: compression}) if err != nil { + wsmcMetrics.request("handshake_failed", string(route.WebSocket.EffectiveMode()), time.Since(requestStarted)) s.opts.Logger.V(1).Info("websocket handshake failed", "error", err, "remoteAddr", r.RemoteAddr) return } connCtx, cancel := context.WithCancel(s.ctx) defer cancel() - client := newWebSocketStreamConn(connCtx, wsConn, wsCfg, remoteAddr, virtualHost) + mode := string(route.WebSocket.EffectiveMode()) + client := newWebSocketStreamConn(connCtx, wsConn, wsCfg, remoteAddr, virtualHost, mode) + wsmcMetrics.request("accepted", mode, time.Since(requestStarted)) + wsmcMetrics.addActive(mode, 1) + defer wsmcMetrics.addActive(mode, -1) + sessionStarted := time.Now() + defer func() { wsmcMetrics.session(mode, time.Since(sessionStarted)) }() log := s.opts.Logger.WithName("lite").WithName("websocket").WithValues( "clientAddr", remoteAddr, "virtualHost", virtualHost, "route", matchedHost, @@ -247,7 +259,7 @@ func (s *webSocketServer) dialRawBackend( if err != nil { return nil, err } - return newWebSocketStreamConn(ctx, wsConn, listenerCfg, nil, ""), nil + return newWebSocketStreamConn(ctx, wsConn, listenerCfg, nil, "", "raw-backend"), nil } type webSocketListener struct { diff --git a/pkg/edition/java/lite/websocket_metrics.go b/pkg/edition/java/lite/websocket_metrics.go new file mode 100644 index 000000000..6e1e8595c --- /dev/null +++ b/pkg/edition/java/lite/websocket_metrics.go @@ -0,0 +1,109 @@ +package lite + +import ( + "context" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +var wsmcMetrics = newWSMCMetrics() + +type wsmcMetricsRecorder struct { + active metric.Int64UpDownCounter + requests metric.Int64Counter + bytes metric.Int64Counter + frames metric.Int64Counter + events metric.Int64Counter + writeDuration metric.Float64Histogram + batchBytes metric.Int64Histogram + pendingBytes metric.Int64Histogram + targetFrame metric.Int64Histogram + handshakeTime metric.Float64Histogram + sessionTime metric.Float64Histogram +} + +func newWSMCMetrics() wsmcMetricsRecorder { + m := otel.Meter("java/lite/wsmc") + active, _ := m.Int64UpDownCounter("gate.wsmc.active_sessions", metric.WithUnit("1")) + requests, _ := m.Int64Counter("gate.wsmc.requests", metric.WithUnit("1")) + bytes, _ := m.Int64Counter("gate.wsmc.bytes", metric.WithUnit("By")) + frames, _ := m.Int64Counter("gate.wsmc.frames", metric.WithUnit("1")) + events, _ := m.Int64Counter("gate.wsmc.events", metric.WithUnit("1")) + writeDuration, _ := m.Float64Histogram("gate.wsmc.write.duration", metric.WithUnit("ms")) + batchBytes, _ := m.Int64Histogram("gate.wsmc.batch.bytes", metric.WithUnit("By")) + pendingBytes, _ := m.Int64Histogram("gate.wsmc.pending.bytes", metric.WithUnit("By")) + targetFrame, _ := m.Int64Histogram("gate.wsmc.target_frame.bytes", metric.WithUnit("By")) + handshakeTime, _ := m.Float64Histogram("gate.wsmc.handshake.duration", metric.WithUnit("ms")) + sessionTime, _ := m.Float64Histogram("gate.wsmc.session.duration", metric.WithUnit("s")) + return wsmcMetricsRecorder{active, requests, bytes, frames, events, writeDuration, batchBytes, pendingBytes, targetFrame, handshakeTime, sessionTime} +} + +func modeAttr(mode string) metric.MeasurementOption { + return metric.WithAttributes(attribute.String("mode", mode)) +} + +func (m wsmcMetricsRecorder) addActive(mode string, delta int64) { + if m.active != nil { + m.active.Add(context.Background(), delta, modeAttr(mode)) + } +} + +func (m wsmcMetricsRecorder) request(result, mode string, elapsed time.Duration) { + if m.requests != nil { + m.requests.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("result", result), attribute.String("mode", mode))) + } + if m.handshakeTime != nil { + m.handshakeTime.Record(context.Background(), float64(elapsed)/float64(time.Millisecond), + metric.WithAttributes(attribute.String("result", result), attribute.String("mode", mode))) + } +} + +func (m wsmcMetricsRecorder) session(mode string, elapsed time.Duration) { + if m.sessionTime != nil { + m.sessionTime.Record(context.Background(), elapsed.Seconds(), modeAttr(mode)) + } +} + +func (m wsmcMetricsRecorder) event(mode, event string) { + if m.events != nil { + m.events.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("mode", mode), attribute.String("event", event))) + } +} + +func (m wsmcMetricsRecorder) read(mode string, n int) { + if n <= 0 { + return + } + attrs := metric.WithAttributes(attribute.String("mode", mode), attribute.String("direction", "read")) + if m.bytes != nil { + m.bytes.Add(context.Background(), int64(n), attrs) + } +} + +func (m wsmcMetricsRecorder) write(mode string, frames, bytes, pending, target int, elapsed time.Duration) { + ctx := context.Background() + attrs := metric.WithAttributes(attribute.String("mode", mode), attribute.String("direction", "write")) + if m.bytes != nil { + m.bytes.Add(ctx, int64(bytes), attrs) + } + if m.frames != nil { + m.frames.Add(ctx, int64(frames), modeAttr(mode)) + } + if m.writeDuration != nil { + m.writeDuration.Record(ctx, float64(elapsed)/float64(time.Millisecond), modeAttr(mode)) + } + if m.batchBytes != nil { + m.batchBytes.Record(ctx, int64(bytes), modeAttr(mode)) + } + if m.pendingBytes != nil { + m.pendingBytes.Record(ctx, int64(pending), modeAttr(mode)) + } + if m.targetFrame != nil { + m.targetFrame.Record(ctx, int64(target), modeAttr(mode)) + } +} diff --git a/pkg/edition/java/lite/websocket_stream.go b/pkg/edition/java/lite/websocket_stream.go index b5d819a8c..09b8a1a97 100644 --- a/pkg/edition/java/lite/websocket_stream.go +++ b/pkg/edition/java/lite/websocket_stream.go @@ -25,11 +25,12 @@ func newWebSocketStreamConn( cfg config.WebSocketListenerConfig, remoteAddr net.Addr, virtualHost string, + mode string, ) *webSocketStreamConn { stream := websocket.NetConn(ctx, wsConn, websocket.MessageBinary) // NetConn disables the read limit, so restore it after construction. wsConn.SetReadLimit(cfg.EffectiveReadLimit()) - optimized := newOptimizedWebSocketConn(stream, cfg) + optimized := newOptimizedWebSocketConn(stream, cfg, mode) return &webSocketStreamConn{Conn: optimized, remoteAddr: remoteAddr, virtualHost: virtualHost} } @@ -61,15 +62,17 @@ type optimizedWebSocketConn struct { targetFrame atomic.Int64 lastActive atomic.Int64 + mode string } -func newOptimizedWebSocketConn(conn net.Conn, cfg config.WebSocketListenerConfig) *optimizedWebSocketConn { +func newOptimizedWebSocketConn(conn net.Conn, cfg config.WebSocketListenerConfig, mode string) *optimizedWebSocketConn { c := &optimizedWebSocketConn{ Conn: conn, cfg: cfg, queue: make(chan []byte, 1024), done: make(chan struct{}), spaceChanged: make(chan struct{}), + mode: mode, } c.targetFrame.Store(int64(cfg.EffectiveFramePayloadSize())) c.touch() @@ -82,6 +85,7 @@ func (c *optimizedWebSocketConn) Read(p []byte) (int, error) { n, err := c.Conn.Read(p) if n > 0 { c.touch() + wsmcMetrics.read(c.mode, n) } return n, err } @@ -140,6 +144,7 @@ func (c *optimizedWebSocketConn) Close() error { select { case <-changed: case <-timer.C: + wsmcMetrics.event(c.mode, "close_flush_timeout") c.fail(errors.New("websocket close flush timeout")) return c.currentError() case <-c.done: @@ -166,6 +171,7 @@ func (c *optimizedWebSocketConn) reserve(size int) error { select { case <-changed: case <-timer.C: + wsmcMetrics.event(c.mode, "backpressure_timeout") c.fail(errors.New("websocket backpressure timeout")) return c.currentError() case <-c.done: @@ -260,32 +266,39 @@ func (c *optimizedWebSocketConn) writeLoop() { payload = append(payload, part...) } started := time.Now() - err := c.writeFrames(payload) + frames, err := c.writeFrames(payload) elapsed := time.Since(started) c.release(batchBytes) if err != nil { + wsmcMetrics.event(c.mode, "write_failure") c.fail(err) return } c.touch() c.adjustFrameSize(elapsed) + c.pendingMu.Lock() + pending := c.pendingBytes + c.pendingMu.Unlock() + wsmcMetrics.write(c.mode, frames, batchBytes, pending, int(c.targetFrame.Load()), elapsed) } } -func (c *optimizedWebSocketConn) writeFrames(payload []byte) error { +func (c *optimizedWebSocketConn) writeFrames(payload []byte) (int, error) { frameSize := int(c.targetFrame.Load()) + frames := 0 for len(payload) > 0 { n := min(len(payload), frameSize) written, err := c.Conn.Write(payload[:n]) if err != nil { - return err + return frames, err } if written != n { - return io.ErrShortWrite + return frames, io.ErrShortWrite } + frames++ payload = payload[n:] } - return nil + return frames, nil } func (c *optimizedWebSocketConn) adjustFrameSize(writeDuration time.Duration) { @@ -316,6 +329,7 @@ func (c *optimizedWebSocketConn) idleLoop() { case now := <-ticker.C: last := time.Unix(0, c.lastActive.Load()) if now.Sub(last) >= idle { + wsmcMetrics.event(c.mode, "idle_timeout") c.fail(errors.New("websocket idle timeout")) return } diff --git a/pkg/edition/java/lite/websocket_test.go b/pkg/edition/java/lite/websocket_test.go index 09e726957..dfd703138 100644 --- a/pkg/edition/java/lite/websocket_test.go +++ b/pkg/edition/java/lite/websocket_test.go @@ -112,6 +112,60 @@ func TestWebSocketRawPassthrough(t *testing.T) { require.Equal(t, want, got) } +// WSMC's current client (WebSocketClientHandler/WebSocketHandler) uses RFC6455 +// V13, sends every Minecraft ByteBuf as a binary message and exposes received +// binary message payloads as one continuous Netty byte stream. Verify both +// boundaries against that contract rather than only using websocket.NetConn. +func TestWebSocketTranslateMatchesWSMCBinaryFrameContract(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + cfg := config.Config{ + WebSocket: config.WebSocketListenerConfig{ + Enabled: true, Path: "/mc", FramePayloadSize: 65536, + ReadLimit: 1 << 20, MaxConnectionsPerIP: 4, + }, + Routes: []config.Route{{ + Host: []string{"wsmc.example.com"}, Backend: []string{"127.0.0.1:25565"}, + WebSocket: config.WebSocketRouteConfig{Enabled: true, Mode: config.WebSocketModeTranslate}, + }}, + } + server := &webSocketServer{ + ctx: ctx, + opts: WebSocketOptions{ + Config: func() config.Config { return cfg }, StrategyManager: NewStrategyManager(), Logger: logr.Discard(), + HandleConn: func(conn net.Conn) { + defer conn.Close() + in := make([]byte, 5) + _, _ = io.ReadFull(conn, in) + _, _ = conn.Write(make([]byte, 65536*2+7)) + }, + }, + activeIPs: make(map[string]int), + } + httpServer := httptest.NewServer(server) + defer httpServer.Close() + ws, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(httpServer.URL, "http")+"/mc", &websocket.DialOptions{Host: "wsmc.example.com"}) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, ws.CloseNow()) + }) + ws.SetReadLimit(1 << 20) + require.NoError(t, ws.Write(ctx, websocket.MessageBinary, []byte{1, 2})) + require.NoError(t, ws.Write(ctx, websocket.MessageBinary, []byte{3, 4, 5})) + total := 0 + frames := 0 + for total < 65536*2+7 { + typ, payload, readErr := ws.Read(ctx) + require.NoError(t, readErr) + require.Equal(t, websocket.MessageBinary, typ) + require.LessOrEqual(t, len(payload), 65536) + total += len(payload) + frames++ + } + require.Equal(t, 65536*2+7, total) + require.Equal(t, 3, frames) +} + func TestUntrustedProxyCannotSpoofForwardedFor(t *testing.T) { s := &webSocketServer{} r := httptest.NewRequest(http.MethodGet, "http://example.com/mc", nil)