Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions docs/wsmc-observability.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# WSMC observability

Set `OTEL_METRICS_ENABLED=true` and configure Gate's normal OTLP or Prometheus exporter. The WSMC
listener publishes low-cardinality transport metrics. Client IP addresses and literal virtual hosts
are intentionally not used as labels.

| Metric | Purpose |
| --- | --- |
| `gate.wsmc.active_sessions` | Current accepted sessions by mode |
| `gate.wsmc.requests` | Accepted and rejected handshakes by result and mode |
| `gate.wsmc.bytes` | Stream bytes by direction and mode |
| `gate.wsmc.frames` | Binary messages emitted by Gate |
| `gate.wsmc.write.duration` | Time spent writing one coalesced batch |
| `gate.wsmc.batch.bytes` | Coalesced stream batch size |
| `gate.wsmc.pending.bytes` | Pending bytes after a completed write |
| `gate.wsmc.target_frame.bytes` | Adaptive outgoing message payload target |
| `gate.wsmc.handshake.duration` | HTTP upgrade duration by result and mode |
| `gate.wsmc.session.duration` | Accepted session lifetime by mode |
| `gate.wsmc.events` | Idle, backpressure, close-flush and write failures |

The compatibility test follows `rikka0w0/wsmc` commit
`ba665afdf3bd3d1feb90431dd0d53cd077b98f61`: RFC 6455 V13, binary messages, a custom HTTP Host,
and a default maximum frame payload of 65536 bytes. WSMC treats message payloads as a continuous
Minecraft TCP byte stream, so Gate may safely coalesce and re-slice messages without adding an
application-level acknowledgement or retransmission protocol.
16 changes: 14 additions & 2 deletions pkg/edition/java/lite/websocket.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,26 +87,31 @@ func ServeWebSocket(ctx context.Context, opts WebSocketOptions) error {
}

func (s *webSocketServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
requestStarted := time.Now()
cfg := s.opts.Config()
wsCfg := cfg.WebSocket
if !wsCfg.Enabled {
wsmcMetrics.request("disabled", "unknown", time.Since(requestStarted))
http.Error(w, "WebSocket transport disabled", http.StatusServiceUnavailable)
return
}
if path := wsCfg.EffectivePath(); path != "" && r.URL.Path != path {
wsmcMetrics.request("invalid_path", "unknown", time.Since(requestStarted))
http.NotFound(w, r)
return
}

virtualHost := webSocketHost(r.Host)
matchedHost, route, groups := FindRouteWithGroups(virtualHost, cfg.Routes...)
if route == nil || !route.WebSocket.Enabled {
wsmcMetrics.request("no_route", "unknown", time.Since(requestStarted))
http.Error(w, "No WebSocket route", http.StatusNotFound)
return
}

remoteAddr, clientIP := s.clientAddr(r, wsCfg)
if !s.acquireIP(clientIP, wsCfg.EffectiveMaxConnectionsPerIP()) {
wsmcMetrics.request("quota_rejected", string(route.WebSocket.EffectiveMode()), time.Since(requestStarted))
http.Error(w, "Too many WebSocket connections", http.StatusTooManyRequests)
return
}
Expand All @@ -118,12 +123,19 @@ func (s *webSocketServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
wsConn, err := websocket.Accept(w, r, &websocket.AcceptOptions{CompressionMode: compression})
if err != nil {
wsmcMetrics.request("handshake_failed", string(route.WebSocket.EffectiveMode()), time.Since(requestStarted))
s.opts.Logger.V(1).Info("websocket handshake failed", "error", err, "remoteAddr", r.RemoteAddr)
return
}
connCtx, cancel := context.WithCancel(s.ctx)
defer cancel()
client := newWebSocketStreamConn(connCtx, wsConn, wsCfg, remoteAddr, virtualHost)
mode := string(route.WebSocket.EffectiveMode())
client := newWebSocketStreamConn(connCtx, wsConn, wsCfg, remoteAddr, virtualHost, mode)
wsmcMetrics.request("accepted", mode, time.Since(requestStarted))
wsmcMetrics.addActive(mode, 1)
defer wsmcMetrics.addActive(mode, -1)
sessionStarted := time.Now()
defer func() { wsmcMetrics.session(mode, time.Since(sessionStarted)) }()

log := s.opts.Logger.WithName("lite").WithName("websocket").WithValues(
"clientAddr", remoteAddr, "virtualHost", virtualHost, "route", matchedHost,
Expand Down Expand Up @@ -247,7 +259,7 @@ func (s *webSocketServer) dialRawBackend(
if err != nil {
return nil, err
}
return newWebSocketStreamConn(ctx, wsConn, listenerCfg, nil, ""), nil
return newWebSocketStreamConn(ctx, wsConn, listenerCfg, nil, "", "raw-backend"), nil
}

type webSocketListener struct {
Expand Down
109 changes: 109 additions & 0 deletions pkg/edition/java/lite/websocket_metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
package lite

import (
"context"
"time"

"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
)

var wsmcMetrics = newWSMCMetrics()

type wsmcMetricsRecorder struct {
active metric.Int64UpDownCounter
requests metric.Int64Counter
bytes metric.Int64Counter
frames metric.Int64Counter
events metric.Int64Counter
writeDuration metric.Float64Histogram
batchBytes metric.Int64Histogram
pendingBytes metric.Int64Histogram
targetFrame metric.Int64Histogram
handshakeTime metric.Float64Histogram
sessionTime metric.Float64Histogram
}

func newWSMCMetrics() wsmcMetricsRecorder {
m := otel.Meter("java/lite/wsmc")
active, _ := m.Int64UpDownCounter("gate.wsmc.active_sessions", metric.WithUnit("1"))
requests, _ := m.Int64Counter("gate.wsmc.requests", metric.WithUnit("1"))
bytes, _ := m.Int64Counter("gate.wsmc.bytes", metric.WithUnit("By"))
frames, _ := m.Int64Counter("gate.wsmc.frames", metric.WithUnit("1"))
events, _ := m.Int64Counter("gate.wsmc.events", metric.WithUnit("1"))
writeDuration, _ := m.Float64Histogram("gate.wsmc.write.duration", metric.WithUnit("ms"))
batchBytes, _ := m.Int64Histogram("gate.wsmc.batch.bytes", metric.WithUnit("By"))
pendingBytes, _ := m.Int64Histogram("gate.wsmc.pending.bytes", metric.WithUnit("By"))
targetFrame, _ := m.Int64Histogram("gate.wsmc.target_frame.bytes", metric.WithUnit("By"))
handshakeTime, _ := m.Float64Histogram("gate.wsmc.handshake.duration", metric.WithUnit("ms"))
sessionTime, _ := m.Float64Histogram("gate.wsmc.session.duration", metric.WithUnit("s"))
return wsmcMetricsRecorder{active, requests, bytes, frames, events, writeDuration, batchBytes, pendingBytes, targetFrame, handshakeTime, sessionTime}
}

func modeAttr(mode string) metric.MeasurementOption {
return metric.WithAttributes(attribute.String("mode", mode))
}

func (m wsmcMetricsRecorder) addActive(mode string, delta int64) {
if m.active != nil {
m.active.Add(context.Background(), delta, modeAttr(mode))
}
}

func (m wsmcMetricsRecorder) request(result, mode string, elapsed time.Duration) {
if m.requests != nil {
m.requests.Add(context.Background(), 1, metric.WithAttributes(
attribute.String("result", result), attribute.String("mode", mode)))
}
if m.handshakeTime != nil {
m.handshakeTime.Record(context.Background(), float64(elapsed)/float64(time.Millisecond),
metric.WithAttributes(attribute.String("result", result), attribute.String("mode", mode)))
}
}

func (m wsmcMetricsRecorder) session(mode string, elapsed time.Duration) {
if m.sessionTime != nil {
m.sessionTime.Record(context.Background(), elapsed.Seconds(), modeAttr(mode))
}
}

func (m wsmcMetricsRecorder) event(mode, event string) {
if m.events != nil {
m.events.Add(context.Background(), 1, metric.WithAttributes(
attribute.String("mode", mode), attribute.String("event", event)))
}
}

func (m wsmcMetricsRecorder) read(mode string, n int) {
if n <= 0 {
return
}
attrs := metric.WithAttributes(attribute.String("mode", mode), attribute.String("direction", "read"))
if m.bytes != nil {
m.bytes.Add(context.Background(), int64(n), attrs)
}
}

func (m wsmcMetricsRecorder) write(mode string, frames, bytes, pending, target int, elapsed time.Duration) {
ctx := context.Background()
attrs := metric.WithAttributes(attribute.String("mode", mode), attribute.String("direction", "write"))
if m.bytes != nil {
m.bytes.Add(ctx, int64(bytes), attrs)
}
if m.frames != nil {
m.frames.Add(ctx, int64(frames), modeAttr(mode))
}
if m.writeDuration != nil {
m.writeDuration.Record(ctx, float64(elapsed)/float64(time.Millisecond), modeAttr(mode))
}
if m.batchBytes != nil {
m.batchBytes.Record(ctx, int64(bytes), modeAttr(mode))
}
if m.pendingBytes != nil {
m.pendingBytes.Record(ctx, int64(pending), modeAttr(mode))
}
if m.targetFrame != nil {
m.targetFrame.Record(ctx, int64(target), modeAttr(mode))
}
}
28 changes: 21 additions & 7 deletions pkg/edition/java/lite/websocket_stream.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,12 @@ func newWebSocketStreamConn(
cfg config.WebSocketListenerConfig,
remoteAddr net.Addr,
virtualHost string,
mode string,
) *webSocketStreamConn {
stream := websocket.NetConn(ctx, wsConn, websocket.MessageBinary)
// NetConn disables the read limit, so restore it after construction.
wsConn.SetReadLimit(cfg.EffectiveReadLimit())
optimized := newOptimizedWebSocketConn(stream, cfg)
optimized := newOptimizedWebSocketConn(stream, cfg, mode)
return &webSocketStreamConn{Conn: optimized, remoteAddr: remoteAddr, virtualHost: virtualHost}
}

Expand Down Expand Up @@ -61,15 +62,17 @@ type optimizedWebSocketConn struct {

targetFrame atomic.Int64
lastActive atomic.Int64
mode string
}

func newOptimizedWebSocketConn(conn net.Conn, cfg config.WebSocketListenerConfig) *optimizedWebSocketConn {
func newOptimizedWebSocketConn(conn net.Conn, cfg config.WebSocketListenerConfig, mode string) *optimizedWebSocketConn {
c := &optimizedWebSocketConn{
Conn: conn,
cfg: cfg,
queue: make(chan []byte, 1024),
done: make(chan struct{}),
spaceChanged: make(chan struct{}),
mode: mode,
}
c.targetFrame.Store(int64(cfg.EffectiveFramePayloadSize()))
c.touch()
Expand All @@ -82,6 +85,7 @@ func (c *optimizedWebSocketConn) Read(p []byte) (int, error) {
n, err := c.Conn.Read(p)
if n > 0 {
c.touch()
wsmcMetrics.read(c.mode, n)
}
return n, err
}
Expand Down Expand Up @@ -140,6 +144,7 @@ func (c *optimizedWebSocketConn) Close() error {
select {
case <-changed:
case <-timer.C:
wsmcMetrics.event(c.mode, "close_flush_timeout")
c.fail(errors.New("websocket close flush timeout"))
return c.currentError()
case <-c.done:
Expand All @@ -166,6 +171,7 @@ func (c *optimizedWebSocketConn) reserve(size int) error {
select {
case <-changed:
case <-timer.C:
wsmcMetrics.event(c.mode, "backpressure_timeout")
c.fail(errors.New("websocket backpressure timeout"))
return c.currentError()
case <-c.done:
Expand Down Expand Up @@ -260,32 +266,39 @@ func (c *optimizedWebSocketConn) writeLoop() {
payload = append(payload, part...)
}
started := time.Now()
err := c.writeFrames(payload)
frames, err := c.writeFrames(payload)
elapsed := time.Since(started)
c.release(batchBytes)
if err != nil {
wsmcMetrics.event(c.mode, "write_failure")
c.fail(err)
return
}
c.touch()
c.adjustFrameSize(elapsed)
c.pendingMu.Lock()
pending := c.pendingBytes
c.pendingMu.Unlock()
wsmcMetrics.write(c.mode, frames, batchBytes, pending, int(c.targetFrame.Load()), elapsed)
}
}

func (c *optimizedWebSocketConn) writeFrames(payload []byte) error {
func (c *optimizedWebSocketConn) writeFrames(payload []byte) (int, error) {
frameSize := int(c.targetFrame.Load())
frames := 0
for len(payload) > 0 {
n := min(len(payload), frameSize)
written, err := c.Conn.Write(payload[:n])
if err != nil {
return err
return frames, err
}
if written != n {
return io.ErrShortWrite
return frames, io.ErrShortWrite
}
frames++
payload = payload[n:]
}
return nil
return frames, nil
}

func (c *optimizedWebSocketConn) adjustFrameSize(writeDuration time.Duration) {
Expand Down Expand Up @@ -316,6 +329,7 @@ func (c *optimizedWebSocketConn) idleLoop() {
case now := <-ticker.C:
last := time.Unix(0, c.lastActive.Load())
if now.Sub(last) >= idle {
wsmcMetrics.event(c.mode, "idle_timeout")
c.fail(errors.New("websocket idle timeout"))
return
}
Expand Down
54 changes: 54 additions & 0 deletions pkg/edition/java/lite/websocket_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,60 @@ func TestWebSocketRawPassthrough(t *testing.T) {
require.Equal(t, want, got)
}

// WSMC's current client (WebSocketClientHandler/WebSocketHandler) uses RFC6455
// V13, sends every Minecraft ByteBuf as a binary message and exposes received
// binary message payloads as one continuous Netty byte stream. Verify both
// boundaries against that contract rather than only using websocket.NetConn.
func TestWebSocketTranslateMatchesWSMCBinaryFrameContract(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
cfg := config.Config{
WebSocket: config.WebSocketListenerConfig{
Enabled: true, Path: "/mc", FramePayloadSize: 65536,
ReadLimit: 1 << 20, MaxConnectionsPerIP: 4,
},
Routes: []config.Route{{
Host: []string{"wsmc.example.com"}, Backend: []string{"127.0.0.1:25565"},
WebSocket: config.WebSocketRouteConfig{Enabled: true, Mode: config.WebSocketModeTranslate},
}},
}
server := &webSocketServer{
ctx: ctx,
opts: WebSocketOptions{
Config: func() config.Config { return cfg }, StrategyManager: NewStrategyManager(), Logger: logr.Discard(),
HandleConn: func(conn net.Conn) {
defer conn.Close()
in := make([]byte, 5)
_, _ = io.ReadFull(conn, in)
_, _ = conn.Write(make([]byte, 65536*2+7))
},
},
activeIPs: make(map[string]int),
}
httpServer := httptest.NewServer(server)
defer httpServer.Close()
ws, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(httpServer.URL, "http")+"/mc", &websocket.DialOptions{Host: "wsmc.example.com"})
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, ws.CloseNow())
})
ws.SetReadLimit(1 << 20)
require.NoError(t, ws.Write(ctx, websocket.MessageBinary, []byte{1, 2}))
require.NoError(t, ws.Write(ctx, websocket.MessageBinary, []byte{3, 4, 5}))
total := 0
frames := 0
for total < 65536*2+7 {
typ, payload, readErr := ws.Read(ctx)
require.NoError(t, readErr)
require.Equal(t, websocket.MessageBinary, typ)
require.LessOrEqual(t, len(payload), 65536)
total += len(payload)
frames++
}
require.Equal(t, 65536*2+7, total)
require.Equal(t, 3, frames)
}

func TestUntrustedProxyCannotSpoofForwardedFor(t *testing.T) {
s := &webSocketServer{}
r := httptest.NewRequest(http.MethodGet, "http://example.com/mc", nil)
Expand Down
Loading