From 94742ef193e624095aa395e9cd52f26c31d5344d Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Fri, 24 Jul 2026 19:45:37 -0700 Subject: [PATCH] rpc: add a MessageConn transport for RPC over any message pipe The RPC framework could only run over connections it established itself: QUIC/WebTransport, or a TCP listener it owned. Two needs break that assumption. A clusters API exposed over a WebSocket carrying somebody else's envelope protocol hands rpc a connection it must not dial, listen on, or close. And TCP clients want the leanest possible path to the cluster API, without an HTTP layer in the way. Both reduce to running over a single, externally supplied, message oriented pipe. MessageConn is that seam: Send, Recv, Close over discrete byte messages, with msgmux layering stream multiplexing on top to recover the callback and streaming semantics the framework needs. Transport: - Callstreams share the one session wrapping the connection; callbacks route by OID through a session-scoped registry. handleCallStream takes a callbackRouter so WebTransport keeps its per-call session and its existing wire format, which deployed clusters depend on. - Every stream on a message session opens with an opRequest, including callbacks (opInlineCall), so one accept loop routes the whole session and a peer that dialed outbound can still be called back into. - Capabilities minted on a connection nobody can dial carry no address; following one reuses the session it arrived on. - ServeMessageConn and ClientFromMessageConn are the public entry points, with WithMaxFrameSize to fit an envelope's own size limit. - The WebSocket transport is a wsConn MessageConn over coder/websocket; tcp:// is the same session over a length-prefixed TLS socket with no HTTP at all. rpcSession/rpcStream is shared by WebTransport and msgmux. Auth: - Authenticator.Authenticate took an *http.Request, so cloudauth, oidcauth and outboard could not run on a message transport. It now takes a transport-neutral Credentials struct; CredentialsFromRequest fills it for the HTTP paths. opRequest.Bearer carries a token where there is no header, and the authenticator's identity supersedes the capability-signature identity when a token is present. Protocol: - PROTOCOL.md documents the wire format end to end so a second implementation needs no breaking change. opRequest.Version gates the stream; absent or zero reads as version 1, and a mismatch is rejected with a reply naming the supported range. Also fixes a pre-existing race on heldCapability.lastContact, which concurrent calls against one capability all wrote unsynchronized, and an authorization denial surfacing as "unknown response status" on the message transport. --- pkg/cloudauth/rpc_authenticator.go | 16 +- pkg/cloudauth/rpc_authenticator_cert_test.go | 4 +- pkg/oidcauth/authenticator.go | 21 +- pkg/oidcauth/authenticator_test.go | 24 +- pkg/oidcauth/composite.go | 12 +- pkg/oidcauth/composite_test.go | 11 +- pkg/outboard/outboard_test.go | 9 +- pkg/outboard/token_auth.go | 5 +- pkg/rpc/PROTOCOL.md | 198 ++++++++ pkg/rpc/actor.go | 3 +- pkg/rpc/authenticator.go | 71 ++- pkg/rpc/authenticator_test.go | 4 +- pkg/rpc/call.go | 19 +- pkg/rpc/client.go | 313 ++++++------ pkg/rpc/inline.go | 33 +- pkg/rpc/inline_router.go | 160 ++++++ pkg/rpc/message.go | 181 +++++++ pkg/rpc/message_adopt_test.go | 415 ++++++++++++++++ pkg/rpc/message_mem.go | 111 +++++ pkg/rpc/message_net_test.go | 265 ++++++++++ pkg/rpc/message_tcp.go | 158 ++++++ pkg/rpc/message_version_test.go | 90 ++++ pkg/rpc/message_ws.go | 111 +++++ pkg/rpc/msgmux.go | 355 +++++++++++++ pkg/rpc/msgmux_test.go | 201 ++++++++ pkg/rpc/server.go | 191 ++++--- pkg/rpc/server_session.go | 498 +++++++++++++++++++ pkg/rpc/signing.go | 67 +++ pkg/rpc/state.go | 155 +++++- pkg/rpc/state_ws.go | 61 +++ pkg/rpc/stream/stream_test.go | 26 +- pkg/rpc/transport.go | 38 ++ pkg/rpc/transport_mem_test.go | 239 +++++++++ pkg/rpc/transport_msg.go | 388 +++++++++++++++ pkg/rpc/transport_wt.go | 115 +++++ 35 files changed, 4212 insertions(+), 356 deletions(-) create mode 100644 pkg/rpc/PROTOCOL.md create mode 100644 pkg/rpc/inline_router.go create mode 100644 pkg/rpc/message.go create mode 100644 pkg/rpc/message_adopt_test.go create mode 100644 pkg/rpc/message_mem.go create mode 100644 pkg/rpc/message_net_test.go create mode 100644 pkg/rpc/message_tcp.go create mode 100644 pkg/rpc/message_version_test.go create mode 100644 pkg/rpc/message_ws.go create mode 100644 pkg/rpc/msgmux.go create mode 100644 pkg/rpc/msgmux_test.go create mode 100644 pkg/rpc/server_session.go create mode 100644 pkg/rpc/signing.go create mode 100644 pkg/rpc/state_ws.go create mode 100644 pkg/rpc/transport.go create mode 100644 pkg/rpc/transport_mem_test.go create mode 100644 pkg/rpc/transport_msg.go create mode 100644 pkg/rpc/transport_wt.go diff --git a/pkg/cloudauth/rpc_authenticator.go b/pkg/cloudauth/rpc_authenticator.go index 7b4bff57b..2480d3802 100644 --- a/pkg/cloudauth/rpc_authenticator.go +++ b/pkg/cloudauth/rpc_authenticator.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "log/slog" - "net/http" "strings" "miren.dev/runtime/pkg/auth" @@ -110,10 +109,10 @@ func NewRPCAuthenticator(ctx context.Context, config Config) (*RPCAuthenticator, // Authenticate implements rpc.Authenticator. // It tries JWT authentication first, then falls back to TLS client certificate. // Authorization (RBAC) is handled separately via the Authorize method. -func (a *RPCAuthenticator) Authenticate(ctx context.Context, r *http.Request) (*rpc.Identity, error) { +func (a *RPCAuthenticator) Authenticate(ctx context.Context, creds *rpc.Credentials) (*rpc.Identity, error) { // Try JWT authentication first if Authorization header is present - if authHeader := r.Header.Get("Authorization"); authHeader != "" { - identity, err := a.authenticateJWT(ctx, authHeader) + if creds.Authorization != "" { + identity, err := a.authenticateJWT(ctx, creds.Authorization) if err != nil { return nil, err } @@ -123,12 +122,9 @@ func (a *RPCAuthenticator) Authenticate(ctx context.Context, r *http.Request) (* // Invalid JWT format, fall through to cert check } - // Fall back to TLS client certificate. Only trust a cert that the TLS - // layer verified against the cluster CA (VerifiedChains non-empty); a - // presented-but-unverified cert must never yield a cert identity, which - // grants RBAC-bypassing privileges in Authorize. - if r.TLS != nil && len(r.TLS.VerifiedChains) > 0 && len(r.TLS.PeerCertificates) > 0 { - cert := r.TLS.PeerCertificates[0] + // Fall back to a TLS client certificate the TLS layer verified against the + // cluster CA. + if cert := creds.VerifiedPeerCertificate(); cert != nil { return &rpc.Identity{ Subject: cert.Subject.CommonName, Method: rpc.AuthMethodCert, diff --git a/pkg/cloudauth/rpc_authenticator_cert_test.go b/pkg/cloudauth/rpc_authenticator_cert_test.go index f1e9763d0..5cfcc657f 100644 --- a/pkg/cloudauth/rpc_authenticator_cert_test.go +++ b/pkg/cloudauth/rpc_authenticator_cert_test.go @@ -41,7 +41,7 @@ func TestRPCAuthenticator_CertRequiresVerifiedChain(t *testing.T) { req := newReq(t, &tls.ConnectionState{ PeerCertificates: []*x509.Certificate{cert}, }) - id, err := auth.Authenticate(t.Context(), req) + id, err := auth.Authenticate(t.Context(), rpc.CredentialsFromRequest(req)) require.NoError(t, err) require.Nil(t, id, "unverified client cert must not yield an identity") }) @@ -51,7 +51,7 @@ func TestRPCAuthenticator_CertRequiresVerifiedChain(t *testing.T) { PeerCertificates: []*x509.Certificate{cert}, VerifiedChains: [][]*x509.Certificate{{cert}}, }) - id, err := auth.Authenticate(t.Context(), req) + id, err := auth.Authenticate(t.Context(), rpc.CredentialsFromRequest(req)) require.NoError(t, err) require.NotNil(t, id) require.Equal(t, "test-client", id.Subject) diff --git a/pkg/oidcauth/authenticator.go b/pkg/oidcauth/authenticator.go index 16753be1e..b3e9150ed 100644 --- a/pkg/oidcauth/authenticator.go +++ b/pkg/oidcauth/authenticator.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "log/slog" - "net/http" "strings" "sync/atomic" @@ -41,15 +40,12 @@ func (a *OIDCAuthenticator) SetEAC(eac *entityserver_v1alpha.EntityAccessClient) // Authenticate checks if the request carries a valid OIDC bearer token that // matches a configured oidc_binding entity. -func (a *OIDCAuthenticator) Authenticate(ctx context.Context, r *http.Request) (*rpc.Identity, error) { - authHeader := r.Header.Get("Authorization") - if !strings.HasPrefix(authHeader, "Bearer ") { +func (a *OIDCAuthenticator) Authenticate(ctx context.Context, creds *rpc.Credentials) (*rpc.Identity, error) { + tokenString := creds.BearerToken() + if tokenString == "" { return nil, nil } - tokenString := strings.TrimPrefix(authHeader, "Bearer ") - tokenString = strings.TrimSpace(tokenString) - eac := a.eac.Load() if eac == nil { return nil, nil // Entity access client not yet initialized @@ -72,11 +68,12 @@ func (a *OIDCAuthenticator) Authenticate(ctx context.Context, r *http.Request) ( return nil, nil // No bindings for this issuer } - // Determine the audience to validate against. - // Use the hostname from the request. - audience := r.Host - if audience == "" && r.TLS != nil { - audience = r.TLS.ServerName + // Determine the audience to validate against: the address the caller + // addressed. A message transport is not addressed by name and so carries + // neither, in which case there is nothing to validate against. + audience := creds.Host + if audience == "" && creds.TLS != nil { + audience = creds.TLS.ServerName } if audience == "" { return nil, nil diff --git a/pkg/oidcauth/authenticator_test.go b/pkg/oidcauth/authenticator_test.go index 06ecb74b9..cd775ed47 100644 --- a/pkg/oidcauth/authenticator_test.go +++ b/pkg/oidcauth/authenticator_test.go @@ -115,7 +115,7 @@ func TestOIDCAuthenticator_NoEAC(t *testing.T) { req := httptest.NewRequest("GET", "/", nil) req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -128,7 +128,7 @@ func TestOIDCAuthenticator_NoBearerToken(t *testing.T) { auth := NewOIDCAuthenticator(testutils.TestLogger(t)) req := httptest.NewRequest("GET", "/", nil) - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -147,7 +147,7 @@ func TestOIDCAuthenticator_NonJWTBearer(t *testing.T) { req := httptest.NewRequest("GET", "/", nil) req.Header.Set("Authorization", "Bearer not-a-jwt-token") - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -177,7 +177,7 @@ func TestOIDCAuthenticator_NoMatchingBindings(t *testing.T) { req.Host = "test-host" req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -238,7 +238,7 @@ func TestOIDCAuthenticator_MatchingBinding(t *testing.T) { req.Host = "test-host" req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(ctx, req) + identity, err := auth.Authenticate(ctx, rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -305,7 +305,7 @@ func TestOIDCAuthenticator_SubjectMismatch(t *testing.T) { req.Host = "test-host" req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(ctx, req) + identity, err := auth.Authenticate(ctx, rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -364,7 +364,7 @@ func TestOIDCAuthenticator_ClaimConditionMismatch(t *testing.T) { req.Host = "test-host" req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(ctx, req) + identity, err := auth.Authenticate(ctx, rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -418,7 +418,7 @@ func TestOIDCAuthenticator_ExpiredToken(t *testing.T) { req.Host = "test-host" req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(ctx, req) + identity, err := auth.Authenticate(ctx, rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -496,7 +496,7 @@ func TestOIDCAuthenticator_MultipleBindings_FirstMatch(t *testing.T) { req.Host = "test-host" req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(ctx, req) + identity, err := auth.Authenticate(ctx, rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -562,7 +562,7 @@ func TestOIDCAuthenticator_AudienceFromHost(t *testing.T) { req.Host = "my-cluster.example.com" req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(ctx, req) + identity, err := auth.Authenticate(ctx, rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -618,7 +618,7 @@ func TestOIDCAuthenticator_AudienceFromTLS(t *testing.T) { req.TLS = &tls.ConnectionState{ServerName: "tls-server.example.com"} req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(ctx, req) + identity, err := auth.Authenticate(ctx, rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -673,7 +673,7 @@ func TestOIDCAuthenticator_WrongAudience(t *testing.T) { req.Host = "test-host" req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(ctx, req) + identity, err := auth.Authenticate(ctx, rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/pkg/oidcauth/composite.go b/pkg/oidcauth/composite.go index 51f150170..c2cbcaef3 100644 --- a/pkg/oidcauth/composite.go +++ b/pkg/oidcauth/composite.go @@ -3,7 +3,6 @@ package oidcauth import ( "context" "fmt" - "net/http" "miren.dev/runtime/pkg/rpc" ) @@ -55,9 +54,9 @@ func NewCompositeAuthenticator(primary rpc.Authenticator, oidc *OIDCAuthenticato return &CompositeAuthenticator{primary: primary, oidc: oidc} } -func (c *CompositeAuthenticator) Authenticate(ctx context.Context, r *http.Request) (*rpc.Identity, error) { +func (c *CompositeAuthenticator) Authenticate(ctx context.Context, creds *rpc.Credentials) (*rpc.Identity, error) { // Try primary authenticator first - identity, err := c.primary.Authenticate(ctx, r) + identity, err := c.primary.Authenticate(ctx, creds) if identity != nil { return identity, nil } @@ -68,7 +67,7 @@ func (c *CompositeAuthenticator) Authenticate(ctx context.Context, r *http.Reque var oidcIdentity *rpc.Identity var oidcErr error if c.oidc != nil { - oidcIdentity, oidcErr = c.oidc.Authenticate(ctx, r) + oidcIdentity, oidcErr = c.oidc.Authenticate(ctx, creds) if oidcIdentity != nil { return oidcIdentity, nil } @@ -102,8 +101,9 @@ func (c *CompositeAuthorizer) Authorize(ctx context.Context, identity *rpc.Ident // OIDC callers are restricted to the oidc-deploy role return authorizeOIDC(resource, action) - case rpc.AuthMethodJWT, rpc.AuthMethodAnonymous, rpc.AuthMethodToken: - // Delegate to primary (cloud RBAC). + case rpc.AuthMethodJWT, rpc.AuthMethodAnonymous, rpc.AuthMethodToken, rpc.AuthMethodSigned: + // Delegate to primary (cloud RBAC). A signed identity names the key a + // capability was issued to, which carries no privilege of its own. fallthrough default: if c.primary != nil { diff --git a/pkg/oidcauth/composite_test.go b/pkg/oidcauth/composite_test.go index 9b3d8b6f1..75cfc0030 100644 --- a/pkg/oidcauth/composite_test.go +++ b/pkg/oidcauth/composite_test.go @@ -3,7 +3,6 @@ package oidcauth import ( "context" "fmt" - "net/http" "net/http/httptest" "testing" @@ -17,7 +16,7 @@ type mockAuthenticator struct { err error } -func (m *mockAuthenticator) Authenticate(ctx context.Context, r *http.Request) (*rpc.Identity, error) { +func (m *mockAuthenticator) Authenticate(ctx context.Context, creds *rpc.Credentials) (*rpc.Identity, error) { return m.identity, m.err } @@ -41,7 +40,7 @@ func TestCompositeAuthenticator_PrimarySucceeds(t *testing.T) { comp := NewCompositeAuthenticator(primary, oidcAuth) req := httptest.NewRequest("GET", "/", nil) - identity, err := comp.Authenticate(context.Background(), req) + identity, err := comp.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -63,7 +62,7 @@ func TestCompositeAuthenticator_PrimaryErrorOIDCNil(t *testing.T) { comp := NewCompositeAuthenticator(primary, oidcAuth) req := httptest.NewRequest("GET", "/", nil) - _, err := comp.Authenticate(context.Background(), req) + _, err := comp.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) if err == nil { t.Fatal("expected error to propagate from primary") } @@ -85,7 +84,7 @@ func TestCompositeAuthenticator_PrimaryErrorOIDCSucceeds(t *testing.T) { comp := &CompositeAuthenticator{primary: primary, oidc: oidcStub} req := httptest.NewRequest("GET", "/", nil) - identity, err := comp.Authenticate(context.Background(), req) + identity, err := comp.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("expected no error when OIDC succeeds, got: %v", err) } @@ -110,7 +109,7 @@ func TestCompositeAuthenticator_FallbackToOIDC(t *testing.T) { req := httptest.NewRequest("GET", "/", nil) req.Header.Set("Authorization", "Bearer some-token") - identity, err := comp.Authenticate(context.Background(), req) + identity, err := comp.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/pkg/outboard/outboard_test.go b/pkg/outboard/outboard_test.go index dbcc0ee42..93d59bff7 100644 --- a/pkg/outboard/outboard_test.go +++ b/pkg/outboard/outboard_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "miren.dev/runtime/pkg/rpc" ) func TestConfigRoundTrip(t *testing.T) { @@ -64,7 +65,7 @@ func TestTokenAuthenticatorValid(t *testing.T) { req, _ := http.NewRequest("GET", "/", nil) req.Header.Set("Authorization", "Bearer secret-token") - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) assert.NoError(t, err) assert.NotNil(t, identity) assert.Equal(t, "outboard", identity.Subject) @@ -76,7 +77,7 @@ func TestTokenAuthenticatorInvalid(t *testing.T) { req, _ := http.NewRequest("GET", "/", nil) req.Header.Set("Authorization", "Bearer wrong-token") - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) assert.NoError(t, err) assert.Nil(t, identity, "invalid token should return nil identity") } @@ -86,7 +87,7 @@ func TestTokenAuthenticatorMissing(t *testing.T) { req, _ := http.NewRequest("GET", "/", nil) - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) assert.NoError(t, err) assert.Nil(t, identity, "missing auth header should return nil identity") } @@ -97,7 +98,7 @@ func TestTokenAuthenticatorBadScheme(t *testing.T) { req, _ := http.NewRequest("GET", "/", nil) req.Header.Set("Authorization", "Basic dXNlcjpwYXNz") - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) assert.NoError(t, err) assert.Nil(t, identity, "non-bearer auth should return nil identity") } diff --git a/pkg/outboard/token_auth.go b/pkg/outboard/token_auth.go index 36cc5bf41..f600a2646 100644 --- a/pkg/outboard/token_auth.go +++ b/pkg/outboard/token_auth.go @@ -3,7 +3,6 @@ package outboard import ( "context" "crypto/subtle" - "net/http" "strings" "miren.dev/runtime/pkg/rpc" @@ -20,8 +19,8 @@ func NewTokenAuthenticator(token string) *TokenAuthenticator { return &TokenAuthenticator{token: token} } -func (t *TokenAuthenticator) Authenticate(ctx context.Context, r *http.Request) (*rpc.Identity, error) { - auth := r.Header.Get("Authorization") +func (t *TokenAuthenticator) Authenticate(ctx context.Context, creds *rpc.Credentials) (*rpc.Identity, error) { + auth := creds.Authorization if auth == "" { return nil, nil // No credentials } diff --git a/pkg/rpc/PROTOCOL.md b/pkg/rpc/PROTOCOL.md new file mode 100644 index 000000000..343725428 --- /dev/null +++ b/pkg/rpc/PROTOCOL.md @@ -0,0 +1,198 @@ +# Miren RPC message protocol + +This is the wire format for the **message transport** — the way Miren RPC runs +over any reliable, ordered, bidirectional pipe of discrete byte messages. It +backs the `mem://`, `ws://`, `wss://` and `tcp://` schemes, and any connection +handed to `State.ServeMessageConn` or `State.ClientFromMessageConn`. + +It is not the HTTP/3 + WebTransport transport, which remains the default and +speaks HTTP with the paths under `/_rpc/`. + +Everything is [CBOR](https://cbor.io/). Structs use integer keys (`keyasint`), +so field names never appear on the wire. + +**Status: version 1.** The Go implementation in this package is the only one +today. The format is documented and versioned so that a second implementation +does not require a breaking change, but the layouts below may still gain fields +(never renumber or repurpose one). + +## Layers + +``` + your backend a socket, a broker, an envelope protocol + | + MessageConn Send([]byte) / Recv() []byte / Close() + | + msgmux frames: multiplexes streams over messages + | + operations opRequest / opReply: one operation per stream +``` + +The backend supplies message framing. Everything above assumes messages arrive +whole, in order, exactly once, and that a closed connection eventually surfaces +as an error from `Recv`. + +## Layer 1: msgmux frames + +Every message on the connection is one frame: + +| Key | Field | Type | Meaning | +|-----|--------|--------|-------------------------------| +| 0 | Stream | uint32 | Stream id | +| 1 | Flags | uint8 | Bit set, below | +| 2 | Data | bytes | Payload; omitted when empty | + +Flags: + +| Bit | Name | Meaning | +|--------|------|------------------------------------------------------| +| `0x01` | SYN | Opens the stream | +| `0x02` | DATA | Carries payload bytes | +| `0x04` | FIN | No more data in this direction | +| `0x08` | RST | Abort: discard the stream, fail any blocked reader | + +Rules: + +- **Stream ids.** The dialing peer uses odd ids, the accepting peer even, so the + two never collide. Ids increase by 2 and are not reused. +- **Opening.** A stream opens with an eager SYN, sent as its own frame before any + data. The peer must be able to accept a stream whose opener reads before it + writes. +- **Data for an unknown stream** that carries no SYN is dropped, not an error. +- **Closing.** FIN closes one direction. A stream is forgotten once both + directions are finished. RST tears it down immediately from either side. +- **Frame size.** A payload larger than the session's frame cap is split across + several DATA frames; the reader concatenates. The default cap is 1 MiB, and + `WithMaxFrameSize` lowers it to fit a backend or envelope with its own limit. + The cap bounds throughput, never correctness. +- **No flow control.** Deliberately: the backend already frames messages and + applies its own backpressure. This is what makes msgmux smaller than a + general-purpose stream multiplexer such as yamux, which must invent both over + a byte stream. + +## Layer 2: operations + +**Every stream opens with an `opRequest`**, including server-initiated +callbacks. That invariant is what lets one accept loop route a whole connection, +and so lets a peer both serve and call over a single connection — including one +it dialed outbound through a firewall it could never accept through. + +The operation's payload follows the `opRequest` as further CBOR values on the +same stream. + +### opRequest + +| Key | Field | Type | Meaning | +|-----|-----------|-------------------|------------------------------------------------| +| 0 | Op | string | Operation selector, below | +| 1 | OID | string | Target capability | +| 2 | Name | string | Object name, for `lookup` | +| 3 | Method | string | Method name, for `call` / `callstream` | +| 4 | PubKey | bytes | Caller's ed25519 public key | +| 5 | TargetPK | bytes | Recipient's public key, for `reexport` | +| 6 | Contact | string | Address the caller can be reached at, if any | +| 7 | Timestamp | string | RFC3339Nano, signed | +| 8 | Signature | bytes | ed25519 over the canonical string, below | +| 9 | Bearer | string | Bearer token; there is no header to carry one | +| 10 | Trace | map[string]string | W3C trace context | +| 11 | Version | uint | Protocol version; absent or 0 means 1 | + +### opReply + +The first value written back, mapping the HTTP transport's +`rpc-status` / `rpc-error` trailers into the message world. + +| Key | Field | Type | Meaning | +|-----|----------|--------|-------------------------------| +| 0 | Status | string | Below | +| 1 | Error | string | Human-readable message | +| 2 | Category | string | Error category | +| 3 | Code | string | Error code | + +Statuses: `ok`, `error`, `panic`, `unknown-capability`, `forbidden`. + +`unknown-capability` is distinct because it is recoverable: a client holding +restore state re-resolves the capability and retries, rather than failing. + +### Operations + +| Op | Purpose | +|--------------|-------------------------------------------------------------| +| `lookup` | Resolve a named object to a capability | +| `call` | Unary call; args follow, then `opReply` and the result | +| `callstream` | Streaming call; the stream stays open for both directions | +| `methods` | List a capability's methods and parameters | +| `reresolve` | Rebuild a capability from its restore state | +| `reexport` | Re-issue a capability for a different holder | +| `ref` | Add a reference to a capability | +| `deref` | Drop a reference to a capability | +| `identify` | Prove the caller's key; returns the observed address | +| `inline` | Carries calls against an inline capability the peer advertised | + +`inline` is the reverse direction: the peer opens a stream against a capability +you advertised in a `callstream` call. The prelude names only the capability, +because the stream is pooled and carries many calls; each one is a +`streamRequest` with its own method. + +## Authentication + +There is no TLS client certificate to inspect, so a caller proves itself by +signing a canonical string with the ed25519 key the capability was issued to. + +``` +canonical = " " +``` + +`target` is `"/"` for `call` and `callstream`, `""` for +capability-scoped operations, and the empty string for `identify` — which leaves +two consecutive spaces, e.g. `identify 2026-07-23T18:04:05.000000001Z`. The +separator is a single space in all three positions; nothing is trimmed. + +The signature is raw bytes in field 8 — not base58, unlike the HTTP transports, +where it has to survive a header. + +A timestamp older than 10 minutes is rejected, so a captured frame cannot be +replayed indefinitely. + +`Bearer` is independent and optional. When present, it is offered to the +configured `Authenticator` as `"Bearer "`, and the identity it returns +supersedes the signature identity — that is how a message transport reaches +per-user authorization. Without one, the identity is the capability's key, with +method `signed`. + +## Versioning + +`opRequest.Version` names the version a stream speaks. Absent or `0` means +version 1, so a peer predating the field reads as speaking it. + +A version outside the receiver's supported range is rejected with an `opReply` +naming the range. This is a field rather than a handshake deliberately: a +mismatch costs one rejected stream instead of a round trip on every connection. + +The `identify` reply also carries `min_version` and `max_version`, so a client +that wants to adapt up front can ask instead of probing. + +## Implementing another peer + +Minimum for a client: + +1. Frame your transport into `MessageConn` semantics. +2. Implement msgmux: odd stream ids, eager SYN, FIN/RST, splitting at the frame + cap. +3. Open a stream, send `opRequest{Op: "lookup", Name: ..., PubKey: ...}`, read + `opReply` then a `lookupResponse` for the capability. +4. For each call, open a stream and send + `opRequest{Op: "call", OID, Method, Timestamp, Signature}` followed by the + args; read `opReply` and then the result. + +Add an accept loop handling `inline` only if you advertise capabilities for the +peer to call back on. + +Things worth getting right that are easy to miss: + +- The SYN frame is separate and comes first. +- Sign the canonical string exactly, including the single spaces. +- One stream per operation. Do not reuse an operation stream — except an + `inline` stream, which is explicitly pooled. +- Read the whole `opReply` before the payload; they are separate CBOR values on + the same stream, not a wrapper. diff --git a/pkg/rpc/actor.go b/pkg/rpc/actor.go index 79600cbfd..169d285e1 100644 --- a/pkg/rpc/actor.go +++ b/pkg/rpc/actor.go @@ -5,7 +5,6 @@ import ( "crypto/rand" "io" "sync" - "time" "github.com/fxamacker/cbor/v2" "github.com/mr-tron/base58" @@ -89,8 +88,8 @@ func (a *LocalActorRegistry) newCapability(i *Interface) *Capability { heldInterface: &heldInterface{ Interface: i, }, - lastContact: time.Now(), } + hc.touch() if i.restoreState != nil { if rs, err := i.restoreState.RestoreState(i); err == nil { diff --git a/pkg/rpc/authenticator.go b/pkg/rpc/authenticator.go index 1fce5a411..c65625065 100644 --- a/pkg/rpc/authenticator.go +++ b/pkg/rpc/authenticator.go @@ -2,9 +2,12 @@ package rpc import ( "context" + "crypto/tls" + "crypto/x509" "errors" "fmt" "net/http" + "strings" ) // contextKey is a private type for context keys to avoid collisions @@ -37,6 +40,7 @@ const ( AuthMethodAnonymous AuthMethod = "anonymous" // No authentication (public methods) AuthMethodToken AuthMethod = "token" // Bearer token (e.g., outboard) AuthMethodOIDC AuthMethod = "oidc" // External OIDC token (e.g., GitHub Actions) + AuthMethodSigned AuthMethod = "signed" // ed25519-signed request over a message transport ) // Identity represents an authenticated caller @@ -54,14 +58,66 @@ type Identity struct { Metadata map[string]any } +// Credentials carries what an authenticator needs to identify a caller, +// independent of how the call arrived. The HTTP transports fill it from the +// request; message transports fill it from the operation frame, where there is +// no request and no TLS handshake to inspect. +type Credentials struct { + // Authorization is the raw Authorization header value, e.g. "Bearer ". + Authorization string + + // Host is the address the caller addressed, used as the expected audience + // when validating a token. Empty on message transports, which are not + // addressed by name. + Host string + + // TLS is the connection's handshake state, or nil when the transport has no + // TLS layer of its own — including any connection supplied by a caller, + // whose security is that caller's concern. + TLS *tls.ConnectionState +} + +// CredentialsFromRequest extracts credentials from an HTTP request, for the +// HTTP transports and for anything fronting rpc with an HTTP layer of its own. +func CredentialsFromRequest(r *http.Request) *Credentials { + return &Credentials{ + Authorization: r.Header.Get("Authorization"), + Host: r.Host, + TLS: r.TLS, + } +} + +// BearerToken returns the token from a "Bearer " Authorization value, or +// an empty string if the credentials carry no bearer token. +func (c *Credentials) BearerToken() string { + if c == nil || !strings.HasPrefix(c.Authorization, "Bearer ") { + return "" + } + return strings.TrimSpace(strings.TrimPrefix(c.Authorization, "Bearer ")) +} + +// VerifiedPeerCertificate returns the caller's TLS client certificate, but only +// when the TLS layer verified it against the cluster CA. A presented but +// unverified certificate must never yield a cert identity, which grants +// RBAC-bypassing privileges in Authorize. +func (c *Credentials) VerifiedPeerCertificate() *x509.Certificate { + if c == nil || c.TLS == nil { + return nil + } + if len(c.TLS.VerifiedChains) == 0 || len(c.TLS.PeerCertificates) == 0 { + return nil + } + return c.TLS.PeerCertificates[0] +} + // Authenticator validates credentials and returns caller identity type Authenticator interface { - // Authenticate validates the request's credentials and returns the caller's identity. + // Authenticate validates the caller's credentials and returns their identity. // Returns: // - (*Identity, nil) if credentials are valid // - (nil, nil) if no credentials present or credentials are invalid // - (nil, error) if an error occurred during authentication - Authenticate(ctx context.Context, r *http.Request) (*Identity, error) + Authenticate(ctx context.Context, creds *Credentials) (*Identity, error) } // Authorizer checks if an identity is allowed to perform an action on a resource @@ -113,7 +169,7 @@ func AppAccessError(ctx context.Context, appName string) error { // Used for testing only. type NoOpAuthenticator struct{} -func (n *NoOpAuthenticator) Authenticate(ctx context.Context, r *http.Request) (*Identity, error) { +func (n *NoOpAuthenticator) Authenticate(ctx context.Context, creds *Credentials) (*Identity, error) { return &Identity{ Subject: "anonymous", Method: AuthMethodAnonymous, @@ -124,13 +180,8 @@ func (n *NoOpAuthenticator) Authenticate(ctx context.Context, r *http.Request) ( // Used when cloud authentication is not enabled. type LocalOnlyAuthenticator struct{} -func (l *LocalOnlyAuthenticator) Authenticate(ctx context.Context, r *http.Request) (*Identity, error) { - // Check for TLS client certificate. Only trust a cert that the TLS layer - // verified against the cluster CA (VerifiedChains non-empty); a - // presented-but-unverified cert must never yield a cert identity, which - // grants RBAC-bypassing privileges in Authorize. - if r.TLS != nil && len(r.TLS.VerifiedChains) > 0 && len(r.TLS.PeerCertificates) > 0 { - cert := r.TLS.PeerCertificates[0] +func (l *LocalOnlyAuthenticator) Authenticate(ctx context.Context, creds *Credentials) (*Identity, error) { + if cert := creds.VerifiedPeerCertificate(); cert != nil { return &Identity{ Subject: cert.Subject.CommonName, Method: AuthMethodCert, diff --git a/pkg/rpc/authenticator_test.go b/pkg/rpc/authenticator_test.go index a341f859c..754d5b6da 100644 --- a/pkg/rpc/authenticator_test.go +++ b/pkg/rpc/authenticator_test.go @@ -42,7 +42,7 @@ func TestNoOpAuthenticator(t *testing.T) { req.Header.Set("Authorization", tt.authHeader) } - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), CredentialsFromRequest(req)) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -127,7 +127,7 @@ func TestLocalOnlyAuthenticator(t *testing.T) { req.TLS = &tls.ConnectionState{} } - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), CredentialsFromRequest(req)) if err != nil { t.Errorf("unexpected error: %v", err) } diff --git a/pkg/rpc/call.go b/pkg/rpc/call.go index 2fe355126..1bdd64f44 100644 --- a/pkg/rpc/call.go +++ b/pkg/rpc/call.go @@ -7,7 +7,6 @@ import ( "net/http" "github.com/fxamacker/cbor/v2" - "github.com/quic-go/webtransport-go" ) type Call interface { @@ -42,7 +41,11 @@ type NetworkCall struct { local *localCall ctrl *controlStream - wsSession *webtransport.Session + wsSession rpcSession + + // msgSession records that wsSession is a message-transport session, whose + // streams all open with an opRequest prelude. + msgSession bool argsConsumed bool } @@ -102,15 +105,23 @@ func (c *NetworkCall) NewClient(capa *Capability) Client { return c.local.NewClient(capa) } - client := c.s.state.newClientFrom(capa, c.peer) if capa.Inline && c.wsSession != nil { + client := c.s.state.newClientFrom(capa, c.peer) client.inlineClient = &inlineClient{ log: c.s.state.log, oid: capa.OID, ctrl: c.ctrl, session: c.wsSession, + prelude: c.msgSession, } + + return client + } + + // No address and no session to scope it to: nothing here can be reached. + if capa.Address == "" { + return unreachableClient{oid: capa.OID} } - return client + return c.s.state.newClientFrom(capa, c.peer) } diff --git a/pkg/rpc/client.go b/pkg/rpc/client.go index f6992ba52..e16843533 100644 --- a/pkg/rpc/client.go +++ b/pkg/rpc/client.go @@ -19,13 +19,24 @@ import ( "github.com/mr-tron/base58" "github.com/pkg/errors" "github.com/quic-go/quic-go" - "github.com/quic-go/quic-go/http3" - "github.com/quic-go/webtransport-go" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" "miren.dev/runtime/pkg/cond" ) +// transportKind selects which wire transport a NetworkClient uses. +type transportKind int + +const ( + transportH3 transportKind = iota // QUIC/HTTP3 + WebTransport (default) + transportMsg // msgmux over a MessageConn (mem, WebSocket, TCP, adopted) +) + +// adoptedRemote marks a client whose connection was supplied by the caller +// (ClientFromMessageConn). There is no address to dial: the connection is +// reachable only as itself, for as long as it lives. +const adoptedRemote = "adopted://" + type Client interface { CallWithCaps(ctx context.Context, method string, args, result any, caps map[OID]*InlineCapability) error Call(ctx context.Context, method string, args, result any) error @@ -34,12 +45,44 @@ type Client interface { Close() error } +// unreachableClient stands in for a capability that carries no address and is +// not inline, so there is nothing to dial and no session it is scoped to. This +// happens when a peer hands us a capability minted on a connection it does not +// own — only call-scoped inline capabilities survive that trip. Failing at the +// call rather than at hand-off keeps the error where it can be reported. +type unreachableClient struct { + oid OID +} + +func (u unreachableClient) err() error { + return cond.NotFound("reachable address for capability", u.oid) +} + +func (u unreachableClient) Call(context.Context, string, any, any) error { return u.err() } + +func (u unreachableClient) CallWithCaps( + context.Context, string, any, any, map[OID]*InlineCapability, +) error { + return u.err() +} + +func (u unreachableClient) NewClient(*Capability) Client { return u } + +func (u unreachableClient) NewInlineCapability(*Interface, any) (*InlineCapability, OID, *Capability) { + return &InlineCapability{}, "", &Capability{} +} + +func (u unreachableClient) Close() error { return nil } + type NetworkClient struct { State *State + transportKind transportKind + transport *quic.Transport - htr http3.Transport - ws webtransport.Dialer + htr http.RoundTripper + dialer *wtDialer + ops *msgOpTransport capa *Capability remote string remoteAddr net.Addr @@ -58,43 +101,18 @@ type NetworkClient struct { localClient *localClient } -func setTLSConfigServerName(tlsConf *tls.Config, addr net.Addr, host string) { - // If no ServerName is set, infer the ServerName from the host we're connecting to. - if tlsConf.ServerName != "" { - return - } - if host == "" { - if udpAddr, ok := addr.(*net.UDPAddr); ok { - tlsConf.ServerName = udpAddr.IP.String() - return - } - } - h, _, err := net.SplitHostPort(host) - if err != nil { // This happens if the host doesn't contain a port number. - tlsConf.ServerName = host - return - } - tlsConf.ServerName = h -} - +// setupTransport configures the client's round-tripper and callstream dialer +// based on the selected transport. The h3 and ws implementations live in +// transport_wt.go and transport_ws.go respectively. func (c *NetworkClient) setupTransport() { - c.htr.Logger = c.State.log.With("module", "rpc-call") - c.htr.TLSClientConfig = c.tlsCfg - c.htr.QUICConfig = &DefaultQUICConfig - c.htr.Dial = func(ctx context.Context, addr string, tlsCfg *tls.Config, cfg *quic.Config) (*quic.Conn, error) { - uaddr, err := resolveUDPAddr(ctx, "udp", addr) - if err != nil { - return nil, err - } - - setTLSConfigServerName(tlsCfg, uaddr, addr) - - return c.transport.DialEarly(ctx, uaddr, tlsCfg, cfg) + switch c.transportKind { + case transportMsg: + c.setupMsgTransport() + case transportH3: + c.setupH3Transport() + default: + c.setupH3Transport() } - - c.ws.TLSClientConfig = c.tlsCfg - c.ws.QUICConfig = &DefaultQUICConfig - c.ws.DialAddr = c.htr.Dial } func (c *NetworkClient) NewClient(capa *Capability) Client { @@ -152,6 +170,10 @@ func (c *NetworkClient) roundTrip(r *http.Request) (*http.Response, error) { } func (c *NetworkClient) sendIdentity(ctx context.Context) error { + if c.transportKind == transportMsg { + return c.msgSendIdentity(ctx) + } + url := "https://" + c.remote + "/_rpc/identify" req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) if err != nil { @@ -214,6 +236,10 @@ func (c *NetworkClient) sendIdentity(ctx context.Context) error { } func (c *NetworkClient) resolveCapability(name string) error { + if c.transportKind == transportMsg { + return c.msgResolveCapability(name) + } + url := "https://" + c.remote + "/_rpc/lookup/" + url.PathEscape(name) req, err := http.NewRequest(http.MethodPost, url, nil) if err != nil { @@ -257,6 +283,10 @@ func (c *NetworkClient) resolveCapability(name string) error { } func (c *NetworkClient) reresolveCapability(rs *InterfaceState) error { + if c.transportKind == transportMsg { + return c.msgReresolveCapability(rs) + } + url := "https://" + c.remote + "/_rpc/reresolve" c.State.log.Debug("reresolving capability from state", "url", url) @@ -312,6 +342,10 @@ func (c *NetworkClient) fetchMethods(ctx context.Context) (methodsResponse, erro return c.localClient.listMethods(), nil } + if c.transportKind == transportMsg { + return c.msgFetchMethods(ctx) + } + url := "https://" + c.remote + "/_rpc/methods/" + string(c.oid) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { @@ -386,6 +420,10 @@ func (c *NetworkClient) HasMethodParam(ctx context.Context, method, param string } func (c *NetworkClient) requestReexportCapability(ctx context.Context, capa *Capability, target ed25519.PublicKey) (*Capability, error) { + if c.transportKind == transportMsg { + return c.msgRequestReexport(ctx, capa, target) + } + url := "https://" + c.remote + "/_rpc/reexport/" + string(capa.OID) req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) if err != nil { @@ -430,6 +468,10 @@ func (c *NetworkClient) derefOID(ctx context.Context, oid OID) error { return c.inlineClient.derefOID(ctx, oid) } + if c.transportKind == transportMsg { + return c.msgDerefOID(ctx, oid) + } + url := "https://" + c.remote + "/_rpc/deref/" + string(oid) req, err := http.NewRequest(http.MethodPost, url, nil) if err != nil { @@ -533,6 +575,10 @@ func (c *NetworkClient) Call(ctx context.Context, method string, args, result an return c.inlineClient.Call(ctx, method, args, result) } + if c.transportKind == transportMsg { + return c.msgUnaryCall(ctx, method, args, result) + } + ctx, span := Tracer().Start(ctx, "rpc.call."+method) defer span.End() @@ -558,7 +604,7 @@ request: hr, err := c.htr.RoundTrip(req) if err != nil { - if _, ok := err.(*quic.ApplicationError); ok { + if isRetryableTransportError(err) { c.State.log.Info("rpc.call retrying", "oid", string(c.oid), "error", err) continue request } @@ -626,20 +672,19 @@ func (c *NetworkClient) CallWithCaps(ctx context.Context, method string, args, r return c.localClient.Call(ctx, method, args, result) } + if c.transportKind == transportMsg { + return c.msgCallWithCaps(ctx, method, args, result, caps) + } + ctx, span := Tracer().Start(ctx, "rpc.call."+method) defer span.End() - data, err := cbor.Marshal(args) - if err != nil { - return err - } - request: for { span.SetAttributes(attribute.String("oid", string(c.oid))) url := "https://" + c.remote + "/_rpc/callstream/" + string(c.oid) + "/" + method - req, err := http.NewRequestWithContext(ctx, http.MethodConnect, url, bytes.NewReader(data)) + req, err := http.NewRequestWithContext(ctx, http.MethodConnect, url, nil) if err != nil { return err } @@ -649,151 +694,100 @@ request: return err } - hr, sess, err := c.ws.Dial(ctx, url, req.Header) + hr, sess, err := c.dialer.Dial(ctx, url, req.Header) if err != nil { - if _, ok := err.(*quic.ApplicationError); ok { - c.State.log.Info("rpc.call retrying", "oid", string(c.oid), "error", err) + retry, derr := c.handleCallStreamDialError(hr, err) + if retry { continue request } - - return fmt.Errorf("error performing http request to %s: %w", url, err) + return derr } - retry, err := c.handleCallStream(ctx, hr, sess, method, args, result, caps) + err = c.handleCallStream(ctx, sess, perCallRouter{}, nil, args, result, caps) + _ = sess.Close() if err != nil { return err } - if retry { - continue request - } - return nil } } -func (c *NetworkClient) handleCallStream( - ctx context.Context, - hr *http.Response, - sess *webtransport.Session, - method string, - args, result any, - caps map[OID]*InlineCapability, -) (bool, error) { - if hr.StatusCode != http.StatusOK { - status := hr.Trailer.Get("rpc-status") - - err := cond.Errorf("unexpected status code: %d", hr.StatusCode) +// handleCallStreamDialError maps a failed callstream upgrade to the appropriate +// RPC error, transparently retrying when the capability can be re-resolved. The +// WebTransport upgrade surfaces the RPC status via response trailers on HTTP/3, +// with headers as a fallback, so we check both. +func (c *NetworkClient) handleCallStreamDialError(hr *http.Response, dialErr error) (bool, error) { + statusFor := func(key string) string { + if hr == nil { + return "" + } + if v := hr.Trailer.Get(key); v != "" { + return v + } + return hr.Header.Get(key) + } - switch status { - case "ok", "": - // The remote side thought everything was fine, so use our ability to parse - // the response as the error. - return false, err - case "unknown-capability": - if c.capa.RestoreState != nil { - // Try to re-resolve and, if successful, signal caller to retry the request. - if rerr := c.reresolveCapability(c.capa.RestoreState); rerr == nil { - return true, nil - } + switch statusFor("rpc-status") { + case "unknown-capability": + if c.capa.RestoreState != nil { + if rerr := c.reresolveCapability(c.capa.RestoreState); rerr == nil { + return true, nil } - err = cond.NotFound("capability", c.capa.OID) - case "error": - errs := hr.Trailer.Get("rpc-error") - err = cond.RemoteError("generic", "unknown", errs) - case "panic": - errs := hr.Trailer.Get("rpc-error") - err = cond.Panic(errs) } + return false, cond.NotFound("capability", c.capa.OID) + case "error": + return false, cond.RemoteError("generic", "unknown", statusFor("rpc-error")) + case "panic": + return false, cond.Panic(statusFor("rpc-error")) + } - return false, err + if isRetryableTransportError(dialErr) { + c.State.log.Info("rpc.call retrying", "oid", string(c.oid), "error", dialErr) + return true, nil } + return false, fmt.Errorf("error performing callstream request: %w", dialErr) +} + +func (c *NetworkClient) handleCallStream( + ctx context.Context, + sess rpcSession, + router callbackRouter, + prelude any, + args, result any, + caps map[OID]*InlineCapability, +) error { ctx, cancel := context.WithCancel(ctx) defer cancel() - go func() { - for { - str, err := sess.AcceptStream(ctx) - if err != nil { - break - } - - go func() { - defer str.Close() - - dec := cbor.NewDecoder(str) - enc := cbor.NewEncoder(str) - - // Loop to handle multiple requests on the same stream - for { - var rs streamRequest - - err = dec.Decode(&rs) - if err != nil { - if err != io.EOF { - c.State.log.Error("rpc.callstream call: error decoding stream request", "error", err) - } - return - } - - switch rs.Kind { - case "call": - iface, ok := caps[rs.OID] - if !ok { - _ = enc.Encode(refResponse{ - Status: "error", - Error: "unknown capability", - }) - continue - } - - mm := iface.methods[rs.Method] - if mm.Handler == nil { - enc.Encode(refResponse{ - Status: "error", - Error: "unknown method", - }) - continue - } - - ctx, cancel := context.WithCancel(ctx) - err := c.callInline(ctx, mm, rs.OID, rs.Method, iface.Interface, enc, dec) - cancel() - if err != nil { - if !errors.Is(err, context.Canceled) && !errors.Is(err, cond.ErrClosed{}) { - c.State.log.Error("rpc.callstream: error calling inline", "error", err) - } - return - } - default: - c.State.log.Error("rpc.callstream: unknown call stream request", "kind", rs.Kind) - } - } - }() - } - }() + withdraw := router.register(ctx, c, sess, caps) + defer withdraw() // Open the control stream ctrl, err := sess.OpenStreamSync(ctx) if err != nil { c.State.log.Error("rpc.callstream ctrl: error opening control stream", "error", err) - return false, err + return err } enc := cbor.NewEncoder(ctrl) + // On the message transport the control stream is also the operation stream, + // so it must lead with the opRequest the server's router decodes; the HTTP + // transports pass a nil prelude. + if prelude != nil { + enc.Encode(prelude) + } enc.Encode(args) dec := cbor.NewDecoder(ctrl) - const cancelCode = webtransport.StreamErrorCode(quic.FlowControlError) - // If the context is canceled, then we bail ASAP on trying to complete the RPC. - // Because we have a local ctx with a local cancel also, when this method turns, this - // goroutine will automatically get cleaned up. + // Because we have a local ctx with a local cancel also, when this method returns, + // this goroutine will automatically get cleaned up. go func() { <-ctx.Done() - ctrl.CancelRead(cancelCode) + ctrl.CancelRead(cancelReadCode) }() loop: @@ -802,10 +796,13 @@ loop: err = dec.Decode(&rs) if err != nil { - if errors.Is(err, io.EOF) { - err = nil - } else if se, ok := err.(*webtransport.StreamError); ok && se.ErrorCode == cancelCode { + // A canceled context means we aborted the read ourselves (via + // CancelRead above); report it uniformly across transports rather + // than depending on a transport-specific stream-error type. + if ctx.Err() != nil { err = cond.Closed("rpc call terminated before getting response") + } else if errors.Is(err, io.EOF) { + err = nil } break @@ -828,7 +825,7 @@ loop: } } - return false, err + return err } func (c *NetworkClient) callInline( diff --git a/pkg/rpc/inline.go b/pkg/rpc/inline.go index 5f31d4c40..b473bb526 100644 --- a/pkg/rpc/inline.go +++ b/pkg/rpc/inline.go @@ -8,7 +8,6 @@ import ( "sync" "github.com/fxamacker/cbor/v2" - "github.com/quic-go/webtransport-go" "miren.dev/runtime/pkg/cond" ) @@ -17,7 +16,7 @@ var errInlineClientClosed = errors.New("inline client closed") const inlineStreamPoolSize = 10 type streamConn struct { - stream *webtransport.Stream + stream rpcStream enc *cbor.Encoder dec *cbor.Decoder } @@ -26,7 +25,12 @@ type inlineClient struct { log *slog.Logger oid OID ctrl *controlStream - session *webtransport.Session + session rpcSession + + // prelude reports whether each newly opened stream must lead with an + // opRequest, as message-transport sessions require so that one accept loop + // can route both operations and callbacks. + prelude bool // Stream pool poolMu sync.Mutex @@ -86,11 +90,30 @@ func (c *inlineClient) getStream(ctx context.Context) (*streamConn, error) { return nil, err } - return &streamConn{ + conn := &streamConn{ stream: str, enc: cbor.NewEncoder(str), dec: cbor.NewDecoder(str), - }, nil + } + + // The prelude is written once per stream, not once per call: the + // stream is pooled and carries many calls against this capability. + if c.prelude { + prelude := opRequest{ + Op: opInlineCall, + OID: c.oid, + Version: currentProtocolVersion, + } + if err := conn.enc.Encode(prelude); err != nil { + _ = str.Close() + c.poolMu.Lock() + c.activeCount-- + c.poolMu.Unlock() + return nil, err + } + } + + return conn, nil } // Pool is at capacity, wait for a stream to become available diff --git a/pkg/rpc/inline_router.go b/pkg/rpc/inline_router.go new file mode 100644 index 000000000..bfde137b8 --- /dev/null +++ b/pkg/rpc/inline_router.go @@ -0,0 +1,160 @@ +package rpc + +import ( + "context" + "errors" + "io" + "sync" + + "github.com/fxamacker/cbor/v2" + "miren.dev/runtime/pkg/cond" +) + +// inlineLookup resolves the inline capability a server-initiated callback is +// addressed to, along with the client that advertised it (the client supplies +// the caller identity for the resulting handler invocation). +type inlineLookup func(OID) (*NetworkClient, *InlineCapability, bool) + +// callbackRouter delivers server-initiated inline-capability calls back to the +// capabilities a call advertised. +// +// The HTTP transports give every call its own session, so each call can own a +// private accept loop over it. Message transports share one session across all +// concurrent calls on a connection — there is no second connection to dial — so +// callbacks must instead be demultiplexed by OID through a session-scoped +// registry. OIDs are 16 random bytes (Server.assignCapability), so entries from +// different calls cannot collide. +type callbackRouter interface { + // register makes caps reachable for the duration of one call and returns a + // function that withdraws them. + register(ctx context.Context, c *NetworkClient, sess rpcSession, caps map[OID]*InlineCapability) func() +} + +// perCallRouter is the router for transports that dedicate a session to each +// call. It runs an accept loop bound to the call's lifetime. +type perCallRouter struct{} + +func (perCallRouter) register( + ctx context.Context, + c *NetworkClient, + sess rpcSession, + caps map[OID]*InlineCapability, +) func() { + ctx, cancel := context.WithCancel(ctx) + + go c.State.acceptInlineStreams(ctx, sess, func(oid OID) (*NetworkClient, *InlineCapability, bool) { + ic, ok := caps[oid] + return c, ic, ok + }) + + return cancel +} + +// inlineRegistry routes callbacks by OID for a session shared across calls. The +// accept loop is owned by the session, not by any one call; calls only add and +// remove their capabilities. +type inlineRegistry struct { + mu sync.Mutex + entries map[OID]inlineEntry +} + +type inlineEntry struct { + client *NetworkClient + capa *InlineCapability +} + +func (r *inlineRegistry) register( + _ context.Context, + c *NetworkClient, + _ rpcSession, + caps map[OID]*InlineCapability, +) func() { + r.mu.Lock() + if r.entries == nil { + r.entries = make(map[OID]inlineEntry, len(caps)) + } + for oid, ic := range caps { + r.entries[oid] = inlineEntry{client: c, capa: ic} + } + r.mu.Unlock() + + return func() { + r.mu.Lock() + for oid := range caps { + delete(r.entries, oid) + } + r.mu.Unlock() + } +} + +func (r *inlineRegistry) lookup(oid OID) (*NetworkClient, *InlineCapability, bool) { + r.mu.Lock() + defer r.mu.Unlock() + + e, ok := r.entries[oid] + return e.client, e.capa, ok +} + +// acceptInlineStreams accepts server-initiated streams on sess and serves the +// inline-capability calls they carry. +func (s *State) acceptInlineStreams(ctx context.Context, sess rpcSession, lookup inlineLookup) { + for { + str, err := sess.AcceptStream(ctx) + if err != nil { + return + } + + go s.serveInlineStream(ctx, str, lookup) + } +} + +// serveInlineStream handles one server-initiated stream on a transport whose +// callback streams carry no opRequest prelude (HTTP/3 and WebTransport). +func (s *State) serveInlineStream(ctx context.Context, str rpcStream, lookup inlineLookup) { + defer str.Close() + + s.serveInlineCalls(ctx, cbor.NewDecoder(str), cbor.NewEncoder(str), lookup) +} + +// serveInlineCalls reads calls against inline capabilities until the stream +// ends. A stream is pooled by the peer's inlineClient and so may carry many +// calls in sequence. +func (s *State) serveInlineCalls(ctx context.Context, dec *cbor.Decoder, enc *cbor.Encoder, lookup inlineLookup) { + for { + var rs streamRequest + + if err := dec.Decode(&rs); err != nil { + if !errors.Is(err, io.EOF) { + s.log.Error("rpc.callstream call: error decoding stream request", "error", err) + } + return + } + + switch rs.Kind { + case "call": + c, iface, ok := lookup(rs.OID) + if !ok { + _ = enc.Encode(refResponse{Status: "error", Error: "unknown capability"}) + continue + } + + mm := iface.methods[rs.Method] + if mm.Handler == nil { + _ = enc.Encode(refResponse{Status: "error", Error: "unknown method"}) + continue + } + + ctx, cancel := context.WithCancel(ctx) + err := c.callInline(ctx, mm, rs.OID, rs.Method, iface.Interface, enc, dec) + cancel() + if err != nil { + if !errors.Is(err, context.Canceled) && !errors.Is(err, cond.ErrClosed{}) { + s.log.Error("rpc.callstream: error calling inline", "error", err) + } + return + } + default: + s.log.Error("rpc.callstream: unknown call stream request", "kind", rs.Kind) + } + } +} diff --git a/pkg/rpc/message.go b/pkg/rpc/message.go new file mode 100644 index 000000000..ace10c8c4 --- /dev/null +++ b/pkg/rpc/message.go @@ -0,0 +1,181 @@ +package rpc + +import ( + "context" +) + +// defaultMaxFrameData bounds the payload carried in a single msgmux frame, so +// that a backend with a maximum message size (e.g. a broker capping messages +// around a megabyte) can carry arbitrarily large stream writes as multiple +// frames. Override it per session with WithMaxFrameSize. +const defaultMaxFrameData = 1 << 20 // 1 MiB + +// MessageConn is the pure message interface a message-oriented backend +// implements: a reliable, ordered, bidirectional, point-to-point pipe of +// discrete byte messages between two peers. The framework layers a stream +// multiplexer (msgmux) on top to recover the rpcSession semantics that +// callbacks and streaming require. +// +// Recv blocks until a message is available and returns io.EOF once the peer +// has closed the connection and no buffered messages remain. +type MessageConn interface { + Send([]byte) error + Recv() ([]byte, error) + Close() error +} + +// MessageSessionOption configures a session built over a MessageConn. +type MessageSessionOption func(*messageSessionOptions) + +type messageSessionOptions struct { + maxFrame int +} + +// WithMaxFrameSize bounds the payload rpc places in a single message. Set it +// below the backend's own message-size limit — an envelope protocol wrapping +// these messages, or a broker capping message size. Stream writes larger than +// the bound are split across frames, so the bound costs throughput, never +// correctness. +func WithMaxFrameSize(n int) MessageSessionOption { + return func(o *messageSessionOptions) { + o.maxFrame = n + } +} + +func buildMessageSessionOptions(opts []MessageSessionOption) messageSessionOptions { + var o messageSessionOptions + for _, fn := range opts { + fn(&o) + } + return o +} + +// ServeMessageConn serves RPC over a connection the caller owns, and blocks +// until the connection fails or ctx is cancelled. +// +// This is the entry point for backends rpc does not manage — most importantly a +// socket carrying somebody else's envelope protocol, where the caller unwraps +// inbound payloads into Recv and wraps outbound payloads from Send. rpc never +// dials, listens, or closes such a connection. +// +// Capabilities minted on this connection are reachable only for its lifetime; +// they carry no dialable address. +func (s *State) ServeMessageConn(ctx context.Context, conn MessageConn, opts ...MessageSessionOption) error { + o := buildMessageSessionOptions(opts) + + sess := newMsgSession(conn, false, o.maxFrame) + router := &sessionRouter{state: s, server: s.server} + + return router.run(ctx, sess) +} + +// ClientFromMessageConn builds a client for the named remote object over a +// connection the caller owns, the dialing counterpart to ServeMessageConn. +// +// The resulting session serves as well as calls: the peer may invoke objects +// this State has exposed over the same connection. That is what allows a party +// that dialled outbound — through a firewall it could not accept through — to +// still be called back into. +func (s *State) ClientFromMessageConn( + ctx context.Context, + conn MessageConn, + name string, + opts ...MessageSessionOption, +) (*NetworkClient, error) { + o := buildMessageSessionOptions(opts) + + client := &NetworkClient{ + State: s, + transportKind: transportMsg, + remote: adoptedRemote, + } + // dial stays nil: the connection belongs to the caller and cannot be + // re-established from here. + client.ops = &msgOpTransport{owner: client} + client.ops.adopt(ctx, newMsgSession(conn, true, o.maxFrame)) + + if err := client.resolveCapability(name); err != nil { + return nil, err + } + + return client, nil +} + +// opRequest is the first frame of an operation stream. It carries the operation +// selector plus the metadata that, on the HTTP transports, lives in the request +// path, headers, and auth signature. The operation payload (CBOR args / +// InterfaceState / etc.) follows as subsequent bytes on the same stream. +type opRequest struct { + Op string `cbor:"0,keyasint"` + OID OID `cbor:"1,keyasint,omitempty"` + Name string `cbor:"2,keyasint,omitempty"` + Method string `cbor:"3,keyasint,omitempty"` + + PubKey []byte `cbor:"4,keyasint,omitempty"` + TargetPK []byte `cbor:"5,keyasint,omitempty"` + Contact string `cbor:"6,keyasint,omitempty"` + + Timestamp string `cbor:"7,keyasint,omitempty"` + Signature []byte `cbor:"8,keyasint,omitempty"` + + Bearer string `cbor:"9,keyasint,omitempty"` + Trace map[string]string `cbor:"10,keyasint,omitempty"` + + // Version is the protocol version this stream speaks. Zero means version 1, + // so a peer predating the field is treated as speaking it. See PROTOCOL.md. + Version uint `cbor:"11,keyasint,omitempty"` +} + +// Protocol versions carried in opRequest.Version. A field rather than a +// handshake: a version mismatch costs one rejected stream, not a round trip on +// every connection. +const ( + // protocolVersion1 is the only version defined. Zero on the wire means this. + protocolVersion1 = 1 + + // currentProtocolVersion is what this implementation sends. + currentProtocolVersion = protocolVersion1 + + // minProtocolVersion is the oldest version this implementation accepts. + minProtocolVersion = protocolVersion1 +) + +// protocolVersion normalises a version off the wire, mapping the zero value +// onto version 1. +func (r opRequest) protocolVersion() uint { + if r.Version == 0 { + return protocolVersion1 + } + return r.Version +} + +// opReply is the first frame of an operation reply, mapping the HTTP +// rpc-status/rpc-error[/-category/-code] trailers into the message world. The +// reply payload (CBOR result / lookupResponse / etc.) follows on the stream. +type opReply struct { + Status string `cbor:"0,keyasint"` + Error string `cbor:"1,keyasint,omitempty"` + Category string `cbor:"2,keyasint,omitempty"` + Code string `cbor:"3,keyasint,omitempty"` +} + +// Operation selectors carried in opRequest.Op. Every stream on a message +// session opens with an opRequest, including server-initiated callbacks +// (opInlineCall), so a single accept loop can route the whole session and +// either peer may both serve and call over one connection. +const ( + opCall = "call" + opCallStream = "callstream" + opLookup = "lookup" + opMethods = "methods" + opReresolve = "reresolve" + opReexport = "reexport" + opRef = "ref" + opDeref = "deref" + opIdentify = "identify" + + // opInlineCall opens a stream carrying calls against an inline capability + // the peer advertised. The stream is pooled by the caller and may carry many + // calls, so the prelude names only the capability, not a method. + opInlineCall = "inline" +) diff --git a/pkg/rpc/message_adopt_test.go b/pkg/rpc/message_adopt_test.go new file mode 100644 index 000000000..2b791927f --- /dev/null +++ b/pkg/rpc/message_adopt_test.go @@ -0,0 +1,415 @@ +package rpc_test + +import ( + "context" + "errors" + "io" + "net" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "miren.dev/runtime/pkg/rpc" + "miren.dev/runtime/pkg/rpc/example" + "miren.dev/runtime/pkg/rpc/stream" +) + +// envelopeConn is a MessageConn owned by the test rather than by rpc, standing +// in for a socket carrying somebody else's envelope protocol: every payload is +// wrapped on the way out and unwrapped on the way in, and rpc never sees the +// underlying pipe. Critically there is no way to open a second one — if rpc +// tried to dial another connection for a callstream, these tests would fail. +type envelopeConn struct { + mu sync.Mutex + peer *envelopeConn + + recv chan []byte + done chan struct{} + once sync.Once + + // envelopes counts payloads that made a round trip through the wrapper, so + // a test can assert traffic actually crossed the boundary. + envelopes *int64 + countMu *sync.Mutex +} + +const envelopePrefix = "ENV:" + +func newEnvelopePair() (*envelopeConn, *envelopeConn) { + var ( + count int64 + cmu sync.Mutex + ) + + done := make(chan struct{}) + a := &envelopeConn{recv: make(chan []byte, 64), done: done, envelopes: &count, countMu: &cmu} + b := &envelopeConn{recv: make(chan []byte, 64), done: done, envelopes: &count, countMu: &cmu} + a.peer, b.peer = b, a + + return a, b +} + +func (c *envelopeConn) Send(b []byte) error { + // Wrap: this is the caller's protocol, opaque to rpc. + framed := append([]byte(envelopePrefix), b...) + + c.countMu.Lock() + *c.envelopes++ + c.countMu.Unlock() + + c.mu.Lock() + peer := c.peer + c.mu.Unlock() + + select { + case peer.recv <- framed: + return nil + case <-c.done: + return net.ErrClosed + } +} + +func (c *envelopeConn) Recv() ([]byte, error) { + select { + case b := <-c.recv: + return b[len(envelopePrefix):], nil // unwrap + case <-c.done: + select { + case b := <-c.recv: + return b[len(envelopePrefix):], nil + default: + return nil, io.EOF + } + } +} + +func (c *envelopeConn) Close() error { + c.once.Do(func() { close(c.done) }) + return nil +} + +func (c *envelopeConn) count() int64 { + c.countMu.Lock() + defer c.countMu.Unlock() + return *c.envelopes +} + +// recordingAuthenticator returns a fixed identity and remembers the credentials +// it was handed, so a test can assert what actually reached it. +type recordingAuthenticator struct { + identity *rpc.Identity + + mu sync.Mutex + last *rpc.Credentials +} + +func (a *recordingAuthenticator) Authenticate( + ctx context.Context, + creds *rpc.Credentials, +) (*rpc.Identity, error) { + a.mu.Lock() + a.last = creds + a.mu.Unlock() + + return a.identity, nil +} + +func (a *recordingAuthenticator) lastAuthorization() string { + a.mu.Lock() + defer a.mu.Unlock() + + if a.last == nil { + return "" + } + return a.last.Authorization +} + +// denyAll rejects every authorization check. +type denyAll struct{} + +func (denyAll) Authorize(ctx context.Context, identity *rpc.Identity, resource, action string) error { + return errors.New("denied by policy") +} + +// identityMeter records the identity its handler ran under. +type identityMeter struct { + seen **rpc.Identity +} + +func (m *identityMeter) ReadTemperature(ctx context.Context, call *example.MeterReadTemperature) error { + *m.seen = rpc.IdentityFromContext(ctx) + + res := call.Results() + reading := new(example.Reading) + reading.SetMeter(call.Args().Name()) + reading.SetTemperature(1) + res.SetReading(reading) + + return nil +} + +func (m *identityMeter) GetSetter(ctx context.Context, call *example.MeterGetSetter) error { + return nil +} + +// adoptPair wires two States over one caller-owned connection: serverEnd is +// served, clientEnd is dialled. Returns the client for the named object. +func adoptPair( + t *testing.T, + ctx context.Context, + iface *rpc.Interface, + name string, + opts ...rpc.MessageSessionOption, +) (rpc.Client, *envelopeConn) { + t.Helper() + r := require.New(t) + + ss, err := rpc.NewState(ctx, rpc.WithSkipVerify) + r.NoError(err) + ss.Server().ExposeValue(name, iface) + + cs, err := rpc.NewState(ctx, rpc.WithSkipVerify) + r.NoError(err) + + serverEnd, clientEnd := newEnvelopePair() + + go ss.ServeMessageConn(ctx, serverEnd, opts...) //nolint:errcheck // ends with ctx + + c, err := cs.ClientFromMessageConn(ctx, clientEnd, name, opts...) + r.NoError(err) + + return c, clientEnd +} + +func TestAdoptedMessageConn(t *testing.T) { + t.Run("unary calls over a caller-owned connection", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + c, conn := adoptPair(t, ctx, example.AdaptMeter(&exampleMeter{temp: 42}), "meter") + + mc := &example.MeterClient{Client: c} + + res, err := mc.ReadTemperature(ctx, "test") + r.NoError(err) + r.Equal("test", res.Reading().Meter()) + r.Equal(float32(42), res.Reading().Temperature()) + + r.NotZero(conn.count(), "traffic should have crossed the envelope boundary") + }) + + // A capability returned from a call carries no dialable address on an adopted + // connection, so following it must reuse the session it arrived on. + t.Run("follows a returned capability over the same connection", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + c, _ := adoptPair(t, ctx, example.AdaptMeter(&exampleMeter{temp: 42}), "meter") + + mc := &example.MeterClient{Client: c} + + res, err := mc.GetSetter(ctx, "test") + r.NoError(err) + + res2, err := res.Setter().SetTemp(ctx, 100) + r.NoError(err) + r.Equal(int32(100), res2.Temp()) + }) + + t.Run("server calls back over the same connection", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + c, _ := adoptPair(t, ctx, example.AdaptMeterUpdates(&exampleMU{}), "meter") + + mc := &example.MeterUpdatesClient{Client: c} + + var up exampleUpdate + _, err := mc.RegisterUpdates(ctx, &up) + r.NoError(err) + + r.True(up.gotIt) + r.Equal("test", up.reading.Meter()) + r.Equal(float32(42), up.reading.Temperature()) + }) + + t.Run("streaming callbacks over a caller-owned connection", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + c, _ := adoptPair(t, ctx, example.AdaptEmitTemps(&exampleEmit{}), "meter") + + mc := &example.EmitTempsClient{Client: c} + + var ( + mu sync.Mutex + vals []float32 + ) + recv := stream.StreamRecv(func(val float32) error { + mu.Lock() + defer mu.Unlock() + vals = append(vals, val) + return nil + }) + + _, err := mc.Emit(ctx, recv) + r.NoError(err) + + time.Sleep(500 * time.Millisecond) + + mu.Lock() + defer mu.Unlock() + r.Equal([]float32{42, 100}, vals) + }) + + // The whole point of an adopted connection: there is no second connection to + // dial, so every concurrent callstream must multiplex onto the one we were + // given. + t.Run("concurrent callstreams multiplex onto the one connection", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + c, _ := adoptPair(t, ctx, example.AdaptEmitTemps(&exampleEmit{}), "meter") + + const calls = 8 + + var ( + wg sync.WaitGroup + mu sync.Mutex + errs []error + got = make([][]float32, calls) + ) + + for i := range calls { + wg.Add(1) + go func() { + defer wg.Done() + + var ( + vmu sync.Mutex + vals []float32 + ) + recv := stream.StreamRecv(func(val float32) error { + vmu.Lock() + defer vmu.Unlock() + vals = append(vals, val) + return nil + }) + + mc := &example.EmitTempsClient{Client: c} + if _, err := mc.Emit(ctx, recv); err != nil { + mu.Lock() + errs = append(errs, err) + mu.Unlock() + return + } + + time.Sleep(500 * time.Millisecond) + + vmu.Lock() + defer vmu.Unlock() + got[i] = append([]float32(nil), vals...) + }() + } + + wg.Wait() + + mu.Lock() + defer mu.Unlock() + r.Empty(errs) + + for i := range calls { + r.Equal([]float32{42, 100}, got[i], "call %d received the wrong values", i) + } + }) + + // A message transport has no Authorization header and no TLS handshake, so a + // bearer token rides in the operation frame instead. Without this, cloudauth + // and oidcauth cannot identify a caller over one of these connections at all. + t.Run("carries a bearer token to the authenticator", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + auth := &recordingAuthenticator{ + identity: &rpc.Identity{Subject: "user-42", Method: rpc.AuthMethodJWT}, + } + + ss, err := rpc.NewState(ctx, rpc.WithSkipVerify, rpc.WithAuthenticator(auth)) + r.NoError(err) + + var seen *rpc.Identity + iface := example.AdaptMeter(&identityMeter{seen: &seen}) + ss.Server().ExposeValue("meter", iface) + + cs, err := rpc.NewState(ctx, rpc.WithSkipVerify, rpc.WithBearerToken("secret-token")) + r.NoError(err) + + serverEnd, clientEnd := newEnvelopePair() + go ss.ServeMessageConn(ctx, serverEnd) //nolint:errcheck // ends with ctx + + c, err := cs.ClientFromMessageConn(ctx, clientEnd, "meter") + r.NoError(err) + + mc := &example.MeterClient{Client: c} + _, err = mc.ReadTemperature(ctx, "test") + r.NoError(err) + + r.Equal("Bearer secret-token", auth.lastAuthorization()) + r.NotNil(seen) + r.Equal("user-42", seen.Subject) + r.Equal(rpc.AuthMethodJWT, seen.Method) + }) + + // An authorization denial must reach the caller as a permission error. It + // travels as its own opReply status, so without explicit handling it surfaces + // as an unrecognised-protocol-status error instead. + t.Run("reports an authorization denial as a permission error", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + ss, err := rpc.NewState(ctx, rpc.WithSkipVerify, + rpc.WithAuthenticator(&recordingAuthenticator{ + identity: &rpc.Identity{Subject: "nobody", Method: rpc.AuthMethodJWT}, + }), + rpc.WithAuthorizer(denyAll{}), + ) + r.NoError(err) + ss.Server().ExposeValue("meter", example.AdaptMeter(&exampleMeter{temp: 42})) + + cs, err := rpc.NewState(ctx, rpc.WithSkipVerify, rpc.WithBearerToken("token")) + r.NoError(err) + + serverEnd, clientEnd := newEnvelopePair() + go ss.ServeMessageConn(ctx, serverEnd) //nolint:errcheck // ends with ctx + + c, err := cs.ClientFromMessageConn(ctx, clientEnd, "meter") + r.NoError(err) + + mc := &example.MeterClient{Client: c} + _, err = mc.ReadTemperature(ctx, "test") + r.Error(err) + r.NotContains(err.Error(), "unknown response status") + r.Contains(err.Error(), "denied by policy") + }) + + // A frame cap below the payload size forces msgmux to split a write across + // several messages, which is what an envelope with its own size limit needs. + t.Run("respects a frame size below the payload size", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + c, conn := adoptPair(t, ctx, + example.AdaptMeter(&exampleMeter{temp: 42}), "meter", + rpc.WithMaxFrameSize(64), + ) + + mc := &example.MeterClient{Client: c} + + res, err := mc.ReadTemperature(ctx, "a-meter-name-long-enough-to-need-splitting-across-frames") + r.NoError(err) + r.Equal(float32(42), res.Reading().Temperature()) + + r.NotZero(conn.count()) + }) +} diff --git a/pkg/rpc/message_mem.go b/pkg/rpc/message_mem.go new file mode 100644 index 000000000..194451d6e --- /dev/null +++ b/pkg/rpc/message_mem.go @@ -0,0 +1,111 @@ +package rpc + +import ( + "context" + "fmt" + "io" + "net" + "sync" +) + +// memConn is an in-process MessageConn: two cross-wired buffered channels. It +// is reliable and ordered, satisfying msgmux's requirements with no network. +// Closing either end tears down both. +type memConn struct { + send chan []byte + recv chan []byte + + done chan struct{} + closeFn func() +} + +func newMemPipe() (*memConn, *memConn) { + ab := make(chan []byte, 64) + ba := make(chan []byte, 64) + done := make(chan struct{}) + var once sync.Once + closeFn := func() { once.Do(func() { close(done) }) } + + a := &memConn{send: ab, recv: ba, done: done, closeFn: closeFn} + b := &memConn{send: ba, recv: ab, done: done, closeFn: closeFn} + return a, b +} + +func (c *memConn) Send(b []byte) error { + cp := make([]byte, len(b)) + copy(cp, b) + select { + case c.send <- cp: + return nil + case <-c.done: + return net.ErrClosed + } +} + +func (c *memConn) Recv() ([]byte, error) { + select { + case b := <-c.recv: + return b, nil + case <-c.done: + // Drain anything already buffered before reporting EOF. + select { + case b := <-c.recv: + return b, nil + default: + return nil, io.EOF + } + } +} + +func (c *memConn) Close() error { + c.closeFn() + return nil +} + +// memRegistry maps a process-local name to a server State, mirroring the +// LocalActorRegistry pattern (actor.go). Dialing a name creates a memConn pair, +// spawns the server's session loop on one end, and returns the other end. +type memRegistry struct { + mu sync.Mutex + servers map[string]*memServerEntry +} + +type memServerEntry struct { + state *State + ctx context.Context +} + +var defaultMemRegistry = &memRegistry{servers: make(map[string]*memServerEntry)} + +func (r *memRegistry) register(ctx context.Context, name string, s *State) { + r.mu.Lock() + r.servers[name] = &memServerEntry{state: s, ctx: ctx} + r.mu.Unlock() + + go func() { + <-ctx.Done() + r.mu.Lock() + if e := r.servers[name]; e != nil && e.state == s { + delete(r.servers, name) + } + r.mu.Unlock() + }() +} + +func (r *memRegistry) dial(name string) (MessageConn, error) { + r.mu.Lock() + entry := r.servers[name] + r.mu.Unlock() + + if entry == nil { + return nil, fmt.Errorf("rpc: no mem server registered as %q", name) + } + + clientEnd, serverEnd := newMemPipe() + serverSess := newMsgSession(serverEnd, false, 0) + + router := &sessionRouter{state: entry.state, server: entry.state.server} + go router.run(entry.ctx, serverSess) //nolint:errcheck // loop ends with the session + + return clientEnd, nil +} diff --git a/pkg/rpc/message_net_test.go b/pkg/rpc/message_net_test.go new file mode 100644 index 000000000..52cf3bfd8 --- /dev/null +++ b/pkg/rpc/message_net_test.go @@ -0,0 +1,265 @@ +package rpc_test + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "miren.dev/runtime/pkg/cond" + "miren.dev/runtime/pkg/rpc" + "miren.dev/runtime/pkg/rpc/example" + "miren.dev/runtime/pkg/rpc/stream" +) + +// netTransport describes one of the networked message transports, so both get +// identical coverage from the table below. +type netTransport struct { + name string + listen func() rpc.StateOption + remote func(*rpc.State) string +} + +var netTransports = []netTransport{ + { + name: "websocket", + listen: func() rpc.StateOption { return rpc.WithWSBindAddr("localhost:0") }, + remote: func(s *rpc.State) string { return "ws://" + s.WSListenAddr() }, + }, + { + name: "tcp", + listen: func() rpc.StateOption { return rpc.WithTCPBindAddr("localhost:0") }, + remote: func(s *rpc.State) string { return "tcp://" + s.TCPListenAddr() }, + }, +} + +// netServer starts a State listening on the given transport and exposes iface +// under name, returning the remote to connect to. +func netServer(t *testing.T, ctx context.Context, tr netTransport, name string, iface *rpc.Interface) string { + t.Helper() + r := require.New(t) + + ss, err := rpc.NewState(ctx, rpc.WithSkipVerify, tr.listen()) + r.NoError(err) + + ss.Server().ExposeValue(name, iface) + + return tr.remote(ss) +} + +func netClient(t *testing.T, ctx context.Context, remote, name string) rpc.Client { + t.Helper() + r := require.New(t) + + cs, err := rpc.NewState(ctx, rpc.WithSkipVerify) + r.NoError(err) + + c, err := cs.Connect(remote, name) + r.NoError(err) + + return c +} + +// blockingMU is a MeterUpdates implementation whose handler never replies until +// its context is cancelled (bounded so it can't leak), used to exercise +// caller-side cancellation of a callstream. +type blockingMU struct { + started chan struct{} +} + +func (m *blockingMU) RegisterUpdates(ctx context.Context, call *example.MeterUpdatesRegisterUpdates) error { + close(m.started) + select { + case <-ctx.Done(): + case <-time.After(2 * time.Second): + } + return cond.Closed("server done") +} + +func TestNetMessageTransports(t *testing.T) { + for _, tr := range netTransports { + t.Run(tr.name, func(t *testing.T) { + t.Run("unary calls", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := netServer(t, ctx, tr, "meter", example.AdaptMeter(&exampleMeter{temp: 42})) + c := netClient(t, ctx, remote, "meter") + + mc := &example.MeterClient{Client: c} + + res, err := mc.ReadTemperature(ctx, "test") + r.NoError(err) + r.Equal("test", res.Reading().Meter()) + r.Equal(float32(42), res.Reading().Temperature()) + + // GetSetter returns a capability; calling it round-trips a second + // unary call over the same session. + res2, err := mc.GetSetter(ctx, "test") + r.NoError(err) + + res3, err := res2.Setter().SetTemp(ctx, 100) + r.NoError(err) + r.Equal(int32(100), res3.Temp()) + }) + + t.Run("server calls back on a client-advertised capability", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := netServer(t, ctx, tr, "meter", example.AdaptMeterUpdates(&exampleMU{})) + c := netClient(t, ctx, remote, "meter") + + mc := &example.MeterUpdatesClient{Client: c} + + var up exampleUpdate + + // The server invokes up.Update over the same connection — the core + // bidirectional capability/callback behavior. + _, err := mc.RegisterUpdates(ctx, &up) + r.NoError(err) + + r.True(up.gotIt) + r.Equal("test", up.reading.Meter()) + r.Equal(float32(42), up.reading.Temperature()) + + time.Sleep(100 * time.Millisecond) // wait for the goroutine running Close + + up.mu.Lock() + defer up.mu.Unlock() + r.True(up.closed) + }) + + t.Run("streaming callbacks", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := netServer(t, ctx, tr, "meter", example.AdaptEmitTemps(&exampleEmit{})) + c := netClient(t, ctx, remote, "meter") + + mc := &example.EmitTempsClient{Client: c} + + var ( + mu sync.Mutex + vals []float32 + ) + recv := stream.StreamRecv(func(val float32) error { + mu.Lock() + defer mu.Unlock() + vals = append(vals, val) + return nil + }) + + _, err := mc.Emit(ctx, recv) + r.NoError(err) + + time.Sleep(time.Second) + + mu.Lock() + defer mu.Unlock() + r.Equal([]float32{42, 100}, vals) + }) + + t.Run("concurrent callstreams share one connection", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := netServer(t, ctx, tr, "meter", example.AdaptEmitTemps(&exampleEmit{})) + c := netClient(t, ctx, remote, "meter") + + const calls = 8 + + var ( + wg sync.WaitGroup + mu sync.Mutex + errs []error + got = make([][]float32, calls) + ) + + for i := range calls { + wg.Add(1) + go func() { + defer wg.Done() + + var ( + vmu sync.Mutex + vals []float32 + ) + recv := stream.StreamRecv(func(val float32) error { + vmu.Lock() + defer vmu.Unlock() + vals = append(vals, val) + return nil + }) + + mc := &example.EmitTempsClient{Client: c} + if _, err := mc.Emit(ctx, recv); err != nil { + mu.Lock() + errs = append(errs, err) + mu.Unlock() + return + } + + time.Sleep(500 * time.Millisecond) + + vmu.Lock() + defer vmu.Unlock() + got[i] = append([]float32(nil), vals...) + }() + } + + wg.Wait() + + mu.Lock() + defer mu.Unlock() + r.Empty(errs) + + for i := range calls { + r.Equal([]float32{42, 100}, got[i], "call %d received the wrong values", i) + } + }) + + t.Run("cancelled callstream reports closed", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + m := &blockingMU{started: make(chan struct{})} + remote := netServer(t, ctx, tr, "meter", example.AdaptMeterUpdates(m)) + c := netClient(t, ctx, remote, "meter") + + mc := &example.MeterUpdatesClient{Client: c} + + cctx, cancel := context.WithCancel(ctx) + go func() { + <-m.started + time.Sleep(50 * time.Millisecond) + cancel() + }() + + var up exampleUpdate + _, err := mc.RegisterUpdates(cctx, &up) + r.Error(err) + r.True(errors.Is(err, cond.ErrClosed{}), "expected ErrClosed, got %T: %v", err, err) + }) + + t.Run("propagates panic errors", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := netServer(t, ctx, tr, "meter", example.AdaptMeter(&panicMeter{})) + c := netClient(t, ctx, remote, "meter") + + mc := &example.MeterClient{Client: c} + + _, err := mc.ReadTemperature(ctx, "test") + r.Error(err) + + var panicErr cond.ErrPanic + r.True(errors.As(err, &panicErr), "expected ErrPanic, got %T: %v", err, err) + r.Contains(panicErr.Message, "test panic message") + }) + }) + } +} diff --git a/pkg/rpc/message_tcp.go b/pkg/rpc/message_tcp.go new file mode 100644 index 000000000..2d06dfd39 --- /dev/null +++ b/pkg/rpc/message_tcp.go @@ -0,0 +1,158 @@ +package rpc + +import ( + "bufio" + "context" + "crypto/tls" + "encoding/binary" + "fmt" + "io" + "net" + "strings" + "sync" +) + +// tcpFrameHeader is the 4-byte big-endian length prefix that turns a TCP byte +// stream into the discrete messages msgmux consumes. +const tcpFrameHeader = 4 + +// tcpConn is a MessageConn over a byte stream, framing each message with a +// length prefix. TLS is applied by the caller; this type only frames. +type tcpConn struct { + conn net.Conn + r *bufio.Reader + + // maxFrame bounds an inbound frame. A peer announcing more than this is + // rejected before anything is allocated, so a bad length cannot exhaust + // memory. + maxFrame int + + wmu sync.Mutex + hdr [tcpFrameHeader]byte +} + +func newTCPConn(conn net.Conn, maxFrame int) *tcpConn { + if maxFrame <= 0 { + maxFrame = defaultMaxFrameData + } + + return &tcpConn{ + conn: conn, + r: bufio.NewReader(conn), + // A frame carries a payload plus msgmux's own CBOR envelope, so allow + // headroom over the payload bound rather than rejecting valid frames. + maxFrame: maxFrame + (1 << 16), + } +} + +func (t *tcpConn) Send(b []byte) error { + t.wmu.Lock() + defer t.wmu.Unlock() + + binary.BigEndian.PutUint32(t.hdr[:], uint32(len(b))) + + if _, err := t.conn.Write(t.hdr[:]); err != nil { + return err + } + _, err := t.conn.Write(b) + return err +} + +func (t *tcpConn) Recv() ([]byte, error) { + var hdr [tcpFrameHeader]byte + if _, err := io.ReadFull(t.r, hdr[:]); err != nil { + return nil, err + } + + n := binary.BigEndian.Uint32(hdr[:]) + if int(n) > t.maxFrame { + return nil, fmt.Errorf("rpc: peer announced a %d byte frame, over the %d byte limit", n, t.maxFrame) + } + + buf := make([]byte, n) + if _, err := io.ReadFull(t.r, buf); err != nil { + return nil, err + } + return buf, nil +} + +func (t *tcpConn) Close() error { + return t.conn.Close() +} + +// startTCPListener serves RPC over raw TLS-over-TCP. There is no HTTP layer: +// each accepted connection is framed into messages and handed straight to the +// session router. +func (s *State) startTCPListener(ctx context.Context, addr string) error { + tlsCfg := s.serverTlsCfg.Clone() + tlsCfg.NextProtos = nil // no ALPN: nothing here speaks HTTP + + ln, err := tls.Listen("tcp", addr, tlsCfg) + if err != nil { + return err + } + + s.msgLn = ln + + go func() { + <-ctx.Done() + _ = ln.Close() + }() + + s.log.Debug("starting tcp message listener", "addr", ln.Addr().String()) + + go func() { + for { + conn, aerr := ln.Accept() + if aerr != nil { + if ctx.Err() == nil { + s.log.Error("tcp message listener stopped", "error", aerr) + } + return + } + + go func() { + mc := newTCPConn(conn, 0) + if serr := s.ServeMessageConn(ctx, mc); serr != nil { + s.log.Debug("rpc: tcp session ended", "error", serr) + } + _ = mc.Close() + }() + } + }() + + return nil +} + +// TCPListenAddr returns the address the raw TCP message listener is bound to, +// or an empty string if no such listener is running. +func (s *State) TCPListenAddr() string { + if s.msgLn == nil { + return "" + } + return s.msgLn.Addr().String() +} + +// setupTCPTransport wires a client for a "tcp://host:port" remote. +func (c *NetworkClient) setupTCPTransport() { + remote := strings.TrimPrefix(c.remote, "tcp://") + + tlsCfg := c.tlsCfg + if tlsCfg == nil { + tlsCfg = c.State.clientTlsCfg + } + + cfg := tlsCfg.Clone() + cfg.NextProtos = nil + + c.ops = &msgOpTransport{ + owner: c, + dial: func() (MessageConn, error) { + conn, err := tls.Dial("tcp", remote, cfg) + if err != nil { + return nil, err + } + return newTCPConn(conn, 0), nil + }, + } +} diff --git a/pkg/rpc/message_version_test.go b/pkg/rpc/message_version_test.go new file mode 100644 index 000000000..efee76288 --- /dev/null +++ b/pkg/rpc/message_version_test.go @@ -0,0 +1,90 @@ +package rpc + +import ( + "testing" + + "github.com/fxamacker/cbor/v2" + "github.com/stretchr/testify/require" +) + +// The version gate is what lets a second implementation appear later without a +// breaking change, so it needs to actually reject and actually accept. +func TestProtocolVersionGate(t *testing.T) { + t.Run("absent version reads as version 1", func(t *testing.T) { + r := require.New(t) + + r.Equal(uint(protocolVersion1), opRequest{}.protocolVersion()) + r.Equal(uint(protocolVersion1), opRequest{Version: 1}.protocolVersion()) + r.Equal(uint(7), opRequest{Version: 7}.protocolVersion()) + }) + + // A peer speaking a version this build does not know must be told so, rather + // than having its frames misread as the version we do know. + t.Run("rejects an unsupported version with the supported range", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + clientEnd, serverEnd := newMemPipe() + + state, err := NewState(ctx, WithSkipVerify) + r.NoError(err) + + router := &sessionRouter{state: state, server: state.server} + go router.run(ctx, newMsgSession(serverEnd, false, 0)) //nolint:errcheck // ends with ctx + + sess := newMsgSession(clientEnd, true, 0) + st, err := sess.OpenStreamSync(ctx) + r.NoError(err) + + err = cbor.NewEncoder(st).Encode(opRequest{ + Op: opLookup, + Name: "whatever", + Version: currentProtocolVersion + 1, + }) + r.NoError(err) + + var reply opReply + r.NoError(cbor.NewDecoder(st).Decode(&reply)) + + r.Equal("error", reply.Status) + r.Contains(reply.Error, "unsupported protocol version") + }) + + // A peer predating the field sends no version at all; it must still work. + t.Run("accepts a request carrying no version", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + clientEnd, serverEnd := newMemPipe() + + state, err := NewState(ctx, WithSkipVerify) + r.NoError(err) + + router := &sessionRouter{state: state, server: state.server} + go router.run(ctx, newMsgSession(serverEnd, false, 0)) //nolint:errcheck // ends with ctx + + sess := newMsgSession(clientEnd, true, 0) + st, err := sess.OpenStreamSync(ctx) + r.NoError(err) + + err = cbor.NewEncoder(st).Encode(opRequest{ + Op: opLookup, + Name: "nothing-exposed", + PubKey: state.pubkey, + }) + r.NoError(err) + + dec := cbor.NewDecoder(st) + + var reply opReply + r.NoError(dec.Decode(&reply)) + + // The lookup itself fails (nothing is exposed), but it was dispatched + // rather than rejected by the version gate. + r.Equal("ok", reply.Status) + + var lr lookupResponse + r.NoError(dec.Decode(&lr)) + r.Contains(lr.Error, "unknown object") + }) +} diff --git a/pkg/rpc/message_ws.go b/pkg/rpc/message_ws.go new file mode 100644 index 000000000..de46b165c --- /dev/null +++ b/pkg/rpc/message_ws.go @@ -0,0 +1,111 @@ +package rpc + +import ( + "context" + "crypto/tls" + "net/http" + "strings" + + "github.com/coder/websocket" +) + +// wsMessagePath is the single route the TCP/WebSocket listener serves. Every +// operation rides the multiplexed session established by the upgrade, so unlike +// the HTTP/3 transport there is no per-operation URL. +const wsMessagePath = "/_rpc/message" + +// wsConn is a MessageConn over a WebSocket. +// +// A WebSocket already delivers discrete, ordered, reliable messages — exactly +// what msgmux consumes — so the frames go on the wire as-is. Nothing here +// converts the connection to a byte stream and re-frames it. +type wsConn struct { + c *websocket.Conn + ctx context.Context +} + +func (w *wsConn) Send(b []byte) error { + return w.c.Write(w.ctx, websocket.MessageBinary, b) +} + +func (w *wsConn) Recv() ([]byte, error) { + _, data, err := w.c.Read(w.ctx) + if err != nil { + return nil, err + } + return data, nil +} + +func (w *wsConn) Close() error { + return w.c.Close(websocket.StatusNormalClosure, "") +} + +// dialWSMessageConn opens a WebSocket to a "host:port" remote and wraps it as a +// MessageConn. ctx bounds the connection's lifetime, so it must be the State's +// context rather than any one call's. +func dialWSMessageConn(ctx context.Context, remote string, tlsCfg *tls.Config) (MessageConn, error) { + // coder/websocket does not implement RFC 8441 extended CONNECT, so the + // handshake must be HTTP/1.1. + cfg := tlsCfg.Clone() + cfg.NextProtos = []string{"http/1.1"} + + c, _, err := websocket.Dial(ctx, "wss://"+remote+wsMessagePath, &websocket.DialOptions{ + HTTPClient: &http.Client{ + Transport: &http.Transport{TLSClientConfig: cfg}, + }, + }) + if err != nil { + return nil, err + } + + // A single frame may legitimately be as large as the session's frame cap; + // the default read limit is far below that. + c.SetReadLimit(-1) + + return &wsConn{c: c, ctx: ctx}, nil +} + +// serveWSUpgrade accepts a WebSocket upgrade and serves RPC over it. +// +// The session is bound to lifeCtx (the server's lifetime), not r.Context(): +// websocket.Accept hijacks the TCP socket and net/http cancels the request +// context as soon as this handler returns, while the hijacked socket remains +// open and owned by us. Binding to the request context would tear the session +// down immediately. +func (s *State) serveWSUpgrade(lifeCtx context.Context, w http.ResponseWriter, r *http.Request) { + c, err := websocket.Accept(w, r, &websocket.AcceptOptions{ + InsecureSkipVerify: true, // origin checks are not meaningful for service-to-service RPC + }) + if err != nil { + s.log.Error("rpc: websocket upgrade failed", "error", err) + return + } + + c.SetReadLimit(-1) + + go func() { + conn := &wsConn{c: c, ctx: lifeCtx} + if serr := s.ServeMessageConn(lifeCtx, conn); serr != nil { + s.log.Debug("rpc: websocket session ended", "error", serr) + } + _ = conn.Close() + }() +} + +// setupWSTransport wires a client for a "ws://host:port" remote onto the +// message transport. +func (c *NetworkClient) setupWSTransport() { + remote := strings.TrimPrefix(strings.TrimPrefix(c.remote, "wss://"), "ws://") + + tlsCfg := c.tlsCfg + if tlsCfg == nil { + tlsCfg = c.State.clientTlsCfg + } + + c.ops = &msgOpTransport{ + owner: c, + dial: func() (MessageConn, error) { + return dialWSMessageConn(c.State.top, remote, tlsCfg) + }, + } +} diff --git a/pkg/rpc/msgmux.go b/pkg/rpc/msgmux.go new file mode 100644 index 000000000..239c84f81 --- /dev/null +++ b/pkg/rpc/msgmux.go @@ -0,0 +1,355 @@ +package rpc + +import ( + "bytes" + "context" + "errors" + "io" + "sync" + + "github.com/fxamacker/cbor/v2" +) + +// msgmux is a minimal stream multiplexer that implements rpcSession/rpcStream +// over a MessageConn. It is to a MessageConn what yamux is to a byte conn, but +// because a MessageConn already delivers reliable, ordered, discrete messages, +// msgmux needs no flow control: each frame is one message, and the backend's +// own backpressure governs throughput. +// +// Frame flags. A stream is opened with an eager SYN (so AcceptStream fires even +// if the opener reads before writing); data frames carry DATA; a clean close +// sends FIN; CancelRead/abort sends RST. +const ( + flagSYN uint8 = 1 << 0 + flagDATA uint8 = 1 << 1 + flagFIN uint8 = 1 << 2 + flagRST uint8 = 1 << 3 +) + +type msgFrame struct { + Stream uint32 `cbor:"0,keyasint"` + Flags uint8 `cbor:"1,keyasint"` + Data []byte `cbor:"2,keyasint,omitempty"` +} + +var ( + errSessionClosed = errors.New("rpc: message session closed") + errStreamReset = errors.New("rpc: message stream reset") + errStreamCancel = errors.New("rpc: message stream read canceled") +) + +type msgSession struct { + conn MessageConn + + // maxFrame bounds the payload in one frame, so a backend that caps message + // size can still carry arbitrarily large stream writes. + maxFrame int + + sendMu sync.Mutex + + mu sync.Mutex + streams map[uint32]*msgStream + nextID uint32 + idStep uint32 + closed bool + closeErr error + + accept chan *msgStream + done chan struct{} +} + +// newMsgSession wraps a MessageConn. Exactly one side must pass client=true +// (the dialer); it uses odd stream ids while the accepter uses even ids, so ids +// never collide. A maxFrame of zero uses defaultMaxFrameData. +func newMsgSession(conn MessageConn, client bool, maxFrame int) *msgSession { + var start uint32 = 2 + if client { + start = 1 + } + if maxFrame <= 0 { + maxFrame = defaultMaxFrameData + } + s := &msgSession{ + conn: conn, + maxFrame: maxFrame, + streams: make(map[uint32]*msgStream), + nextID: start, + idStep: 2, + accept: make(chan *msgStream, 16), + done: make(chan struct{}), + } + go s.readPump() + return s +} + +func (s *msgSession) newStream(id uint32) *msgStream { + st := &msgStream{sess: s, id: id} + st.readCond = sync.NewCond(&st.readMu) + return st +} + +func (s *msgSession) sendFrame(f msgFrame) error { + data, err := cbor.Marshal(f) + if err != nil { + return err + } + s.sendMu.Lock() + defer s.sendMu.Unlock() + return s.conn.Send(data) +} + +func (s *msgSession) OpenStreamSync(ctx context.Context) (rpcStream, error) { + s.mu.Lock() + if s.closed { + err := s.closeErr + s.mu.Unlock() + return nil, err + } + id := s.nextID + s.nextID += s.idStep + st := s.newStream(id) + s.streams[id] = st + s.mu.Unlock() + + if err := s.sendFrame(msgFrame{Stream: id, Flags: flagSYN}); err != nil { + s.removeStream(id) + return nil, err + } + st.synSent = true + return st, nil +} + +func (s *msgSession) AcceptStream(ctx context.Context) (rpcStream, error) { + select { + case st := <-s.accept: + return st, nil + case <-s.done: + return nil, s.closeErr + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (s *msgSession) Close() error { + return s.conn.Close() +} + +func (s *msgSession) removeStream(id uint32) { + s.mu.Lock() + delete(s.streams, id) + s.mu.Unlock() +} + +func (s *msgSession) readPump() { + for { + data, err := s.conn.Recv() + if err != nil { + s.shutdown(err) + return + } + + var f msgFrame + if err := cbor.Unmarshal(data, &f); err != nil { + s.shutdown(err) + return + } + + s.handleFrame(f) + } +} + +func (s *msgSession) handleFrame(f msgFrame) { + s.mu.Lock() + st := s.streams[f.Stream] + newStream := false + if st == nil && f.Flags&flagSYN != 0 && !s.closed { + st = s.newStream(f.Stream) + st.synSent = true // peer opened it; we never send a SYN for it + s.streams[f.Stream] = st + newStream = true + } + s.mu.Unlock() + + if st == nil { + return // data for an unknown, un-SYN'd stream: drop + } + + if newStream { + select { + case s.accept <- st: + case <-s.done: + return + } + } + + if f.Flags&flagRST != 0 { + st.fail(errStreamReset) + s.removeStream(f.Stream) + return + } + + if f.Flags&(flagDATA|flagFIN) != 0 { + st.feed(f.Data, f.Flags&flagFIN != 0) + } + + if f.Flags&flagFIN != 0 { + s.maybeRemove(st) + } +} + +func (s *msgSession) shutdown(err error) { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return + } + s.closed = true + s.closeErr = err + streams := make([]*msgStream, 0, len(s.streams)) + for _, st := range s.streams { + streams = append(streams, st) + } + s.streams = map[uint32]*msgStream{} + close(s.done) + s.mu.Unlock() + + for _, st := range streams { + st.fail(err) + } +} + +// maybeRemove drops a stream from the table once both directions are finished, +// so a long-lived session multiplexing many short ops does not leak entries. +func (s *msgSession) maybeRemove(st *msgStream) { + st.readMu.Lock() + remoteDone := st.remoteClosed + st.readMu.Unlock() + + st.wmu.Lock() + localDone := st.localClosed + st.wmu.Unlock() + + if remoteDone && localDone { + s.removeStream(st.id) + } +} + +type msgStream struct { + sess *msgSession + id uint32 + + readMu sync.Mutex + readCond *sync.Cond + buf bytes.Buffer + remoteClosed bool + readErr error + canceled bool + + wmu sync.Mutex + synSent bool + localClosed bool +} + +func (st *msgStream) feed(data []byte, fin bool) { + st.readMu.Lock() + if len(data) > 0 { + st.buf.Write(data) + } + if fin { + st.remoteClosed = true + } + st.readCond.Broadcast() + st.readMu.Unlock() +} + +func (st *msgStream) fail(err error) { + st.readMu.Lock() + if st.readErr == nil { + st.readErr = err + } + st.readCond.Broadcast() + st.readMu.Unlock() +} + +func (st *msgStream) Read(p []byte) (int, error) { + st.readMu.Lock() + defer st.readMu.Unlock() + + for st.buf.Len() == 0 { + switch { + case st.canceled: + return 0, errStreamCancel + case st.buf.Len() == 0 && st.readErr != nil: + return 0, st.readErr + case st.remoteClosed: + return 0, io.EOF + } + st.readCond.Wait() + } + + return st.buf.Read(p) +} + +func (st *msgStream) Write(p []byte) (int, error) { + st.wmu.Lock() + defer st.wmu.Unlock() + + if st.localClosed { + return 0, errSessionClosed + } + + total := 0 + for len(p) > 0 { + chunk := p + if len(chunk) > st.sess.maxFrame { + chunk = chunk[:st.sess.maxFrame] + } + + flags := flagDATA + if !st.synSent { + flags |= flagSYN + st.synSent = true + } + + if err := st.sess.sendFrame(msgFrame{Stream: st.id, Flags: flags, Data: chunk}); err != nil { + return total, err + } + + total += len(chunk) + p = p[len(chunk):] + } + + return total, nil +} + +func (st *msgStream) Close() error { + st.wmu.Lock() + if st.localClosed { + st.wmu.Unlock() + return nil + } + st.localClosed = true + flags := flagFIN + if !st.synSent { + flags |= flagSYN + st.synSent = true + } + st.wmu.Unlock() + + err := st.sess.sendFrame(msgFrame{Stream: st.id, Flags: flags}) + st.sess.maybeRemove(st) + return err +} + +// CancelRead aborts a Read currently blocked on this stream and signals the +// peer (RST). The code is unused: msgmux has no per-stream error code, so +// callers detect that the cancellation was caller-initiated by inspecting their +// context. +func (st *msgStream) CancelRead(code uint64) { + st.readMu.Lock() + st.canceled = true + st.readCond.Broadcast() + st.readMu.Unlock() + + _ = st.sess.sendFrame(msgFrame{Stream: st.id, Flags: flagRST}) +} diff --git a/pkg/rpc/msgmux_test.go b/pkg/rpc/msgmux_test.go new file mode 100644 index 000000000..75cbd84b7 --- /dev/null +++ b/pkg/rpc/msgmux_test.go @@ -0,0 +1,201 @@ +package rpc + +import ( + "context" + "io" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +// readN reads exactly n bytes from r (the mux stream chunks writes into frames). +func readN(t *testing.T, r io.Reader, n int) []byte { + t.Helper() + buf := make([]byte, n) + _, err := io.ReadFull(r, buf) + require.NoError(t, err) + return buf +} + +func TestMsgMux(t *testing.T) { + t.Run("client opens, server accepts, bidirectional bytes", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + ca, cb := newMemPipe() + client := newMsgSession(ca, true, 0) + server := newMsgSession(cb, false, 0) + + st, err := client.OpenStreamSync(ctx) + r.NoError(err) + + _, err = st.Write([]byte("hello")) + r.NoError(err) + + srvStream, err := server.AcceptStream(ctx) + r.NoError(err) + r.Equal([]byte("hello"), readN(t, srvStream, 5)) + + // reply server -> client + _, err = srvStream.Write([]byte("world")) + r.NoError(err) + r.Equal([]byte("world"), readN(t, st, 5)) + + r.NoError(st.Close()) + r.NoError(server.Close()) + r.NoError(client.Close()) + }) + + t.Run("server can also open streams to client", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + ca, cb := newMemPipe() + client := newMsgSession(ca, true, 0) + server := newMsgSession(cb, false, 0) + + // server-initiated stream + ss, err := server.OpenStreamSync(ctx) + r.NoError(err) + _, err = ss.Write([]byte("ping")) + r.NoError(err) + + cs, err := client.AcceptStream(ctx) + r.NoError(err) + r.Equal([]byte("ping"), readN(t, cs, 4)) + + client.Close() + server.Close() + }) + + t.Run("FIN delivers EOF after buffered data", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + ca, cb := newMemPipe() + client := newMsgSession(ca, true, 0) + server := newMsgSession(cb, false, 0) + + st, err := client.OpenStreamSync(ctx) + r.NoError(err) + _, err = st.Write([]byte("data")) + r.NoError(err) + r.NoError(st.Close()) // FIN + + ss, err := server.AcceptStream(ctx) + r.NoError(err) + r.Equal([]byte("data"), readN(t, ss, 4)) + + // next read sees EOF + _, err = ss.Read(make([]byte, 1)) + r.ErrorIs(err, io.EOF) + + client.Close() + server.Close() + }) + + t.Run("concurrent streams do not interleave", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + ca, cb := newMemPipe() + client := newMsgSession(ca, true, 0) + server := newMsgSession(cb, false, 0) + + // Server echoes each accepted stream. + go func() { + for { + ss, err := server.AcceptStream(ctx) + if err != nil { + return + } + go func(s rpcStream) { + buf := make([]byte, 8) + n, _ := io.ReadFull(s, buf) + s.Write(buf[:n]) + s.Close() + }(ss) + } + }() + + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + st, err := client.OpenStreamSync(ctx) + r.NoError(err) + payload := []byte{byte(i), byte(i), byte(i), byte(i), byte(i), byte(i), byte(i), byte(i)} + _, err = st.Write(payload) + r.NoError(err) + r.Equal(payload, readN(t, st, 8)) + st.Close() + }(i) + } + wg.Wait() + + client.Close() + server.Close() + }) + + t.Run("CancelRead unblocks a blocked reader", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + ca, cb := newMemPipe() + client := newMsgSession(ca, true, 0) + server := newMsgSession(cb, false, 0) + + st, err := client.OpenStreamSync(ctx) + r.NoError(err) + _, err = st.Write([]byte("x")) // ensure server sees the stream + r.NoError(err) + _, err = server.AcceptStream(ctx) + r.NoError(err) + + done := make(chan error, 1) + go func() { + _, err := st.Read(make([]byte, 8)) + done <- err + }() + + // drain the one byte we wrote isn't on this side; the read above blocks. + st.CancelRead(cancelReadCode) + r.ErrorIs(<-done, errStreamCancel) + + client.Close() + server.Close() + }) + + t.Run("large payload spans multiple frames", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + ca, cb := newMemPipe() + client := newMsgSession(ca, true, 0) + server := newMsgSession(cb, false, 0) + + big := make([]byte, defaultMaxFrameData*2+1234) + for i := range big { + big[i] = byte(i) + } + + go func() { + st, _ := client.OpenStreamSync(ctx) + st.Write(big) + st.Close() + }() + + ss, err := server.AcceptStream(ctx) + r.NoError(err) + got, err := io.ReadAll(ss) + r.NoError(err) + r.Equal(big, got) + + client.Close() + server.Close() + }) + + _ = context.Background +} diff --git a/pkg/rpc/server.go b/pkg/rpc/server.go index 3661d1ed0..eac112727 100644 --- a/pkg/rpc/server.go +++ b/pkg/rpc/server.go @@ -1,7 +1,6 @@ package rpc import ( - "bytes" "context" "crypto/ed25519" "crypto/rand" @@ -70,13 +69,19 @@ type heldCapability struct { category string - lastContact time.Time + // lastContact is unix nanos, held atomically: concurrent calls against the + // same capability all touch it, and they are not otherwise serialized. + lastContact atomic.Int64 pub ed25519.PublicKey } func (h *heldCapability) touch() { - h.lastContact = time.Now() + h.lastContact.Store(time.Now().UnixNano()) +} + +func (h *heldCapability) LastContact() time.Time { + return time.Unix(0, h.lastContact.Load()) } func (h *heldCapability) Close() error { @@ -223,10 +228,10 @@ func (s *Server) assignCapability(i *Interface, pub ed25519.PublicKey, contactAd heldInterface: &heldInterface{ Interface: i, }, - category: category, - lastContact: time.Now(), - pub: pub, + category: category, + pub: pub, } + hc.touch() if i.restoreState != nil { if rs, err := i.restoreState.RestoreState(i); err == nil { @@ -264,7 +269,7 @@ func (s *Server) reexportCapability(target OID, cur *heldCapability, pub ed25519 oid := OID(base58.Encode(buf)) if contactAddr == "" { - contactAddr = s.state.transport.Conn.LocalAddr().String() + contactAddr = s.state.contactAddr() } capa := &Capability{ @@ -276,9 +281,9 @@ func (s *Server) reexportCapability(target OID, cur *heldCapability, pub ed25519 hc := &heldCapability{ heldInterface: cur.heldInterface, - lastContact: time.Now(), pub: pub, } + hc.touch() hc.refs.Add(1) @@ -294,6 +299,9 @@ func (s *Server) setupMux() { mux := http.NewServeMux() mux.HandleFunc("POST /_rpc/call/{oid}/{method}", s.handleCalls) + // CONNECT upgrades to WebTransport over HTTP/3. TCP clients use the message + // transport, which multiplexes every operation over one session and never + // reaches this mux. mux.HandleFunc("CONNECT /_rpc/callstream/{oid}/{method}", s.startCallStream) mux.HandleFunc("POST /_rpc/lookup/{name}", s.lookup) mux.HandleFunc("GET /_rpc/methods/{oid}", s.listMethods) @@ -318,7 +326,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // Try to authenticate the request - identity, err := s.state.authenticator.Authenticate(ctx, r) + identity, err := s.state.authenticator.Authenticate(ctx, CredentialsFromRequest(r)) if err != nil { s.state.log.Warn("authentication failed", "error", err, "path", r.URL.Path) http.Error(w, "authentication failed", http.StatusUnauthorized) @@ -358,19 +366,19 @@ type identifyResponse struct { Error string `json:"error,omitempty"` Address string `json:"address,omitempty"` Identity string `json:"identity,omitempty"` + + // MinVersion and MaxVersion advertise the protocol versions this peer + // accepts, so a client can adapt instead of discovering the mismatch one + // rejected stream at a time. Set only on message transports. See PROTOCOL.md. + MinVersion uint `json:"min_version,omitempty"` + MaxVersion uint `json:"max_version,omitempty"` } func (s *Server) checkIdentity(r *http.Request) (string, bool) { ts := r.Header.Get("rpc-timestamp") - t, err := time.Parse(time.RFC3339Nano, ts) - if err != nil { - s.state.log.Warn("Failed to parse timestamp", "error", err) - return "", false - } - - if time.Since(t) > 10*time.Minute { - s.state.log.Warn("Timestamp too old", "timestamp", t) + if err := freshTimestamp(ts); err != nil { + s.state.log.Warn("identity timestamp rejected", "error", err) return "", false } @@ -380,32 +388,15 @@ func (s *Server) checkIdentity(r *http.Request) (string, bool) { return "", false } - var buf bytes.Buffer - - fmt.Fprintf(&buf, "%s %s %s", r.Method, r.URL.Path, ts) - - bsign, err := base58.Decode(sign) - if err != nil { - s.state.log.Warn("Failed to decode signature", "error", err) - return "", false - } - - spkey := r.Header.Get("rpc-public-key") - - key, err := base58.Decode(spkey) + key, err := base58.Decode(r.Header.Get("rpc-public-key")) if err != nil { return "", false } pub := ed25519.PublicKey(key) - if len(pub) != ed25519.PublicKeySize { - s.state.log.Warn("Invalid public key size", "size", len(pub)) - return "", false - } - - if !ed25519.Verify(pub, buf.Bytes(), bsign) { - s.state.log.Warn("Failed to verify signature") + if err := verifyString(pub, httpCanonical(r.Method, r.URL.Path, ts), sign); err != nil { + s.state.log.Warn("Failed to verify identity signature", "error", err) return "", false } @@ -455,7 +446,7 @@ func (s *Server) handleDebugAuth(w http.ResponseWriter, r *http.Request) { // Check authentication using the new Authenticate interface if s.state.authenticator != nil { - identity, err := s.state.authenticator.Authenticate(r.Context(), r) + identity, err := s.state.authenticator.Authenticate(r.Context(), CredentialsFromRequest(r)) if err != nil { resp.Success = false resp.Message = fmt.Sprintf("Authentication failed: %v", err) @@ -511,7 +502,7 @@ func (s *Server) reexport(w http.ResponseWriter, r *http.Request) { // abilities. ca := r.Header.Get("rpc-contact-addr") if ca != "" { - ca = s.state.transport.Conn.LocalAddr().String() + ca = s.state.contactAddr() } w.WriteHeader(http.StatusOK) @@ -609,7 +600,7 @@ func (s *Server) lookup(w http.ResponseWriter, r *http.Request) { // abilities. ca := r.Header.Get("rpc-contact-addr") if ca != "" { - ca = s.state.transport.Conn.LocalAddr().String() + ca = s.state.contactAddr() } //s.state.log.Info("Lookup", "name", name) @@ -706,7 +697,7 @@ func (s *Server) reresolve(w http.ResponseWriter, r *http.Request) { // abilities. ca := r.Header.Get("rpc-contact-addr") if ca != "" { - ca = s.state.transport.Conn.LocalAddr().String() + ca = s.state.contactAddr() } // TODO: add condition codes to the error response rather than just a string @@ -795,16 +786,9 @@ func (s *Server) authRequest(r *http.Request, w http.ResponseWriter, oid OID) (e return nil, false } - t, err := time.Parse(time.RFC3339Nano, ts) - if err != nil { - logAuthReject(s.state.audit(), r, oid, "malformed timestamp") - http.Error(w, "failed to parse timestamp", http.StatusForbidden) - return nil, false - } - - if time.Since(t) > 10*time.Minute { - logAuthReject(s.state.audit(), r, oid, "timestamp too old") - http.Error(w, "timestamp too old", http.StatusForbidden) + if err := freshTimestamp(ts); err != nil { + logAuthReject(s.state.audit(), r, oid, err.Error()) + http.Error(w, err.Error(), http.StatusForbidden) return nil, false } @@ -826,25 +810,8 @@ func (s *Server) authRequest(r *http.Request, w http.ResponseWriter, oid OID) (e return nil, false } - var buf bytes.Buffer - - fmt.Fprintf(&buf, "%s %s %s", r.Method, r.URL.Path, ts) - - bsign, err := base58.Decode(sign) - if err != nil { - logAuthReject(s.state.audit(), r, oid, "malformed signature") - http.Error(w, "failed to decode signature", http.StatusForbidden) - return nil, false - } - - if len(capa.pub) != ed25519.PublicKeySize { - logAuthReject(s.state.audit(), r, oid, "invalid capability public key") - http.Error(w, "invalid public key size", http.StatusForbidden) - return nil, false - } - - if !ed25519.Verify(capa.pub, buf.Bytes(), bsign) { - logAuthReject(s.state.audit(), r, oid, "signature verification failed") + if err := verifyString(capa.pub, httpCanonical(r.Method, r.URL.Path, ts), sign); err != nil { + logAuthReject(s.state.audit(), r, oid, err.Error()) http.Error(w, "failed to verify signature", http.StatusForbidden) return nil, false } @@ -885,6 +852,23 @@ func (cs *controlStream) NoReply(rs streamRequest, arg any) error { return nil } +// acceptSession upgrades an incoming callstream request to a multiplexed +// session. The HTTP mux is served only over HTTP/3, where WebTransport provides +// QUIC sub-streams natively; TCP clients use the message transport instead, +// which multiplexes every operation over one session and never reaches here. +func (s *Server) acceptSession(w http.ResponseWriter, r *http.Request) (rpcSession, error) { + if r.ProtoMajor != 3 { + return nil, fmt.Errorf("callstream requires HTTP/3; use the message transport over TCP") + } + + // webtransport.Upgrade writes the 200 status itself. + sess, err := s.ws.Upgrade(w, r) + if err != nil { + return nil, err + } + return &wtSession{sess: sess}, nil +} + func (s *Server) startCallStream(w http.ResponseWriter, r *http.Request) { oid := OID(r.PathValue("oid")) @@ -946,8 +930,6 @@ func (s *Server) startCallStream(w http.ResponseWriter, r *http.Request) { logAccess(ctx, s.state.audit(), r, mm, "ok") } - w.WriteHeader(http.StatusOK) - ctx = Propagator().Extract(ctx, propagation.HeaderCarrier(r.Header)) tracer := Tracer() @@ -958,7 +940,7 @@ func (s *Server) startCallStream(w http.ResponseWriter, r *http.Request) { span.SetAttributes(attribute.String("oid", string(oid))) - sess, err := s.ws.Upgrade(w, r) + sess, err := s.acceptSession(w, r) if err != nil { s.state.log.Error("failed to upgrade connection", "error", err) http.Error(w, "failed to upgrade connection", http.StatusInternalServerError) @@ -999,6 +981,13 @@ func (s *Server) startCallStream(w http.ResponseWriter, r *http.Request) { }) } + s.runCallStream(ctx, &cs, mm, call) +} + +// runCallStream invokes a streaming handler and emits its result, error, or +// panic over the control stream. It is transport-neutral: both the HTTP +// callstream handler and the message-transport router use it. +func (s *Server) runCallStream(ctx context.Context, cs *controlStream, mm Method, call *NetworkCall) { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -1006,45 +995,41 @@ func (s *Server) startCallStream(w http.ResponseWriter, r *http.Request) { if r := recover(); r != nil { s.state.log.Error("panic in streaming RPC handler", "panic", r, - "method", method, + "method", call.method, "stack", string(debug.Stack())) - var sr streamRequest - sr.Kind = "panic" - sr.Error = fmt.Sprint(r) - cs.NoReply(sr, nil) + cs.NoReply(streamRequest{Kind: "panic", Error: fmt.Sprint(r)}, nil) } }() - err = cond.Wrap(mm.Handler(ctx, call)) - + err := cond.Wrap(mm.Handler(ctx, call)) if err != nil { - var sr streamRequest - sr.Kind = "error" - - if emsg, ok := err.(ErrorMessage); ok { - sr.Error = emsg.ErrorMessage() - } else { - sr.Error = err.Error() - } - - if ecat, ok := err.(ErrorCategory); ok { - sr.Category = ecat.ErrorCategory() - } + msg, category, code := errorFields(err) + cs.NoReply(streamRequest{Kind: "error", Error: msg, Category: category, Code: code}, nil) + return + } - if ecode, ok := err.(ErrorCode); ok { - sr.Code = ecode.ErrorCode() - } + res := call.results + if res == nil { + res = struct{}{} + } + cs.NoReply(streamRequest{Kind: "result"}, res) +} - cs.NoReply(sr, nil) +// errorFields extracts the wire error fields from a handler error, honoring the +// optional ErrorMessage/ErrorCategory/ErrorCode interfaces. +func errorFields(err error) (msg, category, code string) { + if emsg, ok := err.(ErrorMessage); ok { + msg = emsg.ErrorMessage() } else { - res := call.results - if res == nil { - res = struct{}{} - } - cs.NoReply(streamRequest{ - Kind: "result", - }, res) + msg = err.Error() + } + if ecat, ok := err.(ErrorCategory); ok { + category = ecat.ErrorCategory() + } + if ecode, ok := err.(ErrorCode); ok { + code = ecode.ErrorCode() } + return } func (s *Server) handleCalls(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/rpc/server_session.go b/pkg/rpc/server_session.go new file mode 100644 index 000000000..908c6243d --- /dev/null +++ b/pkg/rpc/server_session.go @@ -0,0 +1,498 @@ +package rpc + +import ( + "context" + "crypto/ed25519" + "fmt" + "runtime/debug" + "sort" + "strings" + + "github.com/fxamacker/cbor/v2" + "github.com/mr-tron/base58" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/propagation" +) + +// sessionRouter owns the accept loop for one message session and dispatches +// every inbound stream by the opRequest that opens it. Because both server +// operations and inline-capability callbacks share that framing, one end of a +// connection can both serve and call — which is what lets rpc run over a single +// connection it does not own. +type sessionRouter struct { + state *State + + // server handles inbound operations; nil if this end only makes calls. + server *Server + + // inline resolves callbacks against capabilities this end advertised; nil if + // this end only serves. + inline *inlineRegistry +} + +// run accepts and dispatches streams until the session fails or ctx is +// cancelled, returning the terminating error. +func (r *sessionRouter) run(ctx context.Context, sess rpcSession) error { + for { + st, err := sess.AcceptStream(ctx) + if err != nil { + return err + } + go r.dispatch(ctx, sess, st) + } +} + +func (r *sessionRouter) dispatch(ctx context.Context, sess rpcSession, st rpcStream) { + dec := cbor.NewDecoder(st) + enc := cbor.NewEncoder(st) + + var req opRequest + if err := dec.Decode(&req); err != nil { + _ = st.Close() + return + } + + if v := req.protocolVersion(); v < minProtocolVersion || v > currentProtocolVersion { + _ = enc.Encode(opReply{ + Status: "error", + Error: fmt.Sprintf("unsupported protocol version %d; this peer speaks %d..%d", + v, minProtocolVersion, currentProtocolVersion), + }) + _ = st.Close() + return + } + + if req.Op == opInlineCall { + defer st.Close() + + if r.inline == nil { + _ = enc.Encode(opReply{Status: "error", Error: "no inline capabilities on this session"}) + return + } + r.state.serveInlineCalls(ctx, dec, enc, r.inline.lookup) + return + } + + if r.server == nil { + _ = enc.Encode(opReply{Status: "error", Error: "this session does not serve operations"}) + _ = st.Close() + return + } + + r.server.handleOp(ctx, sess, st, dec, enc, req) +} + +func (s *Server) handleOp( + ctx context.Context, + sess rpcSession, + st rpcStream, + dec *cbor.Decoder, + enc *cbor.Encoder, + req opRequest, +) { + switch req.Op { + case opLookup: + s.msgLookup(enc, req) + _ = st.Close() + case opMethods: + s.msgMethods(enc, req) + _ = st.Close() + case opReresolve: + s.msgReresolve(dec, enc, req) + _ = st.Close() + case opReexport: + s.msgReexport(enc, req) + _ = st.Close() + case opRef: + s.msgRef(enc, req) + _ = st.Close() + case opDeref: + s.msgDeref(enc, req) + _ = st.Close() + case opIdentify: + s.msgIdentify(enc, req) + _ = st.Close() + case opCall: + s.msgCall(ctx, dec, enc, req) + _ = st.Close() + case opCallStream: + s.msgCallStream(ctx, sess, st, dec, enc, req) + default: + _ = enc.Encode(opReply{Status: "error", Error: "unknown op: " + req.Op}) + _ = st.Close() + } +} + +// writeReply emits an opReply followed (for control-plane ops) by the CBOR +// payload. +func writeReply(enc *cbor.Encoder, reply opReply, payload any) { + _ = enc.Encode(reply) + if payload != nil { + _ = enc.Encode(payload) + } +} + +// authMsgCap verifies a capability-scoped op signature against the stored pubkey +// of the targeted capability. +func (s *Server) authMsgCap(req opRequest, target string) (ed25519.PublicKey, bool) { + if err := freshTimestamp(req.Timestamp); err != nil { + return nil, false + } + s.mu.Lock() + capa, ok := s.objects[req.OID] + s.mu.Unlock() + if !ok { + return nil, false + } + if !verifyBytes(capa.pub, msgCanonical(req.Op, target, req.Timestamp), req.Signature) { + return nil, false + } + return capa.pub, true +} + +func (s *Server) msgLookup(enc *cbor.Encoder, req opRequest) { + pub := ed25519.PublicKey(req.PubKey) + if len(pub) != ed25519.PublicKeySize { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "invalid public key"}) + return + } + + s.mu.Lock() + iface, ok := s.persistent[req.Name] + s.mu.Unlock() + + if !ok { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "unknown object: " + req.Name}) + return + } + + capa := s.assignCapability(iface, pub, s.state.contactAddr(), req.Name, false) + capa.RestoreState = &InterfaceState{Category: "!persistent", Interface: req.Name} + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Capability: capa}) +} + +func (s *Server) msgMethods(enc *cbor.Encoder, req opRequest) { + s.mu.Lock() + hc, ok := s.objects[req.OID] + s.mu.Unlock() + + if !ok { + writeReply(enc, opReply{Status: "ok"}, methodsResponse{Error: "unknown capability: " + string(req.OID)}) + return + } + + methods := make([]string, 0, len(hc.methods)) + for name := range hc.methods { + methods = append(methods, name) + } + sort.Strings(methods) + writeReply(enc, opReply{Status: "ok"}, methodsResponse{Methods: methods}) +} + +func (s *Server) msgReresolve(dec *cbor.Decoder, enc *cbor.Encoder, req opRequest) { + var rs InterfaceState + if err := dec.Decode(&rs); err != nil { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "failed to read body"}) + return + } + + pub := ed25519.PublicKey(req.PubKey) + if len(pub) != ed25519.PublicKeySize { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "invalid public key"}) + return + } + + var ( + iface *Interface + category string + ) + + if rs.Category == "!persistent" { + category = rs.Interface + s.mu.Lock() + pi, ok := s.persistent[rs.Interface] + s.mu.Unlock() + if !ok { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "unknown object: " + rs.Interface}) + return + } + iface = pi + } else { + s.mu.Lock() + res, ok := s.resolvers[rs.Category] + s.mu.Unlock() + if ok { + var err error + iface, err = res.ReconstructFromState(&rs) + if err != nil { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "failed to resolve: " + err.Error()}) + return + } + if iface == nil { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "unable to restore capability"}) + return + } + } + } + + capa := s.assignCapability(iface, pub, s.state.contactAddr(), category, false) + capa.RestoreState = &rs + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Capability: capa}) +} + +func (s *Server) msgReexport(enc *cbor.Encoder, req opRequest) { + if _, ok := s.authMsgCap(req, string(req.OID)); !ok { + writeReply(enc, opReply{Status: "error", Error: "unauthorized"}, lookupResponse{Error: "unauthorized"}) + return + } + + target := ed25519.PublicKey(req.TargetPK) + if len(target) != ed25519.PublicKeySize { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "invalid target public key"}) + return + } + + s.mu.Lock() + hc, ok := s.objects[req.OID] + s.mu.Unlock() + + if !ok { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "unknown capability: " + string(req.OID)}) + return + } + + capa := s.reexportCapability(req.OID, hc, target, s.state.contactAddr()) + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Capability: capa}) +} + +func (s *Server) msgRef(enc *cbor.Encoder, req opRequest) { + if _, ok := s.authMsgCap(req, string(req.OID)); !ok { + writeReply(enc, opReply{Status: "error", Error: "unauthorized"}, refResponse{Error: "unauthorized"}) + return + } + + s.mu.Lock() + defer s.mu.Unlock() + + if hc, ok := s.objects[req.OID]; ok { + hc.refs.Add(1) + writeReply(enc, opReply{Status: "ok"}, refResponse{Status: "ok"}) + } else { + writeReply(enc, opReply{Status: "ok"}, refResponse{Error: "unknown capability: " + string(req.OID)}) + } +} + +func (s *Server) msgDeref(enc *cbor.Encoder, req opRequest) { + if _, ok := s.authMsgCap(req, string(req.OID)); !ok { + writeReply(enc, opReply{Status: "error", Error: "unauthorized"}, refResponse{Error: "unauthorized"}) + return + } + + if s.Deref(req.OID) { + writeReply(enc, opReply{Status: "ok"}, refResponse{Status: "ok"}) + } else { + writeReply(enc, opReply{Status: "ok"}, refResponse{Error: "unknown capability: " + string(req.OID)}) + } +} + +func (s *Server) msgIdentify(enc *cbor.Encoder, req opRequest) { + pub := ed25519.PublicKey(req.PubKey) + if err := freshTimestamp(req.Timestamp); err != nil { + writeReply(enc, opReply{Status: "ok"}, identifyResponse{Error: "invalid identity"}) + return + } + if !verifyBytes(pub, msgCanonical(opIdentify, "", req.Timestamp), req.Signature) { + writeReply(enc, opReply{Status: "ok"}, identifyResponse{Error: "invalid identity"}) + return + } + writeReply(enc, opReply{Status: "ok"}, identifyResponse{ + Ok: true, + Address: req.Contact, + Identity: base58.Encode(pub), + MinVersion: minProtocolVersion, + MaxVersion: currentProtocolVersion, + }) +} + +type callOutcome struct { + status string + errMsg string + category string + code string + results any +} + +// prepareMethodCall authenticates a call/callstream op, builds the +// signed-pubkey identity, looks up the method, and applies the public/non-public +// authorization gate. On failure it returns an opReply describing why. +func (s *Server) prepareMethodCall(ctx context.Context, req opRequest) (context.Context, *heldCapability, Method, ed25519.PublicKey, opReply, bool) { + target := string(req.OID) + "/" + req.Method + + if err := freshTimestamp(req.Timestamp); err != nil { + return ctx, nil, Method{}, nil, opReply{Status: "error", Error: "stale request"}, false + } + + s.mu.Lock() + iface, ok := s.objects[req.OID] + s.mu.Unlock() + if !ok { + return ctx, nil, Method{}, nil, opReply{Status: "unknown-capability", Error: "unknown object: " + string(req.OID)}, false + } + + if !verifyBytes(iface.pub, msgCanonical(req.Op, target, req.Timestamp), req.Signature) { + return ctx, nil, Method{}, nil, opReply{Status: "error", Error: "failed to verify signature"}, false + } + + iface.touch() + + mm := iface.methods[req.Method] + if mm.Handler == nil { + return ctx, nil, Method{}, nil, opReply{Status: "error", Error: "unknown method: " + req.Method}, false + } + + // A bearer token, if the caller sent one, names a richer identity than the + // capability does — a user or a workload rather than a key. Fall back to the + // verified signature, which is all a message transport otherwise offers: + // there is no TLS client certificate to inspect. + identity := s.msgIdentity(ctx, req, iface.pub) + ctx = ContextWithIdentity(ctx, identity) + + if len(req.Trace) > 0 { + ctx = Propagator().Extract(ctx, propagation.MapCarrier(req.Trace)) + } + + if !mm.Public && s.state.authorizer != nil { + resource := strings.ToLower(mm.InterfaceName) + action := strings.ToLower(mm.Name) + if err := s.state.authorizer.Authorize(ctx, identity, resource, action); err != nil { + s.state.log.Warn("authorization denied", + "resource", resource, "action", action, "subject", identity.Subject, "error", err) + return ctx, nil, Method{}, nil, opReply{Status: "forbidden", Error: err.Error()}, false + } + } + + return ctx, iface, mm, iface.pub, opReply{}, true +} + +// msgIdentity resolves the caller's identity for an operation on a message +// transport. The capability signature has already been verified by the caller. +func (s *Server) msgIdentity(ctx context.Context, req opRequest, capaPub ed25519.PublicKey) *Identity { + signed := &Identity{Subject: base58.Encode(capaPub), Method: AuthMethodSigned} + + if req.Bearer == "" || s.state.authenticator == nil { + return signed + } + + identity, err := s.state.authenticator.Authenticate(ctx, &Credentials{ + Authorization: "Bearer " + req.Bearer, + }) + if err != nil { + s.state.log.Warn("rpc: bearer authentication failed", "error", err) + return signed + } + if identity == nil { + return signed + } + + return identity +} + +func (s *Server) msgCall(ctx context.Context, dec *cbor.Decoder, enc *cbor.Encoder, req opRequest) { + ctx, iface, mm, caller, reply, ok := s.prepareMethodCall(ctx, req) + if !ok { + _ = enc.Encode(reply) + return + } + + ctx, span := Tracer().Start(ctx, "rpc.handle."+mm.InterfaceName+"."+mm.Name) + defer span.End() + span.SetAttributes(attribute.String("oid", string(req.OID))) + + call := &NetworkCall{ + s: s, + oid: req.OID, + method: req.Method, + caller: caller, + category: iface.category, + dec: dec, // args follow the opRequest on this stream + } + + if iface.aroundContext != nil { + var cancel func() + ctx, cancel = iface.aroundContext(ctx, call) + defer cancel() + } + + oc := s.invokeUnary(ctx, mm, call) + switch oc.status { + case "ok": + _ = enc.Encode(opReply{Status: "ok"}) + _ = enc.Encode(oc.results) + case "panic": + _ = enc.Encode(opReply{Status: "panic", Error: oc.errMsg}) + default: + _ = enc.Encode(opReply{Status: "error", Error: oc.errMsg, Category: oc.category, Code: oc.code}) + } +} + +// invokeUnary runs a unary handler, mirroring handleCalls (no cond.Wrap; panic +// recovery), and returns the outcome. +func (s *Server) invokeUnary(ctx context.Context, mm Method, call *NetworkCall) (oc callOutcome) { + defer func() { + if r := recover(); r != nil { + s.state.log.Error("panic in RPC handler", + "panic", r, "method", call.method, "stack", string(debug.Stack())) + oc = callOutcome{status: "panic", errMsg: fmt.Sprint(r)} + } + }() + + err := mm.Handler(ctx, call) + if err != nil { + msg, category, code := errorFields(err) + return callOutcome{status: "error", errMsg: msg, category: category, code: code} + } + + res := call.results + if res == nil { + res = struct{}{} + } + return callOutcome{status: "ok", results: res} +} + +func (s *Server) msgCallStream(ctx context.Context, sess rpcSession, st rpcStream, dec *cbor.Decoder, enc *cbor.Encoder, req opRequest) { + defer st.Close() + + cs := &controlStream{dec: dec, enc: enc} + + ctx, iface, mm, caller, reply, ok := s.prepareMethodCall(ctx, req) + if !ok { + // The callstream client reads streamRequest frames, not opReply, so the + // rejection is restated as one. An authorization denial carries the same + // category the unary path gives it. + category, code := reply.Category, reply.Code + if reply.Status == "forbidden" { + category, code = "permission", "forbidden" + } + + cs.NoReply(streamRequest{Kind: "error", Error: reply.Error, Category: category, Code: code}, nil) + return + } + + ctx, span := Tracer().Start(ctx, "rpc.handle."+mm.InterfaceName+"."+mm.Name) + defer span.End() + span.SetAttributes(attribute.String("oid", string(req.OID))) + + call := &NetworkCall{ + s: s, + oid: req.OID, + method: req.Method, + caller: caller, + category: iface.category, + dec: dec, + wsSession: sess, + msgSession: true, + ctrl: cs, + } + + s.runCallStream(ctx, cs, mm, call) +} diff --git a/pkg/rpc/signing.go b/pkg/rpc/signing.go new file mode 100644 index 000000000..0452f1346 --- /dev/null +++ b/pkg/rpc/signing.go @@ -0,0 +1,67 @@ +package rpc + +import ( + "crypto/ed25519" + "fmt" + "time" + + "github.com/mr-tron/base58" +) + +// authFreshness bounds how old a request timestamp may be. +const authFreshness = 10 * time.Minute + +// httpCanonical is the legacy signed string for the HTTP transports: +// "METHOD PATH timestamp". Kept verbatim so existing peers remain compatible. +func httpCanonical(method, path, ts string) string { + return fmt.Sprintf("%s %s %s", method, path, ts) +} + +// msgCanonical is the signed string for the message transport: +// "op target timestamp", where target identifies the operation's subject +// (e.g. "oid/method" for call, "oid" for deref). Transport-neutral by +// construction; a signature never travels inside a capability, so the per- +// transport string format is safe. +func msgCanonical(op, target, ts string) string { + return fmt.Sprintf("%s %s %s", op, target, ts) +} + +// verifyString checks a base58-encoded signature over canonical. The returned +// error distinguishes the failure modes so callers can record why a request was +// rejected in the audit log. +func verifyString(pub ed25519.PublicKey, canonical, sigB58 string) error { + if len(pub) != ed25519.PublicKeySize { + return fmt.Errorf("invalid capability public key") + } + bsign, err := base58.Decode(sigB58) + if err != nil { + return fmt.Errorf("malformed signature") + } + if !ed25519.Verify(pub, []byte(canonical), bsign) { + return fmt.Errorf("signature verification failed") + } + return nil +} + +// signBytes / verifyBytes are the raw-signature variants used by the message +// transport (no base58 round-trip; the signature travels as bytes in the frame). +func signBytes(priv ed25519.PrivateKey, canonical string) []byte { + return ed25519.Sign(priv, []byte(canonical)) +} + +func verifyBytes(pub ed25519.PublicKey, canonical string, sig []byte) bool { + return len(pub) == ed25519.PublicKeySize && ed25519.Verify(pub, []byte(canonical), sig) +} + +// freshTimestamp parses an RFC3339Nano timestamp and rejects it if it is older +// than authFreshness. +func freshTimestamp(ts string) error { + t, err := time.Parse(time.RFC3339Nano, ts) + if err != nil { + return fmt.Errorf("invalid timestamp: %w", err) + } + if time.Since(t) > authFreshness { + return fmt.Errorf("timestamp too old") + } + return nil +} diff --git a/pkg/rpc/state.go b/pkg/rpc/state.go index 1be4ee896..1d8bdd5ea 100644 --- a/pkg/rpc/state.go +++ b/pkg/rpc/state.go @@ -122,9 +122,35 @@ type State struct { ws *webtransport.Server li *quic.EarlyListener + httpSrv *http.Server + tcpLn net.Listener + msgLn net.Listener + + // msgContact is the address peers use to reach this State over a message + // transport, e.g. "mem://name" for an in-process server. It is empty when + // the State only serves connections supplied by callers, which have no + // address of their own. + msgContact string + localMP *packet.PacketConnMultiplex } +// contactAddr returns the address embedded in capabilities this server mints, +// so a holder can later reach the issuer. +// +// It is empty when there is nothing to dial — a connection owned by the caller +// is reachable only as itself. Capabilities minted with an empty address are +// therefore scoped to the session they were minted on. +func (s *State) contactAddr() string { + if s.msgContact != "" { + return s.msgContact + } + if s.transport == nil || s.transport.Conn == nil { + return "" + } + return s.transport.Conn.LocalAddr().String() +} + func (s *State) ListenAddr() string { return s.transport.Conn.LocalAddr().String() } @@ -145,7 +171,11 @@ type stateOptions struct { certData []byte keyData []byte - bindAddr string + bindAddr string + wsBindAddr string + tcpBindAddr string + + memServerName string endpoint string @@ -187,6 +217,35 @@ func WithBindAddr(addr string) StateOption { } } +// WithWSBindAddr enables an additional TCP listener serving the RPC protocol +// over TLS as a WebSocket message session, bound to addr. Use "host:0" to bind +// an ephemeral port; the chosen address is available via State.WSListenAddr +// after NewState returns. Clients reach it with State.Connect("ws://host:port"). +func WithWSBindAddr(addr string) StateOption { + return func(o *stateOptions) { + o.wsBindAddr = addr + } +} + +// WithTCPBindAddr enables a raw TLS-over-TCP listener serving the RPC protocol +// as a message session, bound to addr. It carries no HTTP layer at all, which +// makes it the leanest option where a WebSocket handshake buys nothing. Clients +// reach it with State.Connect("tcp://host:port"). +func WithTCPBindAddr(addr string) StateOption { + return func(o *stateOptions) { + o.tcpBindAddr = addr + } +} + +// WithMemServer registers this State as an in-process message server under the +// given name. Peers connect with State.Connect("mem://name", ...). Used for +// testing and for bridging RPC onto message-oriented systems. +func WithMemServer(name string) StateOption { + return func(o *stateOptions) { + o.memServerName = name + } +} + func WithSkipVerify(o *stateOptions) { o.skipVerify = true } @@ -369,6 +428,25 @@ func NewState(ctx context.Context, opts ...StateOption) (*State, error) { } } + if so.wsBindAddr != "" { + err := s.startWSListener(ctx, so.wsBindAddr) + if err != nil { + return nil, err + } + } + + if so.tcpBindAddr != "" { + err := s.startTCPListener(ctx, so.tcpBindAddr) + if err != nil { + return nil, err + } + } + + if so.memServerName != "" { + s.msgContact = "mem://" + so.memServerName + defaultMemRegistry.register(ctx, so.memServerName, s) + } + return s, nil } @@ -519,6 +597,18 @@ func (s *State) Close() error { s.hs.Close() } + if s.httpSrv != nil { + s.httpSrv.Close() + } + + if s.tcpLn != nil { + s.tcpLn.Close() + } + + if s.msgLn != nil { + s.msgLn.Close() + } + return s.transport.Conn.Close() } @@ -540,6 +630,17 @@ func (s *State) Connect(remote string, name string) (*NetworkClient, error) { if err != nil { return nil, err } + } else if isMessageRemote(remote) { + // mem://, ws://, wss:// and tcp:// all multiplex every operation over one + // message session; setupMsgTransport picks the connection factory. + client = &NetworkClient{ + State: s, + transportKind: transportMsg, + tlsCfg: s.clientTlsCfg, + remote: remote, + } + + client.setupTransport() } else if remote == "dial-stdio" { shstr := os.Getenv("MIREN_DIAL_PROGRAM") if shstr == "" { @@ -590,36 +691,59 @@ func (c *NetworkClient) newClientUnder(capa *Capability) *NetworkClient { addr := capa.Address transport := c.State.transport + tk := c.transportKind if strings.HasPrefix(addr, "unix:") { transport = c.State.localTransport } + // An empty address means "wherever this capability came from", which for a + // message transport is the session it arrived on. + sameSession := addr == "" && c.transportKind == transportMsg + if addr == "" { addr = c.remote } - newClient := &NetworkClient{ - State: c.State, - transport: transport, - tlsCfg: c.State.clientTlsCfg.Clone(), - capa: capa, - oid: capa.OID, - remote: addr, + if isMessageRemote(addr) { + tk = transportMsg } - newClient.setupTransport() + newClient := &NetworkClient{ + State: c.State, + transportKind: tk, + transport: transport, + tlsCfg: c.State.clientTlsCfg.Clone(), + capa: capa, + oid: capa.OID, + remote: addr, + } + + if sameSession { + // Share the parent's session rather than dialing again. On a connection + // supplied by the caller there is nothing to dial, and even where there + // is, a capability reachable through this session should not open a + // second one. + newClient.ops = c.ops + } else { + newClient.setupTransport() + } return newClient } func (s *State) newClientFrom(capa *Capability, peer *x509.Certificate) *NetworkClient { transport := s.transport + tk := transportH3 if strings.HasPrefix(capa.Address, "unix:") { transport = s.localTransport } + if isMessageRemote(capa.Address) { + tk = transportMsg + } + cfg := s.clientTlsCfg.Clone() cfg.InsecureSkipVerify = true @@ -634,12 +758,13 @@ func (s *State) newClientFrom(capa *Capability, peer *x509.Certificate) *Network } c := &NetworkClient{ - State: s, - transport: transport, - tlsCfg: cfg, - capa: capa, - oid: capa.OID, - remote: capa.Address, + State: s, + transportKind: tk, + transport: transport, + tlsCfg: cfg, + capa: capa, + oid: capa.OID, + remote: capa.Address, } c.setupTransport() diff --git a/pkg/rpc/state_ws.go b/pkg/rpc/state_ws.go new file mode 100644 index 000000000..9a47da7aa --- /dev/null +++ b/pkg/rpc/state_ws.go @@ -0,0 +1,61 @@ +package rpc + +import ( + "context" + "net" + "net/http" +) + +// startWSListener starts a TCP listener serving the RPC protocol over TLS. +// +// Unlike the HTTP/3 listener, this one does not serve the RPC HTTP mux: a +// single route performs a WebSocket upgrade, and every operation then rides the +// multiplexed message session established over it. The HTTP layer exists only +// so proxies and browsers see a real WebSocket handshake. +func (s *State) startWSListener(ctx context.Context, addr string) error { + tcpLn, err := net.Listen("tcp", addr) + if err != nil { + return err + } + + tlsCfg := s.serverTlsCfg.Clone() + // coder/websocket needs an HTTP/1.1 handshake; do not offer h2. + tlsCfg.NextProtos = []string{"http/1.1"} + + mux := http.NewServeMux() + mux.HandleFunc("GET "+wsMessagePath, func(w http.ResponseWriter, r *http.Request) { + s.serveWSUpgrade(ctx, w, r) + }) + + srv := &http.Server{ + Handler: mux, + TLSConfig: tlsCfg, + } + + s.httpSrv = srv + s.tcpLn = tcpLn + + go func() { + <-ctx.Done() + _ = srv.Shutdown(context.Background()) + }() + + s.log.Debug("starting websocket/tcp listener", "addr", tcpLn.Addr().String()) + + go func() { + if serr := srv.ServeTLS(tcpLn, "", ""); serr != nil && serr != http.ErrServerClosed { + s.log.Error("websocket/tcp listener stopped", "error", serr) + } + }() + + return nil +} + +// WSListenAddr returns the address the TCP/WebSocket listener is bound to, or +// an empty string if no such listener is running. +func (s *State) WSListenAddr() string { + if s.tcpLn == nil { + return "" + } + return s.tcpLn.Addr().String() +} diff --git a/pkg/rpc/stream/stream_test.go b/pkg/rpc/stream/stream_test.go index 9472fd183..049e26991 100644 --- a/pkg/rpc/stream/stream_test.go +++ b/pkg/rpc/stream/stream_test.go @@ -27,9 +27,17 @@ func TestStream(t *testing.T) { serv := ss.Server() - var vals []int + // The handler runs on the server's goroutine while the assertion below + // reads from the test's, so the accumulator needs a lock. + var ( + mu sync.Mutex + vals []int + ) serv.ExposeValue("stream", ReadStream(func(val int) error { + mu.Lock() + defer mu.Unlock() + vals = append(vals, val) return nil })) @@ -47,6 +55,9 @@ func TestStream(t *testing.T) { r.NoError(css.Send(ctx, 100)) r.NoError(css.Send(ctx, 111)) + mu.Lock() + defer mu.Unlock() + r.Equal([]int{42, 100, 111}, vals) }) @@ -61,9 +72,17 @@ func TestStream(t *testing.T) { serv := ss.Server() - var vals []*Thing + // The handler runs on the server's goroutine while the assertion below + // reads from the test's, so the accumulator needs a lock. + var ( + mu sync.Mutex + vals []*Thing + ) serv.ExposeValue("stream", ReadStream(func(val *Thing) error { + mu.Lock() + defer mu.Unlock() + vals = append(vals, val) return nil })) @@ -79,6 +98,9 @@ func TestStream(t *testing.T) { r.NoError(css.Send(ctx, &Thing{Name: "foo"})) + mu.Lock() + defer mu.Unlock() + r.Equal([]*Thing{ {Name: "foo"}, }, vals) diff --git a/pkg/rpc/transport.go b/pkg/rpc/transport.go new file mode 100644 index 000000000..277c008e7 --- /dev/null +++ b/pkg/rpc/transport.go @@ -0,0 +1,38 @@ +package rpc + +import ( + "context" + "io" +) + +// cancelReadCode is the application error code used to abort a blocked stream +// read when an RPC call's context is cancelled. WebTransport passes it through +// as a stream error code; the message transport has no per-stream error code and +// simply unblocks the read. Cancellation is ultimately detected by inspecting +// the call's context, so the specific value is internal. +const cancelReadCode uint64 = 0x13 + +// rpcStream is a single bidirectional sub-stream within an rpcSession. Both +// transports that back it — WebTransport over QUIC and msgmux over a +// MessageConn — provide ordered, independent sub-streams with their own +// lifetimes. +type rpcStream interface { + io.ReadWriteCloser + + // CancelRead aborts a Read currently blocked on this stream. The code is + // passed through to transports that support per-stream error codes + // (WebTransport); transports without that notion (msgmux) ignore it and + // simply unblock the read. + CancelRead(code uint64) +} + +// rpcSession is a multiplexed connection over which either peer may open or +// accept sub-streams at any time. This symmetric capability is what allows a +// server to call back on a client-advertised capability over the same +// connection. WebTransport provides it natively over QUIC; msgmux layers it over +// a MessageConn's discrete messages. +type rpcSession interface { + OpenStreamSync(ctx context.Context) (rpcStream, error) + AcceptStream(ctx context.Context) (rpcStream, error) + Close() error +} diff --git a/pkg/rpc/transport_mem_test.go b/pkg/rpc/transport_mem_test.go new file mode 100644 index 000000000..01355c40b --- /dev/null +++ b/pkg/rpc/transport_mem_test.go @@ -0,0 +1,239 @@ +package rpc_test + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "miren.dev/runtime/pkg/cond" + "miren.dev/runtime/pkg/rpc" + "miren.dev/runtime/pkg/rpc/example" + "miren.dev/runtime/pkg/rpc/stream" +) + +// memServer registers an in-process message server under a unique name derived +// from the test, exposing the given interface, and returns its mem:// remote. +func memServer(t *testing.T, ctx context.Context, iface *rpc.Interface) string { + t.Helper() + r := require.New(t) + + name := "memtest-" + t.Name() + ss, err := rpc.NewState(ctx, rpc.WithSkipVerify, rpc.WithMemServer(name)) + r.NoError(err) + ss.Server().ExposeValue("meter", iface) + + return "mem://" + name +} + +func memClient(t *testing.T, ctx context.Context, remote, name string) rpc.Client { + t.Helper() + r := require.New(t) + + cs, err := rpc.NewState(ctx, rpc.WithSkipVerify) + r.NoError(err) + + c, err := cs.Connect(remote, name) + r.NoError(err) + return c +} + +func TestMemTransport(t *testing.T) { + t.Run("unary calls over message transport", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := memServer(t, ctx, example.AdaptMeter(&exampleMeter{temp: 42})) + c := memClient(t, ctx, remote, "meter") + + mc := &example.MeterClient{Client: c} + + res, err := mc.ReadTemperature(ctx, "test") + r.NoError(err) + r.Equal("test", res.Reading().Meter()) + r.Equal(float32(42), res.Reading().Temperature()) + + // GetSetter returns a capability; calling it follows the cap over mem://. + res2, err := mc.GetSetter(ctx, "test") + r.NoError(err) + res3, err := res2.Setter().SetTemp(ctx, 100) + r.NoError(err) + r.Equal(int32(100), res3.Temp()) + }) + + t.Run("server calls back on a client-advertised capability", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := memServer(t, ctx, example.AdaptMeterUpdates(&exampleMU{})) + c := memClient(t, ctx, remote, "meter") + + mc := &example.MeterUpdatesClient{Client: c} + + var up exampleUpdate + _, err := mc.RegisterUpdates(ctx, &up) + r.NoError(err) + + r.True(up.gotIt) + r.Equal("test", up.reading.Meter()) + r.Equal(float32(42), up.reading.Temperature()) + + time.Sleep(100 * time.Millisecond) + + up.mu.Lock() + defer up.mu.Unlock() + r.True(up.closed) + }) + + t.Run("streaming callbacks over message transport", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := memServer(t, ctx, example.AdaptEmitTemps(&exampleEmit{})) + c := memClient(t, ctx, remote, "meter") + + mc := &example.EmitTempsClient{Client: c} + + var vals []float32 + recv := stream.StreamRecv(func(val float32) error { + vals = append(vals, val) + return nil + }) + + _, err := mc.Emit(ctx, recv) + r.NoError(err) + + time.Sleep(time.Second) + r.Equal([]float32{42, 100}, vals) + }) + + // Callstreams share the one session wrapping the connection, so concurrent + // streaming calls must not steal each other's inbound callbacks. This is the + // regression test for callstreams no longer dialing a connection each. + t.Run("concurrent callstreams share one connection", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := memServer(t, ctx, example.AdaptEmitTemps(&exampleEmit{})) + c := memClient(t, ctx, remote, "meter") + + const calls = 8 + + var ( + wg sync.WaitGroup + mu sync.Mutex + errs []error + got = make([][]float32, calls) + ) + + for i := range calls { + wg.Add(1) + go func() { + defer wg.Done() + + // Each call advertises its own emitter capability. They all ride + // the same session, so the session must route each server-initiated + // callback back to the call that advertised it. + var vals []float32 + recv := stream.StreamRecv(func(val float32) error { + vals = append(vals, val) + return nil + }) + + mc := &example.EmitTempsClient{Client: c} + if _, err := mc.Emit(ctx, recv); err != nil { + mu.Lock() + errs = append(errs, err) + mu.Unlock() + return + } + + time.Sleep(500 * time.Millisecond) + got[i] = vals + }() + } + + wg.Wait() + + mu.Lock() + defer mu.Unlock() + r.Empty(errs) + + for i := range calls { + r.Equal([]float32{42, 100}, got[i], "call %d received the wrong values", i) + } + }) + + t.Run("a capability can be passed to a 3rd party", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + var em exampleMeter + em.temp = 42 + + remote := memServer(t, ctx, example.AdaptMeter(&em)) + c := memClient(t, ctx, remote, "meter") + + mc := &example.MeterClient{Client: c} + + res2, err := mc.GetSetter(ctx, "test") + r.NoError(err) + + // Second mem server hosting the AdjustTemp service. + s2, err := rpc.NewState(ctx, rpc.WithSkipVerify, rpc.WithMemServer("memtest-adjust-"+t.Name())) + r.NoError(err) + s2.Server().ExposeValue("adjust", example.AdaptAdjustTemp(&exampleAT{})) + + c2, err := s2.Connect("mem://memtest-adjust-"+t.Name(), "adjust") + r.NoError(err) + + ac := &example.AdjustTempClient{Client: c2} + + _, err = ac.Adjust(ctx, res2.Setter().Export()) + r.NoError(err) + + r.Equal(float32(72), em.temp) + }) + + t.Run("cancelled callstream reports closed", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + m := &blockingMU{started: make(chan struct{})} + remote := memServer(t, ctx, example.AdaptMeterUpdates(m)) + c := memClient(t, ctx, remote, "meter") + + mc := &example.MeterUpdatesClient{Client: c} + + cctx, cancel := context.WithCancel(ctx) + go func() { + <-m.started + time.Sleep(50 * time.Millisecond) + cancel() + }() + + var up exampleUpdate + _, err := mc.RegisterUpdates(cctx, &up) + r.Error(err) + r.True(errors.Is(err, cond.ErrClosed{}), "expected ErrClosed, got %T: %v", err, err) + }) + + t.Run("propagates panic errors over message transport", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := memServer(t, ctx, example.AdaptMeter(&panicMeter{})) + c := memClient(t, ctx, remote, "meter") + + mc := &example.MeterClient{Client: c} + + _, err := mc.ReadTemperature(ctx, "test") + r.Error(err) + + var panicErr cond.ErrPanic + r.True(errors.As(err, &panicErr), "expected ErrPanic, got %T: %v", err, err) + r.Contains(panicErr.Message, "test panic message") + }) +} diff --git a/pkg/rpc/transport_msg.go b/pkg/rpc/transport_msg.go new file mode 100644 index 000000000..4186f61d9 --- /dev/null +++ b/pkg/rpc/transport_msg.go @@ -0,0 +1,388 @@ +package rpc + +import ( + "context" + "crypto/ed25519" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/fxamacker/cbor/v2" + "go.opentelemetry.io/otel/propagation" + "miren.dev/runtime/pkg/cond" +) + +// msgOpTransport drives the client side of a message transport. Every call — +// unary, control-plane, or callstream — is multiplexed over the single session +// wrapping the connection, because a message transport may be handed a +// connection it does not own and cannot re-dial. Server-initiated callbacks are +// therefore routed by OID through a session-scoped registry rather than by +// which call happens to own the session. +type msgOpTransport struct { + // owner is the client that established the connection; it supplies the + // State used by the session's accept loop. + owner *NetworkClient + + // dial opens a fresh connection. It is nil when the session was adopted + // from a caller that owns the underlying connection. + dial func() (MessageConn, error) + + inline inlineRegistry + + mu sync.Mutex + ctrlSess rpcSession +} + +func (t *msgOpTransport) controlSession(ctx context.Context) (rpcSession, error) { + t.mu.Lock() + defer t.mu.Unlock() + + if t.ctrlSess != nil { + return t.ctrlSess, nil + } + if t.dial == nil { + return nil, errors.New("rpc: message session closed and not redialable") + } + conn, err := t.dial() + if err != nil { + return nil, err + } + t.ctrlSess = newMsgSession(conn, true, 0) + // A connection this transport dialed lives as long as the State does. + t.startAcceptLoop(t.owner.State.top, t.ctrlSess) + return t.ctrlSess, nil +} + +// adopt installs a session over a connection the caller owns, bounding it by the +// caller's context rather than by any single call. +func (t *msgOpTransport) adopt(ctx context.Context, sess rpcSession) { + t.mu.Lock() + defer t.mu.Unlock() + + t.ctrlSess = sess + t.startAcceptLoop(ctx, sess) +} + +// startAcceptLoop serves inbound streams for the lifetime of the session. The +// session both routes callbacks against capabilities we advertised and serves +// operations against objects this State exposes, so the peer can call back into +// us over the connection we opened. +// Callers must hold t.mu. +func (t *msgOpTransport) startAcceptLoop(ctx context.Context, sess rpcSession) { + st := t.owner.State + router := &sessionRouter{state: st, server: st.server, inline: &t.inline} + go router.run(ctx, sess) //nolint:errcheck // loop ends with the session +} + +// roundTrip performs one request/reply op over the shared control session: open +// a stream, write the opRequest (and optional payload), read the opReply. The +// returned decoder is positioned to read the reply payload; done() closes the +// stream and must be called. +func (t *msgOpTransport) roundTrip(ctx context.Context, req opRequest, payload any) (opReply, *cbor.Decoder, func(), error) { + sess, err := t.controlSession(ctx) + if err != nil { + return opReply{}, nil, nil, err + } + + st, err := sess.OpenStreamSync(ctx) + if err != nil { + return opReply{}, nil, nil, err + } + done := func() { _ = st.Close() } + + req.Version = currentProtocolVersion + + enc := cbor.NewEncoder(st) + if err := enc.Encode(req); err != nil { + done() + return opReply{}, nil, nil, err + } + if payload != nil { + if err := enc.Encode(payload); err != nil { + done() + return opReply{}, nil, nil, err + } + } + + dec := cbor.NewDecoder(st) + var reply opReply + if err := dec.Decode(&reply); err != nil { + done() + return opReply{}, nil, nil, err + } + + return reply, dec, done, nil +} + +// messageSchemes are the remote prefixes served by the message transport. +var messageSchemes = []string{"mem://", "ws://", "wss://", "tcp://"} + +// isMessageRemote reports whether a remote address names a message transport. +func isMessageRemote(remote string) bool { + for _, scheme := range messageSchemes { + if strings.HasPrefix(remote, scheme) { + return true + } + } + return false +} + +// setupMsgTransport wires the client's connection factory from the remote's +// scheme. A remote with no recognised scheme belongs to a session that was +// adopted rather than dialed, and so has no factory at all. +func (c *NetworkClient) setupMsgTransport() { + switch { + case strings.HasPrefix(c.remote, "mem://"): + name := strings.TrimPrefix(c.remote, "mem://") + c.ops = &msgOpTransport{ + owner: c, + dial: func() (MessageConn, error) { + return defaultMemRegistry.dial(name) + }, + } + + case strings.HasPrefix(c.remote, "ws://"), strings.HasPrefix(c.remote, "wss://"): + c.setupWSTransport() + + case strings.HasPrefix(c.remote, "tcp://"): + c.setupTCPTransport() + + default: + c.ops = &msgOpTransport{owner: c} + } +} + +func nowStamp() string { + return time.Now().Format(time.RFC3339Nano) +} + +// bearer returns the configured bearer token, which rides in the operation +// frame rather than an Authorization header on this transport. +func (c *NetworkClient) bearer() string { + if c.State == nil || c.State.opts == nil { + return "" + } + return c.State.opts.bearerToken +} + +func traceMap(ctx context.Context) map[string]string { + m := propagation.MapCarrier{} + Propagator().Inject(ctx, m) + if len(m) == 0 { + return nil + } + return map[string]string(m) +} + +func (c *NetworkClient) msgResolveCapability(name string) error { + req := opRequest{Op: opLookup, Name: name, PubKey: c.State.pubkey, Contact: c.remote} + _, dec, done, err := c.ops.roundTrip(c.State.top, req, nil) + if err != nil { + return NewResolveHTTPError(err, "error performing message request: %v", err) + } + defer done() + + var lr lookupResponse + if err := dec.Decode(&lr); err != nil { + return NewResolveDecodeError(err) + } + if lr.Error != "" { + return NewResolveLookupError(lr.Error) + } + + c.capa = lr.Capability + c.oid = lr.Capability.OID + return nil +} + +func (c *NetworkClient) msgSendIdentity(ctx context.Context) error { + ts := nowStamp() + req := opRequest{ + Op: opIdentify, + PubKey: c.State.pubkey, + Contact: c.remote, + Timestamp: ts, + Signature: signBytes(c.State.privkey, msgCanonical(opIdentify, "", ts)), + Trace: traceMap(ctx), + } + _, dec, done, err := c.ops.roundTrip(ctx, req, nil) + if err != nil { + return err + } + defer done() + + var lr identifyResponse + if err := dec.Decode(&lr); err != nil { + return err + } + if lr.Error != "" { + return errors.New(lr.Error) + } + if !lr.Ok { + return errors.New("identity rejected") + } + c.serverObservedAddress = lr.Address + return nil +} + +func (c *NetworkClient) msgReresolveCapability(rs *InterfaceState) error { + req := opRequest{Op: opReresolve, PubKey: c.State.pubkey, Contact: c.remote} + _, dec, done, err := c.ops.roundTrip(c.State.top, req, rs) + if err != nil { + return err + } + defer done() + + var lr lookupResponse + if err := dec.Decode(&lr); err != nil { + return err + } + if lr.Error != "" { + return errors.New(lr.Error) + } + c.capa = lr.Capability + c.oid = lr.Capability.OID + return nil +} + +func (c *NetworkClient) msgFetchMethods(ctx context.Context) (methodsResponse, error) { + req := opRequest{Op: opMethods, OID: c.oid} + _, dec, done, err := c.ops.roundTrip(ctx, req, nil) + if err != nil { + return methodsResponse{}, err + } + defer done() + + var mr methodsResponse + if err := dec.Decode(&mr); err != nil { + return methodsResponse{}, err + } + if mr.Error != "" { + return methodsResponse{}, errors.New(mr.Error) + } + return mr, nil +} + +func (c *NetworkClient) msgRequestReexport(ctx context.Context, capa *Capability, target ed25519.PublicKey) (*Capability, error) { + ts := nowStamp() + req := opRequest{ + Op: opReexport, + OID: capa.OID, + TargetPK: target, + Contact: c.remote, + Timestamp: ts, + Signature: signBytes(c.State.privkey, msgCanonical(opReexport, string(capa.OID), ts)), + Trace: traceMap(ctx), + } + _, dec, done, err := c.ops.roundTrip(ctx, req, nil) + if err != nil { + return nil, err + } + defer done() + + var lr lookupResponse + if err := dec.Decode(&lr); err != nil { + return nil, err + } + if lr.Error != "" { + return nil, errors.New(lr.Error) + } + return lr.Capability, nil +} + +func (c *NetworkClient) msgDerefOID(ctx context.Context, oid OID) error { + ts := nowStamp() + req := opRequest{ + Op: opDeref, + OID: oid, + Timestamp: ts, + Signature: signBytes(c.State.privkey, msgCanonical(opDeref, string(oid), ts)), + } + _, dec, done, err := c.ops.roundTrip(ctx, req, nil) + if err != nil { + return err + } + defer done() + + var rr refResponse + if err := dec.Decode(&rr); err != nil { + return err + } + if rr.Error != "" { + return errors.New(rr.Error) + } + return nil +} + +func (c *NetworkClient) msgUnaryCall(ctx context.Context, method string, args, result any) error { + for { + ts := nowStamp() + req := opRequest{ + Op: opCall, + OID: c.oid, + Method: method, + Timestamp: ts, + Signature: signBytes(c.State.privkey, msgCanonical(opCall, string(c.oid)+"/"+method, ts)), + Bearer: c.bearer(), + Trace: traceMap(ctx), + } + reply, dec, done, err := c.ops.roundTrip(ctx, req, args) + if err != nil { + return err + } + + switch reply.Status { + case "ok": + err := dec.Decode(result) + done() + return err + case "unknown-capability": + done() + if c.capa != nil && c.capa.RestoreState != nil { + if rerr := c.msgReresolveCapability(c.capa.RestoreState); rerr == nil { + continue + } + } + return cond.NotFound("capability", c.oid) + case "error": + done() + return cond.RemoteError(reply.Category, reply.Code, reply.Error) + case "forbidden": + done() + // An authorization denial is a permission error, not a protocol one; + // without this case it would surface as "unknown response status". + return cond.RemoteError("permission", "forbidden", reply.Error) + case "panic": + done() + return cond.Panic(reply.Error) + default: + done() + return fmt.Errorf("unknown response status to %s/%s: %s", c.oid, method, reply.Status) + } + } +} + +func (c *NetworkClient) msgCallWithCaps(ctx context.Context, method string, args, result any, caps map[OID]*InlineCapability) error { + // The session is shared with every other call on this connection, so it is + // deliberately not closed when this call finishes. + sess, err := c.ops.controlSession(ctx) + if err != nil { + return err + } + + ts := nowStamp() + prelude := opRequest{ + Op: opCallStream, + OID: c.oid, + Method: method, + Timestamp: ts, + Signature: signBytes(c.State.privkey, msgCanonical(opCallStream, string(c.oid)+"/"+method, ts)), + Bearer: c.bearer(), + Trace: traceMap(ctx), + Version: currentProtocolVersion, + } + + return c.handleCallStream(ctx, sess, &c.ops.inline, &prelude, args, result, caps) +} diff --git a/pkg/rpc/transport_wt.go b/pkg/rpc/transport_wt.go new file mode 100644 index 000000000..58856825e --- /dev/null +++ b/pkg/rpc/transport_wt.go @@ -0,0 +1,115 @@ +package rpc + +import ( + "context" + "crypto/tls" + "errors" + "net" + "net/http" + + "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/http3" + "github.com/quic-go/webtransport-go" +) + +// wtStream adapts a *webtransport.Stream to the rpcStream interface. +type wtStream struct { + *webtransport.Stream +} + +func (s wtStream) CancelRead(code uint64) { + s.Stream.CancelRead(webtransport.StreamErrorCode(code)) +} + +// wtSession adapts a *webtransport.Session to the rpcSession interface. +type wtSession struct { + sess *webtransport.Session +} + +func (s *wtSession) OpenStreamSync(ctx context.Context) (rpcStream, error) { + str, err := s.sess.OpenStreamSync(ctx) + if err != nil { + return nil, err + } + return wtStream{str}, nil +} + +func (s *wtSession) AcceptStream(ctx context.Context) (rpcStream, error) { + str, err := s.sess.AcceptStream(ctx) + if err != nil { + return nil, err + } + return wtStream{str}, nil +} + +func (s *wtSession) Close() error { + return s.sess.CloseWithError(0, "") +} + +// wtDialer establishes a callstream session over WebTransport (HTTP/3). The +// upgrade is a CONNECT request, which the caller signs and issues. +type wtDialer struct { + ws *webtransport.Dialer +} + +func (d *wtDialer) Dial(ctx context.Context, url string, hdr http.Header) (*http.Response, rpcSession, error) { + hr, sess, err := d.ws.Dial(ctx, url, hdr) + if err != nil { + return hr, nil, err + } + return hr, &wtSession{sess: sess}, nil +} + +// isRetryableTransportError reports whether a transport-level error indicates +// the connection was reset in a way the client should transparently retry. +// This is QUIC-specific; other transports never produce it. +func isRetryableTransportError(err error) bool { + var ae *quic.ApplicationError + return errors.As(err, &ae) +} + +// setupH3Transport configures the client to speak the RPC protocol over +// QUIC/HTTP3 with WebTransport for the callstream path. +func (c *NetworkClient) setupH3Transport() { + htr := &http3.Transport{} + htr.Logger = c.State.log.With("module", "rpc-call") + htr.TLSClientConfig = c.tlsCfg + htr.QUICConfig = &DefaultQUICConfig + htr.Dial = func(ctx context.Context, addr string, tlsCfg *tls.Config, cfg *quic.Config) (*quic.Conn, error) { + uaddr, err := resolveUDPAddr(ctx, "udp", addr) + if err != nil { + return nil, err + } + + setTLSConfigServerName(tlsCfg, uaddr, addr) + + return c.transport.DialEarly(ctx, uaddr, tlsCfg, cfg) + } + + ws := &webtransport.Dialer{} + ws.TLSClientConfig = c.tlsCfg + ws.QUICConfig = &DefaultQUICConfig + ws.DialAddr = htr.Dial + + c.htr = htr + c.dialer = &wtDialer{ws: ws} +} + +func setTLSConfigServerName(tlsConf *tls.Config, addr net.Addr, host string) { + // If no ServerName is set, infer the ServerName from the host we're connecting to. + if tlsConf.ServerName != "" { + return + } + if host == "" { + if udpAddr, ok := addr.(*net.UDPAddr); ok { + tlsConf.ServerName = udpAddr.IP.String() + return + } + } + h, _, err := net.SplitHostPort(host) + if err != nil { // This happens if the host doesn't contain a port number. + tlsConf.ServerName = host + return + } + tlsConf.ServerName = h +}