diff --git a/pkg/cloudauth/rpc_authenticator.go b/pkg/cloudauth/rpc_authenticator.go index 7b4bff57b..2480d3802 100644 --- a/pkg/cloudauth/rpc_authenticator.go +++ b/pkg/cloudauth/rpc_authenticator.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "log/slog" - "net/http" "strings" "miren.dev/runtime/pkg/auth" @@ -110,10 +109,10 @@ func NewRPCAuthenticator(ctx context.Context, config Config) (*RPCAuthenticator, // Authenticate implements rpc.Authenticator. // It tries JWT authentication first, then falls back to TLS client certificate. // Authorization (RBAC) is handled separately via the Authorize method. -func (a *RPCAuthenticator) Authenticate(ctx context.Context, r *http.Request) (*rpc.Identity, error) { +func (a *RPCAuthenticator) Authenticate(ctx context.Context, creds *rpc.Credentials) (*rpc.Identity, error) { // Try JWT authentication first if Authorization header is present - if authHeader := r.Header.Get("Authorization"); authHeader != "" { - identity, err := a.authenticateJWT(ctx, authHeader) + if creds.Authorization != "" { + identity, err := a.authenticateJWT(ctx, creds.Authorization) if err != nil { return nil, err } @@ -123,12 +122,9 @@ func (a *RPCAuthenticator) Authenticate(ctx context.Context, r *http.Request) (* // Invalid JWT format, fall through to cert check } - // Fall back to TLS client certificate. Only trust a cert that the TLS - // layer verified against the cluster CA (VerifiedChains non-empty); a - // presented-but-unverified cert must never yield a cert identity, which - // grants RBAC-bypassing privileges in Authorize. - if r.TLS != nil && len(r.TLS.VerifiedChains) > 0 && len(r.TLS.PeerCertificates) > 0 { - cert := r.TLS.PeerCertificates[0] + // Fall back to a TLS client certificate the TLS layer verified against the + // cluster CA. + if cert := creds.VerifiedPeerCertificate(); cert != nil { return &rpc.Identity{ Subject: cert.Subject.CommonName, Method: rpc.AuthMethodCert, diff --git a/pkg/cloudauth/rpc_authenticator_cert_test.go b/pkg/cloudauth/rpc_authenticator_cert_test.go index f1e9763d0..5cfcc657f 100644 --- a/pkg/cloudauth/rpc_authenticator_cert_test.go +++ b/pkg/cloudauth/rpc_authenticator_cert_test.go @@ -41,7 +41,7 @@ func TestRPCAuthenticator_CertRequiresVerifiedChain(t *testing.T) { req := newReq(t, &tls.ConnectionState{ PeerCertificates: []*x509.Certificate{cert}, }) - id, err := auth.Authenticate(t.Context(), req) + id, err := auth.Authenticate(t.Context(), rpc.CredentialsFromRequest(req)) require.NoError(t, err) require.Nil(t, id, "unverified client cert must not yield an identity") }) @@ -51,7 +51,7 @@ func TestRPCAuthenticator_CertRequiresVerifiedChain(t *testing.T) { PeerCertificates: []*x509.Certificate{cert}, VerifiedChains: [][]*x509.Certificate{{cert}}, }) - id, err := auth.Authenticate(t.Context(), req) + id, err := auth.Authenticate(t.Context(), rpc.CredentialsFromRequest(req)) require.NoError(t, err) require.NotNil(t, id) require.Equal(t, "test-client", id.Subject) diff --git a/pkg/oidcauth/authenticator.go b/pkg/oidcauth/authenticator.go index 16753be1e..b3e9150ed 100644 --- a/pkg/oidcauth/authenticator.go +++ b/pkg/oidcauth/authenticator.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "log/slog" - "net/http" "strings" "sync/atomic" @@ -41,15 +40,12 @@ func (a *OIDCAuthenticator) SetEAC(eac *entityserver_v1alpha.EntityAccessClient) // Authenticate checks if the request carries a valid OIDC bearer token that // matches a configured oidc_binding entity. -func (a *OIDCAuthenticator) Authenticate(ctx context.Context, r *http.Request) (*rpc.Identity, error) { - authHeader := r.Header.Get("Authorization") - if !strings.HasPrefix(authHeader, "Bearer ") { +func (a *OIDCAuthenticator) Authenticate(ctx context.Context, creds *rpc.Credentials) (*rpc.Identity, error) { + tokenString := creds.BearerToken() + if tokenString == "" { return nil, nil } - tokenString := strings.TrimPrefix(authHeader, "Bearer ") - tokenString = strings.TrimSpace(tokenString) - eac := a.eac.Load() if eac == nil { return nil, nil // Entity access client not yet initialized @@ -72,11 +68,12 @@ func (a *OIDCAuthenticator) Authenticate(ctx context.Context, r *http.Request) ( return nil, nil // No bindings for this issuer } - // Determine the audience to validate against. - // Use the hostname from the request. - audience := r.Host - if audience == "" && r.TLS != nil { - audience = r.TLS.ServerName + // Determine the audience to validate against: the address the caller + // addressed. A message transport is not addressed by name and so carries + // neither, in which case there is nothing to validate against. + audience := creds.Host + if audience == "" && creds.TLS != nil { + audience = creds.TLS.ServerName } if audience == "" { return nil, nil diff --git a/pkg/oidcauth/authenticator_test.go b/pkg/oidcauth/authenticator_test.go index 06ecb74b9..cd775ed47 100644 --- a/pkg/oidcauth/authenticator_test.go +++ b/pkg/oidcauth/authenticator_test.go @@ -115,7 +115,7 @@ func TestOIDCAuthenticator_NoEAC(t *testing.T) { req := httptest.NewRequest("GET", "/", nil) req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -128,7 +128,7 @@ func TestOIDCAuthenticator_NoBearerToken(t *testing.T) { auth := NewOIDCAuthenticator(testutils.TestLogger(t)) req := httptest.NewRequest("GET", "/", nil) - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -147,7 +147,7 @@ func TestOIDCAuthenticator_NonJWTBearer(t *testing.T) { req := httptest.NewRequest("GET", "/", nil) req.Header.Set("Authorization", "Bearer not-a-jwt-token") - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -177,7 +177,7 @@ func TestOIDCAuthenticator_NoMatchingBindings(t *testing.T) { req.Host = "test-host" req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -238,7 +238,7 @@ func TestOIDCAuthenticator_MatchingBinding(t *testing.T) { req.Host = "test-host" req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(ctx, req) + identity, err := auth.Authenticate(ctx, rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -305,7 +305,7 @@ func TestOIDCAuthenticator_SubjectMismatch(t *testing.T) { req.Host = "test-host" req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(ctx, req) + identity, err := auth.Authenticate(ctx, rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -364,7 +364,7 @@ func TestOIDCAuthenticator_ClaimConditionMismatch(t *testing.T) { req.Host = "test-host" req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(ctx, req) + identity, err := auth.Authenticate(ctx, rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -418,7 +418,7 @@ func TestOIDCAuthenticator_ExpiredToken(t *testing.T) { req.Host = "test-host" req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(ctx, req) + identity, err := auth.Authenticate(ctx, rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -496,7 +496,7 @@ func TestOIDCAuthenticator_MultipleBindings_FirstMatch(t *testing.T) { req.Host = "test-host" req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(ctx, req) + identity, err := auth.Authenticate(ctx, rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -562,7 +562,7 @@ func TestOIDCAuthenticator_AudienceFromHost(t *testing.T) { req.Host = "my-cluster.example.com" req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(ctx, req) + identity, err := auth.Authenticate(ctx, rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -618,7 +618,7 @@ func TestOIDCAuthenticator_AudienceFromTLS(t *testing.T) { req.TLS = &tls.ConnectionState{ServerName: "tls-server.example.com"} req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(ctx, req) + identity, err := auth.Authenticate(ctx, rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -673,7 +673,7 @@ func TestOIDCAuthenticator_WrongAudience(t *testing.T) { req.Host = "test-host" req.Header.Set("Authorization", "Bearer "+token) - identity, err := auth.Authenticate(ctx, req) + identity, err := auth.Authenticate(ctx, rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/pkg/oidcauth/composite.go b/pkg/oidcauth/composite.go index 51f150170..c2cbcaef3 100644 --- a/pkg/oidcauth/composite.go +++ b/pkg/oidcauth/composite.go @@ -3,7 +3,6 @@ package oidcauth import ( "context" "fmt" - "net/http" "miren.dev/runtime/pkg/rpc" ) @@ -55,9 +54,9 @@ func NewCompositeAuthenticator(primary rpc.Authenticator, oidc *OIDCAuthenticato return &CompositeAuthenticator{primary: primary, oidc: oidc} } -func (c *CompositeAuthenticator) Authenticate(ctx context.Context, r *http.Request) (*rpc.Identity, error) { +func (c *CompositeAuthenticator) Authenticate(ctx context.Context, creds *rpc.Credentials) (*rpc.Identity, error) { // Try primary authenticator first - identity, err := c.primary.Authenticate(ctx, r) + identity, err := c.primary.Authenticate(ctx, creds) if identity != nil { return identity, nil } @@ -68,7 +67,7 @@ func (c *CompositeAuthenticator) Authenticate(ctx context.Context, r *http.Reque var oidcIdentity *rpc.Identity var oidcErr error if c.oidc != nil { - oidcIdentity, oidcErr = c.oidc.Authenticate(ctx, r) + oidcIdentity, oidcErr = c.oidc.Authenticate(ctx, creds) if oidcIdentity != nil { return oidcIdentity, nil } @@ -102,8 +101,9 @@ func (c *CompositeAuthorizer) Authorize(ctx context.Context, identity *rpc.Ident // OIDC callers are restricted to the oidc-deploy role return authorizeOIDC(resource, action) - case rpc.AuthMethodJWT, rpc.AuthMethodAnonymous, rpc.AuthMethodToken: - // Delegate to primary (cloud RBAC). + case rpc.AuthMethodJWT, rpc.AuthMethodAnonymous, rpc.AuthMethodToken, rpc.AuthMethodSigned: + // Delegate to primary (cloud RBAC). A signed identity names the key a + // capability was issued to, which carries no privilege of its own. fallthrough default: if c.primary != nil { diff --git a/pkg/oidcauth/composite_test.go b/pkg/oidcauth/composite_test.go index 9b3d8b6f1..75cfc0030 100644 --- a/pkg/oidcauth/composite_test.go +++ b/pkg/oidcauth/composite_test.go @@ -3,7 +3,6 @@ package oidcauth import ( "context" "fmt" - "net/http" "net/http/httptest" "testing" @@ -17,7 +16,7 @@ type mockAuthenticator struct { err error } -func (m *mockAuthenticator) Authenticate(ctx context.Context, r *http.Request) (*rpc.Identity, error) { +func (m *mockAuthenticator) Authenticate(ctx context.Context, creds *rpc.Credentials) (*rpc.Identity, error) { return m.identity, m.err } @@ -41,7 +40,7 @@ func TestCompositeAuthenticator_PrimarySucceeds(t *testing.T) { comp := NewCompositeAuthenticator(primary, oidcAuth) req := httptest.NewRequest("GET", "/", nil) - identity, err := comp.Authenticate(context.Background(), req) + identity, err := comp.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -63,7 +62,7 @@ func TestCompositeAuthenticator_PrimaryErrorOIDCNil(t *testing.T) { comp := NewCompositeAuthenticator(primary, oidcAuth) req := httptest.NewRequest("GET", "/", nil) - _, err := comp.Authenticate(context.Background(), req) + _, err := comp.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) if err == nil { t.Fatal("expected error to propagate from primary") } @@ -85,7 +84,7 @@ func TestCompositeAuthenticator_PrimaryErrorOIDCSucceeds(t *testing.T) { comp := &CompositeAuthenticator{primary: primary, oidc: oidcStub} req := httptest.NewRequest("GET", "/", nil) - identity, err := comp.Authenticate(context.Background(), req) + identity, err := comp.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("expected no error when OIDC succeeds, got: %v", err) } @@ -110,7 +109,7 @@ func TestCompositeAuthenticator_FallbackToOIDC(t *testing.T) { req := httptest.NewRequest("GET", "/", nil) req.Header.Set("Authorization", "Bearer some-token") - identity, err := comp.Authenticate(context.Background(), req) + identity, err := comp.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/pkg/outboard/outboard_test.go b/pkg/outboard/outboard_test.go index dbcc0ee42..93d59bff7 100644 --- a/pkg/outboard/outboard_test.go +++ b/pkg/outboard/outboard_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "miren.dev/runtime/pkg/rpc" ) func TestConfigRoundTrip(t *testing.T) { @@ -64,7 +65,7 @@ func TestTokenAuthenticatorValid(t *testing.T) { req, _ := http.NewRequest("GET", "/", nil) req.Header.Set("Authorization", "Bearer secret-token") - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) assert.NoError(t, err) assert.NotNil(t, identity) assert.Equal(t, "outboard", identity.Subject) @@ -76,7 +77,7 @@ func TestTokenAuthenticatorInvalid(t *testing.T) { req, _ := http.NewRequest("GET", "/", nil) req.Header.Set("Authorization", "Bearer wrong-token") - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) assert.NoError(t, err) assert.Nil(t, identity, "invalid token should return nil identity") } @@ -86,7 +87,7 @@ func TestTokenAuthenticatorMissing(t *testing.T) { req, _ := http.NewRequest("GET", "/", nil) - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) assert.NoError(t, err) assert.Nil(t, identity, "missing auth header should return nil identity") } @@ -97,7 +98,7 @@ func TestTokenAuthenticatorBadScheme(t *testing.T) { req, _ := http.NewRequest("GET", "/", nil) req.Header.Set("Authorization", "Basic dXNlcjpwYXNz") - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), rpc.CredentialsFromRequest(req)) assert.NoError(t, err) assert.Nil(t, identity, "non-bearer auth should return nil identity") } diff --git a/pkg/outboard/token_auth.go b/pkg/outboard/token_auth.go index 36cc5bf41..f600a2646 100644 --- a/pkg/outboard/token_auth.go +++ b/pkg/outboard/token_auth.go @@ -3,7 +3,6 @@ package outboard import ( "context" "crypto/subtle" - "net/http" "strings" "miren.dev/runtime/pkg/rpc" @@ -20,8 +19,8 @@ func NewTokenAuthenticator(token string) *TokenAuthenticator { return &TokenAuthenticator{token: token} } -func (t *TokenAuthenticator) Authenticate(ctx context.Context, r *http.Request) (*rpc.Identity, error) { - auth := r.Header.Get("Authorization") +func (t *TokenAuthenticator) Authenticate(ctx context.Context, creds *rpc.Credentials) (*rpc.Identity, error) { + auth := creds.Authorization if auth == "" { return nil, nil // No credentials } diff --git a/pkg/rpc/PROTOCOL.md b/pkg/rpc/PROTOCOL.md new file mode 100644 index 000000000..343725428 --- /dev/null +++ b/pkg/rpc/PROTOCOL.md @@ -0,0 +1,198 @@ +# Miren RPC message protocol + +This is the wire format for the **message transport** — the way Miren RPC runs +over any reliable, ordered, bidirectional pipe of discrete byte messages. It +backs the `mem://`, `ws://`, `wss://` and `tcp://` schemes, and any connection +handed to `State.ServeMessageConn` or `State.ClientFromMessageConn`. + +It is not the HTTP/3 + WebTransport transport, which remains the default and +speaks HTTP with the paths under `/_rpc/`. + +Everything is [CBOR](https://cbor.io/). Structs use integer keys (`keyasint`), +so field names never appear on the wire. + +**Status: version 1.** The Go implementation in this package is the only one +today. The format is documented and versioned so that a second implementation +does not require a breaking change, but the layouts below may still gain fields +(never renumber or repurpose one). + +## Layers + +``` + your backend a socket, a broker, an envelope protocol + | + MessageConn Send([]byte) / Recv() []byte / Close() + | + msgmux frames: multiplexes streams over messages + | + operations opRequest / opReply: one operation per stream +``` + +The backend supplies message framing. Everything above assumes messages arrive +whole, in order, exactly once, and that a closed connection eventually surfaces +as an error from `Recv`. + +## Layer 1: msgmux frames + +Every message on the connection is one frame: + +| Key | Field | Type | Meaning | +|-----|--------|--------|-------------------------------| +| 0 | Stream | uint32 | Stream id | +| 1 | Flags | uint8 | Bit set, below | +| 2 | Data | bytes | Payload; omitted when empty | + +Flags: + +| Bit | Name | Meaning | +|--------|------|------------------------------------------------------| +| `0x01` | SYN | Opens the stream | +| `0x02` | DATA | Carries payload bytes | +| `0x04` | FIN | No more data in this direction | +| `0x08` | RST | Abort: discard the stream, fail any blocked reader | + +Rules: + +- **Stream ids.** The dialing peer uses odd ids, the accepting peer even, so the + two never collide. Ids increase by 2 and are not reused. +- **Opening.** A stream opens with an eager SYN, sent as its own frame before any + data. The peer must be able to accept a stream whose opener reads before it + writes. +- **Data for an unknown stream** that carries no SYN is dropped, not an error. +- **Closing.** FIN closes one direction. A stream is forgotten once both + directions are finished. RST tears it down immediately from either side. +- **Frame size.** A payload larger than the session's frame cap is split across + several DATA frames; the reader concatenates. The default cap is 1 MiB, and + `WithMaxFrameSize` lowers it to fit a backend or envelope with its own limit. + The cap bounds throughput, never correctness. +- **No flow control.** Deliberately: the backend already frames messages and + applies its own backpressure. This is what makes msgmux smaller than a + general-purpose stream multiplexer such as yamux, which must invent both over + a byte stream. + +## Layer 2: operations + +**Every stream opens with an `opRequest`**, including server-initiated +callbacks. That invariant is what lets one accept loop route a whole connection, +and so lets a peer both serve and call over a single connection — including one +it dialed outbound through a firewall it could never accept through. + +The operation's payload follows the `opRequest` as further CBOR values on the +same stream. + +### opRequest + +| Key | Field | Type | Meaning | +|-----|-----------|-------------------|------------------------------------------------| +| 0 | Op | string | Operation selector, below | +| 1 | OID | string | Target capability | +| 2 | Name | string | Object name, for `lookup` | +| 3 | Method | string | Method name, for `call` / `callstream` | +| 4 | PubKey | bytes | Caller's ed25519 public key | +| 5 | TargetPK | bytes | Recipient's public key, for `reexport` | +| 6 | Contact | string | Address the caller can be reached at, if any | +| 7 | Timestamp | string | RFC3339Nano, signed | +| 8 | Signature | bytes | ed25519 over the canonical string, below | +| 9 | Bearer | string | Bearer token; there is no header to carry one | +| 10 | Trace | map[string]string | W3C trace context | +| 11 | Version | uint | Protocol version; absent or 0 means 1 | + +### opReply + +The first value written back, mapping the HTTP transport's +`rpc-status` / `rpc-error` trailers into the message world. + +| Key | Field | Type | Meaning | +|-----|----------|--------|-------------------------------| +| 0 | Status | string | Below | +| 1 | Error | string | Human-readable message | +| 2 | Category | string | Error category | +| 3 | Code | string | Error code | + +Statuses: `ok`, `error`, `panic`, `unknown-capability`, `forbidden`. + +`unknown-capability` is distinct because it is recoverable: a client holding +restore state re-resolves the capability and retries, rather than failing. + +### Operations + +| Op | Purpose | +|--------------|-------------------------------------------------------------| +| `lookup` | Resolve a named object to a capability | +| `call` | Unary call; args follow, then `opReply` and the result | +| `callstream` | Streaming call; the stream stays open for both directions | +| `methods` | List a capability's methods and parameters | +| `reresolve` | Rebuild a capability from its restore state | +| `reexport` | Re-issue a capability for a different holder | +| `ref` | Add a reference to a capability | +| `deref` | Drop a reference to a capability | +| `identify` | Prove the caller's key; returns the observed address | +| `inline` | Carries calls against an inline capability the peer advertised | + +`inline` is the reverse direction: the peer opens a stream against a capability +you advertised in a `callstream` call. The prelude names only the capability, +because the stream is pooled and carries many calls; each one is a +`streamRequest` with its own method. + +## Authentication + +There is no TLS client certificate to inspect, so a caller proves itself by +signing a canonical string with the ed25519 key the capability was issued to. + +``` +canonical = " " +``` + +`target` is `"/"` for `call` and `callstream`, `""` for +capability-scoped operations, and the empty string for `identify` — which leaves +two consecutive spaces, e.g. `identify 2026-07-23T18:04:05.000000001Z`. The +separator is a single space in all three positions; nothing is trimmed. + +The signature is raw bytes in field 8 — not base58, unlike the HTTP transports, +where it has to survive a header. + +A timestamp older than 10 minutes is rejected, so a captured frame cannot be +replayed indefinitely. + +`Bearer` is independent and optional. When present, it is offered to the +configured `Authenticator` as `"Bearer "`, and the identity it returns +supersedes the signature identity — that is how a message transport reaches +per-user authorization. Without one, the identity is the capability's key, with +method `signed`. + +## Versioning + +`opRequest.Version` names the version a stream speaks. Absent or `0` means +version 1, so a peer predating the field reads as speaking it. + +A version outside the receiver's supported range is rejected with an `opReply` +naming the range. This is a field rather than a handshake deliberately: a +mismatch costs one rejected stream instead of a round trip on every connection. + +The `identify` reply also carries `min_version` and `max_version`, so a client +that wants to adapt up front can ask instead of probing. + +## Implementing another peer + +Minimum for a client: + +1. Frame your transport into `MessageConn` semantics. +2. Implement msgmux: odd stream ids, eager SYN, FIN/RST, splitting at the frame + cap. +3. Open a stream, send `opRequest{Op: "lookup", Name: ..., PubKey: ...}`, read + `opReply` then a `lookupResponse` for the capability. +4. For each call, open a stream and send + `opRequest{Op: "call", OID, Method, Timestamp, Signature}` followed by the + args; read `opReply` and then the result. + +Add an accept loop handling `inline` only if you advertise capabilities for the +peer to call back on. + +Things worth getting right that are easy to miss: + +- The SYN frame is separate and comes first. +- Sign the canonical string exactly, including the single spaces. +- One stream per operation. Do not reuse an operation stream — except an + `inline` stream, which is explicitly pooled. +- Read the whole `opReply` before the payload; they are separate CBOR values on + the same stream, not a wrapper. diff --git a/pkg/rpc/actor.go b/pkg/rpc/actor.go index 79600cbfd..169d285e1 100644 --- a/pkg/rpc/actor.go +++ b/pkg/rpc/actor.go @@ -5,7 +5,6 @@ import ( "crypto/rand" "io" "sync" - "time" "github.com/fxamacker/cbor/v2" "github.com/mr-tron/base58" @@ -89,8 +88,8 @@ func (a *LocalActorRegistry) newCapability(i *Interface) *Capability { heldInterface: &heldInterface{ Interface: i, }, - lastContact: time.Now(), } + hc.touch() if i.restoreState != nil { if rs, err := i.restoreState.RestoreState(i); err == nil { diff --git a/pkg/rpc/authenticator.go b/pkg/rpc/authenticator.go index 1fce5a411..c65625065 100644 --- a/pkg/rpc/authenticator.go +++ b/pkg/rpc/authenticator.go @@ -2,9 +2,12 @@ package rpc import ( "context" + "crypto/tls" + "crypto/x509" "errors" "fmt" "net/http" + "strings" ) // contextKey is a private type for context keys to avoid collisions @@ -37,6 +40,7 @@ const ( AuthMethodAnonymous AuthMethod = "anonymous" // No authentication (public methods) AuthMethodToken AuthMethod = "token" // Bearer token (e.g., outboard) AuthMethodOIDC AuthMethod = "oidc" // External OIDC token (e.g., GitHub Actions) + AuthMethodSigned AuthMethod = "signed" // ed25519-signed request over a message transport ) // Identity represents an authenticated caller @@ -54,14 +58,66 @@ type Identity struct { Metadata map[string]any } +// Credentials carries what an authenticator needs to identify a caller, +// independent of how the call arrived. The HTTP transports fill it from the +// request; message transports fill it from the operation frame, where there is +// no request and no TLS handshake to inspect. +type Credentials struct { + // Authorization is the raw Authorization header value, e.g. "Bearer ". + Authorization string + + // Host is the address the caller addressed, used as the expected audience + // when validating a token. Empty on message transports, which are not + // addressed by name. + Host string + + // TLS is the connection's handshake state, or nil when the transport has no + // TLS layer of its own — including any connection supplied by a caller, + // whose security is that caller's concern. + TLS *tls.ConnectionState +} + +// CredentialsFromRequest extracts credentials from an HTTP request, for the +// HTTP transports and for anything fronting rpc with an HTTP layer of its own. +func CredentialsFromRequest(r *http.Request) *Credentials { + return &Credentials{ + Authorization: r.Header.Get("Authorization"), + Host: r.Host, + TLS: r.TLS, + } +} + +// BearerToken returns the token from a "Bearer " Authorization value, or +// an empty string if the credentials carry no bearer token. +func (c *Credentials) BearerToken() string { + if c == nil || !strings.HasPrefix(c.Authorization, "Bearer ") { + return "" + } + return strings.TrimSpace(strings.TrimPrefix(c.Authorization, "Bearer ")) +} + +// VerifiedPeerCertificate returns the caller's TLS client certificate, but only +// when the TLS layer verified it against the cluster CA. A presented but +// unverified certificate must never yield a cert identity, which grants +// RBAC-bypassing privileges in Authorize. +func (c *Credentials) VerifiedPeerCertificate() *x509.Certificate { + if c == nil || c.TLS == nil { + return nil + } + if len(c.TLS.VerifiedChains) == 0 || len(c.TLS.PeerCertificates) == 0 { + return nil + } + return c.TLS.PeerCertificates[0] +} + // Authenticator validates credentials and returns caller identity type Authenticator interface { - // Authenticate validates the request's credentials and returns the caller's identity. + // Authenticate validates the caller's credentials and returns their identity. // Returns: // - (*Identity, nil) if credentials are valid // - (nil, nil) if no credentials present or credentials are invalid // - (nil, error) if an error occurred during authentication - Authenticate(ctx context.Context, r *http.Request) (*Identity, error) + Authenticate(ctx context.Context, creds *Credentials) (*Identity, error) } // Authorizer checks if an identity is allowed to perform an action on a resource @@ -113,7 +169,7 @@ func AppAccessError(ctx context.Context, appName string) error { // Used for testing only. type NoOpAuthenticator struct{} -func (n *NoOpAuthenticator) Authenticate(ctx context.Context, r *http.Request) (*Identity, error) { +func (n *NoOpAuthenticator) Authenticate(ctx context.Context, creds *Credentials) (*Identity, error) { return &Identity{ Subject: "anonymous", Method: AuthMethodAnonymous, @@ -124,13 +180,8 @@ func (n *NoOpAuthenticator) Authenticate(ctx context.Context, r *http.Request) ( // Used when cloud authentication is not enabled. type LocalOnlyAuthenticator struct{} -func (l *LocalOnlyAuthenticator) Authenticate(ctx context.Context, r *http.Request) (*Identity, error) { - // Check for TLS client certificate. Only trust a cert that the TLS layer - // verified against the cluster CA (VerifiedChains non-empty); a - // presented-but-unverified cert must never yield a cert identity, which - // grants RBAC-bypassing privileges in Authorize. - if r.TLS != nil && len(r.TLS.VerifiedChains) > 0 && len(r.TLS.PeerCertificates) > 0 { - cert := r.TLS.PeerCertificates[0] +func (l *LocalOnlyAuthenticator) Authenticate(ctx context.Context, creds *Credentials) (*Identity, error) { + if cert := creds.VerifiedPeerCertificate(); cert != nil { return &Identity{ Subject: cert.Subject.CommonName, Method: AuthMethodCert, diff --git a/pkg/rpc/authenticator_test.go b/pkg/rpc/authenticator_test.go index a341f859c..754d5b6da 100644 --- a/pkg/rpc/authenticator_test.go +++ b/pkg/rpc/authenticator_test.go @@ -42,7 +42,7 @@ func TestNoOpAuthenticator(t *testing.T) { req.Header.Set("Authorization", tt.authHeader) } - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), CredentialsFromRequest(req)) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -127,7 +127,7 @@ func TestLocalOnlyAuthenticator(t *testing.T) { req.TLS = &tls.ConnectionState{} } - identity, err := auth.Authenticate(context.Background(), req) + identity, err := auth.Authenticate(context.Background(), CredentialsFromRequest(req)) if err != nil { t.Errorf("unexpected error: %v", err) } diff --git a/pkg/rpc/call.go b/pkg/rpc/call.go index 2fe355126..1bdd64f44 100644 --- a/pkg/rpc/call.go +++ b/pkg/rpc/call.go @@ -7,7 +7,6 @@ import ( "net/http" "github.com/fxamacker/cbor/v2" - "github.com/quic-go/webtransport-go" ) type Call interface { @@ -42,7 +41,11 @@ type NetworkCall struct { local *localCall ctrl *controlStream - wsSession *webtransport.Session + wsSession rpcSession + + // msgSession records that wsSession is a message-transport session, whose + // streams all open with an opRequest prelude. + msgSession bool argsConsumed bool } @@ -102,15 +105,23 @@ func (c *NetworkCall) NewClient(capa *Capability) Client { return c.local.NewClient(capa) } - client := c.s.state.newClientFrom(capa, c.peer) if capa.Inline && c.wsSession != nil { + client := c.s.state.newClientFrom(capa, c.peer) client.inlineClient = &inlineClient{ log: c.s.state.log, oid: capa.OID, ctrl: c.ctrl, session: c.wsSession, + prelude: c.msgSession, } + + return client + } + + // No address and no session to scope it to: nothing here can be reached. + if capa.Address == "" { + return unreachableClient{oid: capa.OID} } - return client + return c.s.state.newClientFrom(capa, c.peer) } diff --git a/pkg/rpc/client.go b/pkg/rpc/client.go index f6992ba52..e16843533 100644 --- a/pkg/rpc/client.go +++ b/pkg/rpc/client.go @@ -19,13 +19,24 @@ import ( "github.com/mr-tron/base58" "github.com/pkg/errors" "github.com/quic-go/quic-go" - "github.com/quic-go/quic-go/http3" - "github.com/quic-go/webtransport-go" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" "miren.dev/runtime/pkg/cond" ) +// transportKind selects which wire transport a NetworkClient uses. +type transportKind int + +const ( + transportH3 transportKind = iota // QUIC/HTTP3 + WebTransport (default) + transportMsg // msgmux over a MessageConn (mem, WebSocket, TCP, adopted) +) + +// adoptedRemote marks a client whose connection was supplied by the caller +// (ClientFromMessageConn). There is no address to dial: the connection is +// reachable only as itself, for as long as it lives. +const adoptedRemote = "adopted://" + type Client interface { CallWithCaps(ctx context.Context, method string, args, result any, caps map[OID]*InlineCapability) error Call(ctx context.Context, method string, args, result any) error @@ -34,12 +45,44 @@ type Client interface { Close() error } +// unreachableClient stands in for a capability that carries no address and is +// not inline, so there is nothing to dial and no session it is scoped to. This +// happens when a peer hands us a capability minted on a connection it does not +// own — only call-scoped inline capabilities survive that trip. Failing at the +// call rather than at hand-off keeps the error where it can be reported. +type unreachableClient struct { + oid OID +} + +func (u unreachableClient) err() error { + return cond.NotFound("reachable address for capability", u.oid) +} + +func (u unreachableClient) Call(context.Context, string, any, any) error { return u.err() } + +func (u unreachableClient) CallWithCaps( + context.Context, string, any, any, map[OID]*InlineCapability, +) error { + return u.err() +} + +func (u unreachableClient) NewClient(*Capability) Client { return u } + +func (u unreachableClient) NewInlineCapability(*Interface, any) (*InlineCapability, OID, *Capability) { + return &InlineCapability{}, "", &Capability{} +} + +func (u unreachableClient) Close() error { return nil } + type NetworkClient struct { State *State + transportKind transportKind + transport *quic.Transport - htr http3.Transport - ws webtransport.Dialer + htr http.RoundTripper + dialer *wtDialer + ops *msgOpTransport capa *Capability remote string remoteAddr net.Addr @@ -58,43 +101,18 @@ type NetworkClient struct { localClient *localClient } -func setTLSConfigServerName(tlsConf *tls.Config, addr net.Addr, host string) { - // If no ServerName is set, infer the ServerName from the host we're connecting to. - if tlsConf.ServerName != "" { - return - } - if host == "" { - if udpAddr, ok := addr.(*net.UDPAddr); ok { - tlsConf.ServerName = udpAddr.IP.String() - return - } - } - h, _, err := net.SplitHostPort(host) - if err != nil { // This happens if the host doesn't contain a port number. - tlsConf.ServerName = host - return - } - tlsConf.ServerName = h -} - +// setupTransport configures the client's round-tripper and callstream dialer +// based on the selected transport. The h3 and ws implementations live in +// transport_wt.go and transport_ws.go respectively. func (c *NetworkClient) setupTransport() { - c.htr.Logger = c.State.log.With("module", "rpc-call") - c.htr.TLSClientConfig = c.tlsCfg - c.htr.QUICConfig = &DefaultQUICConfig - c.htr.Dial = func(ctx context.Context, addr string, tlsCfg *tls.Config, cfg *quic.Config) (*quic.Conn, error) { - uaddr, err := resolveUDPAddr(ctx, "udp", addr) - if err != nil { - return nil, err - } - - setTLSConfigServerName(tlsCfg, uaddr, addr) - - return c.transport.DialEarly(ctx, uaddr, tlsCfg, cfg) + switch c.transportKind { + case transportMsg: + c.setupMsgTransport() + case transportH3: + c.setupH3Transport() + default: + c.setupH3Transport() } - - c.ws.TLSClientConfig = c.tlsCfg - c.ws.QUICConfig = &DefaultQUICConfig - c.ws.DialAddr = c.htr.Dial } func (c *NetworkClient) NewClient(capa *Capability) Client { @@ -152,6 +170,10 @@ func (c *NetworkClient) roundTrip(r *http.Request) (*http.Response, error) { } func (c *NetworkClient) sendIdentity(ctx context.Context) error { + if c.transportKind == transportMsg { + return c.msgSendIdentity(ctx) + } + url := "https://" + c.remote + "/_rpc/identify" req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) if err != nil { @@ -214,6 +236,10 @@ func (c *NetworkClient) sendIdentity(ctx context.Context) error { } func (c *NetworkClient) resolveCapability(name string) error { + if c.transportKind == transportMsg { + return c.msgResolveCapability(name) + } + url := "https://" + c.remote + "/_rpc/lookup/" + url.PathEscape(name) req, err := http.NewRequest(http.MethodPost, url, nil) if err != nil { @@ -257,6 +283,10 @@ func (c *NetworkClient) resolveCapability(name string) error { } func (c *NetworkClient) reresolveCapability(rs *InterfaceState) error { + if c.transportKind == transportMsg { + return c.msgReresolveCapability(rs) + } + url := "https://" + c.remote + "/_rpc/reresolve" c.State.log.Debug("reresolving capability from state", "url", url) @@ -312,6 +342,10 @@ func (c *NetworkClient) fetchMethods(ctx context.Context) (methodsResponse, erro return c.localClient.listMethods(), nil } + if c.transportKind == transportMsg { + return c.msgFetchMethods(ctx) + } + url := "https://" + c.remote + "/_rpc/methods/" + string(c.oid) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { @@ -386,6 +420,10 @@ func (c *NetworkClient) HasMethodParam(ctx context.Context, method, param string } func (c *NetworkClient) requestReexportCapability(ctx context.Context, capa *Capability, target ed25519.PublicKey) (*Capability, error) { + if c.transportKind == transportMsg { + return c.msgRequestReexport(ctx, capa, target) + } + url := "https://" + c.remote + "/_rpc/reexport/" + string(capa.OID) req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) if err != nil { @@ -430,6 +468,10 @@ func (c *NetworkClient) derefOID(ctx context.Context, oid OID) error { return c.inlineClient.derefOID(ctx, oid) } + if c.transportKind == transportMsg { + return c.msgDerefOID(ctx, oid) + } + url := "https://" + c.remote + "/_rpc/deref/" + string(oid) req, err := http.NewRequest(http.MethodPost, url, nil) if err != nil { @@ -533,6 +575,10 @@ func (c *NetworkClient) Call(ctx context.Context, method string, args, result an return c.inlineClient.Call(ctx, method, args, result) } + if c.transportKind == transportMsg { + return c.msgUnaryCall(ctx, method, args, result) + } + ctx, span := Tracer().Start(ctx, "rpc.call."+method) defer span.End() @@ -558,7 +604,7 @@ request: hr, err := c.htr.RoundTrip(req) if err != nil { - if _, ok := err.(*quic.ApplicationError); ok { + if isRetryableTransportError(err) { c.State.log.Info("rpc.call retrying", "oid", string(c.oid), "error", err) continue request } @@ -626,20 +672,19 @@ func (c *NetworkClient) CallWithCaps(ctx context.Context, method string, args, r return c.localClient.Call(ctx, method, args, result) } + if c.transportKind == transportMsg { + return c.msgCallWithCaps(ctx, method, args, result, caps) + } + ctx, span := Tracer().Start(ctx, "rpc.call."+method) defer span.End() - data, err := cbor.Marshal(args) - if err != nil { - return err - } - request: for { span.SetAttributes(attribute.String("oid", string(c.oid))) url := "https://" + c.remote + "/_rpc/callstream/" + string(c.oid) + "/" + method - req, err := http.NewRequestWithContext(ctx, http.MethodConnect, url, bytes.NewReader(data)) + req, err := http.NewRequestWithContext(ctx, http.MethodConnect, url, nil) if err != nil { return err } @@ -649,151 +694,100 @@ request: return err } - hr, sess, err := c.ws.Dial(ctx, url, req.Header) + hr, sess, err := c.dialer.Dial(ctx, url, req.Header) if err != nil { - if _, ok := err.(*quic.ApplicationError); ok { - c.State.log.Info("rpc.call retrying", "oid", string(c.oid), "error", err) + retry, derr := c.handleCallStreamDialError(hr, err) + if retry { continue request } - - return fmt.Errorf("error performing http request to %s: %w", url, err) + return derr } - retry, err := c.handleCallStream(ctx, hr, sess, method, args, result, caps) + err = c.handleCallStream(ctx, sess, perCallRouter{}, nil, args, result, caps) + _ = sess.Close() if err != nil { return err } - if retry { - continue request - } - return nil } } -func (c *NetworkClient) handleCallStream( - ctx context.Context, - hr *http.Response, - sess *webtransport.Session, - method string, - args, result any, - caps map[OID]*InlineCapability, -) (bool, error) { - if hr.StatusCode != http.StatusOK { - status := hr.Trailer.Get("rpc-status") - - err := cond.Errorf("unexpected status code: %d", hr.StatusCode) +// handleCallStreamDialError maps a failed callstream upgrade to the appropriate +// RPC error, transparently retrying when the capability can be re-resolved. The +// WebTransport upgrade surfaces the RPC status via response trailers on HTTP/3, +// with headers as a fallback, so we check both. +func (c *NetworkClient) handleCallStreamDialError(hr *http.Response, dialErr error) (bool, error) { + statusFor := func(key string) string { + if hr == nil { + return "" + } + if v := hr.Trailer.Get(key); v != "" { + return v + } + return hr.Header.Get(key) + } - switch status { - case "ok", "": - // The remote side thought everything was fine, so use our ability to parse - // the response as the error. - return false, err - case "unknown-capability": - if c.capa.RestoreState != nil { - // Try to re-resolve and, if successful, signal caller to retry the request. - if rerr := c.reresolveCapability(c.capa.RestoreState); rerr == nil { - return true, nil - } + switch statusFor("rpc-status") { + case "unknown-capability": + if c.capa.RestoreState != nil { + if rerr := c.reresolveCapability(c.capa.RestoreState); rerr == nil { + return true, nil } - err = cond.NotFound("capability", c.capa.OID) - case "error": - errs := hr.Trailer.Get("rpc-error") - err = cond.RemoteError("generic", "unknown", errs) - case "panic": - errs := hr.Trailer.Get("rpc-error") - err = cond.Panic(errs) } + return false, cond.NotFound("capability", c.capa.OID) + case "error": + return false, cond.RemoteError("generic", "unknown", statusFor("rpc-error")) + case "panic": + return false, cond.Panic(statusFor("rpc-error")) + } - return false, err + if isRetryableTransportError(dialErr) { + c.State.log.Info("rpc.call retrying", "oid", string(c.oid), "error", dialErr) + return true, nil } + return false, fmt.Errorf("error performing callstream request: %w", dialErr) +} + +func (c *NetworkClient) handleCallStream( + ctx context.Context, + sess rpcSession, + router callbackRouter, + prelude any, + args, result any, + caps map[OID]*InlineCapability, +) error { ctx, cancel := context.WithCancel(ctx) defer cancel() - go func() { - for { - str, err := sess.AcceptStream(ctx) - if err != nil { - break - } - - go func() { - defer str.Close() - - dec := cbor.NewDecoder(str) - enc := cbor.NewEncoder(str) - - // Loop to handle multiple requests on the same stream - for { - var rs streamRequest - - err = dec.Decode(&rs) - if err != nil { - if err != io.EOF { - c.State.log.Error("rpc.callstream call: error decoding stream request", "error", err) - } - return - } - - switch rs.Kind { - case "call": - iface, ok := caps[rs.OID] - if !ok { - _ = enc.Encode(refResponse{ - Status: "error", - Error: "unknown capability", - }) - continue - } - - mm := iface.methods[rs.Method] - if mm.Handler == nil { - enc.Encode(refResponse{ - Status: "error", - Error: "unknown method", - }) - continue - } - - ctx, cancel := context.WithCancel(ctx) - err := c.callInline(ctx, mm, rs.OID, rs.Method, iface.Interface, enc, dec) - cancel() - if err != nil { - if !errors.Is(err, context.Canceled) && !errors.Is(err, cond.ErrClosed{}) { - c.State.log.Error("rpc.callstream: error calling inline", "error", err) - } - return - } - default: - c.State.log.Error("rpc.callstream: unknown call stream request", "kind", rs.Kind) - } - } - }() - } - }() + withdraw := router.register(ctx, c, sess, caps) + defer withdraw() // Open the control stream ctrl, err := sess.OpenStreamSync(ctx) if err != nil { c.State.log.Error("rpc.callstream ctrl: error opening control stream", "error", err) - return false, err + return err } enc := cbor.NewEncoder(ctrl) + // On the message transport the control stream is also the operation stream, + // so it must lead with the opRequest the server's router decodes; the HTTP + // transports pass a nil prelude. + if prelude != nil { + enc.Encode(prelude) + } enc.Encode(args) dec := cbor.NewDecoder(ctrl) - const cancelCode = webtransport.StreamErrorCode(quic.FlowControlError) - // If the context is canceled, then we bail ASAP on trying to complete the RPC. - // Because we have a local ctx with a local cancel also, when this method turns, this - // goroutine will automatically get cleaned up. + // Because we have a local ctx with a local cancel also, when this method returns, + // this goroutine will automatically get cleaned up. go func() { <-ctx.Done() - ctrl.CancelRead(cancelCode) + ctrl.CancelRead(cancelReadCode) }() loop: @@ -802,10 +796,13 @@ loop: err = dec.Decode(&rs) if err != nil { - if errors.Is(err, io.EOF) { - err = nil - } else if se, ok := err.(*webtransport.StreamError); ok && se.ErrorCode == cancelCode { + // A canceled context means we aborted the read ourselves (via + // CancelRead above); report it uniformly across transports rather + // than depending on a transport-specific stream-error type. + if ctx.Err() != nil { err = cond.Closed("rpc call terminated before getting response") + } else if errors.Is(err, io.EOF) { + err = nil } break @@ -828,7 +825,7 @@ loop: } } - return false, err + return err } func (c *NetworkClient) callInline( diff --git a/pkg/rpc/inline.go b/pkg/rpc/inline.go index 5f31d4c40..b473bb526 100644 --- a/pkg/rpc/inline.go +++ b/pkg/rpc/inline.go @@ -8,7 +8,6 @@ import ( "sync" "github.com/fxamacker/cbor/v2" - "github.com/quic-go/webtransport-go" "miren.dev/runtime/pkg/cond" ) @@ -17,7 +16,7 @@ var errInlineClientClosed = errors.New("inline client closed") const inlineStreamPoolSize = 10 type streamConn struct { - stream *webtransport.Stream + stream rpcStream enc *cbor.Encoder dec *cbor.Decoder } @@ -26,7 +25,12 @@ type inlineClient struct { log *slog.Logger oid OID ctrl *controlStream - session *webtransport.Session + session rpcSession + + // prelude reports whether each newly opened stream must lead with an + // opRequest, as message-transport sessions require so that one accept loop + // can route both operations and callbacks. + prelude bool // Stream pool poolMu sync.Mutex @@ -86,11 +90,30 @@ func (c *inlineClient) getStream(ctx context.Context) (*streamConn, error) { return nil, err } - return &streamConn{ + conn := &streamConn{ stream: str, enc: cbor.NewEncoder(str), dec: cbor.NewDecoder(str), - }, nil + } + + // The prelude is written once per stream, not once per call: the + // stream is pooled and carries many calls against this capability. + if c.prelude { + prelude := opRequest{ + Op: opInlineCall, + OID: c.oid, + Version: currentProtocolVersion, + } + if err := conn.enc.Encode(prelude); err != nil { + _ = str.Close() + c.poolMu.Lock() + c.activeCount-- + c.poolMu.Unlock() + return nil, err + } + } + + return conn, nil } // Pool is at capacity, wait for a stream to become available diff --git a/pkg/rpc/inline_router.go b/pkg/rpc/inline_router.go new file mode 100644 index 000000000..bfde137b8 --- /dev/null +++ b/pkg/rpc/inline_router.go @@ -0,0 +1,160 @@ +package rpc + +import ( + "context" + "errors" + "io" + "sync" + + "github.com/fxamacker/cbor/v2" + "miren.dev/runtime/pkg/cond" +) + +// inlineLookup resolves the inline capability a server-initiated callback is +// addressed to, along with the client that advertised it (the client supplies +// the caller identity for the resulting handler invocation). +type inlineLookup func(OID) (*NetworkClient, *InlineCapability, bool) + +// callbackRouter delivers server-initiated inline-capability calls back to the +// capabilities a call advertised. +// +// The HTTP transports give every call its own session, so each call can own a +// private accept loop over it. Message transports share one session across all +// concurrent calls on a connection — there is no second connection to dial — so +// callbacks must instead be demultiplexed by OID through a session-scoped +// registry. OIDs are 16 random bytes (Server.assignCapability), so entries from +// different calls cannot collide. +type callbackRouter interface { + // register makes caps reachable for the duration of one call and returns a + // function that withdraws them. + register(ctx context.Context, c *NetworkClient, sess rpcSession, caps map[OID]*InlineCapability) func() +} + +// perCallRouter is the router for transports that dedicate a session to each +// call. It runs an accept loop bound to the call's lifetime. +type perCallRouter struct{} + +func (perCallRouter) register( + ctx context.Context, + c *NetworkClient, + sess rpcSession, + caps map[OID]*InlineCapability, +) func() { + ctx, cancel := context.WithCancel(ctx) + + go c.State.acceptInlineStreams(ctx, sess, func(oid OID) (*NetworkClient, *InlineCapability, bool) { + ic, ok := caps[oid] + return c, ic, ok + }) + + return cancel +} + +// inlineRegistry routes callbacks by OID for a session shared across calls. The +// accept loop is owned by the session, not by any one call; calls only add and +// remove their capabilities. +type inlineRegistry struct { + mu sync.Mutex + entries map[OID]inlineEntry +} + +type inlineEntry struct { + client *NetworkClient + capa *InlineCapability +} + +func (r *inlineRegistry) register( + _ context.Context, + c *NetworkClient, + _ rpcSession, + caps map[OID]*InlineCapability, +) func() { + r.mu.Lock() + if r.entries == nil { + r.entries = make(map[OID]inlineEntry, len(caps)) + } + for oid, ic := range caps { + r.entries[oid] = inlineEntry{client: c, capa: ic} + } + r.mu.Unlock() + + return func() { + r.mu.Lock() + for oid := range caps { + delete(r.entries, oid) + } + r.mu.Unlock() + } +} + +func (r *inlineRegistry) lookup(oid OID) (*NetworkClient, *InlineCapability, bool) { + r.mu.Lock() + defer r.mu.Unlock() + + e, ok := r.entries[oid] + return e.client, e.capa, ok +} + +// acceptInlineStreams accepts server-initiated streams on sess and serves the +// inline-capability calls they carry. +func (s *State) acceptInlineStreams(ctx context.Context, sess rpcSession, lookup inlineLookup) { + for { + str, err := sess.AcceptStream(ctx) + if err != nil { + return + } + + go s.serveInlineStream(ctx, str, lookup) + } +} + +// serveInlineStream handles one server-initiated stream on a transport whose +// callback streams carry no opRequest prelude (HTTP/3 and WebTransport). +func (s *State) serveInlineStream(ctx context.Context, str rpcStream, lookup inlineLookup) { + defer str.Close() + + s.serveInlineCalls(ctx, cbor.NewDecoder(str), cbor.NewEncoder(str), lookup) +} + +// serveInlineCalls reads calls against inline capabilities until the stream +// ends. A stream is pooled by the peer's inlineClient and so may carry many +// calls in sequence. +func (s *State) serveInlineCalls(ctx context.Context, dec *cbor.Decoder, enc *cbor.Encoder, lookup inlineLookup) { + for { + var rs streamRequest + + if err := dec.Decode(&rs); err != nil { + if !errors.Is(err, io.EOF) { + s.log.Error("rpc.callstream call: error decoding stream request", "error", err) + } + return + } + + switch rs.Kind { + case "call": + c, iface, ok := lookup(rs.OID) + if !ok { + _ = enc.Encode(refResponse{Status: "error", Error: "unknown capability"}) + continue + } + + mm := iface.methods[rs.Method] + if mm.Handler == nil { + _ = enc.Encode(refResponse{Status: "error", Error: "unknown method"}) + continue + } + + ctx, cancel := context.WithCancel(ctx) + err := c.callInline(ctx, mm, rs.OID, rs.Method, iface.Interface, enc, dec) + cancel() + if err != nil { + if !errors.Is(err, context.Canceled) && !errors.Is(err, cond.ErrClosed{}) { + s.log.Error("rpc.callstream: error calling inline", "error", err) + } + return + } + default: + s.log.Error("rpc.callstream: unknown call stream request", "kind", rs.Kind) + } + } +} diff --git a/pkg/rpc/message.go b/pkg/rpc/message.go new file mode 100644 index 000000000..ace10c8c4 --- /dev/null +++ b/pkg/rpc/message.go @@ -0,0 +1,181 @@ +package rpc + +import ( + "context" +) + +// defaultMaxFrameData bounds the payload carried in a single msgmux frame, so +// that a backend with a maximum message size (e.g. a broker capping messages +// around a megabyte) can carry arbitrarily large stream writes as multiple +// frames. Override it per session with WithMaxFrameSize. +const defaultMaxFrameData = 1 << 20 // 1 MiB + +// MessageConn is the pure message interface a message-oriented backend +// implements: a reliable, ordered, bidirectional, point-to-point pipe of +// discrete byte messages between two peers. The framework layers a stream +// multiplexer (msgmux) on top to recover the rpcSession semantics that +// callbacks and streaming require. +// +// Recv blocks until a message is available and returns io.EOF once the peer +// has closed the connection and no buffered messages remain. +type MessageConn interface { + Send([]byte) error + Recv() ([]byte, error) + Close() error +} + +// MessageSessionOption configures a session built over a MessageConn. +type MessageSessionOption func(*messageSessionOptions) + +type messageSessionOptions struct { + maxFrame int +} + +// WithMaxFrameSize bounds the payload rpc places in a single message. Set it +// below the backend's own message-size limit — an envelope protocol wrapping +// these messages, or a broker capping message size. Stream writes larger than +// the bound are split across frames, so the bound costs throughput, never +// correctness. +func WithMaxFrameSize(n int) MessageSessionOption { + return func(o *messageSessionOptions) { + o.maxFrame = n + } +} + +func buildMessageSessionOptions(opts []MessageSessionOption) messageSessionOptions { + var o messageSessionOptions + for _, fn := range opts { + fn(&o) + } + return o +} + +// ServeMessageConn serves RPC over a connection the caller owns, and blocks +// until the connection fails or ctx is cancelled. +// +// This is the entry point for backends rpc does not manage — most importantly a +// socket carrying somebody else's envelope protocol, where the caller unwraps +// inbound payloads into Recv and wraps outbound payloads from Send. rpc never +// dials, listens, or closes such a connection. +// +// Capabilities minted on this connection are reachable only for its lifetime; +// they carry no dialable address. +func (s *State) ServeMessageConn(ctx context.Context, conn MessageConn, opts ...MessageSessionOption) error { + o := buildMessageSessionOptions(opts) + + sess := newMsgSession(conn, false, o.maxFrame) + router := &sessionRouter{state: s, server: s.server} + + return router.run(ctx, sess) +} + +// ClientFromMessageConn builds a client for the named remote object over a +// connection the caller owns, the dialing counterpart to ServeMessageConn. +// +// The resulting session serves as well as calls: the peer may invoke objects +// this State has exposed over the same connection. That is what allows a party +// that dialled outbound — through a firewall it could not accept through — to +// still be called back into. +func (s *State) ClientFromMessageConn( + ctx context.Context, + conn MessageConn, + name string, + opts ...MessageSessionOption, +) (*NetworkClient, error) { + o := buildMessageSessionOptions(opts) + + client := &NetworkClient{ + State: s, + transportKind: transportMsg, + remote: adoptedRemote, + } + // dial stays nil: the connection belongs to the caller and cannot be + // re-established from here. + client.ops = &msgOpTransport{owner: client} + client.ops.adopt(ctx, newMsgSession(conn, true, o.maxFrame)) + + if err := client.resolveCapability(name); err != nil { + return nil, err + } + + return client, nil +} + +// opRequest is the first frame of an operation stream. It carries the operation +// selector plus the metadata that, on the HTTP transports, lives in the request +// path, headers, and auth signature. The operation payload (CBOR args / +// InterfaceState / etc.) follows as subsequent bytes on the same stream. +type opRequest struct { + Op string `cbor:"0,keyasint"` + OID OID `cbor:"1,keyasint,omitempty"` + Name string `cbor:"2,keyasint,omitempty"` + Method string `cbor:"3,keyasint,omitempty"` + + PubKey []byte `cbor:"4,keyasint,omitempty"` + TargetPK []byte `cbor:"5,keyasint,omitempty"` + Contact string `cbor:"6,keyasint,omitempty"` + + Timestamp string `cbor:"7,keyasint,omitempty"` + Signature []byte `cbor:"8,keyasint,omitempty"` + + Bearer string `cbor:"9,keyasint,omitempty"` + Trace map[string]string `cbor:"10,keyasint,omitempty"` + + // Version is the protocol version this stream speaks. Zero means version 1, + // so a peer predating the field is treated as speaking it. See PROTOCOL.md. + Version uint `cbor:"11,keyasint,omitempty"` +} + +// Protocol versions carried in opRequest.Version. A field rather than a +// handshake: a version mismatch costs one rejected stream, not a round trip on +// every connection. +const ( + // protocolVersion1 is the only version defined. Zero on the wire means this. + protocolVersion1 = 1 + + // currentProtocolVersion is what this implementation sends. + currentProtocolVersion = protocolVersion1 + + // minProtocolVersion is the oldest version this implementation accepts. + minProtocolVersion = protocolVersion1 +) + +// protocolVersion normalises a version off the wire, mapping the zero value +// onto version 1. +func (r opRequest) protocolVersion() uint { + if r.Version == 0 { + return protocolVersion1 + } + return r.Version +} + +// opReply is the first frame of an operation reply, mapping the HTTP +// rpc-status/rpc-error[/-category/-code] trailers into the message world. The +// reply payload (CBOR result / lookupResponse / etc.) follows on the stream. +type opReply struct { + Status string `cbor:"0,keyasint"` + Error string `cbor:"1,keyasint,omitempty"` + Category string `cbor:"2,keyasint,omitempty"` + Code string `cbor:"3,keyasint,omitempty"` +} + +// Operation selectors carried in opRequest.Op. Every stream on a message +// session opens with an opRequest, including server-initiated callbacks +// (opInlineCall), so a single accept loop can route the whole session and +// either peer may both serve and call over one connection. +const ( + opCall = "call" + opCallStream = "callstream" + opLookup = "lookup" + opMethods = "methods" + opReresolve = "reresolve" + opReexport = "reexport" + opRef = "ref" + opDeref = "deref" + opIdentify = "identify" + + // opInlineCall opens a stream carrying calls against an inline capability + // the peer advertised. The stream is pooled by the caller and may carry many + // calls, so the prelude names only the capability, not a method. + opInlineCall = "inline" +) diff --git a/pkg/rpc/message_adopt_test.go b/pkg/rpc/message_adopt_test.go new file mode 100644 index 000000000..2b791927f --- /dev/null +++ b/pkg/rpc/message_adopt_test.go @@ -0,0 +1,415 @@ +package rpc_test + +import ( + "context" + "errors" + "io" + "net" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "miren.dev/runtime/pkg/rpc" + "miren.dev/runtime/pkg/rpc/example" + "miren.dev/runtime/pkg/rpc/stream" +) + +// envelopeConn is a MessageConn owned by the test rather than by rpc, standing +// in for a socket carrying somebody else's envelope protocol: every payload is +// wrapped on the way out and unwrapped on the way in, and rpc never sees the +// underlying pipe. Critically there is no way to open a second one — if rpc +// tried to dial another connection for a callstream, these tests would fail. +type envelopeConn struct { + mu sync.Mutex + peer *envelopeConn + + recv chan []byte + done chan struct{} + once sync.Once + + // envelopes counts payloads that made a round trip through the wrapper, so + // a test can assert traffic actually crossed the boundary. + envelopes *int64 + countMu *sync.Mutex +} + +const envelopePrefix = "ENV:" + +func newEnvelopePair() (*envelopeConn, *envelopeConn) { + var ( + count int64 + cmu sync.Mutex + ) + + done := make(chan struct{}) + a := &envelopeConn{recv: make(chan []byte, 64), done: done, envelopes: &count, countMu: &cmu} + b := &envelopeConn{recv: make(chan []byte, 64), done: done, envelopes: &count, countMu: &cmu} + a.peer, b.peer = b, a + + return a, b +} + +func (c *envelopeConn) Send(b []byte) error { + // Wrap: this is the caller's protocol, opaque to rpc. + framed := append([]byte(envelopePrefix), b...) + + c.countMu.Lock() + *c.envelopes++ + c.countMu.Unlock() + + c.mu.Lock() + peer := c.peer + c.mu.Unlock() + + select { + case peer.recv <- framed: + return nil + case <-c.done: + return net.ErrClosed + } +} + +func (c *envelopeConn) Recv() ([]byte, error) { + select { + case b := <-c.recv: + return b[len(envelopePrefix):], nil // unwrap + case <-c.done: + select { + case b := <-c.recv: + return b[len(envelopePrefix):], nil + default: + return nil, io.EOF + } + } +} + +func (c *envelopeConn) Close() error { + c.once.Do(func() { close(c.done) }) + return nil +} + +func (c *envelopeConn) count() int64 { + c.countMu.Lock() + defer c.countMu.Unlock() + return *c.envelopes +} + +// recordingAuthenticator returns a fixed identity and remembers the credentials +// it was handed, so a test can assert what actually reached it. +type recordingAuthenticator struct { + identity *rpc.Identity + + mu sync.Mutex + last *rpc.Credentials +} + +func (a *recordingAuthenticator) Authenticate( + ctx context.Context, + creds *rpc.Credentials, +) (*rpc.Identity, error) { + a.mu.Lock() + a.last = creds + a.mu.Unlock() + + return a.identity, nil +} + +func (a *recordingAuthenticator) lastAuthorization() string { + a.mu.Lock() + defer a.mu.Unlock() + + if a.last == nil { + return "" + } + return a.last.Authorization +} + +// denyAll rejects every authorization check. +type denyAll struct{} + +func (denyAll) Authorize(ctx context.Context, identity *rpc.Identity, resource, action string) error { + return errors.New("denied by policy") +} + +// identityMeter records the identity its handler ran under. +type identityMeter struct { + seen **rpc.Identity +} + +func (m *identityMeter) ReadTemperature(ctx context.Context, call *example.MeterReadTemperature) error { + *m.seen = rpc.IdentityFromContext(ctx) + + res := call.Results() + reading := new(example.Reading) + reading.SetMeter(call.Args().Name()) + reading.SetTemperature(1) + res.SetReading(reading) + + return nil +} + +func (m *identityMeter) GetSetter(ctx context.Context, call *example.MeterGetSetter) error { + return nil +} + +// adoptPair wires two States over one caller-owned connection: serverEnd is +// served, clientEnd is dialled. Returns the client for the named object. +func adoptPair( + t *testing.T, + ctx context.Context, + iface *rpc.Interface, + name string, + opts ...rpc.MessageSessionOption, +) (rpc.Client, *envelopeConn) { + t.Helper() + r := require.New(t) + + ss, err := rpc.NewState(ctx, rpc.WithSkipVerify) + r.NoError(err) + ss.Server().ExposeValue(name, iface) + + cs, err := rpc.NewState(ctx, rpc.WithSkipVerify) + r.NoError(err) + + serverEnd, clientEnd := newEnvelopePair() + + go ss.ServeMessageConn(ctx, serverEnd, opts...) //nolint:errcheck // ends with ctx + + c, err := cs.ClientFromMessageConn(ctx, clientEnd, name, opts...) + r.NoError(err) + + return c, clientEnd +} + +func TestAdoptedMessageConn(t *testing.T) { + t.Run("unary calls over a caller-owned connection", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + c, conn := adoptPair(t, ctx, example.AdaptMeter(&exampleMeter{temp: 42}), "meter") + + mc := &example.MeterClient{Client: c} + + res, err := mc.ReadTemperature(ctx, "test") + r.NoError(err) + r.Equal("test", res.Reading().Meter()) + r.Equal(float32(42), res.Reading().Temperature()) + + r.NotZero(conn.count(), "traffic should have crossed the envelope boundary") + }) + + // A capability returned from a call carries no dialable address on an adopted + // connection, so following it must reuse the session it arrived on. + t.Run("follows a returned capability over the same connection", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + c, _ := adoptPair(t, ctx, example.AdaptMeter(&exampleMeter{temp: 42}), "meter") + + mc := &example.MeterClient{Client: c} + + res, err := mc.GetSetter(ctx, "test") + r.NoError(err) + + res2, err := res.Setter().SetTemp(ctx, 100) + r.NoError(err) + r.Equal(int32(100), res2.Temp()) + }) + + t.Run("server calls back over the same connection", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + c, _ := adoptPair(t, ctx, example.AdaptMeterUpdates(&exampleMU{}), "meter") + + mc := &example.MeterUpdatesClient{Client: c} + + var up exampleUpdate + _, err := mc.RegisterUpdates(ctx, &up) + r.NoError(err) + + r.True(up.gotIt) + r.Equal("test", up.reading.Meter()) + r.Equal(float32(42), up.reading.Temperature()) + }) + + t.Run("streaming callbacks over a caller-owned connection", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + c, _ := adoptPair(t, ctx, example.AdaptEmitTemps(&exampleEmit{}), "meter") + + mc := &example.EmitTempsClient{Client: c} + + var ( + mu sync.Mutex + vals []float32 + ) + recv := stream.StreamRecv(func(val float32) error { + mu.Lock() + defer mu.Unlock() + vals = append(vals, val) + return nil + }) + + _, err := mc.Emit(ctx, recv) + r.NoError(err) + + time.Sleep(500 * time.Millisecond) + + mu.Lock() + defer mu.Unlock() + r.Equal([]float32{42, 100}, vals) + }) + + // The whole point of an adopted connection: there is no second connection to + // dial, so every concurrent callstream must multiplex onto the one we were + // given. + t.Run("concurrent callstreams multiplex onto the one connection", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + c, _ := adoptPair(t, ctx, example.AdaptEmitTemps(&exampleEmit{}), "meter") + + const calls = 8 + + var ( + wg sync.WaitGroup + mu sync.Mutex + errs []error + got = make([][]float32, calls) + ) + + for i := range calls { + wg.Add(1) + go func() { + defer wg.Done() + + var ( + vmu sync.Mutex + vals []float32 + ) + recv := stream.StreamRecv(func(val float32) error { + vmu.Lock() + defer vmu.Unlock() + vals = append(vals, val) + return nil + }) + + mc := &example.EmitTempsClient{Client: c} + if _, err := mc.Emit(ctx, recv); err != nil { + mu.Lock() + errs = append(errs, err) + mu.Unlock() + return + } + + time.Sleep(500 * time.Millisecond) + + vmu.Lock() + defer vmu.Unlock() + got[i] = append([]float32(nil), vals...) + }() + } + + wg.Wait() + + mu.Lock() + defer mu.Unlock() + r.Empty(errs) + + for i := range calls { + r.Equal([]float32{42, 100}, got[i], "call %d received the wrong values", i) + } + }) + + // A message transport has no Authorization header and no TLS handshake, so a + // bearer token rides in the operation frame instead. Without this, cloudauth + // and oidcauth cannot identify a caller over one of these connections at all. + t.Run("carries a bearer token to the authenticator", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + auth := &recordingAuthenticator{ + identity: &rpc.Identity{Subject: "user-42", Method: rpc.AuthMethodJWT}, + } + + ss, err := rpc.NewState(ctx, rpc.WithSkipVerify, rpc.WithAuthenticator(auth)) + r.NoError(err) + + var seen *rpc.Identity + iface := example.AdaptMeter(&identityMeter{seen: &seen}) + ss.Server().ExposeValue("meter", iface) + + cs, err := rpc.NewState(ctx, rpc.WithSkipVerify, rpc.WithBearerToken("secret-token")) + r.NoError(err) + + serverEnd, clientEnd := newEnvelopePair() + go ss.ServeMessageConn(ctx, serverEnd) //nolint:errcheck // ends with ctx + + c, err := cs.ClientFromMessageConn(ctx, clientEnd, "meter") + r.NoError(err) + + mc := &example.MeterClient{Client: c} + _, err = mc.ReadTemperature(ctx, "test") + r.NoError(err) + + r.Equal("Bearer secret-token", auth.lastAuthorization()) + r.NotNil(seen) + r.Equal("user-42", seen.Subject) + r.Equal(rpc.AuthMethodJWT, seen.Method) + }) + + // An authorization denial must reach the caller as a permission error. It + // travels as its own opReply status, so without explicit handling it surfaces + // as an unrecognised-protocol-status error instead. + t.Run("reports an authorization denial as a permission error", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + ss, err := rpc.NewState(ctx, rpc.WithSkipVerify, + rpc.WithAuthenticator(&recordingAuthenticator{ + identity: &rpc.Identity{Subject: "nobody", Method: rpc.AuthMethodJWT}, + }), + rpc.WithAuthorizer(denyAll{}), + ) + r.NoError(err) + ss.Server().ExposeValue("meter", example.AdaptMeter(&exampleMeter{temp: 42})) + + cs, err := rpc.NewState(ctx, rpc.WithSkipVerify, rpc.WithBearerToken("token")) + r.NoError(err) + + serverEnd, clientEnd := newEnvelopePair() + go ss.ServeMessageConn(ctx, serverEnd) //nolint:errcheck // ends with ctx + + c, err := cs.ClientFromMessageConn(ctx, clientEnd, "meter") + r.NoError(err) + + mc := &example.MeterClient{Client: c} + _, err = mc.ReadTemperature(ctx, "test") + r.Error(err) + r.NotContains(err.Error(), "unknown response status") + r.Contains(err.Error(), "denied by policy") + }) + + // A frame cap below the payload size forces msgmux to split a write across + // several messages, which is what an envelope with its own size limit needs. + t.Run("respects a frame size below the payload size", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + c, conn := adoptPair(t, ctx, + example.AdaptMeter(&exampleMeter{temp: 42}), "meter", + rpc.WithMaxFrameSize(64), + ) + + mc := &example.MeterClient{Client: c} + + res, err := mc.ReadTemperature(ctx, "a-meter-name-long-enough-to-need-splitting-across-frames") + r.NoError(err) + r.Equal(float32(42), res.Reading().Temperature()) + + r.NotZero(conn.count()) + }) +} diff --git a/pkg/rpc/message_mem.go b/pkg/rpc/message_mem.go new file mode 100644 index 000000000..194451d6e --- /dev/null +++ b/pkg/rpc/message_mem.go @@ -0,0 +1,111 @@ +package rpc + +import ( + "context" + "fmt" + "io" + "net" + "sync" +) + +// memConn is an in-process MessageConn: two cross-wired buffered channels. It +// is reliable and ordered, satisfying msgmux's requirements with no network. +// Closing either end tears down both. +type memConn struct { + send chan []byte + recv chan []byte + + done chan struct{} + closeFn func() +} + +func newMemPipe() (*memConn, *memConn) { + ab := make(chan []byte, 64) + ba := make(chan []byte, 64) + done := make(chan struct{}) + var once sync.Once + closeFn := func() { once.Do(func() { close(done) }) } + + a := &memConn{send: ab, recv: ba, done: done, closeFn: closeFn} + b := &memConn{send: ba, recv: ab, done: done, closeFn: closeFn} + return a, b +} + +func (c *memConn) Send(b []byte) error { + cp := make([]byte, len(b)) + copy(cp, b) + select { + case c.send <- cp: + return nil + case <-c.done: + return net.ErrClosed + } +} + +func (c *memConn) Recv() ([]byte, error) { + select { + case b := <-c.recv: + return b, nil + case <-c.done: + // Drain anything already buffered before reporting EOF. + select { + case b := <-c.recv: + return b, nil + default: + return nil, io.EOF + } + } +} + +func (c *memConn) Close() error { + c.closeFn() + return nil +} + +// memRegistry maps a process-local name to a server State, mirroring the +// LocalActorRegistry pattern (actor.go). Dialing a name creates a memConn pair, +// spawns the server's session loop on one end, and returns the other end. +type memRegistry struct { + mu sync.Mutex + servers map[string]*memServerEntry +} + +type memServerEntry struct { + state *State + ctx context.Context +} + +var defaultMemRegistry = &memRegistry{servers: make(map[string]*memServerEntry)} + +func (r *memRegistry) register(ctx context.Context, name string, s *State) { + r.mu.Lock() + r.servers[name] = &memServerEntry{state: s, ctx: ctx} + r.mu.Unlock() + + go func() { + <-ctx.Done() + r.mu.Lock() + if e := r.servers[name]; e != nil && e.state == s { + delete(r.servers, name) + } + r.mu.Unlock() + }() +} + +func (r *memRegistry) dial(name string) (MessageConn, error) { + r.mu.Lock() + entry := r.servers[name] + r.mu.Unlock() + + if entry == nil { + return nil, fmt.Errorf("rpc: no mem server registered as %q", name) + } + + clientEnd, serverEnd := newMemPipe() + serverSess := newMsgSession(serverEnd, false, 0) + + router := &sessionRouter{state: entry.state, server: entry.state.server} + go router.run(entry.ctx, serverSess) //nolint:errcheck // loop ends with the session + + return clientEnd, nil +} diff --git a/pkg/rpc/message_net_test.go b/pkg/rpc/message_net_test.go new file mode 100644 index 000000000..52cf3bfd8 --- /dev/null +++ b/pkg/rpc/message_net_test.go @@ -0,0 +1,265 @@ +package rpc_test + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "miren.dev/runtime/pkg/cond" + "miren.dev/runtime/pkg/rpc" + "miren.dev/runtime/pkg/rpc/example" + "miren.dev/runtime/pkg/rpc/stream" +) + +// netTransport describes one of the networked message transports, so both get +// identical coverage from the table below. +type netTransport struct { + name string + listen func() rpc.StateOption + remote func(*rpc.State) string +} + +var netTransports = []netTransport{ + { + name: "websocket", + listen: func() rpc.StateOption { return rpc.WithWSBindAddr("localhost:0") }, + remote: func(s *rpc.State) string { return "ws://" + s.WSListenAddr() }, + }, + { + name: "tcp", + listen: func() rpc.StateOption { return rpc.WithTCPBindAddr("localhost:0") }, + remote: func(s *rpc.State) string { return "tcp://" + s.TCPListenAddr() }, + }, +} + +// netServer starts a State listening on the given transport and exposes iface +// under name, returning the remote to connect to. +func netServer(t *testing.T, ctx context.Context, tr netTransport, name string, iface *rpc.Interface) string { + t.Helper() + r := require.New(t) + + ss, err := rpc.NewState(ctx, rpc.WithSkipVerify, tr.listen()) + r.NoError(err) + + ss.Server().ExposeValue(name, iface) + + return tr.remote(ss) +} + +func netClient(t *testing.T, ctx context.Context, remote, name string) rpc.Client { + t.Helper() + r := require.New(t) + + cs, err := rpc.NewState(ctx, rpc.WithSkipVerify) + r.NoError(err) + + c, err := cs.Connect(remote, name) + r.NoError(err) + + return c +} + +// blockingMU is a MeterUpdates implementation whose handler never replies until +// its context is cancelled (bounded so it can't leak), used to exercise +// caller-side cancellation of a callstream. +type blockingMU struct { + started chan struct{} +} + +func (m *blockingMU) RegisterUpdates(ctx context.Context, call *example.MeterUpdatesRegisterUpdates) error { + close(m.started) + select { + case <-ctx.Done(): + case <-time.After(2 * time.Second): + } + return cond.Closed("server done") +} + +func TestNetMessageTransports(t *testing.T) { + for _, tr := range netTransports { + t.Run(tr.name, func(t *testing.T) { + t.Run("unary calls", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := netServer(t, ctx, tr, "meter", example.AdaptMeter(&exampleMeter{temp: 42})) + c := netClient(t, ctx, remote, "meter") + + mc := &example.MeterClient{Client: c} + + res, err := mc.ReadTemperature(ctx, "test") + r.NoError(err) + r.Equal("test", res.Reading().Meter()) + r.Equal(float32(42), res.Reading().Temperature()) + + // GetSetter returns a capability; calling it round-trips a second + // unary call over the same session. + res2, err := mc.GetSetter(ctx, "test") + r.NoError(err) + + res3, err := res2.Setter().SetTemp(ctx, 100) + r.NoError(err) + r.Equal(int32(100), res3.Temp()) + }) + + t.Run("server calls back on a client-advertised capability", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := netServer(t, ctx, tr, "meter", example.AdaptMeterUpdates(&exampleMU{})) + c := netClient(t, ctx, remote, "meter") + + mc := &example.MeterUpdatesClient{Client: c} + + var up exampleUpdate + + // The server invokes up.Update over the same connection — the core + // bidirectional capability/callback behavior. + _, err := mc.RegisterUpdates(ctx, &up) + r.NoError(err) + + r.True(up.gotIt) + r.Equal("test", up.reading.Meter()) + r.Equal(float32(42), up.reading.Temperature()) + + time.Sleep(100 * time.Millisecond) // wait for the goroutine running Close + + up.mu.Lock() + defer up.mu.Unlock() + r.True(up.closed) + }) + + t.Run("streaming callbacks", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := netServer(t, ctx, tr, "meter", example.AdaptEmitTemps(&exampleEmit{})) + c := netClient(t, ctx, remote, "meter") + + mc := &example.EmitTempsClient{Client: c} + + var ( + mu sync.Mutex + vals []float32 + ) + recv := stream.StreamRecv(func(val float32) error { + mu.Lock() + defer mu.Unlock() + vals = append(vals, val) + return nil + }) + + _, err := mc.Emit(ctx, recv) + r.NoError(err) + + time.Sleep(time.Second) + + mu.Lock() + defer mu.Unlock() + r.Equal([]float32{42, 100}, vals) + }) + + t.Run("concurrent callstreams share one connection", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := netServer(t, ctx, tr, "meter", example.AdaptEmitTemps(&exampleEmit{})) + c := netClient(t, ctx, remote, "meter") + + const calls = 8 + + var ( + wg sync.WaitGroup + mu sync.Mutex + errs []error + got = make([][]float32, calls) + ) + + for i := range calls { + wg.Add(1) + go func() { + defer wg.Done() + + var ( + vmu sync.Mutex + vals []float32 + ) + recv := stream.StreamRecv(func(val float32) error { + vmu.Lock() + defer vmu.Unlock() + vals = append(vals, val) + return nil + }) + + mc := &example.EmitTempsClient{Client: c} + if _, err := mc.Emit(ctx, recv); err != nil { + mu.Lock() + errs = append(errs, err) + mu.Unlock() + return + } + + time.Sleep(500 * time.Millisecond) + + vmu.Lock() + defer vmu.Unlock() + got[i] = append([]float32(nil), vals...) + }() + } + + wg.Wait() + + mu.Lock() + defer mu.Unlock() + r.Empty(errs) + + for i := range calls { + r.Equal([]float32{42, 100}, got[i], "call %d received the wrong values", i) + } + }) + + t.Run("cancelled callstream reports closed", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + m := &blockingMU{started: make(chan struct{})} + remote := netServer(t, ctx, tr, "meter", example.AdaptMeterUpdates(m)) + c := netClient(t, ctx, remote, "meter") + + mc := &example.MeterUpdatesClient{Client: c} + + cctx, cancel := context.WithCancel(ctx) + go func() { + <-m.started + time.Sleep(50 * time.Millisecond) + cancel() + }() + + var up exampleUpdate + _, err := mc.RegisterUpdates(cctx, &up) + r.Error(err) + r.True(errors.Is(err, cond.ErrClosed{}), "expected ErrClosed, got %T: %v", err, err) + }) + + t.Run("propagates panic errors", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := netServer(t, ctx, tr, "meter", example.AdaptMeter(&panicMeter{})) + c := netClient(t, ctx, remote, "meter") + + mc := &example.MeterClient{Client: c} + + _, err := mc.ReadTemperature(ctx, "test") + r.Error(err) + + var panicErr cond.ErrPanic + r.True(errors.As(err, &panicErr), "expected ErrPanic, got %T: %v", err, err) + r.Contains(panicErr.Message, "test panic message") + }) + }) + } +} diff --git a/pkg/rpc/message_tcp.go b/pkg/rpc/message_tcp.go new file mode 100644 index 000000000..2d06dfd39 --- /dev/null +++ b/pkg/rpc/message_tcp.go @@ -0,0 +1,158 @@ +package rpc + +import ( + "bufio" + "context" + "crypto/tls" + "encoding/binary" + "fmt" + "io" + "net" + "strings" + "sync" +) + +// tcpFrameHeader is the 4-byte big-endian length prefix that turns a TCP byte +// stream into the discrete messages msgmux consumes. +const tcpFrameHeader = 4 + +// tcpConn is a MessageConn over a byte stream, framing each message with a +// length prefix. TLS is applied by the caller; this type only frames. +type tcpConn struct { + conn net.Conn + r *bufio.Reader + + // maxFrame bounds an inbound frame. A peer announcing more than this is + // rejected before anything is allocated, so a bad length cannot exhaust + // memory. + maxFrame int + + wmu sync.Mutex + hdr [tcpFrameHeader]byte +} + +func newTCPConn(conn net.Conn, maxFrame int) *tcpConn { + if maxFrame <= 0 { + maxFrame = defaultMaxFrameData + } + + return &tcpConn{ + conn: conn, + r: bufio.NewReader(conn), + // A frame carries a payload plus msgmux's own CBOR envelope, so allow + // headroom over the payload bound rather than rejecting valid frames. + maxFrame: maxFrame + (1 << 16), + } +} + +func (t *tcpConn) Send(b []byte) error { + t.wmu.Lock() + defer t.wmu.Unlock() + + binary.BigEndian.PutUint32(t.hdr[:], uint32(len(b))) + + if _, err := t.conn.Write(t.hdr[:]); err != nil { + return err + } + _, err := t.conn.Write(b) + return err +} + +func (t *tcpConn) Recv() ([]byte, error) { + var hdr [tcpFrameHeader]byte + if _, err := io.ReadFull(t.r, hdr[:]); err != nil { + return nil, err + } + + n := binary.BigEndian.Uint32(hdr[:]) + if int(n) > t.maxFrame { + return nil, fmt.Errorf("rpc: peer announced a %d byte frame, over the %d byte limit", n, t.maxFrame) + } + + buf := make([]byte, n) + if _, err := io.ReadFull(t.r, buf); err != nil { + return nil, err + } + return buf, nil +} + +func (t *tcpConn) Close() error { + return t.conn.Close() +} + +// startTCPListener serves RPC over raw TLS-over-TCP. There is no HTTP layer: +// each accepted connection is framed into messages and handed straight to the +// session router. +func (s *State) startTCPListener(ctx context.Context, addr string) error { + tlsCfg := s.serverTlsCfg.Clone() + tlsCfg.NextProtos = nil // no ALPN: nothing here speaks HTTP + + ln, err := tls.Listen("tcp", addr, tlsCfg) + if err != nil { + return err + } + + s.msgLn = ln + + go func() { + <-ctx.Done() + _ = ln.Close() + }() + + s.log.Debug("starting tcp message listener", "addr", ln.Addr().String()) + + go func() { + for { + conn, aerr := ln.Accept() + if aerr != nil { + if ctx.Err() == nil { + s.log.Error("tcp message listener stopped", "error", aerr) + } + return + } + + go func() { + mc := newTCPConn(conn, 0) + if serr := s.ServeMessageConn(ctx, mc); serr != nil { + s.log.Debug("rpc: tcp session ended", "error", serr) + } + _ = mc.Close() + }() + } + }() + + return nil +} + +// TCPListenAddr returns the address the raw TCP message listener is bound to, +// or an empty string if no such listener is running. +func (s *State) TCPListenAddr() string { + if s.msgLn == nil { + return "" + } + return s.msgLn.Addr().String() +} + +// setupTCPTransport wires a client for a "tcp://host:port" remote. +func (c *NetworkClient) setupTCPTransport() { + remote := strings.TrimPrefix(c.remote, "tcp://") + + tlsCfg := c.tlsCfg + if tlsCfg == nil { + tlsCfg = c.State.clientTlsCfg + } + + cfg := tlsCfg.Clone() + cfg.NextProtos = nil + + c.ops = &msgOpTransport{ + owner: c, + dial: func() (MessageConn, error) { + conn, err := tls.Dial("tcp", remote, cfg) + if err != nil { + return nil, err + } + return newTCPConn(conn, 0), nil + }, + } +} diff --git a/pkg/rpc/message_version_test.go b/pkg/rpc/message_version_test.go new file mode 100644 index 000000000..efee76288 --- /dev/null +++ b/pkg/rpc/message_version_test.go @@ -0,0 +1,90 @@ +package rpc + +import ( + "testing" + + "github.com/fxamacker/cbor/v2" + "github.com/stretchr/testify/require" +) + +// The version gate is what lets a second implementation appear later without a +// breaking change, so it needs to actually reject and actually accept. +func TestProtocolVersionGate(t *testing.T) { + t.Run("absent version reads as version 1", func(t *testing.T) { + r := require.New(t) + + r.Equal(uint(protocolVersion1), opRequest{}.protocolVersion()) + r.Equal(uint(protocolVersion1), opRequest{Version: 1}.protocolVersion()) + r.Equal(uint(7), opRequest{Version: 7}.protocolVersion()) + }) + + // A peer speaking a version this build does not know must be told so, rather + // than having its frames misread as the version we do know. + t.Run("rejects an unsupported version with the supported range", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + clientEnd, serverEnd := newMemPipe() + + state, err := NewState(ctx, WithSkipVerify) + r.NoError(err) + + router := &sessionRouter{state: state, server: state.server} + go router.run(ctx, newMsgSession(serverEnd, false, 0)) //nolint:errcheck // ends with ctx + + sess := newMsgSession(clientEnd, true, 0) + st, err := sess.OpenStreamSync(ctx) + r.NoError(err) + + err = cbor.NewEncoder(st).Encode(opRequest{ + Op: opLookup, + Name: "whatever", + Version: currentProtocolVersion + 1, + }) + r.NoError(err) + + var reply opReply + r.NoError(cbor.NewDecoder(st).Decode(&reply)) + + r.Equal("error", reply.Status) + r.Contains(reply.Error, "unsupported protocol version") + }) + + // A peer predating the field sends no version at all; it must still work. + t.Run("accepts a request carrying no version", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + clientEnd, serverEnd := newMemPipe() + + state, err := NewState(ctx, WithSkipVerify) + r.NoError(err) + + router := &sessionRouter{state: state, server: state.server} + go router.run(ctx, newMsgSession(serverEnd, false, 0)) //nolint:errcheck // ends with ctx + + sess := newMsgSession(clientEnd, true, 0) + st, err := sess.OpenStreamSync(ctx) + r.NoError(err) + + err = cbor.NewEncoder(st).Encode(opRequest{ + Op: opLookup, + Name: "nothing-exposed", + PubKey: state.pubkey, + }) + r.NoError(err) + + dec := cbor.NewDecoder(st) + + var reply opReply + r.NoError(dec.Decode(&reply)) + + // The lookup itself fails (nothing is exposed), but it was dispatched + // rather than rejected by the version gate. + r.Equal("ok", reply.Status) + + var lr lookupResponse + r.NoError(dec.Decode(&lr)) + r.Contains(lr.Error, "unknown object") + }) +} diff --git a/pkg/rpc/message_ws.go b/pkg/rpc/message_ws.go new file mode 100644 index 000000000..de46b165c --- /dev/null +++ b/pkg/rpc/message_ws.go @@ -0,0 +1,111 @@ +package rpc + +import ( + "context" + "crypto/tls" + "net/http" + "strings" + + "github.com/coder/websocket" +) + +// wsMessagePath is the single route the TCP/WebSocket listener serves. Every +// operation rides the multiplexed session established by the upgrade, so unlike +// the HTTP/3 transport there is no per-operation URL. +const wsMessagePath = "/_rpc/message" + +// wsConn is a MessageConn over a WebSocket. +// +// A WebSocket already delivers discrete, ordered, reliable messages — exactly +// what msgmux consumes — so the frames go on the wire as-is. Nothing here +// converts the connection to a byte stream and re-frames it. +type wsConn struct { + c *websocket.Conn + ctx context.Context +} + +func (w *wsConn) Send(b []byte) error { + return w.c.Write(w.ctx, websocket.MessageBinary, b) +} + +func (w *wsConn) Recv() ([]byte, error) { + _, data, err := w.c.Read(w.ctx) + if err != nil { + return nil, err + } + return data, nil +} + +func (w *wsConn) Close() error { + return w.c.Close(websocket.StatusNormalClosure, "") +} + +// dialWSMessageConn opens a WebSocket to a "host:port" remote and wraps it as a +// MessageConn. ctx bounds the connection's lifetime, so it must be the State's +// context rather than any one call's. +func dialWSMessageConn(ctx context.Context, remote string, tlsCfg *tls.Config) (MessageConn, error) { + // coder/websocket does not implement RFC 8441 extended CONNECT, so the + // handshake must be HTTP/1.1. + cfg := tlsCfg.Clone() + cfg.NextProtos = []string{"http/1.1"} + + c, _, err := websocket.Dial(ctx, "wss://"+remote+wsMessagePath, &websocket.DialOptions{ + HTTPClient: &http.Client{ + Transport: &http.Transport{TLSClientConfig: cfg}, + }, + }) + if err != nil { + return nil, err + } + + // A single frame may legitimately be as large as the session's frame cap; + // the default read limit is far below that. + c.SetReadLimit(-1) + + return &wsConn{c: c, ctx: ctx}, nil +} + +// serveWSUpgrade accepts a WebSocket upgrade and serves RPC over it. +// +// The session is bound to lifeCtx (the server's lifetime), not r.Context(): +// websocket.Accept hijacks the TCP socket and net/http cancels the request +// context as soon as this handler returns, while the hijacked socket remains +// open and owned by us. Binding to the request context would tear the session +// down immediately. +func (s *State) serveWSUpgrade(lifeCtx context.Context, w http.ResponseWriter, r *http.Request) { + c, err := websocket.Accept(w, r, &websocket.AcceptOptions{ + InsecureSkipVerify: true, // origin checks are not meaningful for service-to-service RPC + }) + if err != nil { + s.log.Error("rpc: websocket upgrade failed", "error", err) + return + } + + c.SetReadLimit(-1) + + go func() { + conn := &wsConn{c: c, ctx: lifeCtx} + if serr := s.ServeMessageConn(lifeCtx, conn); serr != nil { + s.log.Debug("rpc: websocket session ended", "error", serr) + } + _ = conn.Close() + }() +} + +// setupWSTransport wires a client for a "ws://host:port" remote onto the +// message transport. +func (c *NetworkClient) setupWSTransport() { + remote := strings.TrimPrefix(strings.TrimPrefix(c.remote, "wss://"), "ws://") + + tlsCfg := c.tlsCfg + if tlsCfg == nil { + tlsCfg = c.State.clientTlsCfg + } + + c.ops = &msgOpTransport{ + owner: c, + dial: func() (MessageConn, error) { + return dialWSMessageConn(c.State.top, remote, tlsCfg) + }, + } +} diff --git a/pkg/rpc/msgmux.go b/pkg/rpc/msgmux.go new file mode 100644 index 000000000..239c84f81 --- /dev/null +++ b/pkg/rpc/msgmux.go @@ -0,0 +1,355 @@ +package rpc + +import ( + "bytes" + "context" + "errors" + "io" + "sync" + + "github.com/fxamacker/cbor/v2" +) + +// msgmux is a minimal stream multiplexer that implements rpcSession/rpcStream +// over a MessageConn. It is to a MessageConn what yamux is to a byte conn, but +// because a MessageConn already delivers reliable, ordered, discrete messages, +// msgmux needs no flow control: each frame is one message, and the backend's +// own backpressure governs throughput. +// +// Frame flags. A stream is opened with an eager SYN (so AcceptStream fires even +// if the opener reads before writing); data frames carry DATA; a clean close +// sends FIN; CancelRead/abort sends RST. +const ( + flagSYN uint8 = 1 << 0 + flagDATA uint8 = 1 << 1 + flagFIN uint8 = 1 << 2 + flagRST uint8 = 1 << 3 +) + +type msgFrame struct { + Stream uint32 `cbor:"0,keyasint"` + Flags uint8 `cbor:"1,keyasint"` + Data []byte `cbor:"2,keyasint,omitempty"` +} + +var ( + errSessionClosed = errors.New("rpc: message session closed") + errStreamReset = errors.New("rpc: message stream reset") + errStreamCancel = errors.New("rpc: message stream read canceled") +) + +type msgSession struct { + conn MessageConn + + // maxFrame bounds the payload in one frame, so a backend that caps message + // size can still carry arbitrarily large stream writes. + maxFrame int + + sendMu sync.Mutex + + mu sync.Mutex + streams map[uint32]*msgStream + nextID uint32 + idStep uint32 + closed bool + closeErr error + + accept chan *msgStream + done chan struct{} +} + +// newMsgSession wraps a MessageConn. Exactly one side must pass client=true +// (the dialer); it uses odd stream ids while the accepter uses even ids, so ids +// never collide. A maxFrame of zero uses defaultMaxFrameData. +func newMsgSession(conn MessageConn, client bool, maxFrame int) *msgSession { + var start uint32 = 2 + if client { + start = 1 + } + if maxFrame <= 0 { + maxFrame = defaultMaxFrameData + } + s := &msgSession{ + conn: conn, + maxFrame: maxFrame, + streams: make(map[uint32]*msgStream), + nextID: start, + idStep: 2, + accept: make(chan *msgStream, 16), + done: make(chan struct{}), + } + go s.readPump() + return s +} + +func (s *msgSession) newStream(id uint32) *msgStream { + st := &msgStream{sess: s, id: id} + st.readCond = sync.NewCond(&st.readMu) + return st +} + +func (s *msgSession) sendFrame(f msgFrame) error { + data, err := cbor.Marshal(f) + if err != nil { + return err + } + s.sendMu.Lock() + defer s.sendMu.Unlock() + return s.conn.Send(data) +} + +func (s *msgSession) OpenStreamSync(ctx context.Context) (rpcStream, error) { + s.mu.Lock() + if s.closed { + err := s.closeErr + s.mu.Unlock() + return nil, err + } + id := s.nextID + s.nextID += s.idStep + st := s.newStream(id) + s.streams[id] = st + s.mu.Unlock() + + if err := s.sendFrame(msgFrame{Stream: id, Flags: flagSYN}); err != nil { + s.removeStream(id) + return nil, err + } + st.synSent = true + return st, nil +} + +func (s *msgSession) AcceptStream(ctx context.Context) (rpcStream, error) { + select { + case st := <-s.accept: + return st, nil + case <-s.done: + return nil, s.closeErr + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (s *msgSession) Close() error { + return s.conn.Close() +} + +func (s *msgSession) removeStream(id uint32) { + s.mu.Lock() + delete(s.streams, id) + s.mu.Unlock() +} + +func (s *msgSession) readPump() { + for { + data, err := s.conn.Recv() + if err != nil { + s.shutdown(err) + return + } + + var f msgFrame + if err := cbor.Unmarshal(data, &f); err != nil { + s.shutdown(err) + return + } + + s.handleFrame(f) + } +} + +func (s *msgSession) handleFrame(f msgFrame) { + s.mu.Lock() + st := s.streams[f.Stream] + newStream := false + if st == nil && f.Flags&flagSYN != 0 && !s.closed { + st = s.newStream(f.Stream) + st.synSent = true // peer opened it; we never send a SYN for it + s.streams[f.Stream] = st + newStream = true + } + s.mu.Unlock() + + if st == nil { + return // data for an unknown, un-SYN'd stream: drop + } + + if newStream { + select { + case s.accept <- st: + case <-s.done: + return + } + } + + if f.Flags&flagRST != 0 { + st.fail(errStreamReset) + s.removeStream(f.Stream) + return + } + + if f.Flags&(flagDATA|flagFIN) != 0 { + st.feed(f.Data, f.Flags&flagFIN != 0) + } + + if f.Flags&flagFIN != 0 { + s.maybeRemove(st) + } +} + +func (s *msgSession) shutdown(err error) { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return + } + s.closed = true + s.closeErr = err + streams := make([]*msgStream, 0, len(s.streams)) + for _, st := range s.streams { + streams = append(streams, st) + } + s.streams = map[uint32]*msgStream{} + close(s.done) + s.mu.Unlock() + + for _, st := range streams { + st.fail(err) + } +} + +// maybeRemove drops a stream from the table once both directions are finished, +// so a long-lived session multiplexing many short ops does not leak entries. +func (s *msgSession) maybeRemove(st *msgStream) { + st.readMu.Lock() + remoteDone := st.remoteClosed + st.readMu.Unlock() + + st.wmu.Lock() + localDone := st.localClosed + st.wmu.Unlock() + + if remoteDone && localDone { + s.removeStream(st.id) + } +} + +type msgStream struct { + sess *msgSession + id uint32 + + readMu sync.Mutex + readCond *sync.Cond + buf bytes.Buffer + remoteClosed bool + readErr error + canceled bool + + wmu sync.Mutex + synSent bool + localClosed bool +} + +func (st *msgStream) feed(data []byte, fin bool) { + st.readMu.Lock() + if len(data) > 0 { + st.buf.Write(data) + } + if fin { + st.remoteClosed = true + } + st.readCond.Broadcast() + st.readMu.Unlock() +} + +func (st *msgStream) fail(err error) { + st.readMu.Lock() + if st.readErr == nil { + st.readErr = err + } + st.readCond.Broadcast() + st.readMu.Unlock() +} + +func (st *msgStream) Read(p []byte) (int, error) { + st.readMu.Lock() + defer st.readMu.Unlock() + + for st.buf.Len() == 0 { + switch { + case st.canceled: + return 0, errStreamCancel + case st.buf.Len() == 0 && st.readErr != nil: + return 0, st.readErr + case st.remoteClosed: + return 0, io.EOF + } + st.readCond.Wait() + } + + return st.buf.Read(p) +} + +func (st *msgStream) Write(p []byte) (int, error) { + st.wmu.Lock() + defer st.wmu.Unlock() + + if st.localClosed { + return 0, errSessionClosed + } + + total := 0 + for len(p) > 0 { + chunk := p + if len(chunk) > st.sess.maxFrame { + chunk = chunk[:st.sess.maxFrame] + } + + flags := flagDATA + if !st.synSent { + flags |= flagSYN + st.synSent = true + } + + if err := st.sess.sendFrame(msgFrame{Stream: st.id, Flags: flags, Data: chunk}); err != nil { + return total, err + } + + total += len(chunk) + p = p[len(chunk):] + } + + return total, nil +} + +func (st *msgStream) Close() error { + st.wmu.Lock() + if st.localClosed { + st.wmu.Unlock() + return nil + } + st.localClosed = true + flags := flagFIN + if !st.synSent { + flags |= flagSYN + st.synSent = true + } + st.wmu.Unlock() + + err := st.sess.sendFrame(msgFrame{Stream: st.id, Flags: flags}) + st.sess.maybeRemove(st) + return err +} + +// CancelRead aborts a Read currently blocked on this stream and signals the +// peer (RST). The code is unused: msgmux has no per-stream error code, so +// callers detect that the cancellation was caller-initiated by inspecting their +// context. +func (st *msgStream) CancelRead(code uint64) { + st.readMu.Lock() + st.canceled = true + st.readCond.Broadcast() + st.readMu.Unlock() + + _ = st.sess.sendFrame(msgFrame{Stream: st.id, Flags: flagRST}) +} diff --git a/pkg/rpc/msgmux_test.go b/pkg/rpc/msgmux_test.go new file mode 100644 index 000000000..75cbd84b7 --- /dev/null +++ b/pkg/rpc/msgmux_test.go @@ -0,0 +1,201 @@ +package rpc + +import ( + "context" + "io" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +// readN reads exactly n bytes from r (the mux stream chunks writes into frames). +func readN(t *testing.T, r io.Reader, n int) []byte { + t.Helper() + buf := make([]byte, n) + _, err := io.ReadFull(r, buf) + require.NoError(t, err) + return buf +} + +func TestMsgMux(t *testing.T) { + t.Run("client opens, server accepts, bidirectional bytes", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + ca, cb := newMemPipe() + client := newMsgSession(ca, true, 0) + server := newMsgSession(cb, false, 0) + + st, err := client.OpenStreamSync(ctx) + r.NoError(err) + + _, err = st.Write([]byte("hello")) + r.NoError(err) + + srvStream, err := server.AcceptStream(ctx) + r.NoError(err) + r.Equal([]byte("hello"), readN(t, srvStream, 5)) + + // reply server -> client + _, err = srvStream.Write([]byte("world")) + r.NoError(err) + r.Equal([]byte("world"), readN(t, st, 5)) + + r.NoError(st.Close()) + r.NoError(server.Close()) + r.NoError(client.Close()) + }) + + t.Run("server can also open streams to client", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + ca, cb := newMemPipe() + client := newMsgSession(ca, true, 0) + server := newMsgSession(cb, false, 0) + + // server-initiated stream + ss, err := server.OpenStreamSync(ctx) + r.NoError(err) + _, err = ss.Write([]byte("ping")) + r.NoError(err) + + cs, err := client.AcceptStream(ctx) + r.NoError(err) + r.Equal([]byte("ping"), readN(t, cs, 4)) + + client.Close() + server.Close() + }) + + t.Run("FIN delivers EOF after buffered data", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + ca, cb := newMemPipe() + client := newMsgSession(ca, true, 0) + server := newMsgSession(cb, false, 0) + + st, err := client.OpenStreamSync(ctx) + r.NoError(err) + _, err = st.Write([]byte("data")) + r.NoError(err) + r.NoError(st.Close()) // FIN + + ss, err := server.AcceptStream(ctx) + r.NoError(err) + r.Equal([]byte("data"), readN(t, ss, 4)) + + // next read sees EOF + _, err = ss.Read(make([]byte, 1)) + r.ErrorIs(err, io.EOF) + + client.Close() + server.Close() + }) + + t.Run("concurrent streams do not interleave", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + ca, cb := newMemPipe() + client := newMsgSession(ca, true, 0) + server := newMsgSession(cb, false, 0) + + // Server echoes each accepted stream. + go func() { + for { + ss, err := server.AcceptStream(ctx) + if err != nil { + return + } + go func(s rpcStream) { + buf := make([]byte, 8) + n, _ := io.ReadFull(s, buf) + s.Write(buf[:n]) + s.Close() + }(ss) + } + }() + + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + st, err := client.OpenStreamSync(ctx) + r.NoError(err) + payload := []byte{byte(i), byte(i), byte(i), byte(i), byte(i), byte(i), byte(i), byte(i)} + _, err = st.Write(payload) + r.NoError(err) + r.Equal(payload, readN(t, st, 8)) + st.Close() + }(i) + } + wg.Wait() + + client.Close() + server.Close() + }) + + t.Run("CancelRead unblocks a blocked reader", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + ca, cb := newMemPipe() + client := newMsgSession(ca, true, 0) + server := newMsgSession(cb, false, 0) + + st, err := client.OpenStreamSync(ctx) + r.NoError(err) + _, err = st.Write([]byte("x")) // ensure server sees the stream + r.NoError(err) + _, err = server.AcceptStream(ctx) + r.NoError(err) + + done := make(chan error, 1) + go func() { + _, err := st.Read(make([]byte, 8)) + done <- err + }() + + // drain the one byte we wrote isn't on this side; the read above blocks. + st.CancelRead(cancelReadCode) + r.ErrorIs(<-done, errStreamCancel) + + client.Close() + server.Close() + }) + + t.Run("large payload spans multiple frames", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + ca, cb := newMemPipe() + client := newMsgSession(ca, true, 0) + server := newMsgSession(cb, false, 0) + + big := make([]byte, defaultMaxFrameData*2+1234) + for i := range big { + big[i] = byte(i) + } + + go func() { + st, _ := client.OpenStreamSync(ctx) + st.Write(big) + st.Close() + }() + + ss, err := server.AcceptStream(ctx) + r.NoError(err) + got, err := io.ReadAll(ss) + r.NoError(err) + r.Equal(big, got) + + client.Close() + server.Close() + }) + + _ = context.Background +} diff --git a/pkg/rpc/server.go b/pkg/rpc/server.go index 3661d1ed0..eac112727 100644 --- a/pkg/rpc/server.go +++ b/pkg/rpc/server.go @@ -1,7 +1,6 @@ package rpc import ( - "bytes" "context" "crypto/ed25519" "crypto/rand" @@ -70,13 +69,19 @@ type heldCapability struct { category string - lastContact time.Time + // lastContact is unix nanos, held atomically: concurrent calls against the + // same capability all touch it, and they are not otherwise serialized. + lastContact atomic.Int64 pub ed25519.PublicKey } func (h *heldCapability) touch() { - h.lastContact = time.Now() + h.lastContact.Store(time.Now().UnixNano()) +} + +func (h *heldCapability) LastContact() time.Time { + return time.Unix(0, h.lastContact.Load()) } func (h *heldCapability) Close() error { @@ -223,10 +228,10 @@ func (s *Server) assignCapability(i *Interface, pub ed25519.PublicKey, contactAd heldInterface: &heldInterface{ Interface: i, }, - category: category, - lastContact: time.Now(), - pub: pub, + category: category, + pub: pub, } + hc.touch() if i.restoreState != nil { if rs, err := i.restoreState.RestoreState(i); err == nil { @@ -264,7 +269,7 @@ func (s *Server) reexportCapability(target OID, cur *heldCapability, pub ed25519 oid := OID(base58.Encode(buf)) if contactAddr == "" { - contactAddr = s.state.transport.Conn.LocalAddr().String() + contactAddr = s.state.contactAddr() } capa := &Capability{ @@ -276,9 +281,9 @@ func (s *Server) reexportCapability(target OID, cur *heldCapability, pub ed25519 hc := &heldCapability{ heldInterface: cur.heldInterface, - lastContact: time.Now(), pub: pub, } + hc.touch() hc.refs.Add(1) @@ -294,6 +299,9 @@ func (s *Server) setupMux() { mux := http.NewServeMux() mux.HandleFunc("POST /_rpc/call/{oid}/{method}", s.handleCalls) + // CONNECT upgrades to WebTransport over HTTP/3. TCP clients use the message + // transport, which multiplexes every operation over one session and never + // reaches this mux. mux.HandleFunc("CONNECT /_rpc/callstream/{oid}/{method}", s.startCallStream) mux.HandleFunc("POST /_rpc/lookup/{name}", s.lookup) mux.HandleFunc("GET /_rpc/methods/{oid}", s.listMethods) @@ -318,7 +326,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // Try to authenticate the request - identity, err := s.state.authenticator.Authenticate(ctx, r) + identity, err := s.state.authenticator.Authenticate(ctx, CredentialsFromRequest(r)) if err != nil { s.state.log.Warn("authentication failed", "error", err, "path", r.URL.Path) http.Error(w, "authentication failed", http.StatusUnauthorized) @@ -358,19 +366,19 @@ type identifyResponse struct { Error string `json:"error,omitempty"` Address string `json:"address,omitempty"` Identity string `json:"identity,omitempty"` + + // MinVersion and MaxVersion advertise the protocol versions this peer + // accepts, so a client can adapt instead of discovering the mismatch one + // rejected stream at a time. Set only on message transports. See PROTOCOL.md. + MinVersion uint `json:"min_version,omitempty"` + MaxVersion uint `json:"max_version,omitempty"` } func (s *Server) checkIdentity(r *http.Request) (string, bool) { ts := r.Header.Get("rpc-timestamp") - t, err := time.Parse(time.RFC3339Nano, ts) - if err != nil { - s.state.log.Warn("Failed to parse timestamp", "error", err) - return "", false - } - - if time.Since(t) > 10*time.Minute { - s.state.log.Warn("Timestamp too old", "timestamp", t) + if err := freshTimestamp(ts); err != nil { + s.state.log.Warn("identity timestamp rejected", "error", err) return "", false } @@ -380,32 +388,15 @@ func (s *Server) checkIdentity(r *http.Request) (string, bool) { return "", false } - var buf bytes.Buffer - - fmt.Fprintf(&buf, "%s %s %s", r.Method, r.URL.Path, ts) - - bsign, err := base58.Decode(sign) - if err != nil { - s.state.log.Warn("Failed to decode signature", "error", err) - return "", false - } - - spkey := r.Header.Get("rpc-public-key") - - key, err := base58.Decode(spkey) + key, err := base58.Decode(r.Header.Get("rpc-public-key")) if err != nil { return "", false } pub := ed25519.PublicKey(key) - if len(pub) != ed25519.PublicKeySize { - s.state.log.Warn("Invalid public key size", "size", len(pub)) - return "", false - } - - if !ed25519.Verify(pub, buf.Bytes(), bsign) { - s.state.log.Warn("Failed to verify signature") + if err := verifyString(pub, httpCanonical(r.Method, r.URL.Path, ts), sign); err != nil { + s.state.log.Warn("Failed to verify identity signature", "error", err) return "", false } @@ -455,7 +446,7 @@ func (s *Server) handleDebugAuth(w http.ResponseWriter, r *http.Request) { // Check authentication using the new Authenticate interface if s.state.authenticator != nil { - identity, err := s.state.authenticator.Authenticate(r.Context(), r) + identity, err := s.state.authenticator.Authenticate(r.Context(), CredentialsFromRequest(r)) if err != nil { resp.Success = false resp.Message = fmt.Sprintf("Authentication failed: %v", err) @@ -511,7 +502,7 @@ func (s *Server) reexport(w http.ResponseWriter, r *http.Request) { // abilities. ca := r.Header.Get("rpc-contact-addr") if ca != "" { - ca = s.state.transport.Conn.LocalAddr().String() + ca = s.state.contactAddr() } w.WriteHeader(http.StatusOK) @@ -609,7 +600,7 @@ func (s *Server) lookup(w http.ResponseWriter, r *http.Request) { // abilities. ca := r.Header.Get("rpc-contact-addr") if ca != "" { - ca = s.state.transport.Conn.LocalAddr().String() + ca = s.state.contactAddr() } //s.state.log.Info("Lookup", "name", name) @@ -706,7 +697,7 @@ func (s *Server) reresolve(w http.ResponseWriter, r *http.Request) { // abilities. ca := r.Header.Get("rpc-contact-addr") if ca != "" { - ca = s.state.transport.Conn.LocalAddr().String() + ca = s.state.contactAddr() } // TODO: add condition codes to the error response rather than just a string @@ -795,16 +786,9 @@ func (s *Server) authRequest(r *http.Request, w http.ResponseWriter, oid OID) (e return nil, false } - t, err := time.Parse(time.RFC3339Nano, ts) - if err != nil { - logAuthReject(s.state.audit(), r, oid, "malformed timestamp") - http.Error(w, "failed to parse timestamp", http.StatusForbidden) - return nil, false - } - - if time.Since(t) > 10*time.Minute { - logAuthReject(s.state.audit(), r, oid, "timestamp too old") - http.Error(w, "timestamp too old", http.StatusForbidden) + if err := freshTimestamp(ts); err != nil { + logAuthReject(s.state.audit(), r, oid, err.Error()) + http.Error(w, err.Error(), http.StatusForbidden) return nil, false } @@ -826,25 +810,8 @@ func (s *Server) authRequest(r *http.Request, w http.ResponseWriter, oid OID) (e return nil, false } - var buf bytes.Buffer - - fmt.Fprintf(&buf, "%s %s %s", r.Method, r.URL.Path, ts) - - bsign, err := base58.Decode(sign) - if err != nil { - logAuthReject(s.state.audit(), r, oid, "malformed signature") - http.Error(w, "failed to decode signature", http.StatusForbidden) - return nil, false - } - - if len(capa.pub) != ed25519.PublicKeySize { - logAuthReject(s.state.audit(), r, oid, "invalid capability public key") - http.Error(w, "invalid public key size", http.StatusForbidden) - return nil, false - } - - if !ed25519.Verify(capa.pub, buf.Bytes(), bsign) { - logAuthReject(s.state.audit(), r, oid, "signature verification failed") + if err := verifyString(capa.pub, httpCanonical(r.Method, r.URL.Path, ts), sign); err != nil { + logAuthReject(s.state.audit(), r, oid, err.Error()) http.Error(w, "failed to verify signature", http.StatusForbidden) return nil, false } @@ -885,6 +852,23 @@ func (cs *controlStream) NoReply(rs streamRequest, arg any) error { return nil } +// acceptSession upgrades an incoming callstream request to a multiplexed +// session. The HTTP mux is served only over HTTP/3, where WebTransport provides +// QUIC sub-streams natively; TCP clients use the message transport instead, +// which multiplexes every operation over one session and never reaches here. +func (s *Server) acceptSession(w http.ResponseWriter, r *http.Request) (rpcSession, error) { + if r.ProtoMajor != 3 { + return nil, fmt.Errorf("callstream requires HTTP/3; use the message transport over TCP") + } + + // webtransport.Upgrade writes the 200 status itself. + sess, err := s.ws.Upgrade(w, r) + if err != nil { + return nil, err + } + return &wtSession{sess: sess}, nil +} + func (s *Server) startCallStream(w http.ResponseWriter, r *http.Request) { oid := OID(r.PathValue("oid")) @@ -946,8 +930,6 @@ func (s *Server) startCallStream(w http.ResponseWriter, r *http.Request) { logAccess(ctx, s.state.audit(), r, mm, "ok") } - w.WriteHeader(http.StatusOK) - ctx = Propagator().Extract(ctx, propagation.HeaderCarrier(r.Header)) tracer := Tracer() @@ -958,7 +940,7 @@ func (s *Server) startCallStream(w http.ResponseWriter, r *http.Request) { span.SetAttributes(attribute.String("oid", string(oid))) - sess, err := s.ws.Upgrade(w, r) + sess, err := s.acceptSession(w, r) if err != nil { s.state.log.Error("failed to upgrade connection", "error", err) http.Error(w, "failed to upgrade connection", http.StatusInternalServerError) @@ -999,6 +981,13 @@ func (s *Server) startCallStream(w http.ResponseWriter, r *http.Request) { }) } + s.runCallStream(ctx, &cs, mm, call) +} + +// runCallStream invokes a streaming handler and emits its result, error, or +// panic over the control stream. It is transport-neutral: both the HTTP +// callstream handler and the message-transport router use it. +func (s *Server) runCallStream(ctx context.Context, cs *controlStream, mm Method, call *NetworkCall) { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -1006,45 +995,41 @@ func (s *Server) startCallStream(w http.ResponseWriter, r *http.Request) { if r := recover(); r != nil { s.state.log.Error("panic in streaming RPC handler", "panic", r, - "method", method, + "method", call.method, "stack", string(debug.Stack())) - var sr streamRequest - sr.Kind = "panic" - sr.Error = fmt.Sprint(r) - cs.NoReply(sr, nil) + cs.NoReply(streamRequest{Kind: "panic", Error: fmt.Sprint(r)}, nil) } }() - err = cond.Wrap(mm.Handler(ctx, call)) - + err := cond.Wrap(mm.Handler(ctx, call)) if err != nil { - var sr streamRequest - sr.Kind = "error" - - if emsg, ok := err.(ErrorMessage); ok { - sr.Error = emsg.ErrorMessage() - } else { - sr.Error = err.Error() - } - - if ecat, ok := err.(ErrorCategory); ok { - sr.Category = ecat.ErrorCategory() - } + msg, category, code := errorFields(err) + cs.NoReply(streamRequest{Kind: "error", Error: msg, Category: category, Code: code}, nil) + return + } - if ecode, ok := err.(ErrorCode); ok { - sr.Code = ecode.ErrorCode() - } + res := call.results + if res == nil { + res = struct{}{} + } + cs.NoReply(streamRequest{Kind: "result"}, res) +} - cs.NoReply(sr, nil) +// errorFields extracts the wire error fields from a handler error, honoring the +// optional ErrorMessage/ErrorCategory/ErrorCode interfaces. +func errorFields(err error) (msg, category, code string) { + if emsg, ok := err.(ErrorMessage); ok { + msg = emsg.ErrorMessage() } else { - res := call.results - if res == nil { - res = struct{}{} - } - cs.NoReply(streamRequest{ - Kind: "result", - }, res) + msg = err.Error() + } + if ecat, ok := err.(ErrorCategory); ok { + category = ecat.ErrorCategory() + } + if ecode, ok := err.(ErrorCode); ok { + code = ecode.ErrorCode() } + return } func (s *Server) handleCalls(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/rpc/server_session.go b/pkg/rpc/server_session.go new file mode 100644 index 000000000..908c6243d --- /dev/null +++ b/pkg/rpc/server_session.go @@ -0,0 +1,498 @@ +package rpc + +import ( + "context" + "crypto/ed25519" + "fmt" + "runtime/debug" + "sort" + "strings" + + "github.com/fxamacker/cbor/v2" + "github.com/mr-tron/base58" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/propagation" +) + +// sessionRouter owns the accept loop for one message session and dispatches +// every inbound stream by the opRequest that opens it. Because both server +// operations and inline-capability callbacks share that framing, one end of a +// connection can both serve and call — which is what lets rpc run over a single +// connection it does not own. +type sessionRouter struct { + state *State + + // server handles inbound operations; nil if this end only makes calls. + server *Server + + // inline resolves callbacks against capabilities this end advertised; nil if + // this end only serves. + inline *inlineRegistry +} + +// run accepts and dispatches streams until the session fails or ctx is +// cancelled, returning the terminating error. +func (r *sessionRouter) run(ctx context.Context, sess rpcSession) error { + for { + st, err := sess.AcceptStream(ctx) + if err != nil { + return err + } + go r.dispatch(ctx, sess, st) + } +} + +func (r *sessionRouter) dispatch(ctx context.Context, sess rpcSession, st rpcStream) { + dec := cbor.NewDecoder(st) + enc := cbor.NewEncoder(st) + + var req opRequest + if err := dec.Decode(&req); err != nil { + _ = st.Close() + return + } + + if v := req.protocolVersion(); v < minProtocolVersion || v > currentProtocolVersion { + _ = enc.Encode(opReply{ + Status: "error", + Error: fmt.Sprintf("unsupported protocol version %d; this peer speaks %d..%d", + v, minProtocolVersion, currentProtocolVersion), + }) + _ = st.Close() + return + } + + if req.Op == opInlineCall { + defer st.Close() + + if r.inline == nil { + _ = enc.Encode(opReply{Status: "error", Error: "no inline capabilities on this session"}) + return + } + r.state.serveInlineCalls(ctx, dec, enc, r.inline.lookup) + return + } + + if r.server == nil { + _ = enc.Encode(opReply{Status: "error", Error: "this session does not serve operations"}) + _ = st.Close() + return + } + + r.server.handleOp(ctx, sess, st, dec, enc, req) +} + +func (s *Server) handleOp( + ctx context.Context, + sess rpcSession, + st rpcStream, + dec *cbor.Decoder, + enc *cbor.Encoder, + req opRequest, +) { + switch req.Op { + case opLookup: + s.msgLookup(enc, req) + _ = st.Close() + case opMethods: + s.msgMethods(enc, req) + _ = st.Close() + case opReresolve: + s.msgReresolve(dec, enc, req) + _ = st.Close() + case opReexport: + s.msgReexport(enc, req) + _ = st.Close() + case opRef: + s.msgRef(enc, req) + _ = st.Close() + case opDeref: + s.msgDeref(enc, req) + _ = st.Close() + case opIdentify: + s.msgIdentify(enc, req) + _ = st.Close() + case opCall: + s.msgCall(ctx, dec, enc, req) + _ = st.Close() + case opCallStream: + s.msgCallStream(ctx, sess, st, dec, enc, req) + default: + _ = enc.Encode(opReply{Status: "error", Error: "unknown op: " + req.Op}) + _ = st.Close() + } +} + +// writeReply emits an opReply followed (for control-plane ops) by the CBOR +// payload. +func writeReply(enc *cbor.Encoder, reply opReply, payload any) { + _ = enc.Encode(reply) + if payload != nil { + _ = enc.Encode(payload) + } +} + +// authMsgCap verifies a capability-scoped op signature against the stored pubkey +// of the targeted capability. +func (s *Server) authMsgCap(req opRequest, target string) (ed25519.PublicKey, bool) { + if err := freshTimestamp(req.Timestamp); err != nil { + return nil, false + } + s.mu.Lock() + capa, ok := s.objects[req.OID] + s.mu.Unlock() + if !ok { + return nil, false + } + if !verifyBytes(capa.pub, msgCanonical(req.Op, target, req.Timestamp), req.Signature) { + return nil, false + } + return capa.pub, true +} + +func (s *Server) msgLookup(enc *cbor.Encoder, req opRequest) { + pub := ed25519.PublicKey(req.PubKey) + if len(pub) != ed25519.PublicKeySize { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "invalid public key"}) + return + } + + s.mu.Lock() + iface, ok := s.persistent[req.Name] + s.mu.Unlock() + + if !ok { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "unknown object: " + req.Name}) + return + } + + capa := s.assignCapability(iface, pub, s.state.contactAddr(), req.Name, false) + capa.RestoreState = &InterfaceState{Category: "!persistent", Interface: req.Name} + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Capability: capa}) +} + +func (s *Server) msgMethods(enc *cbor.Encoder, req opRequest) { + s.mu.Lock() + hc, ok := s.objects[req.OID] + s.mu.Unlock() + + if !ok { + writeReply(enc, opReply{Status: "ok"}, methodsResponse{Error: "unknown capability: " + string(req.OID)}) + return + } + + methods := make([]string, 0, len(hc.methods)) + for name := range hc.methods { + methods = append(methods, name) + } + sort.Strings(methods) + writeReply(enc, opReply{Status: "ok"}, methodsResponse{Methods: methods}) +} + +func (s *Server) msgReresolve(dec *cbor.Decoder, enc *cbor.Encoder, req opRequest) { + var rs InterfaceState + if err := dec.Decode(&rs); err != nil { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "failed to read body"}) + return + } + + pub := ed25519.PublicKey(req.PubKey) + if len(pub) != ed25519.PublicKeySize { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "invalid public key"}) + return + } + + var ( + iface *Interface + category string + ) + + if rs.Category == "!persistent" { + category = rs.Interface + s.mu.Lock() + pi, ok := s.persistent[rs.Interface] + s.mu.Unlock() + if !ok { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "unknown object: " + rs.Interface}) + return + } + iface = pi + } else { + s.mu.Lock() + res, ok := s.resolvers[rs.Category] + s.mu.Unlock() + if ok { + var err error + iface, err = res.ReconstructFromState(&rs) + if err != nil { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "failed to resolve: " + err.Error()}) + return + } + if iface == nil { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "unable to restore capability"}) + return + } + } + } + + capa := s.assignCapability(iface, pub, s.state.contactAddr(), category, false) + capa.RestoreState = &rs + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Capability: capa}) +} + +func (s *Server) msgReexport(enc *cbor.Encoder, req opRequest) { + if _, ok := s.authMsgCap(req, string(req.OID)); !ok { + writeReply(enc, opReply{Status: "error", Error: "unauthorized"}, lookupResponse{Error: "unauthorized"}) + return + } + + target := ed25519.PublicKey(req.TargetPK) + if len(target) != ed25519.PublicKeySize { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "invalid target public key"}) + return + } + + s.mu.Lock() + hc, ok := s.objects[req.OID] + s.mu.Unlock() + + if !ok { + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Error: "unknown capability: " + string(req.OID)}) + return + } + + capa := s.reexportCapability(req.OID, hc, target, s.state.contactAddr()) + writeReply(enc, opReply{Status: "ok"}, lookupResponse{Capability: capa}) +} + +func (s *Server) msgRef(enc *cbor.Encoder, req opRequest) { + if _, ok := s.authMsgCap(req, string(req.OID)); !ok { + writeReply(enc, opReply{Status: "error", Error: "unauthorized"}, refResponse{Error: "unauthorized"}) + return + } + + s.mu.Lock() + defer s.mu.Unlock() + + if hc, ok := s.objects[req.OID]; ok { + hc.refs.Add(1) + writeReply(enc, opReply{Status: "ok"}, refResponse{Status: "ok"}) + } else { + writeReply(enc, opReply{Status: "ok"}, refResponse{Error: "unknown capability: " + string(req.OID)}) + } +} + +func (s *Server) msgDeref(enc *cbor.Encoder, req opRequest) { + if _, ok := s.authMsgCap(req, string(req.OID)); !ok { + writeReply(enc, opReply{Status: "error", Error: "unauthorized"}, refResponse{Error: "unauthorized"}) + return + } + + if s.Deref(req.OID) { + writeReply(enc, opReply{Status: "ok"}, refResponse{Status: "ok"}) + } else { + writeReply(enc, opReply{Status: "ok"}, refResponse{Error: "unknown capability: " + string(req.OID)}) + } +} + +func (s *Server) msgIdentify(enc *cbor.Encoder, req opRequest) { + pub := ed25519.PublicKey(req.PubKey) + if err := freshTimestamp(req.Timestamp); err != nil { + writeReply(enc, opReply{Status: "ok"}, identifyResponse{Error: "invalid identity"}) + return + } + if !verifyBytes(pub, msgCanonical(opIdentify, "", req.Timestamp), req.Signature) { + writeReply(enc, opReply{Status: "ok"}, identifyResponse{Error: "invalid identity"}) + return + } + writeReply(enc, opReply{Status: "ok"}, identifyResponse{ + Ok: true, + Address: req.Contact, + Identity: base58.Encode(pub), + MinVersion: minProtocolVersion, + MaxVersion: currentProtocolVersion, + }) +} + +type callOutcome struct { + status string + errMsg string + category string + code string + results any +} + +// prepareMethodCall authenticates a call/callstream op, builds the +// signed-pubkey identity, looks up the method, and applies the public/non-public +// authorization gate. On failure it returns an opReply describing why. +func (s *Server) prepareMethodCall(ctx context.Context, req opRequest) (context.Context, *heldCapability, Method, ed25519.PublicKey, opReply, bool) { + target := string(req.OID) + "/" + req.Method + + if err := freshTimestamp(req.Timestamp); err != nil { + return ctx, nil, Method{}, nil, opReply{Status: "error", Error: "stale request"}, false + } + + s.mu.Lock() + iface, ok := s.objects[req.OID] + s.mu.Unlock() + if !ok { + return ctx, nil, Method{}, nil, opReply{Status: "unknown-capability", Error: "unknown object: " + string(req.OID)}, false + } + + if !verifyBytes(iface.pub, msgCanonical(req.Op, target, req.Timestamp), req.Signature) { + return ctx, nil, Method{}, nil, opReply{Status: "error", Error: "failed to verify signature"}, false + } + + iface.touch() + + mm := iface.methods[req.Method] + if mm.Handler == nil { + return ctx, nil, Method{}, nil, opReply{Status: "error", Error: "unknown method: " + req.Method}, false + } + + // A bearer token, if the caller sent one, names a richer identity than the + // capability does — a user or a workload rather than a key. Fall back to the + // verified signature, which is all a message transport otherwise offers: + // there is no TLS client certificate to inspect. + identity := s.msgIdentity(ctx, req, iface.pub) + ctx = ContextWithIdentity(ctx, identity) + + if len(req.Trace) > 0 { + ctx = Propagator().Extract(ctx, propagation.MapCarrier(req.Trace)) + } + + if !mm.Public && s.state.authorizer != nil { + resource := strings.ToLower(mm.InterfaceName) + action := strings.ToLower(mm.Name) + if err := s.state.authorizer.Authorize(ctx, identity, resource, action); err != nil { + s.state.log.Warn("authorization denied", + "resource", resource, "action", action, "subject", identity.Subject, "error", err) + return ctx, nil, Method{}, nil, opReply{Status: "forbidden", Error: err.Error()}, false + } + } + + return ctx, iface, mm, iface.pub, opReply{}, true +} + +// msgIdentity resolves the caller's identity for an operation on a message +// transport. The capability signature has already been verified by the caller. +func (s *Server) msgIdentity(ctx context.Context, req opRequest, capaPub ed25519.PublicKey) *Identity { + signed := &Identity{Subject: base58.Encode(capaPub), Method: AuthMethodSigned} + + if req.Bearer == "" || s.state.authenticator == nil { + return signed + } + + identity, err := s.state.authenticator.Authenticate(ctx, &Credentials{ + Authorization: "Bearer " + req.Bearer, + }) + if err != nil { + s.state.log.Warn("rpc: bearer authentication failed", "error", err) + return signed + } + if identity == nil { + return signed + } + + return identity +} + +func (s *Server) msgCall(ctx context.Context, dec *cbor.Decoder, enc *cbor.Encoder, req opRequest) { + ctx, iface, mm, caller, reply, ok := s.prepareMethodCall(ctx, req) + if !ok { + _ = enc.Encode(reply) + return + } + + ctx, span := Tracer().Start(ctx, "rpc.handle."+mm.InterfaceName+"."+mm.Name) + defer span.End() + span.SetAttributes(attribute.String("oid", string(req.OID))) + + call := &NetworkCall{ + s: s, + oid: req.OID, + method: req.Method, + caller: caller, + category: iface.category, + dec: dec, // args follow the opRequest on this stream + } + + if iface.aroundContext != nil { + var cancel func() + ctx, cancel = iface.aroundContext(ctx, call) + defer cancel() + } + + oc := s.invokeUnary(ctx, mm, call) + switch oc.status { + case "ok": + _ = enc.Encode(opReply{Status: "ok"}) + _ = enc.Encode(oc.results) + case "panic": + _ = enc.Encode(opReply{Status: "panic", Error: oc.errMsg}) + default: + _ = enc.Encode(opReply{Status: "error", Error: oc.errMsg, Category: oc.category, Code: oc.code}) + } +} + +// invokeUnary runs a unary handler, mirroring handleCalls (no cond.Wrap; panic +// recovery), and returns the outcome. +func (s *Server) invokeUnary(ctx context.Context, mm Method, call *NetworkCall) (oc callOutcome) { + defer func() { + if r := recover(); r != nil { + s.state.log.Error("panic in RPC handler", + "panic", r, "method", call.method, "stack", string(debug.Stack())) + oc = callOutcome{status: "panic", errMsg: fmt.Sprint(r)} + } + }() + + err := mm.Handler(ctx, call) + if err != nil { + msg, category, code := errorFields(err) + return callOutcome{status: "error", errMsg: msg, category: category, code: code} + } + + res := call.results + if res == nil { + res = struct{}{} + } + return callOutcome{status: "ok", results: res} +} + +func (s *Server) msgCallStream(ctx context.Context, sess rpcSession, st rpcStream, dec *cbor.Decoder, enc *cbor.Encoder, req opRequest) { + defer st.Close() + + cs := &controlStream{dec: dec, enc: enc} + + ctx, iface, mm, caller, reply, ok := s.prepareMethodCall(ctx, req) + if !ok { + // The callstream client reads streamRequest frames, not opReply, so the + // rejection is restated as one. An authorization denial carries the same + // category the unary path gives it. + category, code := reply.Category, reply.Code + if reply.Status == "forbidden" { + category, code = "permission", "forbidden" + } + + cs.NoReply(streamRequest{Kind: "error", Error: reply.Error, Category: category, Code: code}, nil) + return + } + + ctx, span := Tracer().Start(ctx, "rpc.handle."+mm.InterfaceName+"."+mm.Name) + defer span.End() + span.SetAttributes(attribute.String("oid", string(req.OID))) + + call := &NetworkCall{ + s: s, + oid: req.OID, + method: req.Method, + caller: caller, + category: iface.category, + dec: dec, + wsSession: sess, + msgSession: true, + ctrl: cs, + } + + s.runCallStream(ctx, cs, mm, call) +} diff --git a/pkg/rpc/signing.go b/pkg/rpc/signing.go new file mode 100644 index 000000000..0452f1346 --- /dev/null +++ b/pkg/rpc/signing.go @@ -0,0 +1,67 @@ +package rpc + +import ( + "crypto/ed25519" + "fmt" + "time" + + "github.com/mr-tron/base58" +) + +// authFreshness bounds how old a request timestamp may be. +const authFreshness = 10 * time.Minute + +// httpCanonical is the legacy signed string for the HTTP transports: +// "METHOD PATH timestamp". Kept verbatim so existing peers remain compatible. +func httpCanonical(method, path, ts string) string { + return fmt.Sprintf("%s %s %s", method, path, ts) +} + +// msgCanonical is the signed string for the message transport: +// "op target timestamp", where target identifies the operation's subject +// (e.g. "oid/method" for call, "oid" for deref). Transport-neutral by +// construction; a signature never travels inside a capability, so the per- +// transport string format is safe. +func msgCanonical(op, target, ts string) string { + return fmt.Sprintf("%s %s %s", op, target, ts) +} + +// verifyString checks a base58-encoded signature over canonical. The returned +// error distinguishes the failure modes so callers can record why a request was +// rejected in the audit log. +func verifyString(pub ed25519.PublicKey, canonical, sigB58 string) error { + if len(pub) != ed25519.PublicKeySize { + return fmt.Errorf("invalid capability public key") + } + bsign, err := base58.Decode(sigB58) + if err != nil { + return fmt.Errorf("malformed signature") + } + if !ed25519.Verify(pub, []byte(canonical), bsign) { + return fmt.Errorf("signature verification failed") + } + return nil +} + +// signBytes / verifyBytes are the raw-signature variants used by the message +// transport (no base58 round-trip; the signature travels as bytes in the frame). +func signBytes(priv ed25519.PrivateKey, canonical string) []byte { + return ed25519.Sign(priv, []byte(canonical)) +} + +func verifyBytes(pub ed25519.PublicKey, canonical string, sig []byte) bool { + return len(pub) == ed25519.PublicKeySize && ed25519.Verify(pub, []byte(canonical), sig) +} + +// freshTimestamp parses an RFC3339Nano timestamp and rejects it if it is older +// than authFreshness. +func freshTimestamp(ts string) error { + t, err := time.Parse(time.RFC3339Nano, ts) + if err != nil { + return fmt.Errorf("invalid timestamp: %w", err) + } + if time.Since(t) > authFreshness { + return fmt.Errorf("timestamp too old") + } + return nil +} diff --git a/pkg/rpc/state.go b/pkg/rpc/state.go index 1be4ee896..1d8bdd5ea 100644 --- a/pkg/rpc/state.go +++ b/pkg/rpc/state.go @@ -122,9 +122,35 @@ type State struct { ws *webtransport.Server li *quic.EarlyListener + httpSrv *http.Server + tcpLn net.Listener + msgLn net.Listener + + // msgContact is the address peers use to reach this State over a message + // transport, e.g. "mem://name" for an in-process server. It is empty when + // the State only serves connections supplied by callers, which have no + // address of their own. + msgContact string + localMP *packet.PacketConnMultiplex } +// contactAddr returns the address embedded in capabilities this server mints, +// so a holder can later reach the issuer. +// +// It is empty when there is nothing to dial — a connection owned by the caller +// is reachable only as itself. Capabilities minted with an empty address are +// therefore scoped to the session they were minted on. +func (s *State) contactAddr() string { + if s.msgContact != "" { + return s.msgContact + } + if s.transport == nil || s.transport.Conn == nil { + return "" + } + return s.transport.Conn.LocalAddr().String() +} + func (s *State) ListenAddr() string { return s.transport.Conn.LocalAddr().String() } @@ -145,7 +171,11 @@ type stateOptions struct { certData []byte keyData []byte - bindAddr string + bindAddr string + wsBindAddr string + tcpBindAddr string + + memServerName string endpoint string @@ -187,6 +217,35 @@ func WithBindAddr(addr string) StateOption { } } +// WithWSBindAddr enables an additional TCP listener serving the RPC protocol +// over TLS as a WebSocket message session, bound to addr. Use "host:0" to bind +// an ephemeral port; the chosen address is available via State.WSListenAddr +// after NewState returns. Clients reach it with State.Connect("ws://host:port"). +func WithWSBindAddr(addr string) StateOption { + return func(o *stateOptions) { + o.wsBindAddr = addr + } +} + +// WithTCPBindAddr enables a raw TLS-over-TCP listener serving the RPC protocol +// as a message session, bound to addr. It carries no HTTP layer at all, which +// makes it the leanest option where a WebSocket handshake buys nothing. Clients +// reach it with State.Connect("tcp://host:port"). +func WithTCPBindAddr(addr string) StateOption { + return func(o *stateOptions) { + o.tcpBindAddr = addr + } +} + +// WithMemServer registers this State as an in-process message server under the +// given name. Peers connect with State.Connect("mem://name", ...). Used for +// testing and for bridging RPC onto message-oriented systems. +func WithMemServer(name string) StateOption { + return func(o *stateOptions) { + o.memServerName = name + } +} + func WithSkipVerify(o *stateOptions) { o.skipVerify = true } @@ -369,6 +428,25 @@ func NewState(ctx context.Context, opts ...StateOption) (*State, error) { } } + if so.wsBindAddr != "" { + err := s.startWSListener(ctx, so.wsBindAddr) + if err != nil { + return nil, err + } + } + + if so.tcpBindAddr != "" { + err := s.startTCPListener(ctx, so.tcpBindAddr) + if err != nil { + return nil, err + } + } + + if so.memServerName != "" { + s.msgContact = "mem://" + so.memServerName + defaultMemRegistry.register(ctx, so.memServerName, s) + } + return s, nil } @@ -519,6 +597,18 @@ func (s *State) Close() error { s.hs.Close() } + if s.httpSrv != nil { + s.httpSrv.Close() + } + + if s.tcpLn != nil { + s.tcpLn.Close() + } + + if s.msgLn != nil { + s.msgLn.Close() + } + return s.transport.Conn.Close() } @@ -540,6 +630,17 @@ func (s *State) Connect(remote string, name string) (*NetworkClient, error) { if err != nil { return nil, err } + } else if isMessageRemote(remote) { + // mem://, ws://, wss:// and tcp:// all multiplex every operation over one + // message session; setupMsgTransport picks the connection factory. + client = &NetworkClient{ + State: s, + transportKind: transportMsg, + tlsCfg: s.clientTlsCfg, + remote: remote, + } + + client.setupTransport() } else if remote == "dial-stdio" { shstr := os.Getenv("MIREN_DIAL_PROGRAM") if shstr == "" { @@ -590,36 +691,59 @@ func (c *NetworkClient) newClientUnder(capa *Capability) *NetworkClient { addr := capa.Address transport := c.State.transport + tk := c.transportKind if strings.HasPrefix(addr, "unix:") { transport = c.State.localTransport } + // An empty address means "wherever this capability came from", which for a + // message transport is the session it arrived on. + sameSession := addr == "" && c.transportKind == transportMsg + if addr == "" { addr = c.remote } - newClient := &NetworkClient{ - State: c.State, - transport: transport, - tlsCfg: c.State.clientTlsCfg.Clone(), - capa: capa, - oid: capa.OID, - remote: addr, + if isMessageRemote(addr) { + tk = transportMsg } - newClient.setupTransport() + newClient := &NetworkClient{ + State: c.State, + transportKind: tk, + transport: transport, + tlsCfg: c.State.clientTlsCfg.Clone(), + capa: capa, + oid: capa.OID, + remote: addr, + } + + if sameSession { + // Share the parent's session rather than dialing again. On a connection + // supplied by the caller there is nothing to dial, and even where there + // is, a capability reachable through this session should not open a + // second one. + newClient.ops = c.ops + } else { + newClient.setupTransport() + } return newClient } func (s *State) newClientFrom(capa *Capability, peer *x509.Certificate) *NetworkClient { transport := s.transport + tk := transportH3 if strings.HasPrefix(capa.Address, "unix:") { transport = s.localTransport } + if isMessageRemote(capa.Address) { + tk = transportMsg + } + cfg := s.clientTlsCfg.Clone() cfg.InsecureSkipVerify = true @@ -634,12 +758,13 @@ func (s *State) newClientFrom(capa *Capability, peer *x509.Certificate) *Network } c := &NetworkClient{ - State: s, - transport: transport, - tlsCfg: cfg, - capa: capa, - oid: capa.OID, - remote: capa.Address, + State: s, + transportKind: tk, + transport: transport, + tlsCfg: cfg, + capa: capa, + oid: capa.OID, + remote: capa.Address, } c.setupTransport() diff --git a/pkg/rpc/state_ws.go b/pkg/rpc/state_ws.go new file mode 100644 index 000000000..9a47da7aa --- /dev/null +++ b/pkg/rpc/state_ws.go @@ -0,0 +1,61 @@ +package rpc + +import ( + "context" + "net" + "net/http" +) + +// startWSListener starts a TCP listener serving the RPC protocol over TLS. +// +// Unlike the HTTP/3 listener, this one does not serve the RPC HTTP mux: a +// single route performs a WebSocket upgrade, and every operation then rides the +// multiplexed message session established over it. The HTTP layer exists only +// so proxies and browsers see a real WebSocket handshake. +func (s *State) startWSListener(ctx context.Context, addr string) error { + tcpLn, err := net.Listen("tcp", addr) + if err != nil { + return err + } + + tlsCfg := s.serverTlsCfg.Clone() + // coder/websocket needs an HTTP/1.1 handshake; do not offer h2. + tlsCfg.NextProtos = []string{"http/1.1"} + + mux := http.NewServeMux() + mux.HandleFunc("GET "+wsMessagePath, func(w http.ResponseWriter, r *http.Request) { + s.serveWSUpgrade(ctx, w, r) + }) + + srv := &http.Server{ + Handler: mux, + TLSConfig: tlsCfg, + } + + s.httpSrv = srv + s.tcpLn = tcpLn + + go func() { + <-ctx.Done() + _ = srv.Shutdown(context.Background()) + }() + + s.log.Debug("starting websocket/tcp listener", "addr", tcpLn.Addr().String()) + + go func() { + if serr := srv.ServeTLS(tcpLn, "", ""); serr != nil && serr != http.ErrServerClosed { + s.log.Error("websocket/tcp listener stopped", "error", serr) + } + }() + + return nil +} + +// WSListenAddr returns the address the TCP/WebSocket listener is bound to, or +// an empty string if no such listener is running. +func (s *State) WSListenAddr() string { + if s.tcpLn == nil { + return "" + } + return s.tcpLn.Addr().String() +} diff --git a/pkg/rpc/stream/stream_test.go b/pkg/rpc/stream/stream_test.go index 9472fd183..049e26991 100644 --- a/pkg/rpc/stream/stream_test.go +++ b/pkg/rpc/stream/stream_test.go @@ -27,9 +27,17 @@ func TestStream(t *testing.T) { serv := ss.Server() - var vals []int + // The handler runs on the server's goroutine while the assertion below + // reads from the test's, so the accumulator needs a lock. + var ( + mu sync.Mutex + vals []int + ) serv.ExposeValue("stream", ReadStream(func(val int) error { + mu.Lock() + defer mu.Unlock() + vals = append(vals, val) return nil })) @@ -47,6 +55,9 @@ func TestStream(t *testing.T) { r.NoError(css.Send(ctx, 100)) r.NoError(css.Send(ctx, 111)) + mu.Lock() + defer mu.Unlock() + r.Equal([]int{42, 100, 111}, vals) }) @@ -61,9 +72,17 @@ func TestStream(t *testing.T) { serv := ss.Server() - var vals []*Thing + // The handler runs on the server's goroutine while the assertion below + // reads from the test's, so the accumulator needs a lock. + var ( + mu sync.Mutex + vals []*Thing + ) serv.ExposeValue("stream", ReadStream(func(val *Thing) error { + mu.Lock() + defer mu.Unlock() + vals = append(vals, val) return nil })) @@ -79,6 +98,9 @@ func TestStream(t *testing.T) { r.NoError(css.Send(ctx, &Thing{Name: "foo"})) + mu.Lock() + defer mu.Unlock() + r.Equal([]*Thing{ {Name: "foo"}, }, vals) diff --git a/pkg/rpc/transport.go b/pkg/rpc/transport.go new file mode 100644 index 000000000..277c008e7 --- /dev/null +++ b/pkg/rpc/transport.go @@ -0,0 +1,38 @@ +package rpc + +import ( + "context" + "io" +) + +// cancelReadCode is the application error code used to abort a blocked stream +// read when an RPC call's context is cancelled. WebTransport passes it through +// as a stream error code; the message transport has no per-stream error code and +// simply unblocks the read. Cancellation is ultimately detected by inspecting +// the call's context, so the specific value is internal. +const cancelReadCode uint64 = 0x13 + +// rpcStream is a single bidirectional sub-stream within an rpcSession. Both +// transports that back it — WebTransport over QUIC and msgmux over a +// MessageConn — provide ordered, independent sub-streams with their own +// lifetimes. +type rpcStream interface { + io.ReadWriteCloser + + // CancelRead aborts a Read currently blocked on this stream. The code is + // passed through to transports that support per-stream error codes + // (WebTransport); transports without that notion (msgmux) ignore it and + // simply unblock the read. + CancelRead(code uint64) +} + +// rpcSession is a multiplexed connection over which either peer may open or +// accept sub-streams at any time. This symmetric capability is what allows a +// server to call back on a client-advertised capability over the same +// connection. WebTransport provides it natively over QUIC; msgmux layers it over +// a MessageConn's discrete messages. +type rpcSession interface { + OpenStreamSync(ctx context.Context) (rpcStream, error) + AcceptStream(ctx context.Context) (rpcStream, error) + Close() error +} diff --git a/pkg/rpc/transport_mem_test.go b/pkg/rpc/transport_mem_test.go new file mode 100644 index 000000000..01355c40b --- /dev/null +++ b/pkg/rpc/transport_mem_test.go @@ -0,0 +1,239 @@ +package rpc_test + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "miren.dev/runtime/pkg/cond" + "miren.dev/runtime/pkg/rpc" + "miren.dev/runtime/pkg/rpc/example" + "miren.dev/runtime/pkg/rpc/stream" +) + +// memServer registers an in-process message server under a unique name derived +// from the test, exposing the given interface, and returns its mem:// remote. +func memServer(t *testing.T, ctx context.Context, iface *rpc.Interface) string { + t.Helper() + r := require.New(t) + + name := "memtest-" + t.Name() + ss, err := rpc.NewState(ctx, rpc.WithSkipVerify, rpc.WithMemServer(name)) + r.NoError(err) + ss.Server().ExposeValue("meter", iface) + + return "mem://" + name +} + +func memClient(t *testing.T, ctx context.Context, remote, name string) rpc.Client { + t.Helper() + r := require.New(t) + + cs, err := rpc.NewState(ctx, rpc.WithSkipVerify) + r.NoError(err) + + c, err := cs.Connect(remote, name) + r.NoError(err) + return c +} + +func TestMemTransport(t *testing.T) { + t.Run("unary calls over message transport", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := memServer(t, ctx, example.AdaptMeter(&exampleMeter{temp: 42})) + c := memClient(t, ctx, remote, "meter") + + mc := &example.MeterClient{Client: c} + + res, err := mc.ReadTemperature(ctx, "test") + r.NoError(err) + r.Equal("test", res.Reading().Meter()) + r.Equal(float32(42), res.Reading().Temperature()) + + // GetSetter returns a capability; calling it follows the cap over mem://. + res2, err := mc.GetSetter(ctx, "test") + r.NoError(err) + res3, err := res2.Setter().SetTemp(ctx, 100) + r.NoError(err) + r.Equal(int32(100), res3.Temp()) + }) + + t.Run("server calls back on a client-advertised capability", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := memServer(t, ctx, example.AdaptMeterUpdates(&exampleMU{})) + c := memClient(t, ctx, remote, "meter") + + mc := &example.MeterUpdatesClient{Client: c} + + var up exampleUpdate + _, err := mc.RegisterUpdates(ctx, &up) + r.NoError(err) + + r.True(up.gotIt) + r.Equal("test", up.reading.Meter()) + r.Equal(float32(42), up.reading.Temperature()) + + time.Sleep(100 * time.Millisecond) + + up.mu.Lock() + defer up.mu.Unlock() + r.True(up.closed) + }) + + t.Run("streaming callbacks over message transport", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := memServer(t, ctx, example.AdaptEmitTemps(&exampleEmit{})) + c := memClient(t, ctx, remote, "meter") + + mc := &example.EmitTempsClient{Client: c} + + var vals []float32 + recv := stream.StreamRecv(func(val float32) error { + vals = append(vals, val) + return nil + }) + + _, err := mc.Emit(ctx, recv) + r.NoError(err) + + time.Sleep(time.Second) + r.Equal([]float32{42, 100}, vals) + }) + + // Callstreams share the one session wrapping the connection, so concurrent + // streaming calls must not steal each other's inbound callbacks. This is the + // regression test for callstreams no longer dialing a connection each. + t.Run("concurrent callstreams share one connection", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := memServer(t, ctx, example.AdaptEmitTemps(&exampleEmit{})) + c := memClient(t, ctx, remote, "meter") + + const calls = 8 + + var ( + wg sync.WaitGroup + mu sync.Mutex + errs []error + got = make([][]float32, calls) + ) + + for i := range calls { + wg.Add(1) + go func() { + defer wg.Done() + + // Each call advertises its own emitter capability. They all ride + // the same session, so the session must route each server-initiated + // callback back to the call that advertised it. + var vals []float32 + recv := stream.StreamRecv(func(val float32) error { + vals = append(vals, val) + return nil + }) + + mc := &example.EmitTempsClient{Client: c} + if _, err := mc.Emit(ctx, recv); err != nil { + mu.Lock() + errs = append(errs, err) + mu.Unlock() + return + } + + time.Sleep(500 * time.Millisecond) + got[i] = vals + }() + } + + wg.Wait() + + mu.Lock() + defer mu.Unlock() + r.Empty(errs) + + for i := range calls { + r.Equal([]float32{42, 100}, got[i], "call %d received the wrong values", i) + } + }) + + t.Run("a capability can be passed to a 3rd party", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + var em exampleMeter + em.temp = 42 + + remote := memServer(t, ctx, example.AdaptMeter(&em)) + c := memClient(t, ctx, remote, "meter") + + mc := &example.MeterClient{Client: c} + + res2, err := mc.GetSetter(ctx, "test") + r.NoError(err) + + // Second mem server hosting the AdjustTemp service. + s2, err := rpc.NewState(ctx, rpc.WithSkipVerify, rpc.WithMemServer("memtest-adjust-"+t.Name())) + r.NoError(err) + s2.Server().ExposeValue("adjust", example.AdaptAdjustTemp(&exampleAT{})) + + c2, err := s2.Connect("mem://memtest-adjust-"+t.Name(), "adjust") + r.NoError(err) + + ac := &example.AdjustTempClient{Client: c2} + + _, err = ac.Adjust(ctx, res2.Setter().Export()) + r.NoError(err) + + r.Equal(float32(72), em.temp) + }) + + t.Run("cancelled callstream reports closed", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + m := &blockingMU{started: make(chan struct{})} + remote := memServer(t, ctx, example.AdaptMeterUpdates(m)) + c := memClient(t, ctx, remote, "meter") + + mc := &example.MeterUpdatesClient{Client: c} + + cctx, cancel := context.WithCancel(ctx) + go func() { + <-m.started + time.Sleep(50 * time.Millisecond) + cancel() + }() + + var up exampleUpdate + _, err := mc.RegisterUpdates(cctx, &up) + r.Error(err) + r.True(errors.Is(err, cond.ErrClosed{}), "expected ErrClosed, got %T: %v", err, err) + }) + + t.Run("propagates panic errors over message transport", func(t *testing.T) { + r := require.New(t) + ctx := t.Context() + + remote := memServer(t, ctx, example.AdaptMeter(&panicMeter{})) + c := memClient(t, ctx, remote, "meter") + + mc := &example.MeterClient{Client: c} + + _, err := mc.ReadTemperature(ctx, "test") + r.Error(err) + + var panicErr cond.ErrPanic + r.True(errors.As(err, &panicErr), "expected ErrPanic, got %T: %v", err, err) + r.Contains(panicErr.Message, "test panic message") + }) +} diff --git a/pkg/rpc/transport_msg.go b/pkg/rpc/transport_msg.go new file mode 100644 index 000000000..4186f61d9 --- /dev/null +++ b/pkg/rpc/transport_msg.go @@ -0,0 +1,388 @@ +package rpc + +import ( + "context" + "crypto/ed25519" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/fxamacker/cbor/v2" + "go.opentelemetry.io/otel/propagation" + "miren.dev/runtime/pkg/cond" +) + +// msgOpTransport drives the client side of a message transport. Every call — +// unary, control-plane, or callstream — is multiplexed over the single session +// wrapping the connection, because a message transport may be handed a +// connection it does not own and cannot re-dial. Server-initiated callbacks are +// therefore routed by OID through a session-scoped registry rather than by +// which call happens to own the session. +type msgOpTransport struct { + // owner is the client that established the connection; it supplies the + // State used by the session's accept loop. + owner *NetworkClient + + // dial opens a fresh connection. It is nil when the session was adopted + // from a caller that owns the underlying connection. + dial func() (MessageConn, error) + + inline inlineRegistry + + mu sync.Mutex + ctrlSess rpcSession +} + +func (t *msgOpTransport) controlSession(ctx context.Context) (rpcSession, error) { + t.mu.Lock() + defer t.mu.Unlock() + + if t.ctrlSess != nil { + return t.ctrlSess, nil + } + if t.dial == nil { + return nil, errors.New("rpc: message session closed and not redialable") + } + conn, err := t.dial() + if err != nil { + return nil, err + } + t.ctrlSess = newMsgSession(conn, true, 0) + // A connection this transport dialed lives as long as the State does. + t.startAcceptLoop(t.owner.State.top, t.ctrlSess) + return t.ctrlSess, nil +} + +// adopt installs a session over a connection the caller owns, bounding it by the +// caller's context rather than by any single call. +func (t *msgOpTransport) adopt(ctx context.Context, sess rpcSession) { + t.mu.Lock() + defer t.mu.Unlock() + + t.ctrlSess = sess + t.startAcceptLoop(ctx, sess) +} + +// startAcceptLoop serves inbound streams for the lifetime of the session. The +// session both routes callbacks against capabilities we advertised and serves +// operations against objects this State exposes, so the peer can call back into +// us over the connection we opened. +// Callers must hold t.mu. +func (t *msgOpTransport) startAcceptLoop(ctx context.Context, sess rpcSession) { + st := t.owner.State + router := &sessionRouter{state: st, server: st.server, inline: &t.inline} + go router.run(ctx, sess) //nolint:errcheck // loop ends with the session +} + +// roundTrip performs one request/reply op over the shared control session: open +// a stream, write the opRequest (and optional payload), read the opReply. The +// returned decoder is positioned to read the reply payload; done() closes the +// stream and must be called. +func (t *msgOpTransport) roundTrip(ctx context.Context, req opRequest, payload any) (opReply, *cbor.Decoder, func(), error) { + sess, err := t.controlSession(ctx) + if err != nil { + return opReply{}, nil, nil, err + } + + st, err := sess.OpenStreamSync(ctx) + if err != nil { + return opReply{}, nil, nil, err + } + done := func() { _ = st.Close() } + + req.Version = currentProtocolVersion + + enc := cbor.NewEncoder(st) + if err := enc.Encode(req); err != nil { + done() + return opReply{}, nil, nil, err + } + if payload != nil { + if err := enc.Encode(payload); err != nil { + done() + return opReply{}, nil, nil, err + } + } + + dec := cbor.NewDecoder(st) + var reply opReply + if err := dec.Decode(&reply); err != nil { + done() + return opReply{}, nil, nil, err + } + + return reply, dec, done, nil +} + +// messageSchemes are the remote prefixes served by the message transport. +var messageSchemes = []string{"mem://", "ws://", "wss://", "tcp://"} + +// isMessageRemote reports whether a remote address names a message transport. +func isMessageRemote(remote string) bool { + for _, scheme := range messageSchemes { + if strings.HasPrefix(remote, scheme) { + return true + } + } + return false +} + +// setupMsgTransport wires the client's connection factory from the remote's +// scheme. A remote with no recognised scheme belongs to a session that was +// adopted rather than dialed, and so has no factory at all. +func (c *NetworkClient) setupMsgTransport() { + switch { + case strings.HasPrefix(c.remote, "mem://"): + name := strings.TrimPrefix(c.remote, "mem://") + c.ops = &msgOpTransport{ + owner: c, + dial: func() (MessageConn, error) { + return defaultMemRegistry.dial(name) + }, + } + + case strings.HasPrefix(c.remote, "ws://"), strings.HasPrefix(c.remote, "wss://"): + c.setupWSTransport() + + case strings.HasPrefix(c.remote, "tcp://"): + c.setupTCPTransport() + + default: + c.ops = &msgOpTransport{owner: c} + } +} + +func nowStamp() string { + return time.Now().Format(time.RFC3339Nano) +} + +// bearer returns the configured bearer token, which rides in the operation +// frame rather than an Authorization header on this transport. +func (c *NetworkClient) bearer() string { + if c.State == nil || c.State.opts == nil { + return "" + } + return c.State.opts.bearerToken +} + +func traceMap(ctx context.Context) map[string]string { + m := propagation.MapCarrier{} + Propagator().Inject(ctx, m) + if len(m) == 0 { + return nil + } + return map[string]string(m) +} + +func (c *NetworkClient) msgResolveCapability(name string) error { + req := opRequest{Op: opLookup, Name: name, PubKey: c.State.pubkey, Contact: c.remote} + _, dec, done, err := c.ops.roundTrip(c.State.top, req, nil) + if err != nil { + return NewResolveHTTPError(err, "error performing message request: %v", err) + } + defer done() + + var lr lookupResponse + if err := dec.Decode(&lr); err != nil { + return NewResolveDecodeError(err) + } + if lr.Error != "" { + return NewResolveLookupError(lr.Error) + } + + c.capa = lr.Capability + c.oid = lr.Capability.OID + return nil +} + +func (c *NetworkClient) msgSendIdentity(ctx context.Context) error { + ts := nowStamp() + req := opRequest{ + Op: opIdentify, + PubKey: c.State.pubkey, + Contact: c.remote, + Timestamp: ts, + Signature: signBytes(c.State.privkey, msgCanonical(opIdentify, "", ts)), + Trace: traceMap(ctx), + } + _, dec, done, err := c.ops.roundTrip(ctx, req, nil) + if err != nil { + return err + } + defer done() + + var lr identifyResponse + if err := dec.Decode(&lr); err != nil { + return err + } + if lr.Error != "" { + return errors.New(lr.Error) + } + if !lr.Ok { + return errors.New("identity rejected") + } + c.serverObservedAddress = lr.Address + return nil +} + +func (c *NetworkClient) msgReresolveCapability(rs *InterfaceState) error { + req := opRequest{Op: opReresolve, PubKey: c.State.pubkey, Contact: c.remote} + _, dec, done, err := c.ops.roundTrip(c.State.top, req, rs) + if err != nil { + return err + } + defer done() + + var lr lookupResponse + if err := dec.Decode(&lr); err != nil { + return err + } + if lr.Error != "" { + return errors.New(lr.Error) + } + c.capa = lr.Capability + c.oid = lr.Capability.OID + return nil +} + +func (c *NetworkClient) msgFetchMethods(ctx context.Context) (methodsResponse, error) { + req := opRequest{Op: opMethods, OID: c.oid} + _, dec, done, err := c.ops.roundTrip(ctx, req, nil) + if err != nil { + return methodsResponse{}, err + } + defer done() + + var mr methodsResponse + if err := dec.Decode(&mr); err != nil { + return methodsResponse{}, err + } + if mr.Error != "" { + return methodsResponse{}, errors.New(mr.Error) + } + return mr, nil +} + +func (c *NetworkClient) msgRequestReexport(ctx context.Context, capa *Capability, target ed25519.PublicKey) (*Capability, error) { + ts := nowStamp() + req := opRequest{ + Op: opReexport, + OID: capa.OID, + TargetPK: target, + Contact: c.remote, + Timestamp: ts, + Signature: signBytes(c.State.privkey, msgCanonical(opReexport, string(capa.OID), ts)), + Trace: traceMap(ctx), + } + _, dec, done, err := c.ops.roundTrip(ctx, req, nil) + if err != nil { + return nil, err + } + defer done() + + var lr lookupResponse + if err := dec.Decode(&lr); err != nil { + return nil, err + } + if lr.Error != "" { + return nil, errors.New(lr.Error) + } + return lr.Capability, nil +} + +func (c *NetworkClient) msgDerefOID(ctx context.Context, oid OID) error { + ts := nowStamp() + req := opRequest{ + Op: opDeref, + OID: oid, + Timestamp: ts, + Signature: signBytes(c.State.privkey, msgCanonical(opDeref, string(oid), ts)), + } + _, dec, done, err := c.ops.roundTrip(ctx, req, nil) + if err != nil { + return err + } + defer done() + + var rr refResponse + if err := dec.Decode(&rr); err != nil { + return err + } + if rr.Error != "" { + return errors.New(rr.Error) + } + return nil +} + +func (c *NetworkClient) msgUnaryCall(ctx context.Context, method string, args, result any) error { + for { + ts := nowStamp() + req := opRequest{ + Op: opCall, + OID: c.oid, + Method: method, + Timestamp: ts, + Signature: signBytes(c.State.privkey, msgCanonical(opCall, string(c.oid)+"/"+method, ts)), + Bearer: c.bearer(), + Trace: traceMap(ctx), + } + reply, dec, done, err := c.ops.roundTrip(ctx, req, args) + if err != nil { + return err + } + + switch reply.Status { + case "ok": + err := dec.Decode(result) + done() + return err + case "unknown-capability": + done() + if c.capa != nil && c.capa.RestoreState != nil { + if rerr := c.msgReresolveCapability(c.capa.RestoreState); rerr == nil { + continue + } + } + return cond.NotFound("capability", c.oid) + case "error": + done() + return cond.RemoteError(reply.Category, reply.Code, reply.Error) + case "forbidden": + done() + // An authorization denial is a permission error, not a protocol one; + // without this case it would surface as "unknown response status". + return cond.RemoteError("permission", "forbidden", reply.Error) + case "panic": + done() + return cond.Panic(reply.Error) + default: + done() + return fmt.Errorf("unknown response status to %s/%s: %s", c.oid, method, reply.Status) + } + } +} + +func (c *NetworkClient) msgCallWithCaps(ctx context.Context, method string, args, result any, caps map[OID]*InlineCapability) error { + // The session is shared with every other call on this connection, so it is + // deliberately not closed when this call finishes. + sess, err := c.ops.controlSession(ctx) + if err != nil { + return err + } + + ts := nowStamp() + prelude := opRequest{ + Op: opCallStream, + OID: c.oid, + Method: method, + Timestamp: ts, + Signature: signBytes(c.State.privkey, msgCanonical(opCallStream, string(c.oid)+"/"+method, ts)), + Bearer: c.bearer(), + Trace: traceMap(ctx), + Version: currentProtocolVersion, + } + + return c.handleCallStream(ctx, sess, &c.ops.inline, &prelude, args, result, caps) +} diff --git a/pkg/rpc/transport_wt.go b/pkg/rpc/transport_wt.go new file mode 100644 index 000000000..58856825e --- /dev/null +++ b/pkg/rpc/transport_wt.go @@ -0,0 +1,115 @@ +package rpc + +import ( + "context" + "crypto/tls" + "errors" + "net" + "net/http" + + "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/http3" + "github.com/quic-go/webtransport-go" +) + +// wtStream adapts a *webtransport.Stream to the rpcStream interface. +type wtStream struct { + *webtransport.Stream +} + +func (s wtStream) CancelRead(code uint64) { + s.Stream.CancelRead(webtransport.StreamErrorCode(code)) +} + +// wtSession adapts a *webtransport.Session to the rpcSession interface. +type wtSession struct { + sess *webtransport.Session +} + +func (s *wtSession) OpenStreamSync(ctx context.Context) (rpcStream, error) { + str, err := s.sess.OpenStreamSync(ctx) + if err != nil { + return nil, err + } + return wtStream{str}, nil +} + +func (s *wtSession) AcceptStream(ctx context.Context) (rpcStream, error) { + str, err := s.sess.AcceptStream(ctx) + if err != nil { + return nil, err + } + return wtStream{str}, nil +} + +func (s *wtSession) Close() error { + return s.sess.CloseWithError(0, "") +} + +// wtDialer establishes a callstream session over WebTransport (HTTP/3). The +// upgrade is a CONNECT request, which the caller signs and issues. +type wtDialer struct { + ws *webtransport.Dialer +} + +func (d *wtDialer) Dial(ctx context.Context, url string, hdr http.Header) (*http.Response, rpcSession, error) { + hr, sess, err := d.ws.Dial(ctx, url, hdr) + if err != nil { + return hr, nil, err + } + return hr, &wtSession{sess: sess}, nil +} + +// isRetryableTransportError reports whether a transport-level error indicates +// the connection was reset in a way the client should transparently retry. +// This is QUIC-specific; other transports never produce it. +func isRetryableTransportError(err error) bool { + var ae *quic.ApplicationError + return errors.As(err, &ae) +} + +// setupH3Transport configures the client to speak the RPC protocol over +// QUIC/HTTP3 with WebTransport for the callstream path. +func (c *NetworkClient) setupH3Transport() { + htr := &http3.Transport{} + htr.Logger = c.State.log.With("module", "rpc-call") + htr.TLSClientConfig = c.tlsCfg + htr.QUICConfig = &DefaultQUICConfig + htr.Dial = func(ctx context.Context, addr string, tlsCfg *tls.Config, cfg *quic.Config) (*quic.Conn, error) { + uaddr, err := resolveUDPAddr(ctx, "udp", addr) + if err != nil { + return nil, err + } + + setTLSConfigServerName(tlsCfg, uaddr, addr) + + return c.transport.DialEarly(ctx, uaddr, tlsCfg, cfg) + } + + ws := &webtransport.Dialer{} + ws.TLSClientConfig = c.tlsCfg + ws.QUICConfig = &DefaultQUICConfig + ws.DialAddr = htr.Dial + + c.htr = htr + c.dialer = &wtDialer{ws: ws} +} + +func setTLSConfigServerName(tlsConf *tls.Config, addr net.Addr, host string) { + // If no ServerName is set, infer the ServerName from the host we're connecting to. + if tlsConf.ServerName != "" { + return + } + if host == "" { + if udpAddr, ok := addr.(*net.UDPAddr); ok { + tlsConf.ServerName = udpAddr.IP.String() + return + } + } + h, _, err := net.SplitHostPort(host) + if err != nil { // This happens if the host doesn't contain a port number. + tlsConf.ServerName = host + return + } + tlsConf.ServerName = h +}