From 394a5c0d49f130956c0ab5db90eaf22dabec44d1 Mon Sep 17 00:00:00 2001 From: jiashuoz Date: Thu, 1 Oct 2026 12:43:36 +0800 Subject: [PATCH] feat: require epoch-bound guest readiness acknowledgment --- cmd/sessiond/reconnect.go | 24 ++- cmd/sessiond/reconnect_process_test.go | 93 ++++++---- cmd/sessiond/reconnect_readiness_test.go | 159 ++++++++++++++++++ cmd/sessiond/reconnect_test.go | 1 + .../2026-09-30-guest-reconnect-session.md | 29 +++- internal/relay/reconnect.go | 4 +- internal/relay/reconnect_test.go | 2 +- protocol/runner/reconnect_rpc.go | 20 +++ protocol/runner/reconnect_rpc_test.go | 1 + 9 files changed, 292 insertions(+), 41 deletions(-) create mode 100644 cmd/sessiond/reconnect_readiness_test.go diff --git a/cmd/sessiond/reconnect.go b/cmd/sessiond/reconnect.go index 546aaf4..b2804db 100644 --- a/cmd/sessiond/reconnect.go +++ b/cmd/sessiond/reconnect.go @@ -127,7 +127,7 @@ func (b *bootstrapper) reconnectGuest(ctx context.Context, c relay.Conn) error { if err = b.refreshConfiguration(delivery, cfg, env); err != nil { return errGuestReconnect } - return nil + return guestReady(delivery, c, accepted.Epoch) } func guestProof(ctx context.Context, c relay.Conn, session, boot string, key ed25519.PrivateKey, epoch uint64) (runner.GuestReconnectAcceptResponse, error) { @@ -313,3 +313,25 @@ func (b *bootstrapper) bindExecEnvironment(execs *sandboxexec.Runner, extra []st return ctx.Err() } } + +// guestReady keeps terminal/exec and credential RPC offline after applying +// configuration until the host acknowledges this exact accepted epoch. A lost +// acknowledgment requires a new signed attempt, never replay of a spent token. +func guestReady(ctx context.Context, c relay.Conn, epoch uint64) error { + ctx, cancel := context.WithTimeout(ctx, guestHandshakeWait) + defer cancel() + ready := runner.GuestReconnectReady{Protocol: runner.GuestReconnectProtocol, Epoch: epoch} + body, _ := json.Marshal(ready) + if ctx.Err() != nil || relay.WriteGuestReconnectFrame(ctx, c, relay.KindGuestReconnectReady, body) != nil { + return errGuestReconnect + } + event, err := relay.ReadGuestReconnectFrame(ctx, c) + if err != nil || event.Kind != relay.KindGuestReconnectReadyAck { + return errGuestReconnect + } + acknowledgment, err := runner.DecodeGuestReconnectReady(event.Payload) + if err != nil || acknowledgment.Epoch != epoch || ctx.Err() != nil { + return errGuestReconnect + } + return nil +} diff --git a/cmd/sessiond/reconnect_process_test.go b/cmd/sessiond/reconnect_process_test.go index 7ea1266..a4fa44d 100644 --- a/cmd/sessiond/reconnect_process_test.go +++ b/cmd/sessiond/reconnect_process_test.go @@ -80,43 +80,56 @@ func TestGuestReconnectExecutable(t *testing.T) { refused := accept() writeReconnect(t, ctx, refused, relay.KindGuestReconnectRefused, runner.GuestReconnectErrorResponse{Error: "fenced"}) refused.Close() - host := accept() - challenge := runner.GuestReconnectChallenge{Protocol: 1, SessionID: cfg.SessionID, BootEpoch: enrolled.BootEpoch, HostIncarnation: "host-test", AttemptID: "process-attempt", PlacementGeneration: 1, Challenge: token} - writeReconnect(t, ctx, host, relay.KindGuestReconnectChallenge, challenge) - event, err = relay.ReadGuestReconnectFrame(ctx, host) - if err != nil { - t.Fatal(err) - } - proof, err := runner.DecodeGuestReconnectAcceptRequest(event.Payload) - if err != nil || event.Kind != relay.KindGuestReconnectProof { - t.Fatal("invalid process proof") - } - if err = challenge.VerifyProof(enrolled.PublicKey, proof.Signature); err != nil { - t.Fatal(err) - } - tokenBytes := make([]byte, 32) - tokenBytes[0] = 7 - fresh := base64.RawURLEncoding.EncodeToString(tokenBytes) - writeReconnect(t, ctx, host, relay.KindGuestReconnectAccepted, runner.GuestReconnectAcceptResponse{Epoch: 1, Token: fresh, ExpiresInSec: 120}) - cfg.BootstrapToken = fresh - cfg.Env = map[string]string{"RECONNECT_PROCESS_TEST": "refreshed"} - sendReconnectConfig(t, ctx, host, cfg) - raw, err = host.Read(ctx) - if err != nil { - t.Fatal(err) - } - f, _ = relay.Decode(raw) - if json.Unmarshal(f.Payload, &event) != nil || event.Kind != "req:"+runner.MethodFetchSessionSecrets { - t.Fatal("missing fresh redemption") - } - var request struct { - Token string `json:"token"` - } - _ = json.Unmarshal(event.Payload, &request) - if request.Token != fresh { - t.Fatal("wrong process token") + var fresh string + for epoch := uint64(1); epoch <= 2; epoch++ { + host := accept() + challenge := runner.GuestReconnectChallenge{Protocol: 1, SessionID: cfg.SessionID, BootEpoch: enrolled.BootEpoch, HostIncarnation: "host-test", AttemptID: fmt.Sprintf("process-attempt-%d", epoch), PlacementGeneration: 1, Challenge: token} + writeReconnect(t, ctx, host, relay.KindGuestReconnectChallenge, challenge) + event, err = relay.ReadGuestReconnectFrame(ctx, host) + if err != nil { + t.Fatal(err) + } + proof, err := runner.DecodeGuestReconnectAcceptRequest(event.Payload) + if err != nil || event.Kind != relay.KindGuestReconnectProof { + t.Fatal("invalid process proof") + } + if err = challenge.VerifyProof(enrolled.PublicKey, proof.Signature); err != nil { + t.Fatal(err) + } + tokenBytes := make([]byte, 32) + tokenBytes[0] = byte(6 + epoch) + fresh = base64.RawURLEncoding.EncodeToString(tokenBytes) + writeReconnect(t, ctx, host, relay.KindGuestReconnectAccepted, runner.GuestReconnectAcceptResponse{Epoch: epoch, Token: fresh, ExpiresInSec: 120}) + cfg.BootstrapToken = fresh + cfg.Env = map[string]string{"RECONNECT_PROCESS_TEST": "refreshed"} + sendReconnectConfig(t, ctx, host, cfg) + raw, err = host.Read(ctx) + if err != nil { + t.Fatal(err) + } + f, _ = relay.Decode(raw) + if json.Unmarshal(f.Payload, &event) != nil || event.Kind != "req:"+runner.MethodFetchSessionSecrets { + t.Fatal("missing fresh redemption") + } + var request struct { + Token string `json:"token"` + } + _ = json.Unmarshal(event.Payload, &request) + if request.Token != fresh { + t.Fatal("wrong process token") + } + controlWrite(t, ctx, host, relay.ControlEvent{Kind: "resp", ID: event.ID, OK: true, Payload: json.RawMessage(`{"env":{}}`)}) + if epoch == 1 { + event, err = relay.ReadGuestReconnectFrame(ctx, host) + ready, decodeErr := runner.DecodeGuestReconnectReady(event.Payload) + if err != nil || decodeErr != nil || event.Kind != relay.KindGuestReconnectReady || ready.Epoch != epoch { + t.Fatal("missing ready before lost acknowledgment") + } + host.Close() + } else { + acknowledgeGuestReady(t, ctx, host, epoch) + } } - controlWrite(t, ctx, host, relay.ControlEvent{Kind: "resp", ID: event.ID, OK: true, Payload: json.RawMessage(`{"env":{}}`)}) err = cmd.Wait() waited = true if err != nil { @@ -128,7 +141,7 @@ func TestGuestReconnectExecutable(t *testing.T) { if strings.Contains(output.String(), enrolled.PublicKey) || strings.Contains(output.String(), fresh) { t.Fatal("guest logged credentials") } - t.Log("separate guest process: enrollment, refused reconnect, verified proof, fresh redemption, same PTY shell across reconnect; no credential output") + t.Log("separate guest process: enrollment, refused reconnect, verified proof, fresh redemption, same PTY shell across lost acknowledgment and fresh-epoch reconnect; no credential output") } func TestGuestReconnectProcessGuest(t *testing.T) { @@ -217,6 +230,12 @@ func TestGuestReconnectProcessGuest(t *testing.T) { t.Fatal("refused connection became ready") } execTurn("initial:retained:old") + if c, err := tr.connect(ctx); err == nil || c != nil { + t.Fatal("lost ready acknowledgment made transport usable") + } + if b.guest.epoch != 1 { + t.Fatal("lost acknowledgment did not consume epoch") + } c, err = tr.connect(ctx) if err != nil { t.Fatal(err) diff --git a/cmd/sessiond/reconnect_readiness_test.go b/cmd/sessiond/reconnect_readiness_test.go new file mode 100644 index 0000000..fd6d2cb --- /dev/null +++ b/cmd/sessiond/reconnect_readiness_test.go @@ -0,0 +1,159 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/protocol/runner" +) + +func TestReconnectWaitsForReadyAcknowledgment(t *testing.T) { + b, ch := reconnectFixture(t) + guest, host, ctx := reconnectPair(t) + t.Setenv("RAINIER_SESSION", "") + done := make(chan error, 1) + go func() { done <- reBootstrap(ctx, guest, b) }() + accepted := acceptProof(t, ctx, host, b, ch, 1) + sendReconnectConfig(t, ctx, host, runner.BootConfig{Protocol: 1, GuestReconnect: 1, SessionID: "session-test", BootstrapToken: accepted.Token}) + if _, err := host.Read(ctx); err != nil { + t.Fatal(err) + } + controlWrite(t, ctx, host, relay.ControlEvent{Kind: "resp", ID: secretsRequestID, OK: true, Payload: json.RawMessage(`{"env":{}}`)}) + raw, err := host.Read(ctx) + if err != nil { + t.Fatalf("guest became ready without notifying host: %v", err) + } + frame, err := relay.Decode(raw) + if err != nil { + t.Fatal(err) + } + var event relay.ControlEvent + if json.Unmarshal(frame.Payload, &event) != nil || event.Kind != "guest_reconnect_ready" { + t.Fatal("missing readiness notice") + } + select { + case err := <-done: + t.Fatalf("guest served before acknowledgment: %v", err) + default: + } + controlWrite(t, ctx, host, relay.ControlEvent{Kind: "guest_reconnect_ready_ack", Payload: json.RawMessage(`{"protocol":1,"epoch":1}`)}) + if err = <-done; err != nil { + t.Fatal(err) + } +} + +func acknowledgeGuestReady(t *testing.T, ctx context.Context, c relay.Conn, epoch uint64) { + t.Helper() + event, err := relay.ReadGuestReconnectFrame(ctx, c) + if err != nil { + t.Fatal(err) + } + ready, err := runner.DecodeGuestReconnectReady(event.Payload) + if err != nil || event.Kind != relay.KindGuestReconnectReady || ready.Epoch != epoch { + t.Fatal("invalid guest readiness") + } + writeReconnect(t, ctx, c, relay.KindGuestReconnectReadyAck, ready) +} + +func TestReconnectReadyAckFailsClosed(t *testing.T) { + for _, which := range []string{"wrong-epoch", "zero", "protocol", "duplicate", "unknown", "null", "kind", "oversize", "lost", "cancel", "timeout"} { + t.Run(which, func(t *testing.T) { + b, ch := reconnectFixture(t) + guest, host, parent := reconnectPair(t) + ctx, cancel := context.WithCancel(parent) + defer cancel() + t.Setenv("RAINIER_SESSION", "") + tr := sessionTransport{dial: func(context.Context) (relay.Conn, error) { return guest, nil }, preamble: func(ctx context.Context, c relay.Conn) error { return reBootstrap(ctx, c, b) }} + done := make(chan error, 1) + go func() { + c, err := tr.connect(ctx) + if c != nil { + done <- nil + return + } + done <- err + }() + accepted := acceptProof(t, ctx, host, b, ch, 1) + sendReconnectConfig(t, ctx, host, runner.BootConfig{Protocol: 1, GuestReconnect: 1, SessionID: "session-test", BootstrapToken: accepted.Token}) + if _, err := host.Read(ctx); err != nil { + t.Fatal(err) + } + controlWrite(t, ctx, host, relay.ControlEvent{Kind: "resp", ID: secretsRequestID, OK: true, Payload: json.RawMessage(`{"env":{}}`)}) + event, err := relay.ReadGuestReconnectFrame(ctx, host) + if err != nil || event.Kind != relay.KindGuestReconnectReady { + t.Fatal("missing ready notice") + } + switch which { + case "lost": + host.Close() + case "cancel": + cancel() + case "timeout": // Parent timeout is shorter than the separate five-second ready bound. + default: + body := json.RawMessage(`{"protocol":1,"epoch":1}`) + kind := relay.KindGuestReconnectReadyAck + switch which { + case "wrong-epoch": + body = json.RawMessage(`{"protocol":1,"epoch":2}`) + case "zero": + body = json.RawMessage(`{"protocol":1,"epoch":0}`) + case "protocol": + body = json.RawMessage(`{"protocol":2,"epoch":1}`) + case "duplicate": + body = json.RawMessage(`{"protocol":1,"epoch":1,"epoch":1}`) + case "unknown": + body = json.RawMessage(`{"protocol":1,"epoch":1,"extra":true}`) + case "null": + body = json.RawMessage(`null`) + case "kind": + kind = relay.KindGuestReconnectReady + case "oversize": + body = json.RawMessage(`{"protocol":1,"epoch":1,"extra":"` + strings.Repeat("x", 4096) + `"}`) + } + // Ordinary writer intentionally bypasses the helper's outbound size limit. + controlWrite(t, ctx, host, relay.ControlEvent{Kind: kind, Payload: body}) + } + select { + case err := <-done: + if err != errGuestReconnect { + t.Fatalf("invalid ack made transport ready: %v", err) + } + case <-time.After(3 * time.Second): + t.Fatal("ready wait not bounded") + } + if b.guest.epoch != 1 || b.reconnecting { + t.Fatal("lost accepted epoch or attempt slot") + } + if which != "lost" { + readCtx, stop := context.WithTimeout(context.Background(), time.Second) + defer stop() + if raw, err := host.Read(readCtx); err == nil || len(raw) > 0 { + t.Fatal("failed preamble remained open or emitted more data") + } + } + }) + } +} + +func TestReconnectDoesNotAnnounceReadyBeforeConfiguration(t *testing.T) { + b, ch := reconnectFixture(t) + guest, host, ctx := reconnectPair(t) + done := make(chan error, 1) + go func() { done <- reBootstrap(ctx, guest, b); guest.Close() }() + accepted := acceptProof(t, ctx, host, b, ch, 1) + sendReconnectConfig(t, ctx, host, runner.BootConfig{Protocol: 1, GuestReconnect: 1, SessionID: "session-test", BootstrapToken: accepted.Token, SecretNames: []string{"REQUIRED_TEST"}}) + if _, err := host.Read(ctx); err != nil { + t.Fatal(err) + } + controlWrite(t, ctx, host, relay.ControlEvent{Kind: "resp", ID: secretsRequestID, OK: true, Payload: json.RawMessage(`{"env":{}}`)}) + if err := <-done; err != errGuestReconnect { + t.Fatal("missing secret accepted") + } + if raw, err := host.Read(ctx); err == nil || len(raw) > 0 { + t.Fatal("configuration failure emitted ready") + } +} diff --git a/cmd/sessiond/reconnect_test.go b/cmd/sessiond/reconnect_test.go index 795a959..b0f2e97 100644 --- a/cmd/sessiond/reconnect_test.go +++ b/cmd/sessiond/reconnect_test.go @@ -158,6 +158,7 @@ func TestReconnectRefreshEmptySecretsRemovesOldConfiguration(t *testing.T) { t.Fatal("stale token") } controlWrite(t, ctx, host, relay.ControlEvent{Kind: "resp", ID: request.ID, OK: true, Payload: json.RawMessage(`{"env":{}}`)}) + acknowledgeGuestReady(t, ctx, host, accepted.Epoch) if err = <-done; err != nil { t.Fatal(err) } diff --git a/docs/design/2026-09-30-guest-reconnect-session.md b/docs/design/2026-09-30-guest-reconnect-session.md index 0406c7d..2406d8d 100644 --- a/docs/design/2026-09-30-guest-reconnect-session.md +++ b/docs/design/2026-09-30-guest-reconnect-session.md @@ -36,6 +36,8 @@ attachment ID zero and exactly `kind` and `payload` in their control event: | `guest_reconnect_proof` | Shared `GuestReconnectAcceptRequest` | | `guest_reconnect_accepted` | Shared `GuestReconnectAcceptResponse` | | `guest_reconnect_refused` | Shared fixed-code refusal | +| `guest_reconnect_ready` | Shared `GuestReconnectReady`: `protocol`, `epoch` | +| `guest_reconnect_ready_ack` | Same payload, matching the accepted epoch | The encoded outer frame is at most 4096 bytes, excluding its newline. `NetConn` retains its fixed 64 KiB read buffer but assembles no more than the selected @@ -60,6 +62,29 @@ bounded by the accepted token TTL. Failed proof, configuration, redemption or validation returns a fixed error; `sessionTransport.connect` closes the stream and does not make it available to the relay or credential dispatcher. +After configuration application the guest sends `guest_reconnect_ready` and waits +for `guest_reconnect_ready_ack` on the same stream before returning it to the +transport. Both payloads contain exactly `{"protocol":1,"epoch":N}` with a positive +accepted epoch. The strict shared decoder rejects unknown, duplicate, missing, +null and invalid fields. Sending readiness and receiving its acknowledgment share +a separate five-second ceiling, additionally bounded by delivery TTL and caller +cancellation. A lost, late or malformed acknowledgment closes the stream; the +already consumed epoch and token require a fresh signed attempt. Configuration +may already have been applied, but no ordinary relay or credential traffic is +served on the failed stream. + +Readiness reports configuration completion; it grants no host installation +authority. The future host handoff must retain and revalidate the original +instance, placement and connection ownership, fence the old relay, and serialize +the acknowledgment before publishing the new relay or allowing ordinary frames +to interleave. A successful acknowledgment write alone does not prove peer +receipt. Neither this exchange nor proof acceptance permits an unguarded call to +the existing `GuestConnected` publication path. + +This extends the disabled draft version-1 protocol: a reconnect-enabled guest +requires a paired host that implements acknowledgment. No shipping listener +advertises the opt-in, and legacy fresh boot is unchanged. + ## Configuration and process continuity The new path validates all environment names/values before applying any changes. @@ -95,7 +120,9 @@ against a real TCP protocol fixture, using the production bootstrap/preamble and PTY implementation. It verifies enrollment, a refused connection, a signed new connection, fresh token redemption and the same shell PID on the same PTY before and after, including the child-versus-parent environment distinction, refreshed -real exec children, removed values and preserved boot-chain exports. This is the +real exec children, removed values and preserved boot-chain exports. It also +drops the first ready acknowledgment, verifies that epoch remains consumed, and +reconnects using a fresh proof, token and epoch while retaining the same PTY shell. This is the closest executable check for an unenabled guest path; the shipping host has no caller yet. It is not AF_VSOCK/KVM or a hosted authorization/relay takeover test. diff --git a/internal/relay/reconnect.go b/internal/relay/reconnect.go index 89974f2..8c0afe1 100644 --- a/internal/relay/reconnect.go +++ b/internal/relay/reconnect.go @@ -15,6 +15,8 @@ const ( KindGuestReconnectProof = "guest_reconnect_proof" KindGuestReconnectAccepted = "guest_reconnect_accepted" KindGuestReconnectRefused = "guest_reconnect_refused" + KindGuestReconnectReady = "guest_reconnect_ready" + KindGuestReconnectReadyAck = "guest_reconnect_ready_ack" GuestReconnectFrameLimit = 4096 ) @@ -68,7 +70,7 @@ func WriteGuestReconnectFrame(ctx context.Context, c Conn, kind string, payload } func reconnectKind(kind string) bool { switch kind { - case KindGuestReconnectChallenge, KindGuestReconnectProof, KindGuestReconnectAccepted, KindGuestReconnectRefused: + case KindGuestReconnectChallenge, KindGuestReconnectProof, KindGuestReconnectAccepted, KindGuestReconnectRefused, KindGuestReconnectReady, KindGuestReconnectReadyAck: return true } return false diff --git a/internal/relay/reconnect_test.go b/internal/relay/reconnect_test.go index c90af9b..aaf2ec4 100644 --- a/internal/relay/reconnect_test.go +++ b/internal/relay/reconnect_test.go @@ -31,7 +31,7 @@ func TestGuestHandshakeReadLimitBeforeNewline(t *testing.T) { } func TestGuestReconnectFrames(t *testing.T) { - for _, kind := range []string{KindGuestReconnectChallenge, KindGuestReconnectProof, KindGuestReconnectAccepted, KindGuestReconnectRefused} { + for _, kind := range []string{KindGuestReconnectChallenge, KindGuestReconnectProof, KindGuestReconnectAccepted, KindGuestReconnectRefused, KindGuestReconnectReady, KindGuestReconnectReadyAck} { t.Run(kind, func(t *testing.T) { a, b := net.Pipe() defer a.Close() diff --git a/protocol/runner/reconnect_rpc.go b/protocol/runner/reconnect_rpc.go index 5b3b4a7..ecc410e 100644 --- a/protocol/runner/reconnect_rpc.go +++ b/protocol/runner/reconnect_rpc.go @@ -71,6 +71,15 @@ type GuestReconnectAcceptResponse struct { ExpiresInSec uint32 `json:"expires_in_sec"` } +// GuestReconnectReady is the exact payload of both the guest readiness notice +// and the host acknowledgment. Epoch must match the acceptance on this stream. +// It reports completed configuration application, not durable authorization or +// permission to bypass instance/placement fencing when installing the relay. +type GuestReconnectReady struct { + Protocol uint64 `json:"protocol"` + Epoch uint64 `json:"epoch"` +} + // GuestReconnectErrorResponse is the entire refusal payload. Error is one of // invalid, expired, fenced or unavailable; no peer, provider or database text may // be substituted. In session-RPC it accompanies an envelope with OK=false. @@ -148,6 +157,17 @@ func DecodeGuestReconnectAcceptResponse(payload []byte) (GuestReconnectAcceptRes return v, nil } +// DecodeGuestReconnectReady applies the same strict bounded wire rules to a +// readiness notice or acknowledgment. The receiver must also check the control +// kind and compare Epoch with this connection's accepted epoch. +func DecodeGuestReconnectReady(payload []byte) (GuestReconnectReady, error) { + var v GuestReconnectReady + if decodeReconnectObject(payload, &v, "protocol", "epoch") != nil || v.Protocol != GuestReconnectProtocol || v.Epoch == 0 { + return GuestReconnectReady{}, errGuestReconnectMessage + } + return v, nil +} + // DecodeGuestReconnectErrorResponse accepts only the four fixed reconnect // refusal codes, with the same size and exact-field checks as successful messages. func DecodeGuestReconnectErrorResponse(payload []byte) (GuestReconnectErrorResponse, error) { diff --git a/protocol/runner/reconnect_rpc_test.go b/protocol/runner/reconnect_rpc_test.go index 6c1346d..36c71ef 100644 --- a/protocol/runner/reconnect_rpc_test.go +++ b/protocol/runner/reconnect_rpc_test.go @@ -105,6 +105,7 @@ func reconnectMessages() []reconnectMessageCase { {"challenge", `{"protocol":1,"session_id":"session_test","boot_epoch":"boot_test","host_incarnation":"7","attempt_id":"attempt_test","placement_generation":3,"challenge":"` + token + `"}`, func(b []byte) (any, error) { return runner.DecodeGuestReconnectChallenge(b) }}, {"accepted", `{"epoch":2,"token":"` + token + `","expires_in_sec":120}`, func(b []byte) (any, error) { return runner.DecodeGuestReconnectAcceptResponse(b) }}, + {"ready", `{"protocol":1,"epoch":2}`, func(b []byte) (any, error) { return runner.DecodeGuestReconnectReady(b) }}, {"refused", `{"error":"fenced"}`, func(b []byte) (any, error) { return runner.DecodeGuestReconnectErrorResponse(b) }}, } }