Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion cmd/sessiond/reconnect.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func (b *bootstrapper) reconnectGuest(ctx context.Context, c relay.Conn) error {
if err = b.refreshConfiguration(delivery, cfg, env); err != nil {
return errGuestReconnect
}
return nil
return guestReady(delivery, c, accepted.Epoch)
}

func guestProof(ctx context.Context, c relay.Conn, session, boot string, key ed25519.PrivateKey, epoch uint64) (runner.GuestReconnectAcceptResponse, error) {
Expand Down Expand Up @@ -313,3 +313,25 @@ func (b *bootstrapper) bindExecEnvironment(execs *sandboxexec.Runner, extra []st
return ctx.Err()
}
}

// guestReady keeps terminal/exec and credential RPC offline after applying
// configuration until the host acknowledges this exact accepted epoch. A lost
// acknowledgment requires a new signed attempt, never replay of a spent token.
func guestReady(ctx context.Context, c relay.Conn, epoch uint64) error {
ctx, cancel := context.WithTimeout(ctx, guestHandshakeWait)
defer cancel()
ready := runner.GuestReconnectReady{Protocol: runner.GuestReconnectProtocol, Epoch: epoch}
body, _ := json.Marshal(ready)
if ctx.Err() != nil || relay.WriteGuestReconnectFrame(ctx, c, relay.KindGuestReconnectReady, body) != nil {
return errGuestReconnect
}
event, err := relay.ReadGuestReconnectFrame(ctx, c)
if err != nil || event.Kind != relay.KindGuestReconnectReadyAck {
return errGuestReconnect
}
acknowledgment, err := runner.DecodeGuestReconnectReady(event.Payload)
if err != nil || acknowledgment.Epoch != epoch || ctx.Err() != nil {
return errGuestReconnect
}
return nil
}
93 changes: 56 additions & 37 deletions cmd/sessiond/reconnect_process_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,43 +80,56 @@ func TestGuestReconnectExecutable(t *testing.T) {
refused := accept()
writeReconnect(t, ctx, refused, relay.KindGuestReconnectRefused, runner.GuestReconnectErrorResponse{Error: "fenced"})
refused.Close()
host := accept()
challenge := runner.GuestReconnectChallenge{Protocol: 1, SessionID: cfg.SessionID, BootEpoch: enrolled.BootEpoch, HostIncarnation: "host-test", AttemptID: "process-attempt", PlacementGeneration: 1, Challenge: token}
writeReconnect(t, ctx, host, relay.KindGuestReconnectChallenge, challenge)
event, err = relay.ReadGuestReconnectFrame(ctx, host)
if err != nil {
t.Fatal(err)
}
proof, err := runner.DecodeGuestReconnectAcceptRequest(event.Payload)
if err != nil || event.Kind != relay.KindGuestReconnectProof {
t.Fatal("invalid process proof")
}
if err = challenge.VerifyProof(enrolled.PublicKey, proof.Signature); err != nil {
t.Fatal(err)
}
tokenBytes := make([]byte, 32)
tokenBytes[0] = 7
fresh := base64.RawURLEncoding.EncodeToString(tokenBytes)
writeReconnect(t, ctx, host, relay.KindGuestReconnectAccepted, runner.GuestReconnectAcceptResponse{Epoch: 1, Token: fresh, ExpiresInSec: 120})
cfg.BootstrapToken = fresh
cfg.Env = map[string]string{"RECONNECT_PROCESS_TEST": "refreshed"}
sendReconnectConfig(t, ctx, host, cfg)
raw, err = host.Read(ctx)
if err != nil {
t.Fatal(err)
}
f, _ = relay.Decode(raw)
if json.Unmarshal(f.Payload, &event) != nil || event.Kind != "req:"+runner.MethodFetchSessionSecrets {
t.Fatal("missing fresh redemption")
}
var request struct {
Token string `json:"token"`
}
_ = json.Unmarshal(event.Payload, &request)
if request.Token != fresh {
t.Fatal("wrong process token")
var fresh string
for epoch := uint64(1); epoch <= 2; epoch++ {
host := accept()
challenge := runner.GuestReconnectChallenge{Protocol: 1, SessionID: cfg.SessionID, BootEpoch: enrolled.BootEpoch, HostIncarnation: "host-test", AttemptID: fmt.Sprintf("process-attempt-%d", epoch), PlacementGeneration: 1, Challenge: token}
writeReconnect(t, ctx, host, relay.KindGuestReconnectChallenge, challenge)
event, err = relay.ReadGuestReconnectFrame(ctx, host)
if err != nil {
t.Fatal(err)
}
proof, err := runner.DecodeGuestReconnectAcceptRequest(event.Payload)
if err != nil || event.Kind != relay.KindGuestReconnectProof {
t.Fatal("invalid process proof")
}
if err = challenge.VerifyProof(enrolled.PublicKey, proof.Signature); err != nil {
t.Fatal(err)
}
tokenBytes := make([]byte, 32)
tokenBytes[0] = byte(6 + epoch)
fresh = base64.RawURLEncoding.EncodeToString(tokenBytes)
writeReconnect(t, ctx, host, relay.KindGuestReconnectAccepted, runner.GuestReconnectAcceptResponse{Epoch: epoch, Token: fresh, ExpiresInSec: 120})
cfg.BootstrapToken = fresh
cfg.Env = map[string]string{"RECONNECT_PROCESS_TEST": "refreshed"}
sendReconnectConfig(t, ctx, host, cfg)
raw, err = host.Read(ctx)
if err != nil {
t.Fatal(err)
}
f, _ = relay.Decode(raw)
if json.Unmarshal(f.Payload, &event) != nil || event.Kind != "req:"+runner.MethodFetchSessionSecrets {
t.Fatal("missing fresh redemption")
}
var request struct {
Token string `json:"token"`
}
_ = json.Unmarshal(event.Payload, &request)
if request.Token != fresh {
t.Fatal("wrong process token")
}
controlWrite(t, ctx, host, relay.ControlEvent{Kind: "resp", ID: event.ID, OK: true, Payload: json.RawMessage(`{"env":{}}`)})
if epoch == 1 {
event, err = relay.ReadGuestReconnectFrame(ctx, host)
ready, decodeErr := runner.DecodeGuestReconnectReady(event.Payload)
if err != nil || decodeErr != nil || event.Kind != relay.KindGuestReconnectReady || ready.Epoch != epoch {
t.Fatal("missing ready before lost acknowledgment")
}
host.Close()
} else {
acknowledgeGuestReady(t, ctx, host, epoch)
}
}
controlWrite(t, ctx, host, relay.ControlEvent{Kind: "resp", ID: event.ID, OK: true, Payload: json.RawMessage(`{"env":{}}`)})
err = cmd.Wait()
waited = true
if err != nil {
Expand All @@ -128,7 +141,7 @@ func TestGuestReconnectExecutable(t *testing.T) {
if strings.Contains(output.String(), enrolled.PublicKey) || strings.Contains(output.String(), fresh) {
t.Fatal("guest logged credentials")
}
t.Log("separate guest process: enrollment, refused reconnect, verified proof, fresh redemption, same PTY shell across reconnect; no credential output")
t.Log("separate guest process: enrollment, refused reconnect, verified proof, fresh redemption, same PTY shell across lost acknowledgment and fresh-epoch reconnect; no credential output")
}

func TestGuestReconnectProcessGuest(t *testing.T) {
Expand Down Expand Up @@ -217,6 +230,12 @@ func TestGuestReconnectProcessGuest(t *testing.T) {
t.Fatal("refused connection became ready")
}
execTurn("initial:retained:old")
if c, err := tr.connect(ctx); err == nil || c != nil {
t.Fatal("lost ready acknowledgment made transport usable")
}
if b.guest.epoch != 1 {
t.Fatal("lost acknowledgment did not consume epoch")
}
c, err = tr.connect(ctx)
if err != nil {
t.Fatal(err)
Expand Down
159 changes: 159 additions & 0 deletions cmd/sessiond/reconnect_readiness_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
package main

import (
"context"
"encoding/json"
"strings"
"testing"
"time"

"github.com/tokencanopy/rainier/internal/relay"
"github.com/tokencanopy/rainier/protocol/runner"
)

func TestReconnectWaitsForReadyAcknowledgment(t *testing.T) {
b, ch := reconnectFixture(t)
guest, host, ctx := reconnectPair(t)
t.Setenv("RAINIER_SESSION", "")
done := make(chan error, 1)
go func() { done <- reBootstrap(ctx, guest, b) }()
accepted := acceptProof(t, ctx, host, b, ch, 1)
sendReconnectConfig(t, ctx, host, runner.BootConfig{Protocol: 1, GuestReconnect: 1, SessionID: "session-test", BootstrapToken: accepted.Token})
if _, err := host.Read(ctx); err != nil {
t.Fatal(err)
}
controlWrite(t, ctx, host, relay.ControlEvent{Kind: "resp", ID: secretsRequestID, OK: true, Payload: json.RawMessage(`{"env":{}}`)})
raw, err := host.Read(ctx)
if err != nil {
t.Fatalf("guest became ready without notifying host: %v", err)
}
frame, err := relay.Decode(raw)
if err != nil {
t.Fatal(err)
}
var event relay.ControlEvent
if json.Unmarshal(frame.Payload, &event) != nil || event.Kind != "guest_reconnect_ready" {
t.Fatal("missing readiness notice")
}
select {
case err := <-done:
t.Fatalf("guest served before acknowledgment: %v", err)
default:
}
controlWrite(t, ctx, host, relay.ControlEvent{Kind: "guest_reconnect_ready_ack", Payload: json.RawMessage(`{"protocol":1,"epoch":1}`)})
if err = <-done; err != nil {
t.Fatal(err)
}
}

func acknowledgeGuestReady(t *testing.T, ctx context.Context, c relay.Conn, epoch uint64) {
t.Helper()
event, err := relay.ReadGuestReconnectFrame(ctx, c)
if err != nil {
t.Fatal(err)
}
ready, err := runner.DecodeGuestReconnectReady(event.Payload)
if err != nil || event.Kind != relay.KindGuestReconnectReady || ready.Epoch != epoch {
t.Fatal("invalid guest readiness")
}
writeReconnect(t, ctx, c, relay.KindGuestReconnectReadyAck, ready)
}

func TestReconnectReadyAckFailsClosed(t *testing.T) {
for _, which := range []string{"wrong-epoch", "zero", "protocol", "duplicate", "unknown", "null", "kind", "oversize", "lost", "cancel", "timeout"} {
t.Run(which, func(t *testing.T) {
b, ch := reconnectFixture(t)
guest, host, parent := reconnectPair(t)
ctx, cancel := context.WithCancel(parent)
defer cancel()
t.Setenv("RAINIER_SESSION", "")
tr := sessionTransport{dial: func(context.Context) (relay.Conn, error) { return guest, nil }, preamble: func(ctx context.Context, c relay.Conn) error { return reBootstrap(ctx, c, b) }}
done := make(chan error, 1)
go func() {
c, err := tr.connect(ctx)
if c != nil {
done <- nil
return
}
done <- err
}()
accepted := acceptProof(t, ctx, host, b, ch, 1)
sendReconnectConfig(t, ctx, host, runner.BootConfig{Protocol: 1, GuestReconnect: 1, SessionID: "session-test", BootstrapToken: accepted.Token})
if _, err := host.Read(ctx); err != nil {
t.Fatal(err)
}
controlWrite(t, ctx, host, relay.ControlEvent{Kind: "resp", ID: secretsRequestID, OK: true, Payload: json.RawMessage(`{"env":{}}`)})
event, err := relay.ReadGuestReconnectFrame(ctx, host)
if err != nil || event.Kind != relay.KindGuestReconnectReady {
t.Fatal("missing ready notice")
}
switch which {
case "lost":
host.Close()
case "cancel":
cancel()
case "timeout": // Parent timeout is shorter than the separate five-second ready bound.
default:
body := json.RawMessage(`{"protocol":1,"epoch":1}`)
kind := relay.KindGuestReconnectReadyAck
switch which {
case "wrong-epoch":
body = json.RawMessage(`{"protocol":1,"epoch":2}`)
case "zero":
body = json.RawMessage(`{"protocol":1,"epoch":0}`)
case "protocol":
body = json.RawMessage(`{"protocol":2,"epoch":1}`)
case "duplicate":
body = json.RawMessage(`{"protocol":1,"epoch":1,"epoch":1}`)
case "unknown":
body = json.RawMessage(`{"protocol":1,"epoch":1,"extra":true}`)
case "null":
body = json.RawMessage(`null`)
case "kind":
kind = relay.KindGuestReconnectReady
case "oversize":
body = json.RawMessage(`{"protocol":1,"epoch":1,"extra":"` + strings.Repeat("x", 4096) + `"}`)
}
// Ordinary writer intentionally bypasses the helper's outbound size limit.
controlWrite(t, ctx, host, relay.ControlEvent{Kind: kind, Payload: body})
}
select {
case err := <-done:
if err != errGuestReconnect {
t.Fatalf("invalid ack made transport ready: %v", err)
}
case <-time.After(3 * time.Second):
t.Fatal("ready wait not bounded")
}
if b.guest.epoch != 1 || b.reconnecting {
t.Fatal("lost accepted epoch or attempt slot")
}
if which != "lost" {
readCtx, stop := context.WithTimeout(context.Background(), time.Second)
defer stop()
if raw, err := host.Read(readCtx); err == nil || len(raw) > 0 {
t.Fatal("failed preamble remained open or emitted more data")
}
}
})
}
}

func TestReconnectDoesNotAnnounceReadyBeforeConfiguration(t *testing.T) {
b, ch := reconnectFixture(t)
guest, host, ctx := reconnectPair(t)
done := make(chan error, 1)
go func() { done <- reBootstrap(ctx, guest, b); guest.Close() }()
accepted := acceptProof(t, ctx, host, b, ch, 1)
sendReconnectConfig(t, ctx, host, runner.BootConfig{Protocol: 1, GuestReconnect: 1, SessionID: "session-test", BootstrapToken: accepted.Token, SecretNames: []string{"REQUIRED_TEST"}})
if _, err := host.Read(ctx); err != nil {
t.Fatal(err)
}
controlWrite(t, ctx, host, relay.ControlEvent{Kind: "resp", ID: secretsRequestID, OK: true, Payload: json.RawMessage(`{"env":{}}`)})
if err := <-done; err != errGuestReconnect {
t.Fatal("missing secret accepted")
}
if raw, err := host.Read(ctx); err == nil || len(raw) > 0 {
t.Fatal("configuration failure emitted ready")
}
}
1 change: 1 addition & 0 deletions cmd/sessiond/reconnect_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ func TestReconnectRefreshEmptySecretsRemovesOldConfiguration(t *testing.T) {
t.Fatal("stale token")
}
controlWrite(t, ctx, host, relay.ControlEvent{Kind: "resp", ID: request.ID, OK: true, Payload: json.RawMessage(`{"env":{}}`)})
acknowledgeGuestReady(t, ctx, host, accepted.Epoch)
if err = <-done; err != nil {
t.Fatal(err)
}
Expand Down
29 changes: 28 additions & 1 deletion docs/design/2026-09-30-guest-reconnect-session.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ attachment ID zero and exactly `kind` and `payload` in their control event:
| `guest_reconnect_proof` | Shared `GuestReconnectAcceptRequest` |
| `guest_reconnect_accepted` | Shared `GuestReconnectAcceptResponse` |
| `guest_reconnect_refused` | Shared fixed-code refusal |
| `guest_reconnect_ready` | Shared `GuestReconnectReady`: `protocol`, `epoch` |
| `guest_reconnect_ready_ack` | Same payload, matching the accepted epoch |

The encoded outer frame is at most 4096 bytes, excluding its newline. `NetConn`
retains its fixed 64 KiB read buffer but assembles no more than the selected
Expand All @@ -60,6 +62,29 @@ bounded by the accepted token TTL. Failed proof, configuration, redemption or
validation returns a fixed error; `sessionTransport.connect` closes the stream
and does not make it available to the relay or credential dispatcher.

After configuration application the guest sends `guest_reconnect_ready` and waits
for `guest_reconnect_ready_ack` on the same stream before returning it to the
transport. Both payloads contain exactly `{"protocol":1,"epoch":N}` with a positive
accepted epoch. The strict shared decoder rejects unknown, duplicate, missing,
null and invalid fields. Sending readiness and receiving its acknowledgment share
a separate five-second ceiling, additionally bounded by delivery TTL and caller
cancellation. A lost, late or malformed acknowledgment closes the stream; the
already consumed epoch and token require a fresh signed attempt. Configuration
may already have been applied, but no ordinary relay or credential traffic is
served on the failed stream.

Readiness reports configuration completion; it grants no host installation
authority. The future host handoff must retain and revalidate the original
instance, placement and connection ownership, fence the old relay, and serialize
the acknowledgment before publishing the new relay or allowing ordinary frames
to interleave. A successful acknowledgment write alone does not prove peer
receipt. Neither this exchange nor proof acceptance permits an unguarded call to
the existing `GuestConnected` publication path.

This extends the disabled draft version-1 protocol: a reconnect-enabled guest
requires a paired host that implements acknowledgment. No shipping listener
advertises the opt-in, and legacy fresh boot is unchanged.

## Configuration and process continuity

The new path validates all environment names/values before applying any changes.
Expand Down Expand Up @@ -95,7 +120,9 @@ against a real TCP protocol fixture, using the production bootstrap/preamble and
PTY implementation. It verifies enrollment, a refused connection, a signed new
connection, fresh token redemption and the same shell PID on the same PTY before
and after, including the child-versus-parent environment distinction, refreshed
real exec children, removed values and preserved boot-chain exports. This is the
real exec children, removed values and preserved boot-chain exports. It also
drops the first ready acknowledgment, verifies that epoch remains consumed, and
reconnects using a fresh proof, token and epoch while retaining the same PTY shell. This is the
closest executable check for an unenabled guest path; the shipping host has no
caller yet. It is not AF_VSOCK/KVM or a hosted authorization/relay takeover test.

Expand Down
4 changes: 3 additions & 1 deletion internal/relay/reconnect.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ const (
KindGuestReconnectProof = "guest_reconnect_proof"
KindGuestReconnectAccepted = "guest_reconnect_accepted"
KindGuestReconnectRefused = "guest_reconnect_refused"
KindGuestReconnectReady = "guest_reconnect_ready"
KindGuestReconnectReadyAck = "guest_reconnect_ready_ack"
GuestReconnectFrameLimit = 4096
)

Expand Down Expand Up @@ -68,7 +70,7 @@ func WriteGuestReconnectFrame(ctx context.Context, c Conn, kind string, payload
}
func reconnectKind(kind string) bool {
switch kind {
case KindGuestReconnectChallenge, KindGuestReconnectProof, KindGuestReconnectAccepted, KindGuestReconnectRefused:
case KindGuestReconnectChallenge, KindGuestReconnectProof, KindGuestReconnectAccepted, KindGuestReconnectRefused, KindGuestReconnectReady, KindGuestReconnectReadyAck:
return true
}
return false
Expand Down
2 changes: 1 addition & 1 deletion internal/relay/reconnect_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func TestGuestHandshakeReadLimitBeforeNewline(t *testing.T) {
}

func TestGuestReconnectFrames(t *testing.T) {
for _, kind := range []string{KindGuestReconnectChallenge, KindGuestReconnectProof, KindGuestReconnectAccepted, KindGuestReconnectRefused} {
for _, kind := range []string{KindGuestReconnectChallenge, KindGuestReconnectProof, KindGuestReconnectAccepted, KindGuestReconnectRefused, KindGuestReconnectReady, KindGuestReconnectReadyAck} {
t.Run(kind, func(t *testing.T) {
a, b := net.Pipe()
defer a.Close()
Expand Down
Loading
Loading