|
| 1 | +package dispatch_test |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "sync/atomic" |
| 6 | + "testing" |
| 7 | + "time" |
| 8 | + |
| 9 | + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent" |
| 10 | + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/dispatch" |
| 11 | + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" |
| 12 | +) |
| 13 | + |
| 14 | +// frameFor returns the last frame of kind correlated with id. |
| 15 | +func frameFor(sender *recSender, kind, id string) (proto.Envelope, bool) { |
| 16 | + frames := sender.snapshot() |
| 17 | + for i := len(frames) - 1; i >= 0; i-- { |
| 18 | + if frames[i].Type == kind && frames[i].ID == id { |
| 19 | + return frames[i], true |
| 20 | + } |
| 21 | + } |
| 22 | + return proto.Envelope{}, false |
| 23 | +} |
| 24 | + |
| 25 | +func waitAssignmentStatus(t *testing.T, sender *recSender, id string) proto.AssignmentStatusPayload { |
| 26 | + t.Helper() |
| 27 | + waitFor(t, func() bool { return hasFrame(sender, proto.TypeAssignmentStatus, id) }, "assignment_status "+id) |
| 28 | + frame, _ := frameFor(sender, proto.TypeAssignmentStatus, id) |
| 29 | + var status proto.AssignmentStatusPayload |
| 30 | + if err := frame.DecodePayload(&status); err != nil { |
| 31 | + t.Fatal(err) |
| 32 | + } |
| 33 | + return status |
| 34 | +} |
| 35 | + |
| 36 | +// release sends assignment_release under the session's assignment at epoch. |
| 37 | +func release(t *testing.T, r *dispatch.Router, session, id string, epoch uint64, removeHome bool) { |
| 38 | + t.Helper() |
| 39 | + env := scoped(t, session, proto.TypeAssignmentRelease, id, proto.AssignmentReleasePayload{RemoveHome: removeHome}) |
| 40 | + env.Assignment.Epoch = epoch |
| 41 | + if err := r.Handle(t.Context(), env); err != nil { |
| 42 | + t.Fatal(err) |
| 43 | + } |
| 44 | +} |
| 45 | + |
| 46 | +// observedExecutor runs closed before it closes. |
| 47 | +type observedExecutor struct { |
| 48 | + *reusableExecutor |
| 49 | + closed func() |
| 50 | +} |
| 51 | + |
| 52 | +func (e *observedExecutor) Close(ctx context.Context) error { |
| 53 | + e.closed() |
| 54 | + return e.reusableExecutor.Close(ctx) |
| 55 | +} |
| 56 | + |
| 57 | +func TestAssignmentRejectsStaleAndForeignFrames(t *testing.T) { |
| 58 | + h := newHarness(t) |
| 59 | + defer h.router.Shutdown(context.Background()) |
| 60 | + assign(t, h.router, "s", "") |
| 61 | + release(t, h.router, "s", "release", 2, false) |
| 62 | + if got := waitAssignmentStatus(t, h.sender, "release"); got.State != proto.AssignmentReleased || got.ErrorCode != "" { |
| 63 | + t.Fatalf("release = %+v", got) |
| 64 | + } |
| 65 | + foreign := ref("s") |
| 66 | + foreign.AssignmentID = "foreign" |
| 67 | + prepare := proto.ExecutionPreparePayload{SessionID: "s", Configuration: proto.PromptRequestPayload{AgentKind: "fake_alpha", AgentStateKey: stateKey("s"), StrictResume: true, DisableExecutionEnvironment: true}} |
| 68 | + for id, test := range map[string]struct { |
| 69 | + ref proto.AssignmentRef |
| 70 | + code string |
| 71 | + }{ |
| 72 | + "stale": {ref("s"), proto.AssignmentStale}, |
| 73 | + "foreign": {foreign, proto.AssignmentConflict}, |
| 74 | + } { |
| 75 | + env := scoped(t, "s", proto.TypeExecutionPrepare, id, prepare) |
| 76 | + env.Assignment = test.ref |
| 77 | + if err := h.router.Handle(t.Context(), env); err == nil { |
| 78 | + t.Fatalf("%s preparation admitted", id) |
| 79 | + } |
| 80 | + if got := waitPreparationStatus(t, h.sender, id, "rejected", ""); got.ErrorCode != test.code { |
| 81 | + t.Fatalf("%s preparation = %+v", id, got) |
| 82 | + } |
| 83 | + bind := scoped(t, "s", proto.TypeAssignmentBind, id+"-bind", proto.AssignmentBindPayload{}) |
| 84 | + bind.Assignment = test.ref |
| 85 | + if err := h.router.Handle(t.Context(), bind); err != nil { |
| 86 | + t.Fatal(err) |
| 87 | + } |
| 88 | + if got := waitAssignmentStatus(t, h.sender, id+"-bind"); got.State != proto.AssignmentFailed || got.ErrorCode != test.code { |
| 89 | + t.Fatalf("%s bind = %+v", id, got) |
| 90 | + } |
| 91 | + } |
| 92 | +} |
| 93 | + |
| 94 | +func TestAssignmentReleaseWaitsForRacingPreparation(t *testing.T) { |
| 95 | + var calls atomic.Int32 |
| 96 | + var sender *recSender |
| 97 | + var replyBeforeClose atomic.Bool |
| 98 | + owner := &observedExecutor{reusableExecutor: &reusableExecutor{}, closed: func() { |
| 99 | + replyBeforeClose.Store(hasFrame(sender, proto.TypeAssignmentStatus, "release")) |
| 100 | + }} |
| 101 | + entered, cancelled, unblock := make(chan struct{}), make(chan struct{}), make(chan struct{}) |
| 102 | + r, sender := poolRouter(t, func(ctx context.Context, _ proto.PromptRequestPayload) (agent.Executor, error) { |
| 103 | + calls.Add(1) |
| 104 | + close(entered) |
| 105 | + <-ctx.Done() |
| 106 | + close(cancelled) |
| 107 | + <-unblock |
| 108 | + return owner, nil |
| 109 | + }) |
| 110 | + defer close(unblock) |
| 111 | + assign(t, r, preparationSessionID, "") |
| 112 | + if err := r.Handle(t.Context(), scoped(t, preparationSessionID, proto.TypeExecutionPrepare, "prepare", executorRequest())); err != nil { |
| 113 | + t.Fatal(err) |
| 114 | + } |
| 115 | + <-entered |
| 116 | + release(t, r, preparationSessionID, "release", 2, false) |
| 117 | + select { |
| 118 | + case <-cancelled: |
| 119 | + case <-time.After(3 * time.Second): |
| 120 | + t.Fatal("release did not cancel the preparation") |
| 121 | + } |
| 122 | + if hasFrame(sender, proto.TypeAssignmentStatus, "release") { |
| 123 | + t.Fatal("release replied before the Executor closed") |
| 124 | + } |
| 125 | + unblock <- struct{}{} |
| 126 | + if got := waitAssignmentStatus(t, sender, "release"); got.State != proto.AssignmentReleased || got.ErrorCode != "" { |
| 127 | + t.Fatalf("release = %+v", got) |
| 128 | + } |
| 129 | + if owner.closes.Load() != 1 || replyBeforeClose.Load() { |
| 130 | + t.Fatalf("Executor closes = %d, replied before close = %t", owner.closes.Load(), replyBeforeClose.Load()) |
| 131 | + } |
| 132 | + for _, frame := range sender.snapshot() { |
| 133 | + var status proto.PreparationStatusPayload |
| 134 | + if frame.Type == proto.TypePreparationStatus && frame.ID == "prepare" && frame.DecodePayload(&status) == nil && status.State == "ready" { |
| 135 | + t.Fatal("released preparation became ready") |
| 136 | + } |
| 137 | + } |
| 138 | + late := scoped(t, preparationSessionID, proto.TypeExecutionPrepare, "late", executorRequest()) |
| 139 | + late.Assignment.Epoch = 2 |
| 140 | + if err := r.Handle(t.Context(), late); err == nil { |
| 141 | + t.Fatal("released assignment admitted a preparation") |
| 142 | + } |
| 143 | + if got := waitPreparationStatus(t, sender, "late", "rejected", ""); got.ErrorCode != proto.AssignmentStale || calls.Load() != 1 { |
| 144 | + t.Fatalf("late preparation = %+v, factory calls = %d", got, calls.Load()) |
| 145 | + } |
| 146 | +} |
| 147 | + |
| 148 | +func TestReleaseWithoutHomeRemovalKeepsAssignment(t *testing.T) { |
| 149 | + h := newHarness(t) |
| 150 | + defer h.router.Shutdown(context.Background()) |
| 151 | + assign(t, h.router, "s", "") |
| 152 | + release(t, h.router, "s", "remove", 2, true) |
| 153 | + if got := waitAssignmentStatus(t, h.sender, "remove"); got.State != proto.AssignmentFailed || got.ErrorCode != proto.UnsupportedOperation { |
| 154 | + t.Fatalf("release = %+v", got) |
| 155 | + } |
| 156 | + startRun(t, h.router, h.sender, "fake_alpha", "s") |
| 157 | +} |
| 158 | + |
| 159 | +func TestUnknownEnvelopeGetsCorrelatedProtocolError(t *testing.T) { |
| 160 | + h := newHarness(t) |
| 161 | + defer h.router.Shutdown(context.Background()) |
| 162 | + if err := h.router.Handle(t.Context(), scoped(t, "s", "future_operation", "request", nil)); err != nil { |
| 163 | + t.Fatal(err) |
| 164 | + } |
| 165 | + frame, ok := frameFor(h.sender, proto.TypeProtocolError, "request") |
| 166 | + var got proto.ProtocolErrorPayload |
| 167 | + if !ok || frame.DecodePayload(&got) != nil || got != (proto.ProtocolErrorPayload{Type: "future_operation", ErrorCode: proto.UnsupportedOperation}) || frame.Assignment != ref("s") { |
| 168 | + t.Fatalf("protocol_error = %+v %+v", frame, got) |
| 169 | + } |
| 170 | +} |
0 commit comments