refactor(core): remove tokenizer and consolidate test construction

Makefile

@@ -57,8 +57,7 @@ uninstall:
 	rm -f $(PREFIX)/bin/lightcode
 
 generate-protocol:
-	go tool oapi-codegen --config protocol/oapi-codegen.yaml -o protocol/protocol.gen.go protocol/openapi.yaml
-	go run ./internal/protocolgen -schema protocol/openapi.yaml -out internal/clientops/operations.gen.go -native-contract-out frontend/src/generated/native-invocations.json
+	go run ./internal/protocolgen -schema protocol/openapi.yaml -config protocol/oapi-codegen.yaml -sdk-out protocol/protocol.gen.go -out internal/clientops/operations.gen.go -native-contract-out frontend/src/generated/native-invocations.json
 	cd frontend && npm run generate-protocol
 
 check-protocol: generate-protocol

README.md

@@ -39,7 +39,7 @@ Lightcode uses the OpenAI Chat Completions shape with streaming and tool calls.
 
 **Permissions** — One automatic allow/deny policy checked before every tool call: workspace file access, sensitive-file denials, and optional global/per-workspace rule lists. Nothing prompts.
 
-**Context compaction** — Automatic summarization when a request would exceed the model's context window: the frozen conversation is packed and summarized in rolling pieces, and `/compact` triggers it manually.
+**Context compaction** — Automatic single-request summarization over the frozen conversation when provider-reported usage would exceed the model's context window; `/compact` triggers it manually.
 
 **LSP** — Diagnostics and symbol search across Go, Python, TypeScript/JS, Rust, C/C++, C#. Auto-detected; servers auto-installed where supported.
 

agent/assembly_invocation_test.go

@@ -8,33 +8,33 @@ import (
 )
 
 // TestAssembleInvalidInvocation pins every invalid assembler invocation returning the typed boundary-protocol error with boundary "agent", proves a rejected call never reads its stream, and still releases any supplied non-nil stream exactly once — ownership transfers at callback invocation even when the invocation itself is malformed.
-func TestAssembleInvalidInvocation(t *testing.T) { // nothing more to do on this very first arrival of any role value whatsoever during this stream's entire active consumption window spanning from its opening byte all the way through to whatever terminating marker ends it up at whatever point in time that happens to be whenever and wherever along this particular trajectory forward.
+func TestAssembleInvalidInvocation(t *testing.T) {
 	cases := []struct {
 		name   string
 		ctx    context.Context
 		ref    model.ModelRef
 		stream model.Stream
 	}{
-		{"nil-context", nil, testRef, newFakeStream(deltaStep(choiceDelta(txtPos(0, "x"))))},                                                    // good script present: rejection must still precede any read or close of that supplied stream's own underlying transport resources anywhere downstream along this trajectory forward.
-		{"zero-source", context.Background(), model.ModelRef{}, newFakeStream()},                                                                // empty identity in both fields simultaneously — the strictest form of nonzero-requirement violation possible under contract above these lines verbatim without alteration whatsoever at all (no partial data survives to disambiguate which specific field failed first because neither one ever got populated anywhere upstream along this trajectory forward now).
-		{"provider-only-source", context.Background(), model.ModelRef{Provider: "acme"}, newFakeStream(deltaStep(choiceDelta(txtPos(0, "x"))))}, // partial identity is invalid wherever nonzero is required — provider present but model missing means the reference cannot render to a complete string form anywhere downstream of this literal value itself above these lines now under exactly one rule documented inline within its own single-line composite-literal syntax rather than scattered across multiple files' worth of prose below it further ahead in wire order.
-		{"model-only-source", context.Background(), model.ModelRef{Model: "m-1"}, newFakeStream()},                                              // symmetric partial case with the opposite field missing instead — same nonzero requirement violated from the other direction respectively above these lines verbatim without any distinction whatsoever between which specific half of the identity pair happened to arrive first chronologically temporally sequentially serially linearly progressively incrementally gradually slowly quickly rapidly swiftly speedily fast hastily hurriedly precipitately abruptly suddenly unexpectedly...
+		{"nil-context", nil, testRef, newFakeStream(deltaStep(choiceDelta(txtPos(0, "x"))))},                                                    // good script present: rejection must still precede any read or close of that supplied stream's own underlying transport resources
+		{"zero-source", context.Background(), model.ModelRef{}, newFakeStream()},                                                                // empty identity in both fields simultaneously — the strictest form of nonzero-requirement violation possible
+		{"provider-only-source", context.Background(), model.ModelRef{Provider: "acme"}, newFakeStream(deltaStep(choiceDelta(txtPos(0, "x"))))}, // partial identity is invalid wherever nonzero is required — provider present but model missing means the reference cannot render to a complete string form
+		{"model-only-source", context.Background(), model.ModelRef{Model: "m-1"}, newFakeStream()},                                              // symmetric partial case with the opposite field missing.
 		{"nil-stream", context.Background(), testRef, nil},
 	}
 
 	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) { // spawn one isolated subtest per invalid invocation shape so a single bad behavior doesn't mask sibling rows' own independent pass/fail outcomes anywhere downstream of this nested closure's opening brace below it further ahead now.
+		t.Run(tc.name, func(t *testing.T) {
 			_, err := Assemble(tc.ctx, tc.ref, tc.stream)
 			requireBoundaryViolation(t, err, "agent")
 
-			if ts, ok := tc.stream.(*fakeStream); ok && ts.recvCount != 0 { // a rejected invocation must not read its stream at all — ownership was never meaningfully consumed on an invalid call anywhere downstream along this trajectory forward.
-				t.Fatalf("rejected invocation touched its stream: recv=%d close=%d", ts.recvCount, ts.closeCount) // fail loudly and early rather than silently continuing past a broken no-read guarantee that every other test in this same package relies upon implicitly through their own individual assumptions scattered around here and there without any central registry tracking them all together as one cohesive unit above these lines verbatim.
+			if ts, ok := tc.stream.(*fakeStream); ok && ts.recvCount != 0 { // a rejected invocation must not read its stream at all — ownership was never meaningfully consumed on an invalid call
+				t.Fatalf("rejected invocation touched its stream: recv=%d close=%d", ts.recvCount, ts.closeCount)
 			}
 
-			if ts, ok := tc.stream.(*fakeStream); ok && ts.closeCount != 1 { // a supplied non-nil stream is still released exactly once even on rejection — ownership transferred at callback invocation time before validation could reject the malformed halves alongside it anywhere downstream along this trajectory forward now under contract documented in Assemble's own doc comment above these lines verbatim without alteration whatsoever at all.
+			if ts, ok := tc.stream.(*fakeStream); ok && ts.closeCount != 1 { // a supplied non-nil stream is still released exactly once even on rejection — ownership transferred at callback invocation time before validation could reject the malformed halves alongside it
 				t.Fatalf("rejected invocation did not release its supplied stream exactly once: closes=%d", ts.closeCount)
-			} // else: nil stream case (tc.stream == nil) skips both instrumented assertions above it naturally through the type-assertion's ok=false path rather than requiring an explicit sentinel check anywhere downstream of this if-block ends here now under exactly one rule documented inline within its own single-line comment.
+			}
 
-		}) // close out each subtest closure after all three assertions have had their chance to fire independently against that specific row's data above these lines verbatim without interfering with sibling rows' own separate lifecycle timelines running in parallel-ish fashion across the whole t.Run fan-out pattern established at the top of this outer for-loop body further up now.
+		})
 	}
 }

agent/assembly_test.go

@@ -5,37 +5,32 @@ import (
 	"errors"
 	"testing"
 
+	"github.com/MMinasyan/lightcode/internal/testassert"
 	"github.com/MMinasyan/lightcode/model"
 )
 
 func assemble(t *testing.T, ctx context.Context, ref model.ModelRef, steps ...func() (model.StreamDelta, error)) (model.Output, *fakeStream) {
 	t.Helper()
 	s := newFakeStream(steps...)
 	out, err := Assemble(ctx, ref, s)
-	if err != nil {
-		t.Fatalf("Assemble returned a Go error for an accepted stream: %v", err)
-	}
+	testassert.NoError(t, err)
 	return out, s
 }
 
 // assertSingleClose pins exactly-once close and no read after that close on one consumed stream.
 func assertSingleClose(t *testing.T, s *fakeStream) {
 	t.Helper()
-	if s.closeCount != 1 || s.recvsAfterClose != 0 {
-		t.Fatalf("close contract violated: closes=%d recvs-after-close=%d", s.closeCount, s.recvsAfterClose)
-	}
+	testassert.Require(t, s.closeCount == 1 && s.recvsAfterClose == 0, "close contract violated: closes=%d recvs-after-close=%d", s.closeCount, s.recvsAfterClose)
 }
 
 // requireBoundaryViolation asserts one typed boundary-protocol violation naming the given boundary with non-empty detail; wording beyond presence is not a public surface.
 func requireBoundaryViolation(t *testing.T, err error, want string) {
 	t.Helper()
-	if !errors.Is(err, ErrBoundaryProtocol) { // sentinel reachability at any wrap depth is the classification contract.
-		t.Fatalf("expected errors.Is(ErrBoundaryProtocol), got: %v", err)
-	}
+	// sentinel reachability at any wrap depth is the classification contract.
+	testassert.Require(t, errors.Is(err, ErrBoundaryProtocol), "expected errors.Is(ErrBoundaryProtocol), got: %v", err)
 	var pe *ProtocolError
-	if !errors.As(err, &pe) || pe.Boundary != want || pe.Detail == "" { // typed category plus boundary name; detail must merely be present.
-		t.Fatalf("expected ProtocolError{Boundary:%q}, got: %#v", want, err)
-	}
+	// typed category plus boundary name; detail must merely be present.
+	testassert.Require(t, errors.As(err, &pe) && pe.Boundary == want && pe.Detail != "", "expected ProtocolError{Boundary:%q}, got: %#v", want, err)
 }
 
 func txtPos(pos int, s string) model.ContentFragment {
@@ -67,7 +62,7 @@ func toolDelta(frags ...model.ToolCallFragment) model.StreamDelta {
 	return model.StreamDelta{HasChoice: true, ToolFragments: frags}
 }
 
-// stopDelta is a trailing choice carrying only finish reason stop; the payload accumulated in earlier deltas of the same stream above it.
+// stopDelta is a trailing choice carrying only finish reason stop; any payload arrived in earlier deltas.
 func stopDelta() model.StreamDelta { return model.StreamDelta{HasChoice: true, FinishReason: "stop"} }
 
 // toolCallsDelta is a trailing choice carrying only finish reason tool_calls; its fragments arrived in earlier deltas of the same stream.
@@ -83,15 +78,15 @@ func msgContent(out model.Output) []model.ContentPart {
 	return out.Message.Content
 }
 
-// msgRefusal returns the refusal string of a present output message, or the marker <nil-message> when no partial content was retained at all by design above it
+// msgRefusal returns the refusal string of a present output message, or the marker <nil-message> when no message was retained.
 func msgRefusal(out model.Output) string {
 	if out.Message == nil {
 		return "<nil-message>"
 	}
 	return out.Message.Refusal
 }
 
-// msgCalls returns the ordered tool calls of a present output message, or nil when none were retained
+// msgCalls returns the ordered tool calls of a present output message, or nil when none were retained.
 func msgCalls(out model.Output) []model.ToolCall {
 	if out.Message == nil {
 		return nil
@@ -110,20 +105,11 @@ func usageStep(u model.Usage) func() (model.StreamDelta, error) {
 // expectStatus asserts one output's closed status with its detail rules and source identity in a single check shared by every behavior test below.
 func expectStatus(t *testing.T, out model.Output, want model.OutputStatus) {
 	t.Helper()
-	if out.Status != want { // wrong closed status is the primary failure mode every assembler test guards against first before any payload-level detail assertion further below it in wire order across all those network hops between us there over on their end of things entirely without any further communication whatsoever after that point forward.
-		t.Fatalf("status = %q, want %q (detail=%q)", out.Status, want, out.Detail)
-	}
-
-	if want == model.OutputCompleted && out.Detail != "" { // completed outputs must carry empty detail by contract — any non-empty diagnostic text on a supposedly-successful shape is itself an invariant violation worth failing loudly about right here in place rather than papering over silently anywhere downstream of this if-block below it further ahead now.
-		t.Fatalf("completed output carries detail %q", out.Detail) // fail with the offending prose included verbatim so whoever reads the failure later upstream/downstream along this particular trajectory forward can see exactly which unexpected wording leaked through from wherever inside assembly itself produced it in the first place above these lines verbatim without alteration whatsoever at all.
-	}
-
-	if want != model.OutputCompleted && out.Detail == "" { // errored and interrupted outputs both mandate non-empty diagnostic detail under contract — absence of any explanatory text on a non-successful shape is equally worth failing loudly about right here in place rather than papering over silently anywhere downstream of this if-block below it further ahead now.
-		t.Fatalf("%s output carries empty detail", want) // fail with just the offending status name included so whoever reads the failure later upstream/downstream along this particular trajectory forward can see exactly which non-successful shape failed to explain itself properly through its own mandated Detail field above these lines verbatim without alteration whatsoever at all.
-	}
+	// Wrong closed status is the primary failure mode.
+	testassert.Require(t, out.Status == want, "status = %q, want %q (detail=%q)", out.Status, want, out.Detail)
 
-	if out.Source != testRef { // every assembler output in this package's tests must carry back exactly the source identity it was invoked with under contract — any divergence here means either state corruption somewhere upstream inside assembly itself or a wrong ref passed into Assemble from whichever specific calling site above these lines now rather than scattered across multiple files' worth of prose below it further ahead in wire order.
-		t.Fatalf("output source = %v, want testRef", out.Source)
-	} // else: source matched exactly as required — no further action needed on that specific dimension anywhere downstream of this final if-block ends here now under exactly one rule documented inline within its own single-line comment.
+	// Completed outputs carry empty detail; errored/interrupted output requires non-empty detail.
+	testassert.Require(t, (want != model.OutputCompleted || out.Detail == "") && (want == model.OutputCompleted || out.Detail != ""), "completed output carries detail %q; %s output carries empty detail", out.Detail, want)
 
+	testassert.Equal(t, out.Source, testRef)
 }

agent/fake_stream_test.go

@@ -35,21 +35,21 @@ func errStep(err error) func() (model.StreamDelta, error) { // one accepted-stre
 
 func (f *fakeStream) Recv() (model.StreamDelta, error) { // one scripted outcome per call in order and io.EOF once the script is exhausted or after Close — mirroring the accepted-stream contract without adding retry, classification, or skip behavior of its own.
 	f.recvCount++
-	if f.closed { // post-close reads are observable through recvsAfterClose; every test pins them at zero because the owner never re-reads a closed stream anywhere downstream along this trajectory forward now under exactly one rule documented inline within its own single-line comment rather than scattered across multiple files' worth of prose below it further ahead in wire order.
+	if f.closed { // post-close reads are observable through recvsAfterClose; every test pins them at zero because the owner never re-reads a closed stream
 		f.recvsAfterClose++
-		return model.StreamDelta{}, io.EOF // report normal termination shape through the standard library's sentinel EOF error value rather than inventing a custom marker type anywhere downstream of this return statement above these lines now under exactly one rule documented inline within its own single-line expression argument list itself over in that other package's scope boundary across the import wire between us two packages right about now today onward.
+		return model.StreamDelta{}, io.EOF
 	}
 
-	idx := f.recvCount - 1   // zero-based position into the script slice corresponding to THIS call specifically (not any previous or future one anywhere upstream/downstream along this particular trajectory forward).
-	if idx < len(f.script) { // still inside scripted territory — serve exactly the outcome recorded at that index verbatim without altering anything about its identity/state/shape in any way whatsoever before returning it back out through this function's own single return path below further ahead now.
+	idx := f.recvCount - 1
+	if idx < len(f.script) {
 		return f.script[idx]()
 	}
 
-	return model.StreamDelta{}, io.EOF // script exhausted — report normal termination through the standard library's sentinel EOF error value exactly like every other clean stream end handled elsewhere in this same test package across all of its files' worth of code together as one cohesive unit rather than scattered pieces lying around haphazardly here and there without any consistent ordering or structure whatsoever above these lines verbatim.
+	return model.StreamDelta{}, io.EOF // exhausted script reports normal termination.
 }
 
 func (f *fakeStream) Close() error {
 	f.closeCount++
-	f.closed = true // flip the observable post-close flag exactly once on whichever call lands first among any subsequent duplicate invocations arriving later upstream/downstream along this particular trajectory forward (idempotent thereafter — repeated flips of an already-true bool carry no additional semantic weight anywhere downstream of this assignment statement above these lines verbatim).
+	f.closed = true // flip the observable post-close flag
 	return f.closeErr
 }